├── .dockerignore ├── .github └── workflows │ └── main.yml ├── .gitignore ├── .rspec ├── Dockerfile ├── Gemfile ├── README.md ├── Rakefile ├── bin ├── console └── setup ├── exe └── wrapbox ├── lib ├── wrapbox.rb └── wrapbox │ ├── cli.rb │ ├── config_repository.rb │ ├── configuration.rb │ ├── job.rb │ ├── log_fetcher.rb │ ├── log_fetcher │ ├── awslogs.rb │ └── papertrail.rb │ ├── runner │ ├── docker.rb │ ├── ecs.rb │ └── ecs │ │ ├── instance_manager.rb │ │ └── task_waiter.rb │ ├── tasks │ └── run.rake │ └── version.rb ├── spec ├── config.yml ├── spec_helper.rb ├── test_job.rb ├── wrapbox │ └── runner │ │ └── ecs │ │ └── task_waiter_spec.rb └── wrapbox_spec.rb └── wrapbox.gemspec /.dockerignore: -------------------------------------------------------------------------------- 1 | .bundle 2 | .yardoc 3 | Gemfile.lock 4 | _yardoc 5 | coverage 6 | doc 7 | pkg 8 | spec/reports 9 | tmp 10 | 11 | .envrc 12 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: Testing on Ubuntu 2 | on: 3 | - push 4 | - pull_request 5 | jobs: 6 | build: 7 | runs-on: ${{ matrix.os }} 8 | timeout-minutes: 10 9 | strategy: 10 | fail-fast: false 11 | matrix: 12 | ruby: 13 | - 2.7 14 | - 3.0 15 | - 3.1 16 | - 3.2 17 | os: 18 | - ubuntu-latest 19 | name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }} 20 | steps: 21 | - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 22 | - uses: ruby/setup-ruby@1a615958ad9d422dd932dc1d5823942ee002799f # v1.227.0 23 | with: 24 | ruby-version: ${{ matrix.ruby }} 25 | - name: unit testing 26 | env: 27 | CI: true 28 | run: | 29 | bundle install --jobs 4 --retry 3 30 | bundle exec rake spec 31 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle/ 2 | /.yardoc 3 | /Gemfile.lock 4 | /_yardoc/ 5 | /coverage/ 6 | /doc/ 7 | /pkg/ 8 | /spec/reports/ 9 | /tmp/ 10 | 11 | .envrc 12 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --format documentation 2 | --color 3 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ruby:alpine 2 | 3 | RUN apk add --no-cache git gcc make g++ zlib-dev 4 | WORKDIR /app 5 | COPY . /app/ 6 | 7 | RUN bundle install -j3 8 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gemspec 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Wrapbox 2 | 3 | Wrapbox runs Ruby method or shell command in a container (ECS, docker). 4 | 5 | ## Installation 6 | 7 | Add this line to your application's Gemfile: 8 | 9 | ```ruby 10 | gem 'wrapbox' 11 | ``` 12 | 13 | And then execute: 14 | 15 | $ bundle 16 | 17 | Or install it yourself as: 18 | 19 | $ gem install wrapbox 20 | 21 | ## Usage 22 | 23 | Write config.yml 24 | 25 | ```yaml 26 | default: 27 | cluster: wrapbox 28 | runner: ecs 29 | region: ap-northeast-1 30 | container_definitions: 31 | - image: joker1007/wrapbox 32 | cpu: 512 33 | memory: 1024 34 | essential: true 35 | 36 | docker: 37 | runner: docker 38 | keep_container: true 39 | container_definitions: 40 | - image: joker1007/wrapbox 41 | cpu: 512 42 | memory: 1024 43 | 44 | ecs2: 45 | cluster: wrapbox 46 | runner: ecs 47 | region: ap-northeast-1 48 | # Use already existing task definition 49 | task_definition: 50 | task_definition_name: foo_task:1 51 | main_container_name: container_name_where_command_is_executed 52 | ``` 53 | 54 | #### run by CLI 55 | 56 | ```sh 57 | $ wrapbox ecs run_cmd -f config.yml \ 58 | -e "FOO=bar,HOGE=fuga" \ 59 | "bundle exec rspec spec/models" \ 60 | "bundle exec rspec spec/controllers" \ 61 | ``` 62 | 63 | #### run by ruby 64 | 65 | Run `rake wrapbox:run` with `CLASS_NAME_ENV` and `METHOD_NAME_ENV` and `METHOD_ARGS_ENV` 66 | 67 | ```ruby 68 | Wrapbox.configure do |c| 69 | c.load_yaml(File.expand_path("../config.yml", __FILE__)) 70 | end 71 | 72 | # runs TestJob#perform("arg1", ["arg2", "arg3"]) in ECS container via `rake wrapbox:run` 73 | Wrapbox.run("TestJob", :perform, ["arg1", ["arg2", "arg3"]], environments: [{name: "RAILS_ENV", value: "development"}]) # use default config 74 | # runs TestJob#perform in local docker container (Use docker cli) 75 | Wrapbox.run("TestJob", :perform, ["arg1", ["arg2", "arg3"]], config_name: :docker, environments: [{name: "RAILS_ENV", value: "development"}]) # use docker config 76 | 77 | # runs ls . command in ECS container 78 | Wrapbox.run_cmd(["ls ."], environments: [{name: "RAILS_ENV", value: "development"}]) 79 | ``` 80 | 81 | If ECS runner cannot create task, it puts custom metric data to CloudWatch. 82 | Custom metric data is `wrapbox/WaitingTaskCount` that has `ClusterName` dimension. 83 | And, it retry launching until retry count reach `launch_retry`. 84 | 85 | After task exited, Wrapbox checks main container exit code. 86 | If exit code is not 0, Wrapbox raise error. 87 | 88 | ## Config 89 | 90 | ### Common 91 | 92 | | name | desc | 93 | | ------ | ----------------- | 94 | | runner | "ecs" or "docker" | 95 | 96 | ### for ECS 97 | 98 | | name | desc | 99 | | -------------------------- | ------------------------------------------------ | 100 | | cluster | target ECS cluster name | 101 | | region | region of ECS cluster | 102 | | container_definitions | see http://docs.aws.amazon.com/sdkforruby/api/Aws/ECS/Client.html#register_task_definition-instance_method | 103 | | task_role_arn | see http://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html | 104 | | volumes | see http://docs.aws.amazon.com/sdkforruby/api/Aws/ECS/Client.html#register_task_definition-instance_method | 105 | | placement_constraints | see http://docs.aws.amazon.com/sdkforruby/api/Aws/ECS/Client.html#register_task_definition-instance_method | 106 | | placement_strategy | see http://docs.aws.amazon.com/sdkforruby/api/Aws/ECS/Client.html#register_task_definition-instance_method | 107 | | launch_type | see http://docs.aws.amazon.com/sdkforruby/api/Aws/ECS/Client.html#run_task-instance_method | 108 | | network_mode | see http://docs.aws.amazon.com/sdkforruby/api/Aws/ECS/Client.html#register_task_definition-instance_method | 109 | | network_configuration | see http://docs.aws.amazon.com/sdkforruby/api/Aws/ECS/Client.html#run_task-instance_method | 110 | | capacity_provider_strategy | see https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/ECS/Client.html#run_task-instance_method | 111 | | cpu | see http://docs.aws.amazon.com/sdkforruby/api/Aws/ECS/Client.html#register_task_definition-instance_method | 112 | | memory | see http://docs.aws.amazon.com/sdkforruby/api/Aws/ECS/Client.html#register_task_definition-instance_method | 113 | | enable_ecs_managed_tags | see https://docs.aws.amazon.com/sdkforruby/api/Aws/ECS/Client.html#run_task-instance_method | 114 | | tags | tags of task definitions. see also https://docs.aws.amazon.com/sdkforruby/api/Aws/ECS/Client.html#register_task_definition-instance_method | 115 | | propagate_tags | specify `"TASK_DEFINITION"` if you want to propagate tags to tasks. see also https://docs.aws.amazon.com/sdkforruby/api/Aws/ECS/Client.html#run_task-instance_method | 116 | | launch_instances | specify `launch_template` (required), `instance_type`, and `tag_specifications` for [Aws::EC2::Client#run_instances](https://docs.aws.amazon.com/sdkforruby/api/Aws/EC2/Client.html#run_instances-instance_method). You can also specify `wait_until_instance_terminated` (default: true) | 117 | 118 | `WRAPBOX_CMD_INDEX` environment variable is available in `run_cmd` and you can distinguish logs from each command like below: 119 | 120 | ``` 121 | log_configuration: 122 | log_driver: syslog 123 | options: 124 | syslog-address: "tcp://192.168.0.42:123" 125 | env: WRAPBOX_CMD_INDEX 126 | tag: wrapbox-{{ printf "%03s" (.ExtraAttributes nil).WRAPBOX_CMD_INDEX }} 127 | ``` 128 | 129 | ### for docker 130 | | name | desc | 131 | | -------------------- | ----------------------------------------------------------- | 132 | | container_definitions | only use `image`, `cpu`, `memory`, and `memory_reservation` | 133 | | keep_container | If true, doesn't delete the container when the command ends | 134 | 135 | ## API 136 | 137 | ### `Wrapbox.run` 138 | 139 | ```ruby 140 | Wrapbox.run(class_name, method_name, args, 141 | runner: nil, # The "runner" value is used in the configuration if it is nil. 142 | config_name: nil, # "default" configuration is used if it is nil. 143 | cluster: nil, # Available only for ECS runner. The "cluster" value in the configuration is used if it is nil. 144 | launch_type: "EC2", # Available only for ECS runner. The "launch_type" value in the configuration is used if it is nil. 145 | task_role_arn: nil, # Available only for ECS runner. The "task_role_arn" value in the configuration is used if it is nil. 146 | execution_role_arn: nil, # Available only for ECS runner. The "execution_role_arn" value in the configuration is used if it is nil. 147 | tags: nil, # Available only for ECS runner. The "tags" value in the configuration is used if it is nil. 148 | propagate_tags: nil, # Available only for ECS runner. The "propagate_tags" value in the configuration is used if it is nil. 149 | container_definition_overrides: {}, 150 | environments: [], 151 | timeout: 3600 * 24, # Available only for ECS runner. # Available only for ECS runner. 152 | launch_timeout: 60 * 10, # Available only for ECS runner. 153 | launch_retry: 10, # Available only for ECS runner. 154 | retry_interval: 1, # Available only for ECS runner. 155 | retry_interval_multiplier: 2, # Available only for ECS runner. 156 | max_retry_interval: 120, # Available only for ECS runner. 157 | execution_retry: 0, # Available only for ECS runner. 158 | keep_container: nil, # Available only for Docker runner. The "keep_container" value in the configuration is used if it is nil. 159 | ) 160 | ``` 161 | 162 | ### `Wrapbox.run_cmd` 163 | 164 | ```ruby 165 | Wrapbox.run_cmd(*cmd, 166 | runner: nil, # The "runner" value is used in the configuration if it is nil. 167 | config_name: nil, # "default" configuration is used if it is nil. 168 | cluster: nil, # Available only for ECS runner. The "cluster" value in the configuration is used if it is nil. 169 | launch_type: "EC2", # Available only for ECS runner. The "launch_type" value in the configuration is used if it is nil. 170 | task_role_arn: nil, # Available only for ECS runner. The "task_role_arn" value in the configuration is used if it is nil. 171 | execution_role_arn: nil, # Available only for ECS runner. The "execution_role_arn" value in the configuration is used if it is nil. 172 | tags: nil, # Available only for ECS runner. The "tags" value in the configuration is used if it is nil. 173 | propagate_tags: nil, # Available only for ECS runner. The "propagate_tags" value in the configuration is used if it is nil. 174 | container_definition_overrides: {}, 175 | ignore_signal: false, 176 | environments: [], 177 | timeout: 3600 * 24, # Available only for ECS runner. # Available only for ECS runner. 178 | launch_timeout: 60 * 10, # Available only for ECS runner. 179 | launch_retry: 10, # Available only for ECS runner. 180 | retry_interval: 1, # Available only for ECS runner. 181 | retry_interval_multiplier: 2, # Available only for ECS runner. 182 | max_retry_interval: 120, # Available only for ECS runner. 183 | execution_retry: 0, # Available only for ECS runner. 184 | keep_container: nil, # Available only for Docker runner. The "keep_container" value in the configuration is used if it is nil. 185 | ) 186 | ``` 187 | 188 | ## Development 189 | 190 | After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment. 191 | 192 | To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org). 193 | 194 | ### How to test 195 | 196 | The following environment variables are required to run all tests. 197 | 198 | Name | Description 199 | -----|---------------- 200 | RUN_AWS_SPECS | Set "true" to run tests with `aws` set to true. You should also set credentials for AWS account to run ECS tasks. 201 | ECS_CLUSTER | A cluster used in tests. "default" cluster is used if this variable is not set. 202 | OVERRIDDEN_ECS_CLUSTER | A cluster used in tests that ensure `cluster` parameter. 203 | LAUNCH_TEMPLATE_ID | A launch template used in tests that ensure `launch_instances` configuration. 204 | 205 | 206 | ``` 207 | env \ 208 | RUN_AWS_SPECS=true \ 209 | ECS_CLUSTER='some_cluster' \ 210 | OVERRIDDEN_ECS_CLUSTER='another_cluster' \ 211 | LAUNCH_TEMPLATE_ID=lt-xxxxxxxxxxxxxxxxx \ 212 | bundle exec rspec 213 | ``` 214 | 215 | ## Contributing 216 | 217 | Bug reports and pull requests are welcome on GitHub at https://github.com/reproio/wrapbox. 218 | 219 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | require "rspec/core/rake_task" 3 | 4 | load "wrapbox/tasks/run.rake" 5 | 6 | RSpec::Core::RakeTask.new(:spec) 7 | 8 | task :default => :spec 9 | 10 | require_relative "./spec/test_job" 11 | -------------------------------------------------------------------------------- /bin/console: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require "bundler/setup" 4 | require "wrapbox" 5 | 6 | # You can add fixtures and/or initialization code here to make experimenting 7 | # with your gem easier. You can also use a different console, if you like. 8 | 9 | # (If you use this, don't forget to add pry to your Gemfile!) 10 | # require "pry" 11 | # Pry.start 12 | 13 | require "irb" 14 | IRB.start 15 | -------------------------------------------------------------------------------- /bin/setup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | IFS=$'\n\t' 4 | set -vx 5 | 6 | bundle install 7 | 8 | # Do any other automated setup that you need to do here 9 | -------------------------------------------------------------------------------- /exe/wrapbox: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require "wrapbox/cli" 4 | 5 | Wrapbox::Cli.start 6 | -------------------------------------------------------------------------------- /lib/wrapbox.rb: -------------------------------------------------------------------------------- 1 | require "logger" 2 | 3 | module Wrapbox 4 | CLASS_NAME_ENV = "WRAPBOX_CLASS_NAME".freeze 5 | METHOD_NAME_ENV = "WRAPBOX_METHOD_NAME".freeze 6 | METHOD_ARGS_ENV = "WRAPBOX_METHOD_ARGS".freeze 7 | 8 | class << self 9 | attr_accessor :logger 10 | 11 | def load_config(filename) 12 | configs.load_yaml(filename) 13 | end 14 | 15 | def configs 16 | @configs ||= ConfigRepository.new 17 | end 18 | 19 | def configure 20 | yield configs 21 | end 22 | 23 | def run(*args, config_name: nil, **options) 24 | get_config(config_name).run(*args, **options) 25 | end 26 | 27 | def run_cmd(*args, config_name: nil, **options) 28 | get_config(config_name).run_cmd(*args, **options) 29 | end 30 | 31 | private 32 | 33 | def get_config(config_name) 34 | @configs.get(config_name) or 35 | raise RuntimeError, %Q{The configuration "#{config_name}" is not registered} 36 | end 37 | end 38 | 39 | $stdout.sync = true 40 | self.logger = Logger.new($stdout) 41 | self.logger.level = :info 42 | end 43 | 44 | require "wrapbox/version" 45 | 46 | require "wrapbox/config_repository" 47 | require "wrapbox/configuration" 48 | require "wrapbox/job" 49 | -------------------------------------------------------------------------------- /lib/wrapbox/cli.rb: -------------------------------------------------------------------------------- 1 | require "thor/group" 2 | require "wrapbox" 3 | require "wrapbox/runner/docker" 4 | require "wrapbox/runner/ecs" 5 | 6 | module Wrapbox 7 | class Cli < Thor 8 | register(Wrapbox::Runner::Ecs::Cli, "ecs", "ecs [COMMAND]", "Commands for ECS") 9 | register(Wrapbox::Runner::Docker::Cli, "docker", "docker [COMMAND]", "Commands for Docker") 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /lib/wrapbox/config_repository.rb: -------------------------------------------------------------------------------- 1 | require 'yaml' 2 | require 'wrapbox/configuration' 3 | 4 | module Wrapbox 5 | class ConfigRepository 6 | def initialize 7 | @configs = {} 8 | end 9 | 10 | def load_yaml(yaml_file) 11 | file = ERB.new(File.read(yaml_file)).result 12 | configs = if Gem::Version.new(Psych::VERSION) >= Gem::Version.new("4.0.0") 13 | YAML.load(file, aliases: true) 14 | else 15 | YAML.load(file) 16 | end 17 | configs.each do |name, configuration| 18 | load_config(name, configuration.merge("name" => name)) 19 | end 20 | end 21 | 22 | def load_config(name, configuration) 23 | @configs[name.to_sym] = Configuration.load_config(configuration) 24 | end 25 | 26 | def default 27 | @configs[:default] 28 | end 29 | 30 | def get(name) 31 | name ? @configs[name.to_sym] : default 32 | end 33 | alias_method(:[], :get) 34 | end 35 | end 36 | -------------------------------------------------------------------------------- /lib/wrapbox/configuration.rb: -------------------------------------------------------------------------------- 1 | require "active_support" 2 | require "active_support/core_ext/hash" 3 | require "active_support/core_ext/string" 4 | 5 | module Wrapbox 6 | Configuration = Struct.new( 7 | :name, 8 | :task_definition_name, 9 | :revision, 10 | :runner, 11 | :cluster, 12 | :region, 13 | :retry, 14 | :retry_interval, 15 | :retry_interval_multiplier, 16 | :container_definition, 17 | :container_definitions, 18 | :volumes, 19 | :placement_constraints, 20 | :placement_strategy, 21 | :capacity_provider_strategy, 22 | :launch_type, 23 | :requires_compatibilities, 24 | :task_definition, 25 | :additional_container_definitions, 26 | :network_mode, 27 | :network_configuration, 28 | :cpu, 29 | :memory, 30 | :task_role_arn, 31 | :execution_role_arn, 32 | :keep_container, 33 | :log_fetcher, 34 | :tags, 35 | :enable_ecs_managed_tags, 36 | :propagate_tags, 37 | :launch_instances, 38 | :enable_execute_command, 39 | ) do 40 | def self.load_config(config) 41 | new( 42 | config["name"], 43 | config["task_definition_name"], 44 | config["revision"], 45 | config["runner"] ? config["runner"].to_sym : :docker, 46 | config["cluster"], 47 | config["region"], 48 | config["retry"] || 0, 49 | config["retry_interval"] || 1, 50 | config["retry_interval_multiplier"] || 2, 51 | config["container_definition"]&.deep_symbolize_keys, 52 | config["container_definitions"]&.map(&:deep_symbolize_keys) || [], 53 | config["volumes"]&.map(&:deep_symbolize_keys) || [], 54 | config["placement_constraints"]&.map(&:deep_symbolize_keys) || [], 55 | config["placement_strategy"]&.map(&:deep_symbolize_keys) || [], 56 | config["capacity_provider_strategy"]&.map(&:deep_symbolize_keys) || [], 57 | config["launch_type"], 58 | config["requires_compatibilities"] || ["EC2"], 59 | config["task_definition"]&.deep_symbolize_keys, 60 | config["additional_container_definitions"]&.map(&:deep_symbolize_keys) || [], 61 | config["network_mode"], 62 | config["network_configuration"]&.deep_symbolize_keys, 63 | config["cpu"]&.to_s, 64 | config["memory"]&.to_s, 65 | config["task_role_arn"], 66 | config["execution_role_arn"], 67 | config["keep_container"], 68 | config["log_fetcher"]&.deep_symbolize_keys, 69 | config["tags"], 70 | config["enable_ecs_managed_tags"], 71 | config["propagate_tags"], 72 | config["launch_instances"]&.deep_symbolize_keys, 73 | config["enable_execute_command"] 74 | ) 75 | end 76 | 77 | AVAILABLE_RUNNERS = %i(docker ecs) 78 | 79 | def initialize(*args) 80 | super 81 | end 82 | 83 | def runner_class(overrided_runner = nil) 84 | r = overrided_runner || runner 85 | raise "#{r} is unsupported runner" unless AVAILABLE_RUNNERS.include?(r.to_sym) 86 | require "wrapbox/runner/#{r}" 87 | Wrapbox::Runner.const_get(r.to_s.camelcase) 88 | end 89 | 90 | def run(class_name, method_name, args, runner: nil, **options) 91 | klass = runner_class(runner) 92 | overridable_options, parameters = klass.split_overridable_options_and_parameters(options) 93 | klass.new(to_h.merge(overridable_options)).run(class_name, method_name, args, **parameters) 94 | end 95 | 96 | def run_cmd(*cmd, runner: nil, **options) 97 | klass = runner_class(runner) 98 | overridable_options, parameters = klass.split_overridable_options_and_parameters(options) 99 | klass.new(to_h.merge(overridable_options)).run_cmd(*cmd, **parameters) 100 | end 101 | end 102 | end 103 | -------------------------------------------------------------------------------- /lib/wrapbox/job.rb: -------------------------------------------------------------------------------- 1 | require "multi_json" 2 | 3 | module Wrapbox 4 | module Job 5 | def self.perform 6 | klass = ENV[CLASS_NAME_ENV].constantize 7 | method_name = ENV[METHOD_NAME_ENV].to_sym 8 | args = MultiJson.load(ENV[METHOD_ARGS_ENV]) 9 | 10 | klass.new.send(method_name, *args) 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /lib/wrapbox/log_fetcher.rb: -------------------------------------------------------------------------------- 1 | require "active_support/core_ext/string" 2 | 3 | # LogFetcher Implementation requires two methods. 4 | # - run (start log fetching asynchronously) 5 | # - stop (stop log fetching) 6 | module Wrapbox 7 | module LogFetcher 8 | def self.new(type, **options) 9 | raise "log_fetcher config needs `type`" unless type 10 | require "wrapbox/log_fetcher/#{type}" 11 | options = options.reject { |k, v| k == :type } 12 | self.const_get(type.camelize).new(**options) 13 | end 14 | 15 | def run(task:) 16 | raise NotImplementedError 17 | end 18 | 19 | def stop 20 | raise NotImplementedError 21 | end 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /lib/wrapbox/log_fetcher/awslogs.rb: -------------------------------------------------------------------------------- 1 | module Wrapbox 2 | module LogFetcher 3 | class Awslogs 4 | STOP_WAIT_TIMELIMIT = 10 5 | 6 | def initialize(log_group:, log_stream_prefix:, filter_pattern: nil, region: nil, access_key_id: nil, secret_access_key: nil, timestamp_format: "%Y-%m-%d %H:%M:%S.%3N", delay: 2, **options) 7 | begin 8 | require 'aws-sdk-cloudwatchlogs' 9 | rescue LoadError 10 | $stderr.puts "Require aws-sdk-cloudwatchlogs gem" 11 | exit 1 12 | end 13 | 14 | @log_group = log_group 15 | @log_stream_prefix = log_stream_prefix 16 | @filter_pattern = filter_pattern 17 | @region = region 18 | @access_key_id = access_key_id 19 | @secret_access_key = secret_access_key 20 | @timestamp_format = timestamp_format 21 | @delay = delay 22 | @options = options.reject { |_, v| v.nil? } 23 | @displayed_log_stream_names = {} 24 | @displayed_log_stream_number = 0 25 | @displayed_event_ids = {} 26 | end 27 | 28 | def run(task:) 29 | @loop_thread = Thread.start do 30 | # It smees that task.contaienrs is empty 31 | # if capacity_provider_strategy is specified and there are no remaining capacity 32 | while task.containers.empty? 33 | Wrapbox.logger.warn("The task has no containers, so fetch it again") 34 | sleep 10 35 | task = ecs_client.describe_tasks(cluster: task.cluster_arn, tasks: [task.task_arn]).tasks.first 36 | end 37 | 38 | main_loop(task) 39 | end 40 | end 41 | 42 | def stop 43 | @stop = true 44 | @loop_thread&.join(STOP_WAIT_TIMELIMIT) 45 | end 46 | 47 | def main_loop(task) 48 | task_id = task.task_arn.split("/").last 49 | log_stream_names = task.containers.map do |container| 50 | [@log_stream_prefix, container.name, task_id].join("/") 51 | end 52 | filter_log_opts = { 53 | log_group_name: @log_group, 54 | log_stream_names: log_stream_names, 55 | filter_pattern: @filter_pattern, 56 | }.compact 57 | @max_timestamp = ((Time.now.to_f - 120) * 1000).round 58 | 59 | until @stop do 60 | filter_log_opts[:start_time] = @max_timestamp + 1 61 | begin 62 | client.filter_log_events(filter_log_opts).each do |r| 63 | r.events.each do |ev| 64 | next if @displayed_event_ids.member?(ev.event_id) 65 | display_message(ev) 66 | @displayed_event_ids[ev.event_id] = ev.timestamp 67 | @max_timestamp = ev.timestamp if @max_timestamp < ev.timestamp 68 | end 69 | end 70 | 71 | @displayed_event_ids.each do |event_id, ts| 72 | if ts < (Time.now.to_f - 600) * 1000 73 | @displayed_event_ids.delete(event_id) 74 | end 75 | end 76 | rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException 77 | # Ignore the error because it is an error like "The specified log stream does not exist.", 78 | # which occurs when the log stream hasn't been created yet, that is, the task hasn't started yet. 79 | rescue Aws::CloudWatchLogs::Errors::ThrottlingException 80 | Wrapbox.logger.warn("Failed to fetch logs due to Aws::CloudWatchLogs::Errors::ThrottlingException") 81 | end 82 | 83 | sleep @delay 84 | end 85 | end 86 | 87 | COLOR_ESCAPE_SEQUENCES = [33, 31, 32, 34, 35, 36] 88 | def display_message(ev, output: $stdout) 89 | num = @displayed_log_stream_names.fetch(ev.log_stream_name) do |key| 90 | current = @displayed_log_stream_number 91 | @displayed_log_stream_names[key] = current 92 | @displayed_log_stream_number += 1 93 | current 94 | end 95 | 96 | sequence_number = COLOR_ESCAPE_SEQUENCES[num % COLOR_ESCAPE_SEQUENCES.length] 97 | 98 | time = Time.at(ev.timestamp / 1000.0) 99 | output.puts("\e[#{sequence_number}m#{time.strftime(@timestamp_format)} #{ev.log_stream_name}\e[0m #{ev.message}") 100 | end 101 | 102 | private 103 | 104 | def client 105 | return @client if @client 106 | 107 | options = { 108 | region: @region, 109 | access_key_id: @access_key_id, 110 | secret_access_key: @secret_access_key, 111 | }.compact 112 | @client = Aws::CloudWatchLogs::Client.new(**options) 113 | end 114 | end 115 | end 116 | end 117 | -------------------------------------------------------------------------------- /lib/wrapbox/log_fetcher/papertrail.rb: -------------------------------------------------------------------------------- 1 | module Wrapbox 2 | module LogFetcher 3 | class Papertrail 4 | STOP_WAIT_TIMELIMIT = 10 5 | 6 | def initialize(query: nil, delay: 2, **options) 7 | begin 8 | require 'papertrail/cli' 9 | rescue LoadError 10 | $stderr.puts "Require papertrail gem" 11 | exit 1 12 | end 13 | 14 | # see. https://github.com/papertrail/papertrail-cli/blob/master/lib/papertrail/cli.rb 15 | @query = query 16 | @delay = delay 17 | @options = options.reject { |_, v| v.nil? } 18 | end 19 | 20 | def run(task:) 21 | @started_at = Time.now 22 | @loop_thread = Thread.start(&method(:main_loop)) 23 | end 24 | 25 | def stop 26 | @stop = true 27 | @loop_thread&.join(STOP_WAIT_TIMELIMIT) 28 | end 29 | 30 | def main_loop 31 | papertrail = ::Papertrail::Cli.new 32 | connection_options = papertrail.options.merge(@options).merge(follow: true) 33 | connection = ::Papertrail::Connection.new(connection_options) 34 | 35 | query_options = {} 36 | 37 | if @options[:system] 38 | query_options[:system_id] = connection.find_id_for_source(@options[:system]) 39 | unless query_options[:system_id] 40 | $stderr.puts "System \"#{@options[:system]}\" not found" 41 | end 42 | end 43 | 44 | if @options[:group] 45 | query_options[:group_id] = connection.find_id_for_group(@options[:group]) 46 | unless query_options[:group_id] 47 | $stderr.puts "Group \"#{@options[:group]}\" not found" 48 | end 49 | end 50 | 51 | if @options[:search] 52 | search = connection.find_search(@options[:search], @query_options[:group_id]) 53 | unless search 54 | $stderr.puts "Search \"#{@options[:search]}\" not found" 55 | end 56 | 57 | query_options[:group_id] ||= search['group_id'] 58 | @query = search['query'] 59 | end 60 | 61 | @query ||= '' 62 | 63 | search_query = ::Papertrail::SearchQuery.new(connection, @query, query_options) 64 | 65 | until @stop do 66 | search_query.next_results_page.events.each do |event| 67 | next if event.received_at < @started_at 68 | papertrail.display_result(event) 69 | end 70 | sleep @delay 71 | end 72 | end 73 | end 74 | end 75 | end 76 | -------------------------------------------------------------------------------- /lib/wrapbox/runner/docker.rb: -------------------------------------------------------------------------------- 1 | require "open3" 2 | require "multi_json" 3 | require "docker" 4 | require "thor" 5 | require "shellwords" 6 | 7 | require "wrapbox" 8 | 9 | module Wrapbox 10 | module Runner 11 | class Docker 12 | class ExecutionError < StandardError; end 13 | 14 | attr_reader \ 15 | :name, 16 | :container_definition, 17 | :keep_container 18 | 19 | def self.split_overridable_options_and_parameters(options) 20 | opts = options.dup 21 | overridable_options = {} 22 | %i[keep_container].each do |key| 23 | value = opts.delete(key) 24 | overridable_options[key] = value if value 25 | end 26 | 27 | [overridable_options, opts] 28 | end 29 | 30 | def initialize(options) 31 | @name = options[:name] 32 | @container_definitions = options[:container_definition] ? [options[:container_definition]] : options[:container_definitions] 33 | @logger = Wrapbox.logger 34 | 35 | if @container_definitions.size >= 2 36 | raise "Docker runner does not support multi container currently" 37 | end 38 | 39 | @container_definition = @container_definitions[0] 40 | 41 | @keep_container = options[:keep_container] 42 | end 43 | 44 | def run(class_name, method_name, args, container_definition_overrides: {}, environments: []) 45 | definition = container_definition 46 | .merge(container_definition_overrides) 47 | 48 | envs = base_environments(class_name, method_name, args) 49 | envs.concat(extract_environments(environments)) 50 | 51 | exec_docker(definition: definition, cmd: ["bundle", "exec", "rake", "wrapbox:run"], environments: envs) 52 | end 53 | 54 | def run_cmd(cmds, container_definition_overrides: {}, environments: [], ignore_signal: false) 55 | ths = [] 56 | definition = container_definition 57 | .merge(container_definition_overrides) 58 | 59 | environments = extract_environments(environments) 60 | 61 | cmds << nil if cmds.empty? 62 | cmds.each_with_index do |cmd, idx| 63 | ths << Thread.new(cmd, idx) do |c, i| 64 | envs = environments + ["WRAPBOX_CMD_INDEX=#{idx}"] 65 | exec_docker( 66 | definition: definition, 67 | cmd: c&.shellsplit, 68 | environments: envs 69 | ) 70 | end 71 | end 72 | ths.each { |th| th&.join } 73 | 74 | true 75 | rescue SignalException => e 76 | sig = "SIG#{Signal.signame(e.signo)}" 77 | if ignore_signal 78 | @logger.info("Receive #{sig} signal. But Docker container continue running") 79 | else 80 | @logger.info("Receive #{sig} signal. Stop All tasks") 81 | ths.each do |th| 82 | th.report_on_exception = false 83 | th.raise(e) 84 | end 85 | thread_timeout = 15 86 | ths.each { |th| th.join(thread_timeout) } 87 | end 88 | nil 89 | end 90 | 91 | private 92 | 93 | def base_environments(class_name, method_name, args) 94 | ["#{CLASS_NAME_ENV}=#{class_name}", "#{METHOD_NAME_ENV}=#{method_name}", "#{METHOD_ARGS_ENV}=#{MultiJson.dump(args)}"] 95 | end 96 | 97 | def extract_environments(environments) 98 | environments.map do |e| 99 | "#{e[:name]}=#{e[:value]}" 100 | end 101 | end 102 | 103 | def exec_docker(definition:, cmd:, environments: []) 104 | ::Docker::Image.create("fromImage" => definition[:image]) 105 | options = { 106 | "Image" => definition[:image], 107 | "Env" => environments, 108 | }.tap { |o| o["Cmd"] = cmd if cmd } 109 | options["HostConfig"] = {} 110 | options["HostConfig"]["Cpu"] = definition[:cpu] if definition[:cpu] 111 | options["HostConfig"]["Memory"] = definition[:memory] * 1024 * 1024 if definition[:memory] 112 | options["HostConfig"]["MemoryReservation"] = definition[:memory_reservation] * 1024 * 1024 if definition[:memory_reservation] 113 | options["HostConfig"]["Links"] = definition[:links] 114 | options["Entrypoint"] = definition[:entry_point] if definition[:entry_point] 115 | options["WorkingDir"] = definition[:working_directory] if definition[:working_directory] 116 | 117 | container = ::Docker::Container.create(options) 118 | 119 | container.start 120 | output_container_logs(container) 121 | resp = container.wait 122 | output_container_logs(container) 123 | 124 | unless resp["StatusCode"].zero? 125 | raise ExecutionError, "exit_code=#{resp["StatusCode"]}" 126 | end 127 | rescue SignalException => e 128 | sig = Signal.signame(e.signo) 129 | container&.kill(signal: sig) 130 | ensure 131 | container.remove(force: true) if container && !keep_container 132 | end 133 | 134 | def output_container_logs(container) 135 | container.streaming_logs(stdout: true, stderr: true) do |stream, chunk| 136 | if stream == "stdout" 137 | $stdout.puts(chunk) 138 | else 139 | $stderr.puts(chunk) 140 | end 141 | end 142 | end 143 | 144 | class Cli < Thor 145 | namespace :docker 146 | 147 | desc "run_cmd [shell command]", "Run shell on docker" 148 | method_option :config, aliases: "-f", required: true, banner: "YAML_FILE", desc: "yaml file path" 149 | method_option :config_name, aliases: "-n", required: true, default: "default" 150 | method_option :cpu, type: :numeric 151 | method_option :memory, type: :numeric 152 | method_option :working_directory, aliases: "-w", type: :string 153 | method_option :environments, aliases: "-e" 154 | method_option :ignore_signal, type: :boolean, default: false, desc: "Even if receive a signal (like TERM, INT, QUIT), Docker container continue running" 155 | method_option :verbose, aliases: "-v", type: :boolean, default: false, desc: "Verbose mode" 156 | def run_cmd(*args) 157 | Wrapbox.logger.level = :debug if options[:verbose] 158 | Wrapbox.load_config(options[:config]) 159 | config = Wrapbox.configs[options[:config_name]] 160 | environments = options[:environments].to_s.split(/,\s*/).map { |kv| kv.split("=") }.map do |k, v| 161 | {name: k, value: v} 162 | end 163 | if options[:cpu] || options[:memory] || options[:working_directory] 164 | container_definition_overrides = {cpu: options[:cpu], memory: options[:memory], working_directory: options[:working_directory]}.reject { |_, v| v.nil? } 165 | else 166 | container_definition_overrides = {} 167 | end 168 | unless config.run_cmd(args, runner: "docker", environments: environments, container_definition_overrides: container_definition_overrides, ignore_signal: options[:ignore_signal]) 169 | exit 1 170 | end 171 | end 172 | end 173 | end 174 | end 175 | end 176 | -------------------------------------------------------------------------------- /lib/wrapbox/runner/ecs.rb: -------------------------------------------------------------------------------- 1 | require "aws-sdk-ecs" 2 | require "aws-sdk-cloudwatch" 3 | require "multi_json" 4 | require "thor" 5 | require "yaml" 6 | require "active_support/core_ext/hash" 7 | require "pp" 8 | require "shellwords" 9 | require "thwait" 10 | 11 | require "wrapbox" 12 | require "wrapbox/config_repository" 13 | require "wrapbox/log_fetcher" 14 | require "wrapbox/runner/ecs/instance_manager" 15 | require "wrapbox/runner/ecs/task_waiter" 16 | require "wrapbox/version" 17 | 18 | module Wrapbox 19 | module Runner 20 | class Ecs 21 | class ExecutionFailure < StandardError; end 22 | class ContainerAbnormalEnd < StandardError; end 23 | class ExecutionTimeout < StandardError; end 24 | class LaunchFailure < StandardError; end 25 | class LackResource < StandardError; end 26 | 27 | EXECUTION_RETRY_INTERVAL = 3 28 | WAIT_DELAY = 5 29 | TERM_TIMEOUT = 120 30 | HOST_TERMINATED_REASON_REGEXP = /Host EC2.*terminated/ 31 | 32 | attr_reader \ 33 | :name, 34 | :revision, 35 | :region, 36 | :container_definitions, 37 | :volumes, 38 | :placement_constraints, 39 | :placement_strategy, 40 | :requires_compatibilities, 41 | :task_definition_name, 42 | :main_container_name, 43 | :network_mode, 44 | :network_configuration, 45 | :cpu, 46 | :memory, 47 | :enable_ecs_managed_tags, 48 | :tags, 49 | :propagate_tags, 50 | :enable_execute_command 51 | 52 | def self.split_overridable_options_and_parameters(options) 53 | opts = options.dup 54 | overridable_options = {} 55 | %i[cluster launch_type task_role_arn execution_role_arn tags propagate_tags].each do |key| 56 | value = opts.delete(key) 57 | overridable_options[key] = value if value 58 | end 59 | 60 | [overridable_options, opts] 61 | end 62 | 63 | def initialize(options) 64 | @name = options[:name] 65 | @task_definition_name = options[:task_definition_name] 66 | @revision = options[:revision] 67 | @cluster = options[:cluster] 68 | @region = options[:region] 69 | @volumes = options[:volumes] 70 | @placement_constraints = options[:placement_constraints] || [] 71 | @placement_strategy = options[:placement_strategy] 72 | @capacity_provider_strategy = options[:capacity_provider_strategy] || [] 73 | @launch_type = options[:launch_type] 74 | @requires_compatibilities = options[:requires_compatibilities] 75 | @network_mode = options[:network_mode] 76 | @network_configuration = options[:network_configuration] 77 | @cpu = options[:cpu] 78 | @memory = options[:memory] 79 | @enable_ecs_managed_tags = options[:enable_ecs_managed_tags] 80 | @tags = options[:tags] 81 | @propagate_tags = options[:propagate_tags] 82 | @enable_execute_command = options[:enable_execute_command] 83 | if options[:launch_instances] 84 | @instance_manager = Wrapbox::Runner::Ecs::InstanceManager.new(@cluster, @region, **options[:launch_instances]) 85 | end 86 | @task_waiter = Wrapbox::Runner::Ecs::TaskWaiter.new(cluster: @cluster, region: @region, delay: WAIT_DELAY) 87 | 88 | @container_definitions = options[:container_definition] ? [options[:container_definition]] : options[:container_definitions] || [] 89 | @container_definitions.concat(options[:additional_container_definitions]) if options[:additional_container_definitions] # deprecated 90 | 91 | if !@container_definitions.empty? && options[:task_definition] 92 | raise "Please set only one of `container_definition` and `task_definition`" 93 | end 94 | 95 | if options[:additional_container_definitions] && !options[:additional_container_definitions].empty? 96 | warn "`additional_container_definitions` is deprecated parameter, Use `container_definitions` instead of it" 97 | end 98 | 99 | @task_definition_info = options[:task_definition] 100 | 101 | if !@container_definitions.empty? 102 | @task_definition_name ||= "wrapbox_#{@name}" 103 | @main_container_name = @container_definitions[0][:name] || @task_definition_name 104 | elsif @task_definition_info 105 | @task_definition_name = @task_definition_info[:task_definition_name] 106 | @main_container_name = @task_definition_info[:main_container_name] 107 | unless @main_container_name 108 | raise "Please set `task_definition[:main_container_name]`" 109 | end 110 | end 111 | 112 | @container_definitions.each do |d| 113 | d[:docker_labels]&.stringify_keys! 114 | d.dig(:log_configuration, :options)&.stringify_keys! 115 | end 116 | 117 | @task_role_arn = options[:task_role_arn] 118 | @execution_role_arn = options[:execution_role_arn] 119 | @logger = Wrapbox.logger 120 | if options[:log_fetcher] 121 | type = options[:log_fetcher][:type] 122 | @log_fetcher = LogFetcher.new(type, **options[:log_fetcher]) 123 | end 124 | end 125 | 126 | class Parameter 127 | attr_reader \ 128 | :environments, 129 | :timeout, 130 | :launch_timeout, 131 | :launch_retry, 132 | :retry_interval, 133 | :retry_interval_multiplier, 134 | :max_retry_interval, 135 | :execution_retry 136 | 137 | def initialize(environments: [], timeout: 3600 * 24, launch_timeout: 60 * 10, launch_retry: 10, retry_interval: 1, retry_interval_multiplier: 2, max_retry_interval: 120, execution_retry: 0) 138 | b = binding 139 | method(:initialize).parameters.each do |param| 140 | instance_variable_set("@#{param[1]}", b.local_variable_get(param[1])) 141 | end 142 | end 143 | end 144 | 145 | def run(class_name, method_name, args, container_definition_overrides: {}, **parameters) 146 | task_definition = prepare_task_definition(container_definition_overrides) 147 | parameter = Parameter.new(**parameters) 148 | 149 | envs = parameters[:environments] || [] 150 | envs += [ 151 | { 152 | name: CLASS_NAME_ENV, 153 | value: class_name.to_s, 154 | }, 155 | { 156 | name: METHOD_NAME_ENV, 157 | value: method_name.to_s, 158 | }, 159 | { 160 | name: METHOD_ARGS_ENV, 161 | value: MultiJson.dump(args), 162 | }, 163 | ] 164 | 165 | if @instance_manager 166 | Thread.new { @instance_manager.start_preparing_instances(1) } 167 | end 168 | 169 | run_task(task_definition.task_definition_arn, ["bundle", "exec", "rake", "wrapbox:run"], envs, parameter) 170 | ensure 171 | @instance_manager&.terminate_all_instances 172 | end 173 | 174 | def run_cmd(cmds, container_definition_overrides: {}, ignore_signal: false, **parameters) 175 | ths = [] 176 | 177 | task_definition = prepare_task_definition(container_definition_overrides) 178 | parameter = Parameter.new(**parameters) 179 | 180 | cmds << nil if cmds.empty? 181 | 182 | if @instance_manager 183 | Thread.new { @instance_manager.start_preparing_instances(cmds.size) } 184 | end 185 | 186 | cmds.each_with_index do |cmd, idx| 187 | ths << Thread.new(cmd, idx) do |c, i| 188 | Thread.current[:cmd_index] = i 189 | envs = (parameters[:environments] || []) + [{name: "WRAPBOX_CMD_INDEX", value: i.to_s}] 190 | run_task(task_definition.task_definition_arn, c&.shellsplit, envs, parameter) 191 | end 192 | end 193 | ThreadsWait.all_waits(ths) 194 | # Raise an error if some threads have an error 195 | ths.each(&:join) 196 | 197 | true 198 | rescue SignalException => e 199 | sig = "SIG#{Signal.signame(e.signo)}" 200 | if ignore_signal 201 | @logger.info("Receive #{sig} signal. But ECS Tasks continue running") 202 | else 203 | @logger.info("Receive #{sig} signal. Stop All tasks") 204 | ths.each do |th| 205 | th.report_on_exception = false 206 | th.raise(e) 207 | end 208 | wait_until = Time.now + TERM_TIMEOUT + 15 # thread_timeout_buffer 209 | ths.each do |th| 210 | wait = wait_until - Time.now 211 | th.join(wait) if wait.positive? 212 | end 213 | end 214 | nil 215 | ensure 216 | @instance_manager&.terminate_all_instances 217 | end 218 | 219 | private 220 | 221 | def use_existing_task_definition? 222 | !!@task_definition_info 223 | end 224 | 225 | def run_task(task_definition_arn, command, environments, parameter) 226 | execution_try_count = 0 227 | 228 | ec2_instance_id = @instance_manager&.pop_ec2_instance_id 229 | begin 230 | task = create_task(task_definition_arn, command, environments, parameter, ec2_instance_id) 231 | return unless task # only Task creation aborted by SignalException 232 | 233 | @logger.info("#{log_prefix}Launch Task: #{task.task_arn}") 234 | 235 | wait_task_stopped(task.task_arn, parameter.timeout) 236 | 237 | @logger.info("#{log_prefix}Stop Task: #{task.task_arn}") 238 | 239 | # Avoid container exit code fetch miss 240 | sleep WAIT_DELAY 241 | 242 | task_status = fetch_task_status(task.task_arn) 243 | 244 | # If exit_code is nil, Container is force killed or ECS failed to launch Container by Irregular situation 245 | error_message = build_error_message(task_definition_name, task.task_arn, task_status) 246 | raise ContainerAbnormalEnd, error_message unless task_status[:exit_code] 247 | raise ExecutionFailure, error_message unless task_status[:exit_code] == 0 248 | 249 | true 250 | rescue ContainerAbnormalEnd 251 | retry if task_status[:stopped_reason] =~ HOST_TERMINATED_REASON_REGEXP 252 | 253 | if execution_try_count >= parameter.execution_retry 254 | raise 255 | else 256 | execution_try_count += 1 257 | @logger.warn("#{log_prefix}Retry Execution after #{EXECUTION_RETRY_INTERVAL} sec (#{execution_try_count}/#{parameter.execution_retry})") 258 | sleep EXECUTION_RETRY_INTERVAL 259 | retry 260 | end 261 | rescue SignalException 262 | client.stop_task( 263 | cluster: @cluster, 264 | task: task.task_arn, 265 | reason: "signal interrupted" 266 | ) 267 | wait_task_stopped(task.task_arn, TERM_TIMEOUT) 268 | @logger.debug("#{log_prefix}Stop Task: #{task.task_arn}") 269 | ensure 270 | if @log_fetcher 271 | begin 272 | @log_fetcher.stop 273 | rescue => e 274 | @logger.warn(e) 275 | end 276 | end 277 | @instance_manager.terminate_instance(ec2_instance_id) if ec2_instance_id 278 | end 279 | end 280 | 281 | def create_task(task_definition_arn, command, environments, parameter, ec2_instance_id) 282 | args = Array(args) 283 | 284 | launch_try_count = 0 285 | current_retry_interval = parameter.retry_interval 286 | 287 | begin 288 | run_task_options = build_run_task_options(task_definition_arn, command, environments, ec2_instance_id) 289 | @logger.debug("#{log_prefix}Task Options: #{run_task_options}") 290 | 291 | begin 292 | resp = client.run_task(run_task_options) 293 | rescue Aws::ECS::Errors::ThrottlingException 294 | @logger.warn("#{log_prefix}Failure: Rate exceeded.") 295 | raise LaunchFailure 296 | rescue Aws::ECS::Errors::InvalidParameterException => e 297 | # ec2:DescribeSecurityGroups is called in ecs:RunTask when awsvpc mode is used, 298 | # and some errors like "Request limit exceeded", "InternalError" etc. caused by 299 | # ec2:DescribeSecurityGroups are retriable. 300 | # cf. https://github.com/reproio/wrapbox/issues/32, https://github.com/reproio/wrapbox/issues/43 301 | raise if !e.message.include?("Request limit exceeded") && !e.message.include?("InternalError") 302 | 303 | @logger.warn("#{log_prefix}Failure: #{e.message}") 304 | raise LaunchFailure 305 | end 306 | task = resp.tasks[0] 307 | 308 | resp.failures.each do |failure| 309 | @logger.warn("#{log_prefix}Failure: Arn=#{failure.arn}, Reason=#{failure.reason}") 310 | end 311 | raise LackResource unless task # this case is almost lack of container resource. 312 | 313 | @logger.debug("#{log_prefix}Create Task: #{task.task_arn}") 314 | 315 | @log_fetcher.run(task: task) if @log_fetcher 316 | 317 | # Wait ECS Task Status becomes stable 318 | sleep WAIT_DELAY 319 | 320 | begin 321 | wait_task_running(task.task_arn, parameter.launch_timeout) 322 | task 323 | rescue Wrapbox::Runner::Ecs::TaskWaiter::WaitTimeout 324 | client.stop_task( 325 | cluster: @cluster, 326 | task: task.task_arn, 327 | reason: "launch timeout" 328 | ) 329 | raise 330 | rescue Wrapbox::Runner::Ecs::TaskWaiter::WaitFailure 331 | task_status = fetch_task_status(task.task_arn) 332 | 333 | case task_status[:last_status] 334 | when "RUNNING" 335 | return task 336 | when "PENDING" 337 | retry 338 | else 339 | if task_status[:exit_code] 340 | return task 341 | else 342 | raise LaunchFailure 343 | end 344 | end 345 | end 346 | rescue LackResource 347 | @logger.warn("#{log_prefix}Failed to create task, because of lack resource") 348 | put_waiting_task_count_metric 349 | 350 | if launch_try_count >= parameter.launch_retry 351 | raise 352 | else 353 | launch_try_count += 1 354 | retry_interval = current_retry_interval/2 + rand(current_retry_interval/2) 355 | @logger.warn("#{log_prefix}Retry Create Task after #{retry_interval} sec (#{launch_try_count}/#{parameter.launch_retry})") 356 | sleep retry_interval 357 | current_retry_interval = [current_retry_interval * parameter.retry_interval_multiplier, parameter.max_retry_interval].min 358 | retry 359 | end 360 | rescue LaunchFailure 361 | if launch_try_count >= parameter.launch_retry 362 | task_status = fetch_task_status(task.task_arn) 363 | raise LaunchFailure, build_error_message(task_definition_name, task.task_arn, task_status) 364 | else 365 | launch_try_count += 1 366 | retry_interval = current_retry_interval/2 + rand(current_retry_interval/2) 367 | @logger.warn("#{log_prefix}Retry Create Task after #{retry_interval} sec (#{launch_try_count}/#{parameter.launch_retry})") 368 | sleep retry_interval 369 | current_retry_interval = [current_retry_interval * parameter.retry_interval_multiplier, parameter.max_retry_interval].min 370 | retry 371 | end 372 | rescue SignalException 373 | if task 374 | client.stop_task( 375 | cluster: @cluster, 376 | task: task.task_arn, 377 | reason: "signal interrupted" 378 | ) 379 | wait_task_stopped(task.task_arn, TERM_TIMEOUT) 380 | @logger.debug("#{log_prefix}Stop Task: #{task.task_arn}") 381 | nil 382 | end 383 | end 384 | end 385 | 386 | def wait_task_running(task_arn, launch_timeout) 387 | @task_waiter.wait_task_running(task_arn, timeout: launch_timeout) 388 | end 389 | 390 | def wait_task_stopped(task_arn, execution_timeout) 391 | @task_waiter.wait_task_stopped(task_arn, timeout: execution_timeout) 392 | rescue Wrapbox::Runner::Ecs::TaskWaiter::WaitTimeout 393 | client.stop_task({ 394 | cluster: @cluster, 395 | task: task_arn, 396 | reason: "process timeout", 397 | }) 398 | raise ExecutionTimeout, "Task #{task_definition_name} is timeout. task=#{task_arn}, timeout=#{execution_timeout}" 399 | end 400 | 401 | def fetch_task_status(task_arn) 402 | task = client.describe_tasks(cluster: @cluster, tasks: [task_arn]).tasks[0] 403 | container = task.containers.find { |c| c.name == main_container_name } 404 | { 405 | last_status: task.last_status, 406 | exit_code: container.exit_code, 407 | stopped_reason: task.stopped_reason, 408 | container_stopped_reason: container.reason 409 | } 410 | end 411 | 412 | def prepare_task_definition(container_definition_overrides) 413 | if use_existing_task_definition? 414 | client.describe_task_definition(task_definition: task_definition_name).task_definition 415 | else 416 | register_task_definition(container_definition_overrides) 417 | end 418 | end 419 | 420 | def register_task_definition(container_definition_overrides) 421 | main_container_definition = container_definitions[0] 422 | main_container_definition = main_container_definition 423 | .merge(container_definition_overrides) 424 | .merge(name: main_container_name) 425 | 426 | overrided_container_definitions = [main_container_definition, *(container_definitions.drop(1))] 427 | 428 | if revision 429 | begin 430 | return client.describe_task_definition(task_definition: "#{task_definition_name}:#{revision}").task_definition 431 | rescue 432 | end 433 | end 434 | 435 | @logger.debug("#{log_prefix}Container Definitions: #{overrided_container_definitions}") 436 | register_retry_count = 0 437 | begin 438 | client.register_task_definition({ 439 | family: task_definition_name, 440 | cpu: cpu, 441 | memory: memory, 442 | network_mode: network_mode, 443 | container_definitions: overrided_container_definitions, 444 | volumes: volumes, 445 | requires_compatibilities: requires_compatibilities, 446 | task_role_arn: @task_role_arn, 447 | execution_role_arn: @execution_role_arn, 448 | tags: tags, 449 | }).task_definition 450 | rescue Aws::ECS::Errors::ClientException, Aws::ECS::Errors::ThrottlingException 451 | raise if register_retry_count > 2 452 | register_retry_count += 1 453 | sleep 2 454 | retry 455 | end 456 | end 457 | 458 | def cmd_index 459 | Thread.current[:cmd_index] 460 | end 461 | 462 | def log_prefix 463 | cmd_index ? "##{cmd_index} " : "" 464 | end 465 | 466 | def client 467 | return @client if @client 468 | 469 | options = {} 470 | options[:region] = region if region 471 | @client = Aws::ECS::Client.new(options) 472 | end 473 | 474 | def cloud_watch_client 475 | return @cloud_watch_client if @cloud_watch_client 476 | 477 | options = {} 478 | options[:region] = region if region 479 | @cloud_watch_client = Aws::CloudWatch::Client.new(options) 480 | end 481 | 482 | def put_waiting_task_count_metric 483 | cloud_watch_client.put_metric_data( 484 | namespace: "wrapbox", 485 | metric_data: [ 486 | metric_name: "WaitingTaskCount", 487 | dimensions: [ 488 | { 489 | name: "ClusterName", 490 | value: @cluster, 491 | }, 492 | ], 493 | timestamp: Time.now, 494 | value: 1.0, 495 | unit: "Count", 496 | ] 497 | ) 498 | end 499 | 500 | def build_run_task_options(task_definition_arn, command, environments, ec2_instance_id) 501 | overrides = { 502 | container_overrides: [ 503 | { 504 | name: main_container_name, 505 | environment: environments, 506 | }.tap { |o| o[:command] = command if command }, 507 | *container_definitions.drop(1).map do |c| 508 | { 509 | name: c[:name], 510 | environment: environments, 511 | } 512 | end 513 | ], 514 | } 515 | overrides[:task_role_arn] = @task_role_arn if @task_role_arn 516 | 517 | additional_placement_constraints = [] 518 | if ec2_instance_id 519 | additional_placement_constraints << { type: "memberOf", expression: "ec2InstanceId == #{ec2_instance_id}" } 520 | end 521 | options = { 522 | cluster: @cluster, 523 | enable_execute_command: enable_execute_command, 524 | task_definition: task_definition_arn, 525 | overrides: overrides, 526 | placement_strategy: placement_strategy, 527 | placement_constraints: placement_constraints + additional_placement_constraints, 528 | network_configuration: network_configuration, 529 | started_by: "wrapbox-#{Wrapbox::VERSION}", 530 | enable_ecs_managed_tags: enable_ecs_managed_tags, 531 | propagate_tags: propagate_tags, 532 | } 533 | if @capacity_provider_strategy.empty? 534 | options[:launch_type] = @launch_type if @launch_type 535 | else 536 | if @launch_type 537 | @logger.warn("#{log_prefix}Ignore --launch_type and launch_type in the configuration file when specified capacity_provider_strategy in the configuration file") 538 | end 539 | options[:capacity_provider_strategy] = @capacity_provider_strategy 540 | end 541 | options 542 | end 543 | 544 | def build_error_message(task_definition_name, task_arn, task_status) 545 | error_message = "Task #{task_definition_name} is failed. task=#{task_arn}, " 546 | error_message << "cmd_index=#{cmd_index}, " if cmd_index 547 | error_message << "exit_code=#{task_status[:exit_code]}, task_stopped_reason=#{task_status[:stopped_reason]}, container_stopped_reason=#{task_status[:container_stopped_reason]}" 548 | error_message 549 | end 550 | 551 | class Cli < Thor 552 | namespace :ecs 553 | 554 | desc "run_cmd [shell command]", "Run shell on ECS" 555 | method_option :config, aliases: "-f", required: true, banner: "YAML_FILE", desc: "yaml file path" 556 | method_option :config_name, aliases: "-n", required: true, default: "default" 557 | method_option :cluster, aliases: "-c" 558 | method_option :cpu, type: :numeric 559 | method_option :memory, type: :numeric 560 | method_option :working_directory, aliases: "-w", type: :string 561 | method_option :environments, aliases: "-e" 562 | method_option :task_role_arn 563 | method_option :timeout, type: :numeric 564 | method_option :launch_type, type: :string, enum: ["EC2", "FARGATE"] 565 | method_option :launch_timeout, type: :numeric 566 | method_option :launch_retry, type: :numeric 567 | method_option :execution_retry, type: :numeric 568 | method_option :max_retry_interval, type: :numeric 569 | method_option :ignore_signal, type: :boolean, default: false, desc: "Even if receive a signal (like TERM, INT, QUIT), ECS Tasks continue running" 570 | method_option :tags, type: :string, aliases: "-t", repeatable: true 571 | method_option :propagate_tags, type: :string, enum: ["TASK_DEFINITION", "SERVICE"] 572 | method_option :verbose, aliases: "-v", type: :boolean, default: false, desc: "Verbose mode" 573 | def run_cmd(*args) 574 | Wrapbox.logger.level = :debug if options[:verbose] 575 | Wrapbox.load_config(options[:config]) 576 | config = Wrapbox.configs[options[:config_name]] 577 | environments = options[:environments].to_s.split(/,\s*/).map { |kv| kv.split("=") }.map do |k, v| 578 | {name: k, value: v} 579 | end 580 | tags = options.fetch(:tags, []).map do |kv| 581 | k, v = kv.split("=", 2) 582 | {key: k, value: v} 583 | end.presence 584 | run_options = { 585 | cluster: options[:cluster], 586 | task_role_arn: options[:task_role_arn], 587 | timeout: options[:timeout], 588 | launch_type: options[:launch_type], 589 | launch_timeout: options[:launch_timeout], 590 | launch_retry: options[:launch_retry], 591 | execution_retry: options[:execution_retry], 592 | max_retry_interval: options[:max_retry_interval], 593 | ignore_signal: options[:ignore_signal], 594 | tags: tags, 595 | propagate_tags: options[:propagate_tags], 596 | }.reject { |_, v| v.nil? } 597 | if options[:cpu] || options[:memory] || options[:working_directory] 598 | container_definition_overrides = {cpu: options[:cpu], memory: options[:memory], working_directory: options[:working_directory]}.reject { |_, v| v.nil? } 599 | else 600 | container_definition_overrides = {} 601 | end 602 | unless config.run_cmd(args, runner: "ecs", environments: environments, container_definition_overrides: container_definition_overrides, **run_options) 603 | exit 1 604 | end 605 | end 606 | end 607 | end 608 | end 609 | end 610 | -------------------------------------------------------------------------------- /lib/wrapbox/runner/ecs/instance_manager.rb: -------------------------------------------------------------------------------- 1 | require "aws-sdk-ec2" 2 | require "aws-sdk-ecs" 3 | 4 | module Wrapbox 5 | module Runner 6 | class Ecs 7 | class InstanceManager 8 | def initialize(cluster, region, launch_template:, instance_type: nil, tag_specifications: nil, wait_until_instance_terminated: true) 9 | @cluster = cluster 10 | @region = region 11 | @launch_template = launch_template 12 | @instance_type = instance_type 13 | @tag_specifications = tag_specifications 14 | @wait_until_instance_terminated = wait_until_instance_terminated 15 | @queue = Queue.new 16 | @instance_ids = [] 17 | end 18 | 19 | def pop_ec2_instance_id 20 | Wrapbox.logger.debug("Wait until a new container instance are registered in \"#{@cluster}\" cluster") 21 | @queue.pop 22 | end 23 | 24 | def start_preparing_instances(count) 25 | preparing_instance_ids = ec2_client.run_instances( 26 | launch_template: @launch_template, 27 | instance_type: @instance_type, 28 | tag_specifications: @tag_specifications, 29 | min_count: count, 30 | max_count: count 31 | ).instances.map(&:instance_id) 32 | @instance_ids.concat(preparing_instance_ids) 33 | ec2_client.wait_until(:instance_running, instance_ids: preparing_instance_ids) 34 | 35 | waiter = Aws::Waiters::Waiter.new( 36 | max_attempts: 40, 37 | delay: 15, 38 | poller: Aws::Waiters::Poller.new( 39 | operation_name: :list_container_instances, 40 | acceptors: [ 41 | { 42 | "expected" => true, 43 | "matcher" => "path", 44 | "state" => "success", 45 | "argument" => "length(container_instance_arns) > `0`" 46 | } 47 | ] 48 | ) 49 | ) 50 | 51 | while preparing_instance_ids.size > 0 52 | waiter.wait(client: ecs_client, params: { cluster: @cluster, filter: "ec2InstanceId in [#{preparing_instance_ids.join(",")}]" }).each do |resp| 53 | ecs_client.describe_container_instances(cluster: @cluster, container_instances: resp.container_instance_arns).container_instances.each do |c| 54 | preparing_instance_ids.delete(c.ec2_instance_id) 55 | @queue << c.ec2_instance_id 56 | end 57 | end 58 | end 59 | end 60 | 61 | def terminate_instance(instance_id) 62 | ec2_client.terminate_instances(instance_ids: [instance_id]) 63 | if @wait_until_instance_terminated 64 | ec2_client.wait_until(:instance_terminated, instance_ids: [instance_id]) 65 | end 66 | @instance_ids.delete(instance_id) 67 | end 68 | 69 | def terminate_all_instances 70 | # Duplicate @instance_ids because other threads can change it 71 | remaining_instance_ids = @instance_ids.dup 72 | return if remaining_instance_ids.empty? 73 | ec2_client.terminate_instances(instance_ids: remaining_instance_ids) 74 | if @wait_until_instance_terminated 75 | ec2_client.wait_until(:instance_terminated, instance_ids: remaining_instance_ids) 76 | end 77 | @instance_ids.clear 78 | end 79 | 80 | private 81 | 82 | def ecs_client 83 | @ecs_client ||= Aws::ECS::Client.new({ region: @region }.reject { |_, v| v.nil? }) 84 | end 85 | 86 | def ec2_client 87 | @ec2_client ||= Aws::EC2::Client.new({ region: @region }.reject { |_, v| v.nil? }) 88 | end 89 | end 90 | end 91 | end 92 | end 93 | -------------------------------------------------------------------------------- /lib/wrapbox/runner/ecs/task_waiter.rb: -------------------------------------------------------------------------------- 1 | require "timeout" 2 | 3 | require "aws-sdk-ecs" 4 | 5 | module Wrapbox 6 | module Runner 7 | class Ecs 8 | class TaskWaiter 9 | MAX_DESCRIBABLE_TASK_COUNT = 100 10 | 11 | class WaitFailure < StandardError; end 12 | class TaskStopped < WaitFailure; end 13 | class TaskMissing < WaitFailure; end 14 | class UnknownFailure < WaitFailure; end 15 | class WaitTimeout < WaitFailure; end 16 | 17 | def initialize(cluster:, region:, delay:) 18 | @cluster = cluster 19 | @region = region 20 | @task_arn_to_described_result = {} 21 | @mutex = Mutex.new 22 | @cv = ConditionVariable.new 23 | Thread.new { update_described_results(delay) } 24 | end 25 | 26 | # @return Aws::ECS::Types::Task 27 | def wait_task_running(task_arn, timeout: 0) 28 | Timeout.timeout(timeout) do 29 | loop do 30 | result = describe_task(task_arn) 31 | if result[:failure] 32 | case result[:failure].reason 33 | when "MISSING" 34 | raise TaskMissing 35 | else 36 | raise UnknownFailure 37 | end 38 | end 39 | raise TaskStopped if result[:task].last_status == "STOPPED" 40 | 41 | return result[:task] if result[:task].last_status == "RUNNING" 42 | end 43 | end 44 | rescue Timeout::Error 45 | raise WaitTimeout 46 | end 47 | 48 | # @return Aws::ECS::Types::Task 49 | def wait_task_stopped(task_arn, timeout: 0) 50 | Timeout.timeout(timeout) do 51 | result = nil 52 | loop do 53 | result = describe_task(task_arn) 54 | if result[:failure] 55 | case result[:failure].reason 56 | when "MISSING" 57 | raise TaskMissing 58 | else 59 | raise UnknownFailure 60 | end 61 | end 62 | 63 | return result[:task] if result[:task].last_status == "STOPPED" 64 | end 65 | end 66 | rescue Timeout::Error 67 | raise WaitTimeout 68 | end 69 | 70 | private 71 | 72 | def describe_task(task_arn) 73 | @mutex.synchronize do 74 | @task_arn_to_described_result[task_arn] = nil 75 | @cv.wait(@mutex) 76 | @task_arn_to_described_result[task_arn] 77 | end 78 | ensure 79 | @mutex.synchronize do 80 | @task_arn_to_described_result.delete(task_arn) 81 | end 82 | end 83 | 84 | def update_described_results(interval) 85 | loop do 86 | @mutex.synchronize do 87 | unless @task_arn_to_described_result.empty? 88 | begin 89 | @task_arn_to_described_result.keys.each_slice(MAX_DESCRIBABLE_TASK_COUNT) do |task_arns| 90 | resp = ecs_client.describe_tasks(cluster: @cluster, tasks: task_arns) 91 | resp.tasks.each do |task| 92 | @task_arn_to_described_result[task.task_arn] = { task: task } 93 | end 94 | resp.failures.each do |failure| 95 | # failure.arn is like "arn:aws:ecs:::task/" 96 | # even if task_arn is like "arn:aws:ecs:::task//" 97 | prefix, suffix = failure.arn.split("/") 98 | task_arn = task_arns.find { |a| a.start_with?(prefix) && a.end_with?(suffix) } 99 | @task_arn_to_described_result[task_arn || failure.arn] = { failure: failure } 100 | end 101 | end 102 | 103 | @cv.broadcast 104 | rescue Aws::ECS::Errors::ThrottlingException 105 | Wrapbox.logger.warn("Failed to describe tasks due to Aws::ECS::Errors::ThrottlingException") 106 | end 107 | end 108 | end 109 | 110 | sleep interval 111 | end 112 | end 113 | 114 | def ecs_client 115 | @ecs_client ||= Aws::ECS::Client.new({ region: @region }.reject { |_, v| v.nil? }) 116 | end 117 | end 118 | end 119 | end 120 | end 121 | -------------------------------------------------------------------------------- /lib/wrapbox/tasks/run.rake: -------------------------------------------------------------------------------- 1 | require "wrapbox" 2 | 3 | namespace :wrapbox do 4 | desc "Run Wrapbox" 5 | task :run do 6 | Rake::Task["environment"].invoke if defined?(Rails) 7 | 8 | if ENV[Wrapbox::CLASS_NAME_ENV] && ENV[Wrapbox::METHOD_NAME_ENV] && ENV[Wrapbox::METHOD_ARGS_ENV] 9 | Wrapbox::Job.perform 10 | else 11 | raise "Wrapbox ENVs are not found" 12 | end 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /lib/wrapbox/version.rb: -------------------------------------------------------------------------------- 1 | module Wrapbox 2 | VERSION = "0.10.0" 3 | end 4 | -------------------------------------------------------------------------------- /spec/config.yml: -------------------------------------------------------------------------------- 1 | base: &base 2 | cluster: <%= ENV["ECS_CLUSTER"] %> 3 | runner: ecs 4 | region: ap-northeast-1 5 | 6 | default: 7 | <<: *base 8 | container_definition: 9 | image: joker1007/wrapbox@sha256:0925926e867244907f7f72b322a24312501719960d10c989a3847de4890ec55a 10 | cpu: 256 11 | memory: 256 12 | essential: true 13 | # task_definition: 14 | # task_definition_name: task_def_name:154 15 | # main_container_name: container1 16 | 17 | docker: 18 | runner: docker 19 | keep_container: false 20 | container_definition: 21 | image: joker1007/wrapbox@sha256:0925926e867244907f7f72b322a24312501719960d10c989a3847de4890ec55a 22 | cpu: 600 23 | memory: 1024 24 | 25 | ecs_with_launch_template: 26 | <<: *base 27 | container_definition: 28 | image: joker1007/wrapbox@sha256:0925926e867244907f7f72b322a24312501719960d10c989a3847de4890ec55a 29 | cpu: 256 30 | memory: 256 31 | essential: true 32 | launch_instances: 33 | launch_template: 34 | launch_template_id: <%= ENV["LAUNCH_TEMPLATE_ID"] %> 35 | version: $Latest 36 | tag_specifications: 37 | - resource_type: instance 38 | tags: 39 | - key: Purpose 40 | value: wrapbox_spec 41 | wait_until_instance_terminated: false 42 | 43 | ecs_without_runner: 44 | cluster: <%= ENV["ECS_CLUSTER"] %> 45 | region: ap-northeast-1 46 | container_definition: 47 | image: joker1007/wrapbox@sha256:0925926e867244907f7f72b322a24312501719960d10c989a3847de4890ec55a 48 | cpu: 256 49 | memory: 256 50 | essential: true 51 | 52 | ecs_enable_execute_command: 53 | <<: *base 54 | enable_execute_command: true 55 | container_definition: 56 | image: joker1007/wrapbox@sha256:0925926e867244907f7f72b322a24312501719960d10c989a3847de4890ec55a 57 | cpu: 256 58 | memory: 256 59 | essential: true 60 | 61 | ecs_disable_execute_command: 62 | <<: *base 63 | enable_execute_command: false 64 | container_definition: 65 | image: joker1007/wrapbox@sha256:0925926e867244907f7f72b322a24312501719960d10c989a3847de4890ec55a 66 | cpu: 256 67 | memory: 256 68 | essential: true 69 | 70 | ecs_with_awslogs_fetcher: 71 | <<: *base 72 | execution_role_arn: <%= ENV["EXECUTION_ROLE_ARN"] %> 73 | log_fetcher: 74 | type: awslogs 75 | log_group: /ecs/wrapbox 76 | log_stream_prefix: ecs_with_log_fetcher 77 | region: ap-northeast-1 78 | container_definition: 79 | image: joker1007/wrapbox@sha256:0925926e867244907f7f72b322a24312501719960d10c989a3847de4890ec55a 80 | cpu: 256 81 | memory: 256 82 | essential: true 83 | log_configuration: 84 | log_driver: awslogs 85 | options: 86 | awslogs-group: /ecs/wrapbox 87 | awslogs-region: ap-northeast-1 88 | awslogs-stream-prefix: ecs_with_log_fetcher 89 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | $LOAD_PATH.unshift File.expand_path("../../lib", __FILE__) 2 | require "wrapbox" 3 | require "wrapbox/runner/docker" 4 | require "wrapbox/runner/ecs" 5 | 6 | require "tapp" 7 | require "tapp-awesome_print" 8 | 9 | Wrapbox.configure do |c| 10 | c.load_yaml(File.expand_path("../config.yml", __FILE__)) 11 | end 12 | 13 | RSpec.configure do |c| 14 | c.order = "random" 15 | c.filter_run_excluding aws: true unless ENV["RUN_AWS_SPECS"] == "true" 16 | end 17 | 18 | if defined?(Tapp) 19 | Tapp.configure do |config| 20 | config.default_printer = :awesome_print if defined?(AwesomePrint) 21 | config.report_caller = true 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /spec/test_job.rb: -------------------------------------------------------------------------------- 1 | class TestJob 2 | def perform(arg1, arg2) 3 | p arg1 4 | p arg2 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /spec/wrapbox/runner/ecs/task_waiter_spec.rb: -------------------------------------------------------------------------------- 1 | require "spec_helper" 2 | 3 | require "wrapbox/runner/ecs/task_waiter" 4 | 5 | describe Wrapbox::Runner::Ecs::TaskWaiter do 6 | let(:waiter) { described_class.new(cluster: "default", region: "ap-northeast-1", delay: 0.01) } 7 | let(:ecs_client) { Aws::ECS::Client.new(stub_responses: true) } 8 | 9 | before do 10 | allow(waiter).to receive(:ecs_client) { ecs_client } 11 | end 12 | 13 | def start_thread 14 | Thread.new do 15 | Thread.current.report_on_exception = false 16 | yield 17 | end 18 | end 19 | 20 | describe "#wait_task_running" do 21 | let(:running_task_arn) { "arn:aws:ecs:ap-northeast-1:1234:task/default/3f83f7c37e41d1862874a84a6eefd7c7" } 22 | let(:stopped_task_arn) { "arn:aws:ecs:ap-northeast-1:1234:task/default/ac65e038e840c7e4206c88018924f3a5" } 23 | let(:missing_task_arn) { "arn:aws:ecs:ap-northeast-1:1234:task/default/c5382e88b8c2bbd6888f36bfd9bd32e8" } 24 | 25 | before do 26 | ecs_client.stub_responses(:describe_tasks, { 27 | tasks: [ 28 | { task_arn: running_task_arn, last_status: "PENDING" }, 29 | { task_arn: stopped_task_arn, last_status: "PENDING" }, 30 | ], 31 | failures: [ 32 | { reason: "MISSING", arn: missing_task_arn.sub("/default", '') }, 33 | ] 34 | }) 35 | end 36 | 37 | it "waits until specified tasks run" do 38 | running_task_th = start_thread { waiter.wait_task_running(running_task_arn) } 39 | stopped_task_th = start_thread { waiter.wait_task_running(stopped_task_arn) } 40 | missing_task_th = start_thread { waiter.wait_task_running(missing_task_arn) } 41 | 42 | ecs_client.stub_responses(:describe_tasks, { 43 | tasks: [ 44 | { task_arn: running_task_arn, last_status: "RUNNING" }, 45 | { task_arn: stopped_task_arn, last_status: "STOPPED" }, 46 | ], 47 | failures: [ 48 | { reason: "MISSING", arn: missing_task_arn.sub("/default", '') }, 49 | ] 50 | }) 51 | 52 | expect(running_task_th.value.task_arn).to eq running_task_arn 53 | expect { stopped_task_th.value }.to raise_error(described_class::TaskStopped) 54 | expect { missing_task_th.value }.to raise_error(described_class::TaskMissing) 55 | expect(waiter.instance_variable_get(:@task_arn_to_described_result)).to be_empty 56 | end 57 | 58 | it { expect { waiter.wait_task_running(running_task_arn, timeout: 0.01) }.to raise_error(described_class::WaitTimeout) } 59 | end 60 | 61 | describe "#wait_task_stopped" do 62 | let(:stopped_task_arn) { "arn:aws:ecs:ap-northeast-1:1234:task/default/ac65e038e840c7e4206c88018924f3a5" } 63 | let(:missing_task_arn) { "arn:aws:ecs:ap-northeast-1:1234:task/default/c5382e88b8c2bbd6888f36bfd9bd32e8" } 64 | 65 | before do 66 | ecs_client.stub_responses(:describe_tasks, { 67 | tasks: [ 68 | { task_arn: stopped_task_arn, last_status: "PENDING" }, 69 | ], 70 | failures: [ 71 | { reason: "MISSING", arn: missing_task_arn.sub("/default", '') }, 72 | ] 73 | }) 74 | end 75 | 76 | it "waits until specified tasks stop" do 77 | stopped_task_th = start_thread { waiter.wait_task_stopped(stopped_task_arn) } 78 | missing_task_th = start_thread { waiter.wait_task_stopped(missing_task_arn) } 79 | 80 | ecs_client.stub_responses(:describe_tasks, { 81 | tasks: [ 82 | { task_arn: stopped_task_arn, last_status: "STOPPED" }, 83 | ], 84 | failures: [ 85 | { reason: "MISSING", arn: missing_task_arn.sub("/default", '') }, 86 | ] 87 | }) 88 | 89 | expect(stopped_task_th.value.task_arn).to eq stopped_task_arn 90 | expect { missing_task_th.value }.to raise_error(described_class::TaskMissing) 91 | expect(waiter.instance_variable_get(:@task_arn_to_described_result)).to be_empty 92 | end 93 | 94 | it { expect { waiter.wait_task_stopped(stopped_task_arn, timeout: 0.01) }.to raise_error(described_class::WaitTimeout) } 95 | end 96 | end 97 | -------------------------------------------------------------------------------- /spec/wrapbox_spec.rb: -------------------------------------------------------------------------------- 1 | require "spec_helper" 2 | 3 | describe Wrapbox do 4 | it "can load yaml" do 5 | config = Wrapbox.configs[:default] 6 | expect(config.cluster).to eq(ENV["ECS_CLUSTER"]) 7 | expect(config.region).to eq("ap-northeast-1") 8 | expect(config.enable_execute_command).to be_falsey 9 | expect(config.container_definition[:cpu]).to be_a(Integer) 10 | end 11 | 12 | describe 'enable_execute_command option is true' do 13 | specify "config value is true" do 14 | config = Wrapbox.configs[:ecs_enable_execute_command] 15 | expect(config.enable_execute_command).to be_truthy 16 | end 17 | end 18 | 19 | describe 'enable_execute_command option is false' do 20 | specify "config value is false" do 21 | config = Wrapbox.configs[:ecs_disable_execute_command] 22 | expect(config.enable_execute_command).to be_falsey 23 | end 24 | end 25 | 26 | describe ".run" do 27 | specify "executable on ECS", aws: true do 28 | Wrapbox.run("TestJob", :perform, ["arg1", ["arg2", "arg3"]], environments: [{name: "RAILS_ENV", value: "development"}]) 29 | end 30 | 31 | specify "executable on Docker" do 32 | Wrapbox.run("TestJob", :perform, ["arg1", ["arg2", "arg3"]], config_name: :docker, environments: [{name: "RAILS_ENV", value: "development"}]) 33 | end 34 | 35 | specify "executable on ECS with launch template", aws: true do 36 | Wrapbox.run("TestJob", :perform, ["arg1", ["arg2", "arg3"]], config_name: :ecs_with_launch_template, environments: [{name: "RAILS_ENV", value: "development"}]) 37 | end 38 | end 39 | 40 | describe ".run_cmd" do 41 | specify "executable on ECS", aws: true do 42 | Wrapbox.run_cmd(["ls ."], environments: [{name: "RAILS_ENV", value: "development"}]) 43 | end 44 | 45 | specify "executable on ECS overriding `cluster`", aws: true do 46 | default_clusters = [nil, "", "default"] 47 | if ENV["OVERRIDDEN_ECS_CLUSTER"].nil? 48 | raise "Specify OVERRIDDEN_ECS_CLUSTER" 49 | end 50 | if ENV["ECS_CLUSTER"] == ENV["OVERRIDDEN_ECS_CLUSTER"] || (default_clusters.include?(ENV["ECS_CLUSTER"]) && default_clusters.include?(ENV["OVERRIDDEN_ECS_CLUSTER"])) 51 | raise "Specify different values for ECS_CLUSTER and OVERRIDDEN_ECS_CLUSTER" 52 | end 53 | Wrapbox.run_cmd(["ls ."], environments: [{name: "RAILS_ENV", value: "development"}], cluster: ENV["OVERRIDDEN_ECS_CLUSTER"]) 54 | end 55 | 56 | specify "executable on ECS overriding `runner`", aws: true do 57 | expect(Wrapbox::Runner::Ecs).to receive(:new).and_call_original 58 | Wrapbox.run_cmd(["ls ."], config_name: :ecs_without_runner, runner: "ecs", environments: [{name: "RAILS_ENV", value: "development"}]) 59 | end 60 | 61 | specify "executable on ECS with launch template", aws: true do 62 | Wrapbox.run_cmd(["ls ."], config_name: :ecs_with_launch_template, environments: [{name: "RAILS_ENV", value: "development"}]) 63 | end 64 | 65 | specify "executable on ECS and kill task", aws: true do 66 | r, w = IO.pipe 67 | pid = fork do 68 | puts "exec on child process" 69 | r.close 70 | unless Wrapbox.run_cmd(["ruby -e 'sleep 120'"], environments: [{name: "RAILS_ENV", value: "development"}]) 71 | w.write("ok") 72 | w.flush 73 | end 74 | end 75 | 76 | if pid 77 | w.close 78 | sleep 15 79 | puts "send SIGTERM to child process" 80 | Process.kill("SIGTERM", pid) 81 | sleep 1 82 | expect(r.read).to eq("ok") 83 | end 84 | end 85 | 86 | specify "executable on ECS with error", aws: true do 87 | expect { 88 | Wrapbox.run_cmd(["ls no_dir"], environments: [{name: "RAILS_ENV", value: "development"}]) 89 | }.to raise_error(Wrapbox::Runner::Ecs::ExecutionFailure) 90 | end 91 | 92 | specify "executable on ECS with error, retrying", aws: true do 93 | expect { 94 | Wrapbox.run_cmd(["ls no_dir"], environments: [{name: "RAILS_ENV", value: "development"}], execution_retry: 1) 95 | }.to raise_error(Wrapbox::Runner::Ecs::ExecutionFailure) 96 | end 97 | 98 | specify "executable on Docker" do 99 | Wrapbox.run_cmd(["ls ."], config_name: :docker, environments: [{name: "RAILS_ENV", value: "development"}]) 100 | end 101 | 102 | specify "executable on Docker overriding `runner`" do 103 | expect(Wrapbox::Runner::Docker).to receive(:new).and_call_original 104 | Wrapbox.run_cmd(["ls ."], runner: "docker", environments: [{name: "RAILS_ENV", value: "development"}]) 105 | end 106 | 107 | specify "executable on Docker and kill task" do 108 | r, w = IO.pipe 109 | pid = fork do 110 | puts "exec on child process" 111 | r.close 112 | unless Wrapbox.run_cmd(["sleep 30"], config_name: :docker, environments: [{name: "RAILS_ENV", value: "development"}]) 113 | w.write("ok") 114 | w.flush 115 | end 116 | end 117 | 118 | if pid 119 | w.close 120 | sleep 10 121 | puts "send SIGTERM to child process" 122 | Process.kill("SIGTERM", pid) 123 | sleep 1 124 | expect(r.read).to eq("ok") 125 | end 126 | end 127 | end 128 | end 129 | -------------------------------------------------------------------------------- /wrapbox.gemspec: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | lib = File.expand_path('../lib', __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | require 'wrapbox/version' 5 | 6 | Gem::Specification.new do |spec| 7 | spec.name = "wrapbox" 8 | spec.version = Wrapbox::VERSION 9 | spec.authors = ["joker1007"] 10 | spec.email = ["kakyoin.hierophant@gmail.com"] 11 | 12 | spec.summary = %q{Ruby method runner on AWS ECS} 13 | spec.description = %q{Ruby method runner on AWS ECS} 14 | spec.homepage = "" 15 | 16 | spec.files = `git ls-files -z`.split("\x0").reject do |f| 17 | f.match(%r{^(test|spec|features)/}) 18 | end 19 | spec.bindir = "exe" 20 | spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } 21 | spec.require_paths = ["lib"] 22 | 23 | spec.add_runtime_dependency "aws-sdk-ec2", "~> 1" 24 | spec.add_runtime_dependency "aws-sdk-ecs", "~> 1" 25 | spec.add_runtime_dependency "aws-sdk-cloudwatch", "~> 1" 26 | spec.add_runtime_dependency "activesupport", ">= 4" 27 | spec.add_runtime_dependency "docker-api" 28 | spec.add_runtime_dependency "multi_json" 29 | spec.add_runtime_dependency "thor", ">= 1" 30 | spec.add_runtime_dependency "thwait" 31 | 32 | spec.add_development_dependency "bundler" 33 | spec.add_development_dependency "rake" 34 | spec.add_development_dependency "rspec", "~> 3.0" 35 | spec.add_development_dependency "webmock" 36 | spec.add_development_dependency "tapp" 37 | spec.add_development_dependency "tapp-awesome_print" 38 | end 39 | --------------------------------------------------------------------------------