├── tmp └── .gitkeep ├── .ruby-version ├── logstreamer ├── bin │ └── .keep ├── pkg │ └── .keep ├── src │ └── .keep ├── Makefile ├── test_setup.sh ├── README.md ├── logstreamer.go └── logstreamer_test.go ├── .gitignore ├── .travis.yml ├── tasks ├── resque.rake └── kochiku.rake ├── config ├── kochiku-worker.yml ├── kochiku-worker.monit.d.example ├── deploy │ └── production.rb ├── deploy_hosts.yml ├── deploy_hosts.rb ├── kochiku-worker.init.d.example └── deploy.rb ├── lib ├── kochiku │ ├── build_strategies │ │ ├── no_op_strategy.rb │ │ ├── log_and_random_fail_strategy.rb │ │ └── build_all_strategy.rb │ ├── build_strategy_factory.rb │ ├── helpers │ │ └── benchmark.rb │ ├── jobs │ │ ├── shutdown_instance_job.rb │ │ ├── job_base.rb │ │ └── build_attempt_job.rb │ ├── build_strategy.rb │ ├── git_repo.rb │ ├── settings.rb │ ├── worker.rb │ └── git_strategies │ │ ├── shared_cache_strategy.rb │ │ └── local_cache_strategy.rb └── capistrano │ └── tasks │ ├── deploy.cap │ └── kochiku.cap ├── Gemfile ├── Rakefile ├── Capfile ├── spec ├── kochiku │ ├── git_strategies │ │ └── local_cache_strategy_spec.rb │ ├── settings_spec.rb │ └── build_strategies │ │ └── build_all_strategy_spec.rb ├── spec_helper.rb └── jobs │ └── build_attempt_job_spec.rb ├── README.md ├── Gemfile.lock └── LICENSE.txt /tmp/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.ruby-version: -------------------------------------------------------------------------------- 1 | 2.4.1 2 | -------------------------------------------------------------------------------- /logstreamer/bin/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /logstreamer/pkg/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /logstreamer/src/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .bundle/ 2 | tmp/ 3 | log/ 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | cache: bundler 3 | language: ruby 4 | rvm: 5 | - 2.3.6 6 | - 2.4.3 7 | - 2.5.0 8 | script: bundle exec rspec 9 | -------------------------------------------------------------------------------- /tasks/resque.rake: -------------------------------------------------------------------------------- 1 | require 'resque/tasks' 2 | 3 | namespace :resque do 4 | task :setup do 5 | require 'kochiku/worker' 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /config/kochiku-worker.yml: -------------------------------------------------------------------------------- 1 | kochiku_web_server_host: localhost:3000 2 | kochiku_web_server_protocol: http 3 | build_strategy: random 4 | redis_host: localhost 5 | redis_port: 6379 6 | -------------------------------------------------------------------------------- /logstreamer/Makefile: -------------------------------------------------------------------------------- 1 | all: build 2 | 3 | logstreamer: logstreamer.go 4 | go build 5 | setup: 6 | ./test_setup.sh 7 | 8 | build: logstreamer 9 | 10 | test: setup build 11 | go test 12 | 13 | clean: 14 | rm logstreamer 15 | -------------------------------------------------------------------------------- /lib/kochiku/build_strategies/no_op_strategy.rb: -------------------------------------------------------------------------------- 1 | module BuildStrategy 2 | class NoOpStrategy 3 | def execute_build(build_attempt_id, build_kind, test_files, test_command, timeout, options) 4 | true 5 | end 6 | 7 | def log_files_glob 8 | [] 9 | end 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /logstreamer/test_setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | mkdir -p logs/100/ 4 | mkdir -p logs/101/ 5 | 6 | cat > logs/100/stdout.log < logs/101/stdout.log < 3.0', require: false 14 | gem 'capistrano-bundler', '~> 1.1', require: false 15 | gem 'capistrano-rvm', require: false 16 | end 17 | 18 | group :test do 19 | gem "rspec", "~> 3.0" 20 | gem "webmock", require: false 21 | gem "memfs" 22 | end 23 | -------------------------------------------------------------------------------- /config/deploy_hosts.yml: -------------------------------------------------------------------------------- 1 | # set this to the location where Kochiku web is running 2 | kochiku_web_host: "kochiku.example.com" 3 | kochiku_web_protocol: "https" 4 | 5 | # Change this if you want to run Redis on a different machine than the build master 6 | # This needs to be the same as the Redis instance that the Kochiku application uses 7 | redis_host: "kochiku.example.com" 8 | 9 | # Change this to the hostnames for the kochiku worker boxes that you want to deploy to 10 | worker_hosts: 11 | - worker1.example.com 12 | - worker2.example.com 13 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'bundler/setup' 2 | $:.unshift(File.expand_path(File.join(File.dirname(__FILE__), 'lib'))) 3 | 4 | require 'resque/tasks' 5 | 6 | Dir.glob('tasks/*.rake').each { |r| import r } 7 | 8 | begin 9 | require 'rspec/core/rake_task' 10 | desc "run spec tests" 11 | RSpec::Core::RakeTask.new(:spec) do |t| 12 | t.pattern = 'spec/**/*_spec.rb' 13 | end 14 | 15 | task :default => :spec 16 | rescue LoadError => e 17 | # We get this error on the deployed workers because rspec is our test bundle and not deployed 18 | raise unless e.message.include?('rspec/core/rake_task') 19 | end 20 | -------------------------------------------------------------------------------- /lib/capistrano/tasks/deploy.cap: -------------------------------------------------------------------------------- 1 | # Users may choose to: 2 | # 3 | # A) edit the deploy:restart task directly inside this file 4 | # B) create another .cap file and define your deploy:restart task inside 5 | # 6 | # Option B is recommended because it will make merging in upstream Kochiku 7 | # changes easy. 8 | namespace :deploy do 9 | 10 | # desc 'Restart workers' 11 | # task :restart do 12 | # on roles(:all) do 13 | # # Necessary step to restart the Resque workers specific to your 14 | # # deployment 15 | # end 16 | # end 17 | 18 | after :publishing, :restart 19 | end 20 | # vi: filetype=ruby 21 | -------------------------------------------------------------------------------- /Capfile: -------------------------------------------------------------------------------- 1 | # Load DSL and Setup Up Stages 2 | require 'capistrano/setup' 3 | 4 | # Includes default deployment tasks 5 | require 'capistrano/deploy' 6 | 7 | # Includes tasks from other gems included in your Gemfile 8 | require 'capistrano/bundler' 9 | 10 | # If you would like to use a Ruby version manager with kochiku-worker 11 | # require it from a .cap file in lib/capistrano/tasks/. 12 | # 13 | # For more information see: 14 | # http://capistranorb.com/documentation/frameworks/rbenv-rvm-chruby/ 15 | 16 | # Loads custom tasks from `lib/capistrano/tasks' if you have any defined. 17 | Dir.glob('lib/capistrano/tasks/*.cap').each { |r| import r } 18 | -------------------------------------------------------------------------------- /lib/kochiku/jobs/shutdown_instance_job.rb: -------------------------------------------------------------------------------- 1 | # This is run on a Kochiku worker when the auto scaling algorithm wants to 2 | # shrink the number of workers. 3 | # 4 | # It is likely that if you want to use auto scaling you will need to adjust the 5 | # commands run by this job for your setup. 6 | class ShutdownInstanceJob < JobBase 7 | def self.perform 8 | # Tell the parent Resque process to exit after it finishes processing this job 9 | Process.kill("QUIT", Process.ppid) 10 | 11 | # Shutdown the instance 1 minute from now. Requires NOPASSWD sudo 12 | pid = Process.spawn("sudo shutdown -h +1") 13 | Process.detach(pid) 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /lib/kochiku/jobs/job_base.rb: -------------------------------------------------------------------------------- 1 | class JobBase 2 | class << self 3 | def enqueue(*args) 4 | Resque.enqueue(self, *args) 5 | end 6 | 7 | def enqueue_on(build_queue, *args) 8 | Resque::Job.create(build_queue, self, *args) 9 | Resque::Plugin.after_enqueue_hooks(self).each do |hook| 10 | klass.send(hook, *args) 11 | end 12 | end 13 | 14 | def perform(*args) 15 | job = new(*args) 16 | job.perform 17 | rescue => e 18 | if job 19 | job.on_exception(e) 20 | else 21 | raise e 22 | end 23 | end 24 | end 25 | 26 | def on_exception(e) 27 | raise e 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /config/deploy_hosts.rb: -------------------------------------------------------------------------------- 1 | class ConfigAccessor 2 | def initialize(yaml) 3 | @hash = YAML.load(yaml) 4 | end 5 | 6 | def kochiku_web_protocol 7 | @hash['kochiku_web_protocol'] 8 | end 9 | 10 | def kochiku_web_host 11 | @hash['kochiku_web_host'] 12 | end 13 | 14 | def redis_host 15 | @hash['redis_host'] 16 | end 17 | 18 | def worker_hosts 19 | @hash['worker_hosts'] 20 | end 21 | end 22 | 23 | CONF_FILE = File.expand_path('deploy_hosts.yml', File.dirname(__FILE__)) 24 | 25 | if !File.exist?(CONF_FILE) 26 | raise "#{CONF_FILE} is required to deploy kochiku-worker" 27 | else 28 | HostSettings = ConfigAccessor.new(File.read(CONF_FILE)) 29 | end 30 | -------------------------------------------------------------------------------- /config/kochiku-worker.init.d.example: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | WORKER_DIR=/home/kochiku/kochiku-worker/current 3 | export PIDFILE=/home/kochiku/kochiku-worker/shared/pids/kochiku-worker.pid 4 | queues=ci,developer 5 | 6 | cd $WORKER_DIR 7 | 8 | case "$1" in 9 | start) 10 | rm -f $PIDFILE 11 | su -c 'QUEUES='${queues:-"*"}' VERBOSE=1 rake resque:work > '$WORKER_DIR'/log/resque.log 2>&1 &' kochiku 12 | ;; 13 | stop) 14 | kill -s QUIT $(cat $PIDFILE) && rm -f $PIDFILE 15 | exit 0 16 | ;; 17 | pause) 18 | kill -s USR2 $(cat $PIDFILE) 19 | exit 0 20 | ;; 21 | cont) 22 | kill -s CONT $(cat $PIDFILE) 23 | exit 0 24 | ;; 25 | restart) 26 | $0 stop 27 | $0 start 28 | ;; 29 | status) 30 | ps -e -o pid,command | grep [r]esque 31 | ;; 32 | *) 33 | echo "Usage: $0 {start|stop|restart|pause|cont|status}" 34 | exit 1 35 | esac 36 | -------------------------------------------------------------------------------- /spec/kochiku/git_strategies/local_cache_strategy_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | RSpec.describe GitStrategy::LocalCache do 4 | describe "#run_git_fetch" do 5 | before do 6 | Retryable.configure do |config| 7 | config.sleep_method = Proc.new { } # do nothing 8 | end 9 | Retryable.enable 10 | end 11 | after { Retryable.disable } 12 | 13 | it "should throw an exception after the third fetch attempt" do 14 | allow(Kochiku::Worker).to receive(:logger) { double('logger', :warn => nil) } 15 | fetch_double = double('git fetch') 16 | expect(fetch_double).to receive(:run).exactly(3).times.and_raise(Cocaine::ExitStatusError) 17 | expect(Cocaine::CommandLine).to receive(:new).with('git fetch', anything).and_return(fetch_double).exactly(3).times 18 | 19 | expect { GitStrategy::LocalCache.send(:run_git_fetch) }.to raise_error(Cocaine::ExitStatusError) 20 | end 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /lib/kochiku/build_strategy.rb: -------------------------------------------------------------------------------- 1 | module BuildStrategy 2 | 3 | def self.execute_with_timeout(command, timeout, log_file) 4 | exit_status = nil 5 | 6 | dir = File.dirname(log_file) 7 | FileUtils.mkdir_p(dir) unless Dir.exists?(dir) 8 | File.open(log_file, "a") do |file| 9 | file.write(Array(command).join(" ") + "\n") 10 | end 11 | pid = nil 12 | Bundler.with_clean_env do 13 | pid = Process.spawn(*command, :out => [log_file, "a"], :err => [:child, :out], :pgroup => true) 14 | end 15 | 16 | begin 17 | Timeout.timeout(timeout) do 18 | Process.wait(pid) 19 | end 20 | 21 | exit_status = ($?.exitstatus == 0) 22 | rescue Timeout::Error 23 | # Keeps this error out of Resque failures 24 | exit_status = false 25 | end 26 | 27 | return exit_status, pid 28 | end 29 | 30 | # run before forcibly killing a process or child-process 31 | # can be used to get a stack trace, etc. 32 | def self.on_terminate_hook(pid, command) 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /lib/kochiku/git_repo.rb: -------------------------------------------------------------------------------- 1 | require 'cocaine' 2 | 3 | module Kochiku 4 | module Worker 5 | class GitRepo 6 | class RefNotFoundError < StandardError; end 7 | 8 | WORKING_DIR = File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'tmp', 'build-partition')) 9 | 10 | class << self 11 | def inside_copy(repo_url, sha) 12 | dir = case Kochiku::Worker.settings.git_strategy 13 | when 'localcache' 14 | GitStrategy::LocalCache.clone_and_checkout(repo_url, sha) 15 | when 'sharedcache' 16 | GitStrategy::SharedCache.clone_and_checkout(repo_url, sha) 17 | else 18 | raise 'unknown git strategy' 19 | end 20 | 21 | Dir.chdir(dir) do 22 | yield 23 | end 24 | 25 | if Kochiku::Worker.settings.git_strategy == 'localcache' 26 | FileUtils.remove_entry(dir) 27 | end 28 | end 29 | end 30 | end 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Kochiku Worker 2 | ============== 3 | 4 | Kochiku-worker is the builder component of [Kochiku](https://github.com/square/kochiku). The worker code is deployed to each computer that runs [`BuildAttemptJob`s](https://github.com/square/kochiku-worker/blob/master/spec/jobs/build_attempt_job_spec.rb), which are enqueued by [`BuildPartitioningJob`s](https://github.com/square/kochiku/blob/master/spec/jobs/build_partitioning_job_spec.rb) from Kochiku. 5 | 6 | Since Kochiku uses [Resque](https://github.com/resque/resque) jobs, a kochiku-worker is essentially just a Resque worker. All of the techniques that you can use with Resque workers, such as the environment variables and Unix signals, also work with Kochiku workers. 7 | 8 | ### Running in development 9 | 10 | See the [Running Kochiku in development](https://github.com/square/kochiku/wiki/Hacking-on-Kochiku#running-kochiku-in-development) section of the Hacking on Kochiku wiki page. 11 | 12 | ### Deployment 13 | 14 | See the [Installation & Deployment](https://github.com/square/kochiku/wiki/Installation-&-Deployment#installing-workers) page of the Kochiku wiki. 15 | 16 | -------------------------------------------------------------------------------- /lib/capistrano/tasks/kochiku.cap: -------------------------------------------------------------------------------- 1 | namespace :kochiku do 2 | task :setup do 3 | on roles(:all) do 4 | execute :mkdir, '-p', "#{shared_path}/{build-partition,pids}" 5 | end 6 | end 7 | 8 | desc "Symlink the build-partition and pids directory into shared_path" 9 | task :symlinks do 10 | # Intentionally not using linked_dirs for these paths because it would 11 | # break the folder structure that we had in place before Capistrano 3. 12 | on roles(:all) do 13 | execute :ln, '-nfFs', shared_path.join('build-partition'), release_path.join('tmp/build-partition') 14 | execute :ln, '-nfFs', shared_path.join('pids'), release_path.join('tmp/pids') 15 | end 16 | end 17 | 18 | task :create_kochiku_worker_yaml do 19 | on roles(:all) do 20 | config = <<-END.gsub(/^\s*/, '') 21 | kochiku_web_server_host: #{HostSettings.kochiku_web_host} 22 | kochiku_web_server_protocol: #{HostSettings.kochiku_web_protocol} 23 | build_strategy: build_all 24 | redis_host: #{HostSettings.redis_host} 25 | END 26 | upload! StringIO.new(config), release_path.join("config/kochiku-worker.yml") 27 | end 28 | end 29 | end 30 | # vi: filetype=ruby 31 | -------------------------------------------------------------------------------- /logstreamer/README.md: -------------------------------------------------------------------------------- 1 | # Log streamer 2 | This is an optional log streamer component written in go that can be used with kochiku. It will stream your logs in real time as tests are running. 3 | To run, it requires the master kochiku instance to communicate with the worker nodes on an additional port (that you specify). 4 | 5 | ## instructions to use with kochiku 6 | 7 | - upgrade your kochiku master instance to the latest version on github 8 | - follow the build instructions to generate the logstreamer binary 9 | - copy the logstreamer binary onto each kochiku worker. The workers expect this binary to exist in the logstreamer subdirectory (so the binary will be located at $(KOCHIKU_WORKER_ROOT)/logstreamer/logstreamer) 10 | - in your config/kochiku-worker.yml file, add the following line: `logstreaming_port: $(PORT_NUMBER)` (choose any open, accessible port to reach the master) 11 | 12 | ## build instructions 13 | 14 | You'll need to download the go distribution (https://golang.org/doc/install) 15 | 16 | ``` 17 | export GOPATH=$(pwd) 18 | go get github.com/julienschmidt/httprouter 19 | go get github.com/stretchr/testify/assert 20 | make 21 | ``` 22 | 23 | ## test 24 | 25 | ``` 26 | make test 27 | ``` 28 | 29 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | $:.unshift(File.expand_path('../lib', File.dirname(__FILE__))) 2 | 3 | # Setup env variable to use as a trigger for log level during testing 4 | ENV['RACK_ENV'] = 'test' 5 | 6 | require 'rubygems' 7 | require 'bundler/setup' 8 | 9 | require 'rspec' 10 | require 'webmock/rspec' 11 | require 'memfs' 12 | 13 | require 'kochiku/worker' 14 | 15 | Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| load f} 16 | 17 | FileUtils.mkdir_p("log") 18 | test_logger = Logger.new("log/test.log", 2) 19 | test_logger.level = Logger::DEBUG 20 | Kochiku::Worker.logger = test_logger 21 | 22 | RSpec.configure do |config| 23 | config.expose_dsl_globally = false 24 | 25 | config.mock_with :rspec do |mocks| 26 | mocks.syntax = :expect 27 | mocks.patch_marshal_to_support_partial_doubles = false 28 | end 29 | 30 | config.mock_with :rspec do |expectations| 31 | expectations.syntax = :expect 32 | end 33 | 34 | config.disable_monkey_patching! # same as what's above 35 | 36 | config.before :suite do 37 | # Disable Retryable; enable for individual tests if desired. 38 | Retryable.disable 39 | end 40 | 41 | config.before :each do 42 | WebMock.disable_net_connect! 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /config/deploy.rb: -------------------------------------------------------------------------------- 1 | # Lock version to protect against cap command being called without bundle exec 2 | # and executing with another version 3 | lock '3.2.1' 4 | 5 | set :application, "kochiku-worker" 6 | set :repo_url, "https://github.com/square/kochiku-worker.git" 7 | set :user, "kochiku" 8 | 9 | ask :branch, proc { `git rev-parse --abbrev-ref HEAD`.chomp } 10 | 11 | # Default value for :format is :pretty 12 | # set :format, :pretty 13 | 14 | # Default value for :log_level is :debug 15 | # set :log_level, :debug 16 | 17 | # Default value for :pty is false 18 | # set :pty, true 19 | 20 | set :linked_dirs, %w{log} 21 | 22 | # Default value for default_env is {} 23 | # set :default_env, { path: "/opt/ruby/bin:$PATH" } 24 | 25 | # Reference Capistrano's flow diagram for help choosing hooks 26 | # http://capistranorb.com/documentation/getting-started/flow/ 27 | before "deploy:started", "kochiku:setup" 28 | after "deploy:symlink:shared", "kochiku:symlinks" 29 | before "deploy:updated", "kochiku:create_kochiku_worker_yaml" 30 | 31 | # warn if a legacy deploy deploy.custom.rb is in place 32 | if File.exist?(File.expand_path('deploy.custom.rb', File.dirname(__FILE__))) 33 | warn "Kochiku-worker has upgraded to Capistrano 3. Placing custom capistrano config in deploy.custom.rb is no longer supported. Please move Capistrano settings to config/deploy/production.rb and remove deploy.custom.rb to make this message go away." 34 | exit(1) 35 | end 36 | -------------------------------------------------------------------------------- /lib/kochiku/settings.rb: -------------------------------------------------------------------------------- 1 | require "erb" 2 | require "ostruct" 3 | require "yaml" 4 | 5 | module Kochiku 6 | module Worker 7 | class Settings < OpenStruct 8 | def initialize(root_dir) 9 | config_file = File.join(root_dir, "config/kochiku-worker.yml") 10 | user_defined_options = load_config(config_file) || {} 11 | 12 | contents = {} 13 | contents["kochiku_web_server_host"] = user_defined_options["kochiku_web_server_host"] || "localhost" 14 | contents["kochiku_web_server_protocol"] = user_defined_options["kochiku_web_server_protocol"] 15 | contents["build_strategy"] = user_defined_options["build_strategy"] || "no_op" 16 | contents["redis_host"] = user_defined_options["redis_host"] || "localhost" 17 | contents["redis_port"] = user_defined_options["redis_port"] || "6379" 18 | contents["git_strategy"] = user_defined_options["git_strategy"] || "localcache" 19 | contents["git_shared_root"] = user_defined_options["git_shared_root"] 20 | contents["logstreamer_port"] = user_defined_options['logstreamer_port'] 21 | contents["running_on_ec2"] = user_defined_options['running_on_ec2'] || false 22 | 23 | validate!(contents) 24 | 25 | @keys = contents.keys.sort 26 | 27 | super(contents) 28 | end 29 | 30 | def validate!(config) 31 | raise 'git_shared_root required for sharedcache.' if config["git_strategy"] == "sharedcache" && config["git_shared_root"].nil? 32 | raise 'logstreamer_port must be valid port number.' if config['logstreamer_port'] && !(config['logstreamer_port'].is_a?(Integer)) 33 | end 34 | 35 | def inspect 36 | self.class.name + ":\n" + @keys.map{|k|" #{k.ljust(20)} = #{send(k.to_sym)}"}.join("\n") + "\n" 37 | end 38 | 39 | private 40 | 41 | def load_config(config_file) 42 | if File.exist?(config_file) 43 | config_yaml = ERB.new(File.read(config_file)).result 44 | YAML.load(config_yaml) 45 | end 46 | end 47 | end 48 | end 49 | end 50 | -------------------------------------------------------------------------------- /spec/kochiku/settings_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | RSpec.describe Kochiku::Worker::Settings do 4 | describe "#initialize" do 5 | around do |example| 6 | MemFs.activate do 7 | example.run 8 | end 9 | end 10 | 11 | let(:settings) { Kochiku::Worker::Settings.new('.') } 12 | 13 | it 'sets default values when a config file is not present' do 14 | expect(settings.kochiku_web_server_host).to eq "localhost" 15 | expect(settings.kochiku_web_server_protocol).to be_nil 16 | expect(settings.build_strategy).to eq "no_op" 17 | expect(settings.redis_host).to eq "localhost" 18 | expect(settings.redis_port).to eq "6379" 19 | expect(settings.git_strategy).to eq "localcache" 20 | expect(settings.git_shared_root).to be_nil 21 | expect(settings.logstreamer_port).to be_nil 22 | end 23 | 24 | it 'sets default values when the config file is empty' do 25 | write_config_file('---') 26 | expect(settings.kochiku_web_server_host).to eq "localhost" 27 | end 28 | 29 | it 'loads a YAML file with settings' do 30 | write_config_file(<<-YAML) 31 | kochiku_web_server_protocol: https 32 | redis_port: 1234 33 | YAML 34 | 35 | expect(settings.kochiku_web_server_protocol).to eq "https" 36 | expect(settings.redis_host).to eq "localhost" 37 | expect(settings.redis_port).to eq 1234 38 | end 39 | 40 | it 'loads a YAML file with settings' do 41 | write_config_file(<<-YAML) 42 | kochiku_web_server_protocol: https 43 | redis_port: 1234 44 | YAML 45 | 46 | expect(settings.kochiku_web_server_protocol).to eq 'https' 47 | expect(settings.redis_port).to eq 1234 48 | end 49 | 50 | it 'allows interpreting ERB in the config file' do 51 | write_config_file(<<-YAML) 52 | redis_port: <%= 123 * 10 %> 53 | YAML 54 | 55 | expect(settings.redis_port).to eq 1230 56 | end 57 | 58 | it 'raises when there is no git_shared_root for the shared cache strategy' do 59 | write_config_file(<<-YAML) 60 | git_strategy: sharedcache 61 | git_shared_root: 62 | YAML 63 | 64 | expect { settings }.to raise_error( 65 | StandardError, 66 | 'git_shared_root required for sharedcache.' 67 | ) 68 | end 69 | 70 | it 'raises when there is an invalid logstreamer port' do 71 | write_config_file(<<-YAML) 72 | logstreamer_port: foo 73 | YAML 74 | 75 | expect { settings }.to raise_error( 76 | StandardError, 77 | 'logstreamer_port must be valid port number.' 78 | ) 79 | end 80 | 81 | def write_config_file(contents) 82 | MemFs.touch('config/kochiku-worker.yml') 83 | File.open('config/kochiku-worker.yml', 'w') { |f| f.write(contents) } 84 | end 85 | end 86 | end 87 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: https://rubygems.org/ 3 | specs: 4 | activesupport (4.2.5.1) 5 | i18n (~> 0.7) 6 | json (~> 1.7, >= 1.7.7) 7 | minitest (~> 5.1) 8 | thread_safe (~> 0.3, >= 0.3.4) 9 | tzinfo (~> 1.1) 10 | addressable (2.5.1) 11 | public_suffix (~> 2.0, >= 2.0.2) 12 | capistrano (3.2.1) 13 | i18n 14 | rake (>= 10.0.0) 15 | sshkit (~> 1.3) 16 | capistrano-bundler (1.1.2) 17 | capistrano (~> 3.0) 18 | sshkit (~> 1.2) 19 | capistrano-rvm (0.1.1) 20 | capistrano (~> 3.0) 21 | sshkit (~> 1.2) 22 | climate_control (0.0.3) 23 | activesupport (>= 3.0) 24 | cocaine (0.5.8) 25 | climate_control (>= 0.0.3, < 1.0) 26 | colorize (0.7.3) 27 | crack (0.4.3) 28 | safe_yaml (~> 1.0.0) 29 | diff-lcs (1.2.5) 30 | domain_name (0.5.20170404) 31 | unf (>= 0.0.5, < 1.0.0) 32 | hashdiff (0.3.4) 33 | http-cookie (1.0.3) 34 | domain_name (~> 0.5) 35 | i18n (0.7.0) 36 | json (1.8.6) 37 | memfs (1.0.0) 38 | mime-types (3.1) 39 | mime-types-data (~> 3.2015) 40 | mime-types-data (3.2016.0521) 41 | minitest (5.8.4) 42 | mono_logger (1.1.0) 43 | multi_json (1.10.1) 44 | net-scp (1.2.1) 45 | net-ssh (>= 2.6.5) 46 | net-ssh (2.9.1) 47 | netrc (0.11.0) 48 | public_suffix (2.0.5) 49 | rack (1.6.10) 50 | rack-protection (1.5.5) 51 | rack 52 | rake (10.3.2) 53 | redis (3.3.3) 54 | redis-namespace (1.5.0) 55 | redis (~> 3.0, >= 3.0.4) 56 | resque (1.25.2) 57 | mono_logger (~> 1.0) 58 | multi_json (~> 1.0) 59 | redis-namespace (~> 1.3) 60 | sinatra (>= 0.9.2) 61 | vegas (~> 0.1.2) 62 | rest-client (2.0.2) 63 | http-cookie (>= 1.0.2, < 2.0) 64 | mime-types (>= 1.16, < 4.0) 65 | netrc (~> 0.8) 66 | retryable (2.0.4) 67 | rspec (3.2.0) 68 | rspec-core (~> 3.2.0) 69 | rspec-expectations (~> 3.2.0) 70 | rspec-mocks (~> 3.2.0) 71 | rspec-core (3.2.1) 72 | rspec-support (~> 3.2.0) 73 | rspec-expectations (3.2.0) 74 | diff-lcs (>= 1.2.0, < 2.0) 75 | rspec-support (~> 3.2.0) 76 | rspec-mocks (3.2.1) 77 | diff-lcs (>= 1.2.0, < 2.0) 78 | rspec-support (~> 3.2.0) 79 | rspec-support (3.2.2) 80 | safe_yaml (1.0.4) 81 | sinatra (1.4.5) 82 | rack (~> 1.4) 83 | rack-protection (~> 1.4) 84 | tilt (~> 1.3, >= 1.3.4) 85 | sshkit (1.5.1) 86 | colorize 87 | net-scp (>= 1.1.2) 88 | net-ssh (>= 2.8.0) 89 | thread_safe (0.3.5) 90 | tilt (1.4.1) 91 | tzinfo (1.2.2) 92 | thread_safe (~> 0.1) 93 | unf (0.1.4) 94 | unf_ext 95 | unf_ext (0.0.7.4) 96 | vegas (0.1.11) 97 | rack (>= 1.0.0) 98 | webmock (3.0.1) 99 | addressable (>= 2.3.6) 100 | crack (>= 0.3.2) 101 | hashdiff 102 | 103 | PLATFORMS 104 | ruby 105 | 106 | DEPENDENCIES 107 | capistrano (~> 3.0) 108 | capistrano-bundler (~> 1.1) 109 | capistrano-rvm 110 | cocaine 111 | json 112 | memfs 113 | rake 114 | resque 115 | rest-client 116 | retryable 117 | rspec (~> 3.0) 118 | webmock 119 | 120 | BUNDLED WITH 121 | 1.16.1 122 | -------------------------------------------------------------------------------- /lib/kochiku/worker.rb: -------------------------------------------------------------------------------- 1 | require 'bundler/setup' 2 | Bundler.require 3 | 4 | require 'logger' 5 | 6 | require 'resque' 7 | require 'rest-client' 8 | 9 | require 'kochiku/settings' 10 | require 'kochiku/git_repo' 11 | require 'kochiku/helpers/benchmark' 12 | require 'kochiku/git_strategies/local_cache_strategy' 13 | require 'kochiku/git_strategies/shared_cache_strategy' 14 | 15 | require 'kochiku/build_strategy' 16 | require 'kochiku/build_strategies/build_all_strategy' 17 | require 'kochiku/build_strategies/log_and_random_fail_strategy' 18 | require 'kochiku/build_strategies/no_op_strategy' 19 | require 'kochiku/build_strategy_factory' 20 | 21 | require 'kochiku/jobs/job_base' 22 | require 'kochiku/jobs/build_attempt_job' 23 | require 'kochiku/jobs/shutdown_instance_job' 24 | 25 | module Kochiku 26 | module Worker 27 | class << self 28 | def settings 29 | @settings ||= Settings.new(File.expand_path(File.join(File.dirname(__FILE__), '..', '..'))) 30 | end 31 | 32 | def logger 33 | @logger ||= begin 34 | default_logger = Logger.new(STDOUT) 35 | default_logger.formatter = proc do |severity, datetime, progname, msg| 36 | "%5s [%s] %d: %s\n" % [severity, datetime.strftime('%H:%M:%S %Y-%m-%d'), $$, msg2str(msg)] 37 | end 38 | Cocaine::CommandLine.logger = default_logger 39 | default_logger 40 | end 41 | end 42 | 43 | def logger=(logger) 44 | @logger = logger 45 | Cocaine::CommandLine.logger = @logger 46 | end 47 | 48 | def msg2str(msg) 49 | case msg 50 | when ::String 51 | msg 52 | when ::Exception 53 | "#{ msg.message } (#{ msg.class })\n" << 54 | (msg.backtrace || []).join("\n") 55 | else 56 | msg.inspect 57 | end 58 | end 59 | 60 | def build_strategy 61 | @build_strategy ||= BuildStrategyFactory.get_strategy(settings.build_strategy) 62 | end 63 | 64 | def instance_type 65 | @instance_type ||= begin 66 | path = File.join(File.dirname(__FILE__), '..', '..', 'tmp', 'instance_type.txt') 67 | if File.file?(path) 68 | instance_type = File.read(path).chomp 69 | elsif settings.running_on_ec2 70 | instance_type = request_instance_type_from_ec2 71 | File.write(path, instance_type) if instance_type 72 | end 73 | instance_type 74 | end 75 | end 76 | 77 | def request_instance_type_from_ec2 78 | begin 79 | instance_type = RestClient::Request.execute(method: :get, 80 | url: "http://169.254.169.254/latest/meta-data/instance-type", 81 | open_timeout: 5).body 82 | rescue RestClient::Exceptions::OpenTimeout => e 83 | logger.warn "Error requesting ec2 metadata: #{e.inspect}" 84 | return nil 85 | end 86 | instance_type 87 | end 88 | end 89 | end 90 | end 91 | 92 | Resque.redis = Redis.new(:host => Kochiku::Worker.settings.redis_host, :port => Kochiku::Worker.settings.redis_port) 93 | Resque.redis.namespace = "resque:kochiku" 94 | 95 | RestClient.log = Kochiku::Worker.logger unless ENV['RACK_ENV'] == 'test' 96 | -------------------------------------------------------------------------------- /logstreamer/logstreamer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "encoding/json" 6 | "flag" 7 | "fmt" 8 | "log" 9 | "net/http" 10 | "os" 11 | "regexp" 12 | "strconv" 13 | 14 | "github.com/julienschmidt/httprouter" 15 | ) 16 | 17 | func status(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { 18 | w.WriteHeader(http.StatusOK) 19 | } 20 | 21 | type logResponse struct { 22 | Start int 23 | Contents string 24 | BytesRead int 25 | LogName string 26 | } 27 | 28 | // seek to Start, and then read to the last newline before/at maxBytes 29 | // if Start is -1, then tail from end of log 30 | func outputLog(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { 31 | // validate that id is a number 32 | // validate that logName is stdout.log (for now, only whitelisting this log file) 33 | // in the future, may support streaming of arbitrary log files. 34 | idString := ps.ByName("id") 35 | logName := ps.ByName("logName") 36 | 37 | digitRegex := regexp.MustCompile(`^\d+$`) 38 | validID := digitRegex.MatchString(idString) 39 | validLogName := (logName == "stdout.log") 40 | 41 | if validID && validLogName { 42 | start := r.URL.Query().Get("start") 43 | maxBytes := r.URL.Query().Get("maxBytes") 44 | 45 | startInt, err := strconv.Atoi(start) // defaults to 0 on parse error 46 | doTail := startInt < 0 47 | 48 | maxBytesInt, err := strconv.Atoi(maxBytes) 49 | if err != nil || maxBytesInt < 0 { 50 | maxBytesInt = 50000 // arbitrary default limit 51 | } 52 | 53 | // check for existence of file; if DNE return 404 54 | file, err := os.Open("logs/" + idString + "/" + logName) 55 | 56 | if err != nil { 57 | w.WriteHeader(http.StatusNotFound) 58 | return 59 | } 60 | 61 | defer file.Close() 62 | 63 | fi, err := file.Stat() 64 | if err != nil { 65 | w.WriteHeader(http.StatusNotFound) 66 | return 67 | } 68 | 69 | size := fi.Size() // File size is needed for tailing log 70 | 71 | if doTail { 72 | startInt = int(size) - maxBytesInt 73 | if startInt < 0 { 74 | startInt = 0 75 | doTail = false 76 | } 77 | } 78 | 79 | actualStart, err := file.Seek(int64(startInt), 0) 80 | 81 | reader := bufio.NewReader(file) 82 | 83 | if doTail { 84 | // read to next newline beginning from offset (we don't return partial lines) 85 | buf, _ := reader.ReadBytes('\n') 86 | actualStart += int64(len(buf)) 87 | } 88 | 89 | contents := "" 90 | byteCounter := 0 91 | 92 | buffer, err := reader.ReadBytes('\n') 93 | for err == nil { 94 | if byteCounter+len(buffer) > maxBytesInt { 95 | break 96 | } 97 | 98 | byteCounter += len(buffer) 99 | contents += string(buffer) 100 | buffer, err = reader.ReadBytes('\n') 101 | } 102 | 103 | response := logResponse{ 104 | Start: int(actualStart), 105 | Contents: contents, 106 | BytesRead: byteCounter, 107 | LogName: logName, 108 | } 109 | 110 | jsonResponse, err := json.Marshal(response) 111 | 112 | w.Header().Set("Content-Type", "application/json") 113 | w.Write(jsonResponse) 114 | fmt.Fprint(w, "\n") 115 | } else { 116 | http.Error(w, "Invalid params", 422) 117 | return 118 | } 119 | } 120 | 121 | func main() { 122 | portPtr := flag.Int("p", 8080, "port number") 123 | flag.Parse() 124 | port := strconv.Itoa(*portPtr) 125 | 126 | fmt.Println("Listening on port " + port) 127 | router := httprouter.New() 128 | router.GET("/_status", status) 129 | router.GET("/build_attempts/:id/log/:logName", outputLog) 130 | log.Fatal(http.ListenAndServe(":"+port, router)) 131 | } 132 | -------------------------------------------------------------------------------- /lib/kochiku/git_strategies/shared_cache_strategy.rb: -------------------------------------------------------------------------------- 1 | module GitStrategy 2 | # Uses alternate object stores to share object stores across worker nodes. This uses 3 | # git clone --shared for fast clones and checkouts. Repos are cloned 4 | # from a central location, which is typically an NFS mount on the workers. 5 | # 6 | # Scaling: 7 | # Unlike cloning over the git protocol, which is very cpu-intensive, this strategy 8 | # scales with available bandwidth on the server. Luckily, bandwidth use is mitigated somewhat 9 | # when using NFS thanks to client-side buffer cache. If you overwhelm the server, 10 | # get a bigger NIC or consider implementing improvement #1. 11 | # 12 | # When to use: 13 | # Use the shared strategy when you have enough workers to overwhelm your 14 | # normal git server and/or mirrors. 15 | # 16 | # Possible improvements: 17 | # 1. Add multiple shared roots and choose randomly between them. Poor man's client side load balancing. 18 | class SharedCache 19 | extend Benchmark 20 | 21 | class << self 22 | def clone_and_checkout(repo_url, commit) 23 | benchmark "SharedCache.clone_and_checkout(#{repo_url}, #{commit})" do 24 | repo_path = repo_url.match(/.+?([^\/]+\/[^\/]+\.git)\z/)[1] 25 | shared_repo_dir = File.join(Kochiku::Worker.settings.git_shared_root, repo_path) 26 | raise 'cannot find repo in shared repos' unless Dir.exists?(shared_repo_dir) 27 | 28 | # check that commit exists 29 | Dir.chdir(shared_repo_dir) do 30 | begin 31 | Cocaine::CommandLine.new('git', 'rev-list --quiet -n1 :commit').run(commit: commit) 32 | rescue Cocaine::ExitStatusError 33 | raise Kochiku::Worker::GitRepo::RefNotFoundError 34 | end 35 | end 36 | 37 | repo_namespace_and_name = repo_path.chomp('.git') 38 | repo_checkout_path = File.join(Kochiku::Worker::GitRepo::WORKING_DIR, repo_namespace_and_name) 39 | 40 | # No `git fetch` is needed if the clone already exists because the NFS 41 | # origin is continually up to date. However, a `git fetch` is needed if 42 | # you are going to be referencing a branch name and not a commit 43 | unless Dir.exist?(repo_checkout_path) 44 | Cocaine::CommandLine.new('git', 'clone --quiet --shared --no-checkout :repo :dir').run(repo: shared_repo_dir, dir: repo_checkout_path) 45 | end 46 | 47 | Dir.chdir(repo_checkout_path) do 48 | Cocaine::CommandLine.new('git', 'reset --hard').run 49 | Cocaine::CommandLine.new('git', 'clean -dfx -f').run 50 | Cocaine::CommandLine.new('git', 'checkout --quiet :commit').run(commit: commit) 51 | 52 | # init submodules 53 | Cocaine::CommandLine.new('git', 'submodule --quiet init').run 54 | 55 | # update submodules. attempt to use references. best-effort. 56 | submodules = Cocaine::CommandLine.new('git', 'config --get-regexp "^submodule\..*\.url$"', expected_outcodes: [0,1]).run 57 | submodules.each_line do |submodule| 58 | _, path, url = submodule.strip.match(/^submodule\.(.+?)\.url .+?([^\/]+\/[^\/\n]+)$/).to_a 59 | shared_repo_dir = File.join(Kochiku::Worker.settings.git_shared_root, url || 'does-not-exist') 60 | 61 | if Dir.exists?(shared_repo_dir) 62 | Cocaine::CommandLine.new('git', 'config --replace-all submodule.:path.url :shared').run(shared: shared_repo_dir, path: path) 63 | end 64 | Cocaine::CommandLine.new('git', 'submodule update -- :path').run(path: path) 65 | end 66 | end 67 | 68 | repo_checkout_path 69 | end 70 | end 71 | end 72 | end 73 | end 74 | -------------------------------------------------------------------------------- /lib/kochiku/git_strategies/local_cache_strategy.rb: -------------------------------------------------------------------------------- 1 | require 'cocaine' 2 | 3 | module GitStrategy 4 | # Keeps a full clone locally on the worker and uses it as a cache. 5 | # Uses git clone --local for fast clones and checkouts. Since caches are kept 6 | # in the same directory as temporary checkouts, they're guaranteed to be on the 7 | # same filesystem and git can hardlink objects under the hood. 8 | # 9 | # Scaling: 10 | # Git operations are required to update the local cache, and git operations 11 | # are cpu-intensive on the server side. Eventually the server will become constrained 12 | # on cpu. At that point, a read-only mirror can be created to absorb the load of the 13 | # worker cluster, or you can switch to the shared cache strategy. 14 | # 15 | # When to use: 16 | # This is the most basic and default git strategy. Use it when you aren't running 17 | # enough workers to overwhelm your primary git server or git mirror. 18 | class LocalCache 19 | extend Benchmark 20 | 21 | class << self 22 | def clone_and_checkout(repo_url, commit) 23 | benchmark "LocalCache.clone_and_checkout(#{repo_url}, #{commit})" do 24 | tmp_dir = Dir.mktmpdir(nil, Kochiku::Worker::GitRepo::WORKING_DIR) 25 | 26 | repo_path = repo_url.match(/.+?([^:\/]+\/[^\/]+)\.git\z/)[1] 27 | cached_repo_path = File.join(Kochiku::Worker::GitRepo::WORKING_DIR, repo_path) 28 | synchronize_cache_repo(cached_repo_path, repo_url, commit) 29 | 30 | # clone local repo (fast!) 31 | run! "git clone #{cached_repo_path} #{tmp_dir}" 32 | 33 | Dir.chdir(tmp_dir) do 34 | run! "git checkout --quiet #{commit}" 35 | 36 | run! "git submodule --quiet init" 37 | 38 | submodules = `git config --get-regexp "^submodule\\..*\\.url$"` 39 | 40 | unless submodules.empty? 41 | cached_submodules = `git config --get-regexp "^submodule\\..*\\.url$"` 42 | 43 | # Redirect the submodules to the cached_repo 44 | # If the submodule was added after the initial clone of the cache 45 | # repo then it will not be present in the cached_repo and we fall 46 | # back to cloning it for each build. 47 | submodules.each_line do |config_line| 48 | if cached_submodules.include?(config_line) 49 | submodule_path = config_line.match(/submodule\.(.*?)\.url/)[1] 50 | `git config --replace-all submodule.#{submodule_path}.url "#{cached_repo_path}/#{submodule_path}"` 51 | end 52 | end 53 | 54 | run! "git submodule --quiet update" 55 | end 56 | end 57 | 58 | tmp_dir 59 | end 60 | end 61 | 62 | private 63 | 64 | def synchronize_cache_repo(cached_repo_path, repo_url, commit) 65 | if !Dir.exist?(cached_repo_path) 66 | clone_repo(repo_url, cached_repo_path) 67 | end 68 | Dir.chdir(cached_repo_path) do 69 | harmonize_remote_url(repo_url) 70 | run_git_fetch 71 | 72 | if !commit.nil? && !system("git rev-list --quiet -n1 #{commit}") 73 | raise Kochiku::Worker::GitRepo::RefNotFoundError.new("Build Ref #{commit} not found in #{repo_url}") 74 | end 75 | 76 | Cocaine::CommandLine.new("git submodule update", "--init --quiet").run 77 | end 78 | end 79 | 80 | def run!(cmd) 81 | unless system(cmd) 82 | raise "non-0 exit code #{$?} returned from [#{cmd}]" 83 | end 84 | end 85 | 86 | def clone_repo(repo_url, cached_repo_path) 87 | Cocaine::CommandLine.new("git clone", "--recursive #{repo_url} #{cached_repo_path}").run 88 | end 89 | 90 | def run_git_fetch 91 | exception_cb = Proc.new do |exception| 92 | Kochiku::Worker.logger.warn(exception) 93 | end 94 | 95 | # likely caused by another 'git fetch' that is currently in progress. Wait a few seconds and try again 96 | Retryable.retryable(tries: 3, on: Cocaine::ExitStatusError, sleep: lambda { |n| 15*n }, exception_cb: exception_cb) do 97 | Cocaine::CommandLine.new("git fetch", "--quiet --prune --no-tags").run 98 | end 99 | end 100 | 101 | # Update the remote url for the git repository if it has changed 102 | def harmonize_remote_url(expected_url) 103 | remote_url = Cocaine::CommandLine.new("git config --get remote.origin.url").run.chomp 104 | if remote_url != expected_url 105 | puts "#{remote_url.inspect} does not match #{expected_url.inspect}. Updating it." 106 | Cocaine::CommandLine.new("git remote", "set-url origin #{expected_url}").run 107 | end 108 | end 109 | end 110 | end 111 | end 112 | -------------------------------------------------------------------------------- /logstreamer/logstreamer_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io/ioutil" 7 | "net/http" 8 | "net/http/httptest" 9 | "testing" 10 | 11 | "github.com/julienschmidt/httprouter" 12 | "github.com/stretchr/testify/assert" 13 | ) 14 | 15 | func TestStatus(t *testing.T) { 16 | req, err := http.NewRequest("GET", "http://localhost:8080/_status", nil) 17 | if err != nil { 18 | t.FailNow() 19 | } 20 | 21 | w := httptest.NewRecorder() 22 | status(w, req, nil) 23 | 24 | fmt.Println(w.Code) 25 | assert.Equal(t, 200, w.Code, "") 26 | } 27 | 28 | func outputLogHelper(t *testing.T, id string, logName string, querystring string) *httptest.ResponseRecorder { 29 | request_url := "http://localhost:8080/build_attempts/" + id + "/log/" + logName + "?" + querystring 30 | 31 | var ps httprouter.Params 32 | ps = make([]httprouter.Param, 2) 33 | idParam := httprouter.Param{Key: "id", Value: id} 34 | logNameParam := httprouter.Param{Key: "logName", Value: logName} 35 | 36 | ps[0] = idParam 37 | ps[1] = logNameParam 38 | 39 | req, err := http.NewRequest("GET", request_url, nil) 40 | if err != nil { 41 | t.FailNow() 42 | } 43 | 44 | w := httptest.NewRecorder() 45 | outputLog(w, req, ps) 46 | return w 47 | } 48 | 49 | func outputLogHelperJson(t *testing.T, id string, logName string, querystring string) (int, logResponse) { 50 | w := outputLogHelper(t, id, logName, querystring) 51 | 52 | body, err := ioutil.ReadAll(w.Body) 53 | if err != nil { 54 | t.FailNow() 55 | } 56 | 57 | var resp logResponse 58 | err = json.Unmarshal(body, &resp) 59 | if err != nil { 60 | t.FailNow() 61 | } 62 | 63 | return w.Code, resp 64 | } 65 | 66 | func TestOutputLogDoesNotExist(t *testing.T) { 67 | w := outputLogHelper(t, "999", "stdout.log", "") 68 | assert.Equal(t, 404, w.Code, "Returns 404 for log that does not exist") 69 | } 70 | 71 | func TestOutputLogDoesExist(t *testing.T) { 72 | w := outputLogHelper(t, "100", "stdout.log", "") 73 | assert.Equal(t, 200, w.Code, "Returns 200 for log that exists") 74 | } 75 | 76 | func TestOutputLogInvalidInput(t *testing.T) { 77 | w := outputLogHelper(t, "a", "stdout.log", "") 78 | assert.Equal(t, 422, w.Code, "Returns 422 for invalid build attempt name.") 79 | 80 | w = outputLogHelper(t, "a", "not_stdout", "") 81 | assert.Equal(t, 422, w.Code, "Returns 422 for invalid log name") 82 | } 83 | 84 | func TestOutputLogEntireLog(t *testing.T) { 85 | respCode, resp := outputLogHelperJson(t, "100", "stdout.log", "") 86 | assert.Equal(t, 200, respCode, "") 87 | assert.Equal(t, 0, resp.Start, "") 88 | assert.Equal(t, 27, resp.BytesRead, "") 89 | assert.Equal(t, "Hello\nline 2\nline 3\nline 4\n", resp.Contents, "") 90 | } 91 | 92 | func TestOutputLogPartialLogHead(t *testing.T) { 93 | respCode, resp := outputLogHelperJson(t, "100", "stdout.log", "maxBytes=13") 94 | assert.Equal(t, 200, respCode, "") 95 | assert.Equal(t, 0, resp.Start, "") 96 | assert.Equal(t, 13, resp.BytesRead, "") 97 | assert.Equal(t, "Hello\nline 2\n", resp.Contents, "") 98 | } 99 | 100 | func TestOutputLogPartialLogTail(t *testing.T) { 101 | respCode, resp := outputLogHelperJson(t, "100", "stdout.log", "start=1") 102 | assert.Equal(t, 200, respCode, "") 103 | assert.Equal(t, 1, resp.Start, "") 104 | assert.Equal(t, 26, resp.BytesRead, "") 105 | assert.Equal(t, "ello\nline 2\nline 3\nline 4\n", resp.Contents, "") 106 | } 107 | 108 | func TestOutputLogNotEnoughBytes(t *testing.T) { 109 | respCode, resp := outputLogHelperJson(t, "100", "stdout.log", "start=6&maxBytes=5") 110 | assert.Equal(t, 200, respCode, "") 111 | assert.Equal(t, 6, resp.Start, "") 112 | assert.Equal(t, 0, resp.BytesRead, "") 113 | assert.Equal(t, "", resp.Contents, "") 114 | } 115 | 116 | func TestOutputLogEmpty(t *testing.T) { 117 | respCode, resp := outputLogHelperJson(t, "100", "stdout.log", "maxBytes=0") 118 | assert.Equal(t, 200, respCode, "") 119 | assert.Equal(t, 0, resp.Start, "") 120 | assert.Equal(t, 0, resp.BytesRead, "") 121 | assert.Equal(t, "", resp.Contents, "") 122 | } 123 | 124 | func TestOutputLogInvalidStart(t *testing.T) { 125 | respCode, resp := outputLogHelperJson(t, "100", "stdout.log", "start=junk") 126 | assert.Equal(t, 200, respCode, "") 127 | assert.Equal(t, 0, resp.Start, "start should default to 0 on error") 128 | } 129 | 130 | func TestOutputLogInvalidNumLines(t *testing.T) { 131 | respCode, resp := outputLogHelperJson(t, "100", "stdout.log", "maxBytes=junk") 132 | assert.Equal(t, 200, respCode, "") 133 | assert.Equal(t, 27, resp.BytesRead, "should return entire log on invalid numLines") 134 | } 135 | 136 | func TestOutputLogTail(t *testing.T) { 137 | respCode, resp := outputLogHelperJson(t, "100", "stdout.log", "start=-1&maxBytes=8") 138 | assert.Equal(t, 200, respCode, "") 139 | assert.Equal(t, 7, resp.BytesRead, "should return last line") 140 | assert.Equal(t, "line 4\n", resp.Contents, "") 141 | } 142 | -------------------------------------------------------------------------------- /lib/kochiku/build_strategies/build_all_strategy.rb: -------------------------------------------------------------------------------- 1 | require 'fileutils' 2 | 3 | module BuildStrategy 4 | class BuildAllStrategy 5 | class ErrorFoundInLogError < StandardError; end 6 | 7 | LOG_FILE = "log/stdout.log" 8 | KILL_TIMEOUT = 10 9 | 10 | def execute_build(build_attempt_id, build_kind, test_files, test_command, timeout, options) 11 | @build_attempt_id = build_attempt_id 12 | if options['log_file_globs'] 13 | @log_files = options['log_file_globs'] << LOG_FILE 14 | end 15 | if options['logstreamer_enabled'] 16 | hardlink_log(LOG_FILE) 17 | end 18 | execute_with_timeout_and_kill(ci_command(build_kind, test_files, test_command, options), timeout) 19 | end 20 | 21 | # log persistence needed for logstreamer 22 | def hardlink_log(log) 23 | FileUtils.mkdir_p("log") 24 | FileUtils.touch(log) 25 | 26 | kochiku_base_dir = File.join(__dir__, "../../..") 27 | 28 | FileUtils.mkdir_p("#{kochiku_base_dir}/logstreamer/logs/#{@build_attempt_id}/") 29 | FileUtils.ln(log, "#{kochiku_base_dir}/logstreamer/logs/#{@build_attempt_id}/stdout.log") 30 | end 31 | 32 | def log_files_glob 33 | @log_files ||= [LOG_FILE] 34 | end 35 | 36 | def execute_with_timeout_and_kill(command, timeout) 37 | success, pid = BuildStrategy.execute_with_timeout(command, timeout, LOG_FILE) 38 | success 39 | ensure 40 | processes_killed = kill_process_group(pid, 15) 41 | 42 | if processes_killed.length > 0 43 | File.open(LOG_FILE, 'a') do |file| 44 | file.write("\n\n******** The following process(es) taking too long, Kochiku killing NOW ************\n") 45 | file.write(processes_killed.join("\n")) 46 | end 47 | end 48 | 49 | check_log_for_errors! unless success 50 | end 51 | 52 | # returns array of the process commands killed (or empty array if none). 53 | def kill_process_group(pid, sig = 15) 54 | processes_killed = [] 55 | 56 | # Kill the head process 57 | # We cannot kill the process group if head is a zombie process 58 | begin 59 | Timeout.timeout(KILL_TIMEOUT) do 60 | ps_entry = `ps p #{pid} -o pid,state,command | tail -n +2`.strip 61 | 62 | unless ps_entry == "" 63 | parsed_entry = /(?.*?)\s+(?.*?)\s+(?.*)/.match(ps_entry) 64 | ps_pid = parsed_entry["pid"].to_i 65 | ps_state = parsed_entry["state"] 66 | ps_command = parsed_entry["command"] 67 | 68 | # Don't record zombie processes 69 | unless ps_state =~ /Z/ 70 | BuildStrategy.on_terminate_hook(ps_pid, ps_command) 71 | processes_killed << ps_command 72 | end 73 | end 74 | 75 | Process.kill(sig, pid) 76 | Process.wait(pid) 77 | end 78 | rescue Timeout::Error 79 | Process.kill(9, pid) 80 | Process.wait(pid) 81 | rescue Errno::ESRCH, Errno::ECHILD # Process has already exited 82 | end 83 | 84 | # Kill the rest of the process group 85 | begin 86 | Timeout.timeout(KILL_TIMEOUT) do 87 | list_processes = processes_in_same_group(pid) 88 | if list_processes.empty? 89 | return processes_killed 90 | else 91 | list_processes.each do |process_id, command| 92 | BuildStrategy.on_terminate_hook(process_id, command) 93 | end 94 | processes_killed += list_processes.values 95 | end 96 | 97 | # (-sig) sends sig to the entire process group 98 | Process.kill(-sig, pid) 99 | 100 | # wait for all processes in group to exit 101 | while processes_in_same_group(pid).length > 0 do 102 | sleep 1 103 | end 104 | end 105 | rescue Timeout::Error 106 | # The processes did not exit within the timeout 107 | # no more CPU time for the child processes 108 | sleep 1 109 | kill_process_group(pid, 9) 110 | rescue Errno::ESRCH, Errno::ECHILD # Process has already exited 111 | end 112 | 113 | processes_killed 114 | end 115 | 116 | # returns hash of pid => command for processes in group pgid 117 | def processes_in_same_group(pgid) 118 | open_processes = `ps -eo pid,pgid,state,command | tail -n +2`.strip.split("\n").map { |x| x.strip } 119 | parsed_processes = open_processes.map { |x| /(?.*?)\s+(?.*?)\s+(?.*?)\s+(?.*)/.match(x) } 120 | .select { |x| x["pgid"].to_i == pgid && x["state"] !~ /Z/ } 121 | pid_commands = {} 122 | 123 | parsed_processes.each do |process| 124 | pid_commands[process["pid"].to_i] = process["command"] 125 | end 126 | 127 | return pid_commands 128 | end 129 | 130 | private 131 | 132 | def ci_command(build_kind, test_files, test_command, options) 133 | ruby_command = if options["ruby"] 134 | "rvm --install --create use #{options["ruby"]}" 135 | else 136 | "if [ -e .rvmrc ]; then source .rvmrc; elif [ -e .ruby-version ]; then rvm --install --create use $(cat .ruby-version); fi" 137 | end 138 | 139 | java_options = "" 140 | 141 | if build_kind == "maven" && options['total_workers'] && options['worker_chunk'] 142 | java_options = " _JAVA_OPTIONS=\"-Dsquare.test.chunkCount=#{options['total_workers']} -Dsquare.test.runChunk=#{options['worker_chunk']}\"" 143 | end 144 | 145 | ( 146 | "env -i HOME=$HOME" + 147 | " USER=$USER" + 148 | " LANG=en_US.UTF-8" + 149 | " LANGUAGE=en_US.UTF-8" + 150 | " LC_ALL=en_US.UTF-8" + 151 | " PATH=/usr/local/bin:/usr/bin:/bin:/usr/local/sbin:/usr/sbin:/sbin:/usr/X11/bin:$M2" + 152 | " DISPLAY=localhost:1.0" + 153 | " TEST_RUNNER='#{build_kind}'" + 154 | " GIT_COMMIT=#{options["git_commit"]}" + 155 | " GIT_BRANCH=#{options["git_branch"]}" + 156 | " RUN_LIST=$TARGETS" + 157 | " KOCHIKU_ENV=#{options["kochiku_env"]}" + 158 | java_options + 159 | " bash --noprofile --norc -c 'if [ -f ~/.rvm/scripts/rvm ]; then source ~/.rvm/scripts/rvm; elif [ -f /usr/local/rvm/scripts/rvm ]; then source /usr/local/rvm/scripts/rvm; fi; #{ruby_command} ; ruby -v ; #{test_command}'" 160 | ).gsub("$TARGETS", test_files.join(',')) 161 | end 162 | 163 | def check_log_for_errors! 164 | File.open(LOG_FILE, :encoding => 'UTF-8') do |file| 165 | file.each do |line| 166 | raise ErrorFoundInLogError.new(line) if known_error?(line) 167 | end 168 | end 169 | end 170 | 171 | @@known_errors = Regexp.union( 172 | [ 173 | "couldn't find resque worker", 174 | "Resource temporarily unavailable", 175 | "Can't connect to local MySQL server through socket", 176 | "cucumber processes did not come up", 177 | "Mysql timed out, bailing" 178 | ] 179 | ) 180 | def known_error?(line) 181 | line =~ @@known_errors 182 | end 183 | end 184 | end 185 | -------------------------------------------------------------------------------- /spec/kochiku/build_strategies/build_all_strategy_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | RSpec.describe BuildStrategy::BuildAllStrategy do 4 | let(:dev_null) { "2>/dev/null 1>/dev/null" } 5 | subject{ BuildStrategy::BuildAllStrategy.new } 6 | 7 | before do 8 | old_spawn = Process.method(:spawn) 9 | allow(Process).to receive(:spawn) do |*args| 10 | @spawned_pid = old_spawn.call(*args) 11 | end 12 | File.unlink(BuildStrategy::BuildAllStrategy::LOG_FILE) if File.exists?(BuildStrategy::BuildAllStrategy::LOG_FILE) 13 | 14 | stub_const("BuildStrategy::BuildAllStrategy::KILL_TIMEOUT", 1) 15 | end 16 | 17 | describe "#ci_command" do 18 | it "should quote the test_runner variable" do 19 | command = subject.send(:ci_command, "build --parallel", ["i/am/a/spec"], "rspec",{}) 20 | expect( command).to include("TEST_RUNNER='build --parallel'") 21 | end 22 | end 23 | 24 | describe "#execute_with_timeout_and_kill" do 25 | let(:busy_wait) { "while true; do true; done"} 26 | let(:trap_sigterm) { "trap 'echo SIGTERM blocked' 15"} 27 | let(:log) { IO.readlines(BuildStrategy::BuildAllStrategy::LOG_FILE) } 28 | let(:process_kill_message) { "******** The following process(es) taking too long, Kochiku killing NOW ************\n" } 29 | 30 | it "should not claim to have killed when it didn't" do 31 | subject.execute_with_timeout_and_kill "true", 0.1 32 | 33 | expect(log).not_to include(process_kill_message) 34 | end 35 | 36 | it "should return true if it succeeds" do 37 | expect(subject.execute_with_timeout_and_kill("true #{dev_null}", 0.1)).to eq(true) 38 | end 39 | 40 | it "should return false if it fails" do 41 | expect(subject.execute_with_timeout_and_kill("false #{dev_null}", 0.1)).to eq(false) 42 | end 43 | 44 | it "should raise a ErrorFoundInLogError for known errors in output of a failed build" do 45 | expect { 46 | subject.execute_with_timeout_and_kill("false # couldn't find resque worker", 0.5) 47 | }.to raise_error(BuildStrategy::BuildAllStrategy::ErrorFoundInLogError) 48 | end 49 | 50 | it "won't raise ErrorFoundInLogError when the build passes" do 51 | expect { 52 | subject.execute_with_timeout_and_kill("true # couldn't find resque worker", 0.5) 53 | }.to_not raise_error 54 | end 55 | 56 | context "No children processes spawned" do 57 | context "process times out" do 58 | context "SIGTERM not trapped" do 59 | it "should kill the command if it takes too long" do 60 | start_time = Time.now 61 | expect { 62 | subject.execute_with_timeout_and_kill("sleep 300 #{dev_null}", 0.1) 63 | }.to_not raise_error 64 | expect(Time.now - start_time).to be_within(0.3).of(0.1) 65 | expect { 66 | Process.kill(0, @spawned_pid) 67 | }.to raise_error(Errno::ESRCH) 68 | 69 | expect(log).to include(process_kill_message) 70 | expect(log).to include("sleep 300") 71 | end 72 | end 73 | context "SIGTERM trapped" do 74 | it "should kill process even if SIGTERM trapped" do 75 | expect { 76 | subject.execute_with_timeout_and_kill( 77 | " 78 | #{trap_sigterm} 79 | #{busy_wait} 80 | ", 0.1) 81 | }.to_not raise_error 82 | 83 | expect { 84 | Process.kill(-15, @spawned_pid) 85 | }.to raise_error(Errno::ESRCH) 86 | end 87 | end 88 | end 89 | end 90 | 91 | context "Children processses spawned" do 92 | context "head process ends normally" do 93 | context "spawned processes do not trap SIGTERM" do 94 | it "should kill all child processes" do 95 | expect { 96 | subject.execute_with_timeout_and_kill( 97 | " 98 | function2_to_fork() { 99 | #{trap_sigterm} 100 | #{busy_wait} 101 | } 102 | 103 | function3_to_fork() { 104 | #{busy_wait} 105 | } 106 | 107 | function_to_fork() { 108 | function2_to_fork & 109 | function3_to_fork & 110 | sleep 100 111 | } 112 | 113 | function_to_fork & 114 | true 115 | ", 0.1) 116 | }.to_not raise_error 117 | 118 | expect { 119 | Process.kill(-15, @spawned_pid) 120 | }.to raise_error(Errno::ESRCH) 121 | 122 | expect(log).to include(process_kill_message) 123 | expect(log).to include("sleep 100") 124 | end 125 | end 126 | 127 | context "some spawned processes trap SIGTERM" do 128 | it "should kill all child processes" do 129 | expect { 130 | subject.execute_with_timeout_and_kill( 131 | " 132 | function2_to_fork() { 133 | #{trap_sigterm} 134 | #{busy_wait} 135 | } 136 | 137 | function3_to_fork() { 138 | #{busy_wait} 139 | } 140 | 141 | function_to_fork() { 142 | function2_to_fork & 143 | function3_to_fork & 144 | sleep 100 145 | } 146 | 147 | function_to_fork & 148 | true 149 | ", 0.1) 150 | }.to_not raise_error 151 | 152 | expect { 153 | Process.kill(-15, @spawned_pid) 154 | }.to raise_error(Errno::ESRCH) 155 | 156 | expect(log).to include(process_kill_message) 157 | expect(log).to include("sleep 100") 158 | end 159 | end 160 | end 161 | 162 | context "head process times out" do 163 | context "spawned processes do not trap SIGTERM" do 164 | it "should kill all child processes" do 165 | expect { 166 | subject.execute_with_timeout_and_kill( 167 | " 168 | function2_to_fork() { 169 | #{busy_wait} 170 | } 171 | 172 | function3_to_fork() { 173 | #{busy_wait} 174 | } 175 | 176 | function_to_fork() { 177 | function2_to_fork & 178 | function3_to_fork & 179 | sleep 100 180 | } 181 | 182 | function_to_fork & 183 | sleep 300 184 | ", 0.1) 185 | }.to_not raise_error 186 | 187 | expect { 188 | Process.kill(-15, @spawned_pid) 189 | }.to raise_error(Errno::ESRCH) 190 | end 191 | end 192 | context "some spawned processes trap SIGTERM" do 193 | it "should kill all child processes" do 194 | expect { 195 | subject.execute_with_timeout_and_kill( 196 | " 197 | function2_to_fork() { 198 | #{trap_sigterm} 199 | #{busy_wait} 200 | } 201 | 202 | function3_to_fork() { 203 | #{busy_wait} 204 | } 205 | 206 | function_to_fork() { 207 | function2_to_fork & 208 | function3_to_fork & 209 | sleep 100 210 | } 211 | 212 | function_to_fork & 213 | #{busy_wait} 214 | ", 0.1) 215 | }.to_not raise_error 216 | 217 | expect { 218 | Process.kill(-15, @spawned_pid) 219 | }.to raise_error(Errno::ESRCH) 220 | end 221 | end 222 | end 223 | end 224 | end 225 | end 226 | -------------------------------------------------------------------------------- /lib/kochiku/jobs/build_attempt_job.rb: -------------------------------------------------------------------------------- 1 | require 'socket' 2 | require 'rest-client' 3 | require 'retryable' 4 | 5 | class BuildAttemptJob < JobBase 6 | include Benchmark 7 | 8 | def initialize(build_options) 9 | @build_attempt_id = build_options["build_attempt_id"] 10 | @build_ref = build_options["build_ref"] 11 | @build_kind = build_options["build_kind"] 12 | @branch = build_options["branch"] 13 | @test_files = build_options["test_files"] 14 | @test_command = build_options["test_command"] 15 | @repo_url = build_options["repo_url"] 16 | @timeout = build_options["timeout"] 17 | @options = build_options["options"] || {} 18 | @kochiku_env = build_options["kochiku_env"] 19 | end 20 | 21 | def sha 22 | @build_ref 23 | end 24 | 25 | def logger 26 | Kochiku::Worker.logger 27 | end 28 | 29 | def perform 30 | logstreamer_port = Kochiku::Worker.settings['logstreamer_port'] 31 | if logstreamer_port && !launch_logstreamer(logstreamer_port) 32 | logger.info("Launch of logstreamer on port #{logstreamer_port} failed.") 33 | logstreamer_port = nil 34 | end 35 | 36 | logger.info("Build Attempt #{@build_attempt_id} perform starting") 37 | 38 | return if signal_build_is_starting(logstreamer_port) == :aborted 39 | 40 | Retryable.retryable(tries: 5, on: Kochiku::Worker::GitRepo::RefNotFoundError, sleep: 12) do # wait for up to 60 seconds for the sha to be available 41 | Kochiku::Worker::GitRepo.inside_copy(@repo_url, @build_ref) do 42 | begin 43 | options = @options.merge("git_commit" => @build_ref, "git_branch" => @branch, "kochiku_env" => @kochiku_env, "logstreamer_enabled" => !!logstreamer_port) 44 | result = run_tests(@build_kind, @test_files, @test_command, @timeout, options) ? :passed : :failed 45 | signal_build_is_finished(result) 46 | ensure 47 | collect_logs(Kochiku::Worker.build_strategy.log_files_glob) 48 | end 49 | end 50 | end 51 | logger.info("Build Attempt #{@build_attempt_id} perform finished") 52 | end 53 | 54 | # attempts to launch logstreamer on specified port. Returns true on success, false otherwise. 55 | def launch_logstreamer(port) 56 | exception_cb = Proc.new do 57 | Dir.chdir("logstreamer") { system("./logstreamer -p #{port} &") } 58 | end 59 | 60 | begin 61 | Retryable.retryable(sleep: 3, on: [Errno::EHOSTUNREACH, Errno::ECONNREFUSED, RestClient::Exception, SocketError], 62 | exception_cb: exception_cb, tries: 2) do 63 | RestClient.get("http://localhost:#{port}/_status", :timeout => 5).code == 200 64 | end 65 | true 66 | rescue Errno::EHOSTUNREACH, Errno::ECONNREFUSED, RestClient::Exception, SocketError 67 | false 68 | end 69 | end 70 | 71 | def collect_logs(file_glob) 72 | detected_files = Dir.glob(file_glob) 73 | benchmark("collecting logs (#{detected_files.join(', ')}) for Build Attempt #{@build_attempt_id}") do 74 | detected_files.each do |path| 75 | if File.file?(path) && !File.zero?(path) 76 | if path =~ /log$/ 77 | Cocaine::CommandLine.new("gzip", "-f :path").run(path: path) # -f is because we need to break the hardlink 78 | path += '.gz' 79 | end 80 | upload_log_file(File.new(path)) 81 | end 82 | end 83 | end 84 | end 85 | 86 | def on_exception(e) 87 | if e.instance_of?(Kochiku::Worker::GitRepo::RefNotFoundError) || 88 | (e.instance_of?(Cocaine::ExitStatusError) && e.message.include?("Invalid symmetric difference expression")) 89 | handle_git_ref_not_found(e) 90 | # avoid calling super because this does not need to go into the failed queue 91 | return 92 | end 93 | 94 | logger.error("Exception occurred during Build Attempt (#{@build_attempt_id}):") 95 | logger.error(e) 96 | 97 | message = StringIO.new 98 | message.puts(e.message) 99 | message.puts(e.backtrace) 100 | message.rewind 101 | # Need to override path method for RestClient to upload this correctly 102 | def message.path 103 | 'error.txt' 104 | end 105 | 106 | upload_log_file(message) 107 | 108 | # Signal build is errored after error.txt is uploaded so we can 109 | # reference error.txt in the build_attempt observer on the master. 110 | signal_build_is_finished(:errored) 111 | 112 | super 113 | end 114 | 115 | private 116 | 117 | def hostname 118 | Socket.gethostname 119 | end 120 | 121 | def run_tests(build_kind, test_files, test_command, timeout, options) 122 | logger.info("Running tests for #{@build_attempt_id}") 123 | Kochiku::Worker.build_strategy.execute_build(@build_attempt_id, build_kind, test_files, test_command, timeout, options) 124 | end 125 | 126 | def with_http_retries(&block) 127 | # 3 retries; sleep for 15, 45, and 60 seconds between tries 128 | backoff_proc = lambda { |n| [15, 45, 60][n] } 129 | 130 | Retryable.retryable(tries: 4, on: [Errno::ECONNRESET, Errno::EHOSTUNREACH, RestClient::Exception, SocketError], sleep: backoff_proc) do 131 | block.call 132 | end 133 | end 134 | 135 | def signal_build_is_starting(logstreamer_port) 136 | result = nil 137 | benchmark("Signal Build Attempt #{@build_attempt_id} starting") do 138 | build_start_url = "#{url_base}/start" 139 | payload = {:builder => hostname} 140 | payload[:instance_type] = Kochiku::Worker.instance_type if Kochiku::Worker.instance_type 141 | payload[:logstreamer_port] = logstreamer_port if logstreamer_port 142 | 143 | with_http_retries do 144 | result = RestClient::Request.execute(method: :post, 145 | url: build_start_url, 146 | payload: payload, 147 | headers: { accept: :json }) 148 | end 149 | end 150 | JSON.parse(result)["build_attempt"]["state"].to_sym 151 | end 152 | 153 | def signal_build_is_finished(result) 154 | benchmark("Signal Build Attempt #{@build_attempt_id} finished") do 155 | build_finish_url = "#{url_base}/finish" 156 | with_http_retries do 157 | RestClient::Request.execute(method: :post, 158 | url: build_finish_url, 159 | payload: { state: result }, 160 | headers: { accept: :json }, 161 | timeout: 60, 162 | open_timeout: 60) 163 | end 164 | end 165 | end 166 | 167 | def upload_log_file(file) 168 | log_artifact_upload_url = "#{url_base}/build_artifacts" 169 | with_http_retries do 170 | file.rewind 171 | RestClient::Request.execute(method: :post, 172 | url: log_artifact_upload_url, 173 | payload: { build_artifact: { log_file: file.clone } }, 174 | headers: { accept: :xml }, 175 | timeout: 60 * 5) 176 | end 177 | rescue Errno::ECONNRESET, Errno::EHOSTUNREACH, RestClient::Exception, RuntimeError => e 178 | # log exception and continue. A failed log file upload should not interrupt the BuildAttempt 179 | logger.error("Upload of artifact (#{file.to_s}) for Build Attempt #{@build_attempt_id} failed: #{e.message}") 180 | ensure 181 | file.close 182 | end 183 | 184 | def handle_git_ref_not_found(exception) 185 | logger.warn("#{exception.class} during Build Attempt (#{@build_attempt_id}):") 186 | logger.warn(exception.message) 187 | 188 | message = StringIO.new 189 | message.puts(exception.message) 190 | message.puts(exception.backtrace) 191 | message.rewind 192 | # Need to override path method for RestClient to upload this correctly 193 | def message.path 194 | 'aborted.txt' 195 | end 196 | 197 | upload_log_file(message) 198 | 199 | signal_build_is_finished(:aborted) 200 | end 201 | 202 | def url_base 203 | "#{Kochiku::Worker.settings.kochiku_web_server_protocol}://" + 204 | "#{Kochiku::Worker.settings.kochiku_web_server_host}/build_attempts/#{@build_attempt_id}" 205 | end 206 | end 207 | -------------------------------------------------------------------------------- /spec/jobs/build_attempt_job_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'fileutils' 3 | require 'json' 4 | 5 | RSpec.describe BuildAttemptJob do 6 | let(:master_host) { Kochiku::Worker.settings.kochiku_web_server_protocol + 7 | "://" + Kochiku::Worker.settings.kochiku_web_server_host } 8 | let(:build_attempt_id) { "42" } 9 | let(:build_part_kind) { "test" } 10 | let(:build_ref) { "123abc" } 11 | let(:test_files) { ["/foo/1.test", "foo/baz/a.test", "foo/baz/b.test"] } 12 | let(:build_options) { { 13 | "build_attempt_id" => build_attempt_id, 14 | "build_ref" => build_ref, 15 | "build_kind" => build_part_kind, 16 | "test_files" => test_files, 17 | "test_command" => "script/ci worker", 18 | "repo_url" => "git@github.com:square/kochiku-worker.git" 19 | } } 20 | let(:retry_count) { 4 } 21 | 22 | subject { BuildAttemptJob.new(build_options) } 23 | 24 | describe "#perform" do 25 | before do 26 | allow(Kochiku::Worker::GitRepo).to receive(:inside_copy).and_yield 27 | end 28 | 29 | context "logstreamer port specified" do 30 | before do 31 | # directly setting the hash value pollutes the other tests 32 | allow(Kochiku::Worker.settings).to receive(:[]).with('logstreamer_port').and_return(10000) 33 | end 34 | 35 | context "able to launch logstreamer" do 36 | before do 37 | allow(subject).to receive(:launch_logstreamer).and_return(true) 38 | end 39 | 40 | it "should not specify logstreamer port to kochiku master" do 41 | hostname = "i-am-a-compooter" 42 | allow(subject).to receive(:run_tests) 43 | allow(subject).to receive(:hostname).and_return(hostname) 44 | stub_request(:post, "#{master_host}/build_attempts/#{build_attempt_id}/start").to_return(:body => {'build_attempt' => {'state' => 'running'}}.to_json) 45 | stub_request(:post, "#{master_host}/build_attempts/#{build_attempt_id}/finish") 46 | 47 | subject.perform 48 | expect(WebMock).to have_requested(:post, "#{master_host}/build_attempts/#{build_attempt_id}/start").with(:body => "builder=#{hostname}&logstreamer_port=10000") 49 | end 50 | end 51 | 52 | context "unable to launch logstreamer" do 53 | before do 54 | allow(RestClient).to receive(:get).and_raise Errno::ECONNREFUSED 55 | end 56 | 57 | it "should not specify logstreamer port to kochiku master" do 58 | hostname = "i-am-a-compooter" 59 | allow(subject).to receive(:run_tests) 60 | allow(subject).to receive(:hostname).and_return(hostname) 61 | stub_request(:post, "#{master_host}/build_attempts/#{build_attempt_id}/start").to_return(:body => {'build_attempt' => {'state' => 'running'}}.to_json) 62 | stub_request(:post, "#{master_host}/build_attempts/#{build_attempt_id}/finish") 63 | 64 | subject.perform 65 | expect(WebMock).to have_requested(:post, "#{master_host}/build_attempts/#{build_attempt_id}/start").with(:body => {"builder"=> hostname}) 66 | end 67 | end 68 | end 69 | 70 | context "build_attempt has been aborted" do 71 | it "should return without running the tests" do 72 | stub_request(:post, "#{master_host}/build_attempts/#{build_attempt_id}/start").to_return(:body => {'build_attempt' => {'state' => 'aborted'}}.to_json) 73 | 74 | expect(subject).not_to receive(:run_tests) 75 | subject.perform 76 | end 77 | end 78 | 79 | it "sets the builder on its build attempt" do 80 | hostname = "i-am-a-compooter" 81 | allow(subject).to receive(:run_tests) 82 | allow(subject).to receive(:hostname).and_return(hostname) 83 | stub_request(:post, "#{master_host}/build_attempts/#{build_attempt_id}/start").to_return(:body => {'build_attempt' => {'state' => 'running'}}.to_json) 84 | stub_request(:post, "#{master_host}/build_attempts/#{build_attempt_id}/finish") 85 | 86 | subject.perform 87 | expect(WebMock).to have_requested(:post, "#{master_host}/build_attempts/#{build_attempt_id}/start").with(:body => {"builder"=> hostname}) 88 | end 89 | 90 | context "build is successful" do 91 | before { allow(subject).to receive(:run_tests).and_return(true) } 92 | 93 | it "creates a build result with a passed result" do 94 | stub_request(:post, "#{master_host}/build_attempts/#{build_attempt_id}/start").to_return(:body => {'build_attempt' => {'state' => 'running'}}.to_json) 95 | stub_request(:post, "#{master_host}/build_attempts/#{build_attempt_id}/finish") 96 | 97 | subject.perform 98 | 99 | expect(WebMock).to have_requested(:post, "#{master_host}/build_attempts/#{build_attempt_id}/finish").with(:body => {"state"=> "passed"}) 100 | end 101 | end 102 | 103 | context "build is unsuccessful" do 104 | before { allow(subject).to receive(:run_tests).and_return(false) } 105 | 106 | it "creates a build result with a failed result" do 107 | stub_request(:post, "#{master_host}/build_attempts/#{build_attempt_id}/start").to_return(:body => {'build_attempt' => {'state' => 'running'}}.to_json) 108 | stub_request(:post, "#{master_host}/build_attempts/#{build_attempt_id}/finish") 109 | 110 | subject.perform 111 | 112 | expect(WebMock).to have_requested(:post, "#{master_host}/build_attempts/#{build_attempt_id}/finish").with(:body => {"state"=> "failed"}) 113 | end 114 | end 115 | 116 | context "an exception occurs" do 117 | class FakeTestError < StandardError; end 118 | 119 | it "sets the build attempt state to errored" do 120 | stub_request(:post, "#{master_host}/build_attempts/#{build_attempt_id}/start").to_return(:body => {'build_attempt' => {'state' => 'running'}}.to_json) 121 | stub_request(:post, "#{master_host}/build_attempts/#{build_attempt_id}/build_artifacts") 122 | stub_request(:post, "#{master_host}/build_attempts/#{build_attempt_id}/finish") 123 | 124 | expect(subject).to receive(:run_tests).and_raise(FakeTestError.new('something went wrong')) 125 | expect(BuildAttemptJob).to receive(:new).and_return(subject) 126 | allow(Kochiku::Worker.logger).to receive(:error) 127 | 128 | expect { BuildAttemptJob.perform(build_options) }.to raise_error(FakeTestError) 129 | 130 | expect(WebMock).to have_requested(:post, "#{master_host}/build_attempts/#{build_attempt_id}/build_artifacts").with( 131 | :headers => {'Content-Type' => /multipart\/form-data/} 132 | # current version of Webmock does not support matching body for multipart/form-data requests 133 | # https://github.com/bblimke/webmock/issues/623 134 | #:body => /something went wrong/ 135 | ) 136 | expect(WebMock).to have_requested(:post, "#{master_host}/build_attempts/#{build_attempt_id}/finish").with(:body => {"state" => "errored"}) 137 | end 138 | 139 | context "and its GitRepo::RefNotFoundError" do 140 | it "sets the build attempt state to aborted" do 141 | stub_request(:post, "#{master_host}/build_attempts/#{build_attempt_id}/start").to_return(:body => {'build_attempt' => {'state' => 'running'}}.to_json) 142 | stub_request(:post, "#{master_host}/build_attempts/#{build_attempt_id}/build_artifacts") 143 | stub_request(:post, "#{master_host}/build_attempts/#{build_attempt_id}/finish") 144 | 145 | expect(subject).to receive(:run_tests).and_raise(Kochiku::Worker::GitRepo::RefNotFoundError.new) 146 | expect(BuildAttemptJob).to receive(:new).and_return(subject) 147 | allow(Kochiku::Worker.logger).to receive(:warn) 148 | 149 | expect { BuildAttemptJob.perform(build_options) }.to_not raise_error 150 | 151 | expect(WebMock).to have_requested(:post, "#{master_host}/build_attempts/#{build_attempt_id}/build_artifacts") 152 | expect(WebMock).to have_requested(:post, "#{master_host}/build_attempts/#{build_attempt_id}/finish").with(:body => {"state" => "aborted"}) 153 | end 154 | end 155 | end 156 | end 157 | 158 | describe "#collect_logs" do 159 | before do 160 | allow(Cocaine::CommandLine).to receive(:new).with("gzip", anything).and_call_original 161 | stub_request(:any, /#{master_host}.*/) 162 | end 163 | 164 | it "posts the local build logs back to the master server" do 165 | Dir.mktmpdir do |dir| 166 | Dir.chdir(dir) do 167 | wanted_logs = ['a.wantedlog', 'b.wantedlog', 'd/c.wantedlog'] 168 | 169 | FileUtils.mkdir 'd' 170 | (wanted_logs + ['e.unwantedlog']).each do |file_path| 171 | File.open(file_path, 'w') do |file| 172 | file.puts "Carrierwave won't save blank files" 173 | end 174 | end 175 | 176 | subject.collect_logs('**/*.wantedlog') 177 | 178 | wanted_logs.each do |artifact| 179 | log_name = File.basename(artifact) 180 | expect(WebMock).to have_requested(:post, "#{master_host}/build_attempts/#{build_attempt_id}/build_artifacts").with { |req| req.body.include?(log_name) } 181 | end 182 | end 183 | end 184 | end 185 | 186 | it "should not attempt to save blank files" do 187 | Dir.mktmpdir do |dir| 188 | Dir.chdir(dir) do 189 | log_name = 'empty.log' 190 | system("touch #{log_name}") 191 | subject.collect_logs('*.log') 192 | expect(WebMock).not_to have_requested(:post, "#{master_host}/build_attempts/#{build_attempt_id}/build_artifacts").with { |req| req.body.include?(log_name) } 193 | end 194 | end 195 | end 196 | 197 | it "should be able to retry, even if the IO object has been closed" do 198 | Retryable.enable 199 | allow(Kernel).to receive(:sleep) 200 | stub_request(:post, "#{master_host}/build_attempts/#{build_attempt_id}/build_artifacts").to_return(:status => 500, :body => "", :headers => {}) 201 | 202 | Dir.mktmpdir do |dir| 203 | Dir.chdir(dir) do 204 | log_name = 'a.log' 205 | 206 | File.open(log_name, 'w') do |file| 207 | file.puts "Carrierwave won't save blank files" 208 | end 209 | 210 | expect { 211 | subject.collect_logs('**/*.log') 212 | }.not_to raise_error # specifically, IOError 213 | 214 | expect(WebMock).to have_requested(:post, "#{master_host}/build_attempts/#{build_attempt_id}/build_artifacts").times(retry_count) 215 | end 216 | end 217 | 218 | Retryable.disable 219 | end 220 | end 221 | 222 | describe "#with_http_retries" do 223 | it "should raise after retrying" do 224 | allow(Kernel).to receive(:sleep) 225 | Retryable.enable 226 | 227 | expect { 228 | subject.send(:with_http_retries) do 229 | raise Errno::EHOSTUNREACH 230 | end 231 | }.to raise_error(Errno::EHOSTUNREACH) 232 | 233 | Retryable.disable 234 | end 235 | 236 | it "should sleep an increasing amount of time between retries" do 237 | Retryable.enable 238 | 239 | expect(Kernel).to receive(:sleep).with(15).ordered 240 | expect(Kernel).to receive(:sleep).with(45).ordered 241 | expect(Kernel).to receive(:sleep).with(60).ordered 242 | 243 | expect { 244 | subject.send(:with_http_retries) do 245 | raise Errno::EHOSTUNREACH 246 | end 247 | }.to raise_error 248 | 249 | Retryable.disable 250 | end 251 | end 252 | end 253 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | --------------------------------------------------------------------------------