├── test ├── Gemfile ├── data │ ├── dir1 │ │ └── stub │ └── dir2 │ │ └── test ├── hello.sh ├── workers │ ├── file.csv │ ├── wfile_paths │ │ ├── dir1 │ │ │ └── greeting.txt │ │ ├── worker.rb │ │ └── wfile_paths.worker │ ├── full_remote_build │ │ ├── pdftk.sh │ │ └── pdftk.worker │ ├── with_build_command │ │ ├── hello.rb │ │ └── build_command.worker │ ├── config_worker.worker │ ├── fail_worker.rb │ ├── merge_gem_worker │ │ ├── use_iron_core.rb │ │ └── merge_gem_worker.worker │ ├── sleepy.worker │ ├── config_worker.rb │ ├── locale_csv_worker.rb │ ├── ruby_runner_test_class.rb │ ├── ruby_runner_test_script.rb │ └── sleepy_worker.rb ├── stacks │ ├── mono-2.10 │ │ ├── mono.sh │ │ └── mono-2.10.worker │ ├── mono-3.0 │ │ ├── mono3.sh │ │ └── mono-3.0.worker │ ├── php-5.6 │ │ ├── php-5.6.sh │ │ ├── php-5.6.worker │ │ └── php-5.6.php │ ├── crawler │ │ ├── crawler.sh │ │ └── crawler.worker │ ├── selenium │ │ ├── selenium.sh │ │ └── selenium.worker │ ├── phantom-1.9-b │ │ ├── phantom-1.9-b.sh │ │ └── phantom-1.9-b.worker │ ├── scala-2.9 │ │ ├── scala.sh │ │ └── scala-2.9.worker │ ├── php-5.4 │ │ ├── php-5.4.worker │ │ └── php.php │ ├── mono-3.6 │ │ ├── mono-3.6.worker │ │ └── mono3.sh │ ├── node-0.10 │ │ ├── node-0.10.worker │ │ └── node.js │ ├── node-0.11 │ │ ├── node-0.11.worker │ │ └── node.js │ ├── php-5.5 │ │ ├── php-5.5.worker │ │ └── php-5.5.php │ ├── java-1.7 │ │ ├── java-1.7.worker │ │ └── HelloWorldApp.jar │ ├── ffmpeg-2.3 │ │ ├── ffmpeg-2.3.worker │ │ └── ffmpeg-2.3.sh │ ├── newrelic │ │ ├── newrelic.worker │ │ └── newrelic.sh │ ├── phantom-1.9 │ │ ├── phantom-1.9.worker │ │ └── phantom-1.9.sh │ ├── phantom-2.0 │ │ ├── phantom-2.0.worker │ │ └── phantom-2.0.sh │ ├── ruby-1.8 │ │ ├── ruby-1.8.worker │ │ └── ruby-1.8.sh │ ├── go-1.4 │ │ ├── go-1.4.worker │ │ └── go-1.4.sh │ ├── phantom-2.0-b │ │ ├── phantom-2.0-b.worker │ │ └── phantom-2.0-b.sh │ ├── java-1.8 │ │ ├── java-1.8.worker │ │ └── java-1.8.sh │ ├── python-2.7 │ │ ├── python2.py │ │ └── python-2.7.worker │ ├── python-3.2 │ │ ├── python3.py │ │ └── python-3.2.worker │ ├── ruby-1.9 │ │ ├── ruby-1.9.rb │ │ └── ruby-1.9.worker │ └── ruby-2.1 │ │ ├── ruby-2.1.rb │ │ └── ruby-2.1.worker ├── hello.rb ├── sleep.worker ├── hello.worker ├── local_run_with_config.json ├── local_run_with_config.rb ├── sleep.rb ├── worker-test │ ├── stat │ │ ├── stat.worker │ │ └── stat.rb │ ├── mem-kill │ │ ├── mem-kill.worker │ │ └── mem-kill.rb │ └── io-mem-cpu │ │ ├── io-mem-cpu.worker │ │ └── io-mem-cpu.rb ├── local_run_with_config.worker ├── hello.jar ├── Hello.class ├── rake_test_ng │ ├── Gemfile │ ├── webhook_uri │ ├── Workerfile │ ├── Gemfile.lock │ └── rake_test_ng.rb ├── hello.c ├── cli_runner.rb ├── Hello.java ├── ng_tests_worker.rb ├── examples │ ├── example_worker101.rb │ ├── example_hello.rb │ ├── README.md │ ├── example_master_slave.rb │ ├── example_github_webhook.rb │ ├── example_chargify_to_campfire_webhook.rb │ ├── example_actionmailer_standalone.rb │ ├── example_simple.rb │ └── example_email_worker.rb ├── test_full_remote_build.rb ├── test_wfile_paths.rb ├── test_java_worker.rb ├── test_local_run_with_config.rb ├── test_workerfile.rb ├── test_tasks_list.rb ├── test_concurrency.rb ├── test_config.rb ├── test_batch.rb ├── remote_test.rb ├── test_fetcher.rb ├── Rakefile ├── quick_run.rb ├── test_cli.rb ├── test_code_delete.rb ├── test_stacks.rb ├── test_builder.rb ├── test_clusters.rb ├── test_change_runtime.rb ├── iron_io_config.rb ├── test_ruby_merges.rb ├── test_code_create.rb ├── test_retry.rb ├── test_ruby_runner.rb ├── test_quick_run.rb ├── test_basic.rb ├── test_worker_consistency.rb ├── helpers.rb └── test_common_features.rb ├── Gemfile ├── tools ├── gb.json.example ├── gems.json └── gb.rb ├── .document ├── lib ├── iron_worker_ng │ ├── version.rb │ ├── code │ │ ├── go.rb │ │ ├── php.rb │ │ ├── java.rb │ │ ├── mono.rb │ │ ├── node.rb │ │ ├── perl.rb │ │ ├── ruby.rb │ │ ├── binary.rb │ │ ├── python.rb │ │ ├── runtime │ │ │ ├── go.rb │ │ │ ├── mono.rb │ │ │ ├── perl.rb │ │ │ ├── binary.rb │ │ │ ├── python.rb │ │ │ ├── java.rb │ │ │ ├── node.rb │ │ │ ├── php.rb │ │ │ └── ruby.rb │ │ ├── container │ │ │ ├── zip.rb │ │ │ ├── dir.rb │ │ │ └── base.rb │ │ ├── builder.rb │ │ └── base.rb │ ├── compat.rb │ ├── feature │ │ ├── python │ │ │ ├── merge_requirements.rb │ │ │ ├── merge_pip_dependency.rb │ │ │ └── merge_pip.rb │ │ ├── common │ │ │ ├── set_env.rb │ │ │ ├── merge_dir.rb │ │ │ ├── merge_file.rb │ │ │ ├── merge_deb.rb │ │ │ ├── merge_exec.rb │ │ │ └── merge_zip.rb │ │ ├── base.rb │ │ ├── ruby │ │ │ ├── merge_gemfile.rb │ │ │ ├── merge_gem.rb │ │ │ └── merge_gem_dependency.rb │ │ └── java │ │ │ └── merge_jar.rb │ ├── fetcher.rb │ ├── api_client.rb │ └── cli.rb ├── 3rdparty │ └── hashie │ │ ├── merge_initializer.rb │ │ └── indifferent_access.rb └── iron_worker_ng.rb ├── .gitignore ├── LICENSE ├── iron_worker_ng.gemspec ├── Gemfile.lock └── bin └── iron_worker /test/Gemfile: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/data/dir1/stub: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/data/dir2/test: -------------------------------------------------------------------------------- 1 | test -------------------------------------------------------------------------------- /test/hello.sh: -------------------------------------------------------------------------------- 1 | echo "hello" 2 | -------------------------------------------------------------------------------- /test/workers/file.csv: -------------------------------------------------------------------------------- 1 | ыфвфывфыв 2 | -------------------------------------------------------------------------------- /test/stacks/mono-2.10/mono.sh: -------------------------------------------------------------------------------- 1 | mono --version 2 | -------------------------------------------------------------------------------- /test/stacks/mono-3.0/mono3.sh: -------------------------------------------------------------------------------- 1 | mono --version 2 | -------------------------------------------------------------------------------- /test/stacks/php-5.6/php-5.6.sh: -------------------------------------------------------------------------------- 1 | hhvm php-5.6.php -------------------------------------------------------------------------------- /test/hello.rb: -------------------------------------------------------------------------------- 1 | puts "hello" 2 | 3 | #raise "DOH!" 4 | -------------------------------------------------------------------------------- /test/stacks/crawler/crawler.sh: -------------------------------------------------------------------------------- 1 | ps ax|grep selenium -------------------------------------------------------------------------------- /test/stacks/selenium/selenium.sh: -------------------------------------------------------------------------------- 1 | ps ax|grep selenium -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | gemspec 4 | -------------------------------------------------------------------------------- /test/sleep.worker: -------------------------------------------------------------------------------- 1 | runtime 'ruby' 2 | exec 'sleep.rb' 3 | -------------------------------------------------------------------------------- /test/stacks/phantom-1.9-b/phantom-1.9-b.sh: -------------------------------------------------------------------------------- 1 | phantomjs -v -------------------------------------------------------------------------------- /test/workers/wfile_paths/dir1/greeting.txt: -------------------------------------------------------------------------------- 1 | hello there 2 | -------------------------------------------------------------------------------- /test/hello.worker: -------------------------------------------------------------------------------- 1 | runtime 'ruby' 2 | 3 | exec 'hello.rb' 4 | -------------------------------------------------------------------------------- /test/local_run_with_config.json: -------------------------------------------------------------------------------- 1 | {"local_run_config":"present!"} -------------------------------------------------------------------------------- /test/stacks/scala-2.9/scala.sh: -------------------------------------------------------------------------------- 1 | scala -version 2 | printenv 3 | -------------------------------------------------------------------------------- /test/workers/full_remote_build/pdftk.sh: -------------------------------------------------------------------------------- 1 | pdftk --version 2 | -------------------------------------------------------------------------------- /test/workers/with_build_command/hello.rb: -------------------------------------------------------------------------------- 1 | puts "goodbye" 2 | -------------------------------------------------------------------------------- /test/local_run_with_config.rb: -------------------------------------------------------------------------------- 1 | require 'json' 2 | puts config.to_json -------------------------------------------------------------------------------- /test/sleep.rb: -------------------------------------------------------------------------------- 1 | seconds = params['sleep'] 2 | sleep seconds.to_i 3 | -------------------------------------------------------------------------------- /test/worker-test/stat/stat.worker: -------------------------------------------------------------------------------- 1 | runtime "ruby" 2 | exec 'stat.rb' 3 | -------------------------------------------------------------------------------- /test/workers/wfile_paths/worker.rb: -------------------------------------------------------------------------------- 1 | puts File.read('dir1/greeting.txt') 2 | -------------------------------------------------------------------------------- /tools/gb.json.example: -------------------------------------------------------------------------------- 1 | {"s3_access_key_id": "", "s3_secret_access_key": ""} 2 | -------------------------------------------------------------------------------- /test/worker-test/mem-kill/mem-kill.worker: -------------------------------------------------------------------------------- 1 | runtime "ruby" 2 | exec 'mem-kill.rb' 3 | -------------------------------------------------------------------------------- /.document: -------------------------------------------------------------------------------- 1 | lib/**/*.rb 2 | bin/* 3 | - 4 | features/**/*.feature 5 | LICENSE.txt 6 | -------------------------------------------------------------------------------- /test/local_run_with_config.worker: -------------------------------------------------------------------------------- 1 | runtime 'ruby' 2 | 3 | exec 'local_run_with_config.rb' -------------------------------------------------------------------------------- /test/stacks/php-5.4/php-5.4.worker: -------------------------------------------------------------------------------- 1 | runtime 'php' 2 | stack 'php-5.4' 3 | exec 'php.php' -------------------------------------------------------------------------------- /test/workers/config_worker.worker: -------------------------------------------------------------------------------- 1 | runtime 'ruby' 2 | 3 | exec 'config_worker.rb' 4 | -------------------------------------------------------------------------------- /test/hello.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iron-io/iron_worker_ruby_ng/HEAD/test/hello.jar -------------------------------------------------------------------------------- /test/stacks/crawler/crawler.worker: -------------------------------------------------------------------------------- 1 | runtime 'binary' 2 | stack 'crawler' 3 | exec 'crawler.sh' -------------------------------------------------------------------------------- /test/stacks/mono-3.0/mono-3.0.worker: -------------------------------------------------------------------------------- 1 | runtime 'binary' 2 | stack 'mono-3.0' 3 | exec 'mono3.sh' -------------------------------------------------------------------------------- /test/stacks/mono-3.6/mono-3.6.worker: -------------------------------------------------------------------------------- 1 | runtime 'binary' 2 | stack 'mono-3.6' 3 | exec 'mono3.sh' -------------------------------------------------------------------------------- /test/stacks/node-0.10/node-0.10.worker: -------------------------------------------------------------------------------- 1 | runtime 'node' 2 | stack 'node-0.10' 3 | exec 'node.js' -------------------------------------------------------------------------------- /test/stacks/node-0.11/node-0.11.worker: -------------------------------------------------------------------------------- 1 | runtime 'node' 2 | stack 'node-0.11' 3 | exec 'node.js' -------------------------------------------------------------------------------- /test/stacks/php-5.5/php-5.5.worker: -------------------------------------------------------------------------------- 1 | runtime 'php' 2 | stack 'php-5.5' 3 | exec 'php-5.5.php' -------------------------------------------------------------------------------- /test/worker-test/io-mem-cpu/io-mem-cpu.worker: -------------------------------------------------------------------------------- 1 | runtime "ruby" 2 | exec 'io-mem-cpu.rb' 3 | -------------------------------------------------------------------------------- /test/Hello.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iron-io/iron_worker_ruby_ng/HEAD/test/Hello.class -------------------------------------------------------------------------------- /test/stacks/mono-2.10/mono-2.10.worker: -------------------------------------------------------------------------------- 1 | runtime 'binary' 2 | stack 'mono-2.10' 3 | exec 'mono.sh' -------------------------------------------------------------------------------- /test/stacks/scala-2.9/scala-2.9.worker: -------------------------------------------------------------------------------- 1 | runtime 'binary' 2 | stack 'scala-2.9' 3 | exec 'scala.sh' -------------------------------------------------------------------------------- /test/stacks/selenium/selenium.worker: -------------------------------------------------------------------------------- 1 | runtime 'binary' 2 | stack 'selenium' 3 | exec 'selenium.sh' -------------------------------------------------------------------------------- /test/workers/fail_worker.rb: -------------------------------------------------------------------------------- 1 | 2 | puts "I am about to fail..." 3 | 4 | raise "Fail Whale!" 5 | -------------------------------------------------------------------------------- /test/rake_test_ng/Gemfile: -------------------------------------------------------------------------------- 1 | source 'http://rubygems.org' 2 | 3 | gem 'hipchat' 4 | gem 'aws-s3' 5 | 6 | -------------------------------------------------------------------------------- /test/stacks/java-1.7/java-1.7.worker: -------------------------------------------------------------------------------- 1 | runtime 'java' 2 | stack 'java-1.7' 3 | exec 'HelloWorldApp.jar' -------------------------------------------------------------------------------- /test/workers/wfile_paths/wfile_paths.worker: -------------------------------------------------------------------------------- 1 | name 'WfilePaths' 2 | exec 'worker.rb' 3 | dir 'dir1' 4 | -------------------------------------------------------------------------------- /test/stacks/ffmpeg-2.3/ffmpeg-2.3.worker: -------------------------------------------------------------------------------- 1 | runtime 'binary' 2 | stack 'ffmpeg-2.3' 3 | exec 'ffmpeg-2.3.sh' -------------------------------------------------------------------------------- /test/stacks/newrelic/newrelic.worker: -------------------------------------------------------------------------------- 1 | runtime 'binary' 2 | stack 'newrelic' 3 | exec 'newrelic.sh' 4 | 5 | -------------------------------------------------------------------------------- /test/stacks/phantom-1.9/phantom-1.9.worker: -------------------------------------------------------------------------------- 1 | runtime 'binary' 2 | stack 'phantom-1.9' 3 | exec 'phantom-1.9.sh' -------------------------------------------------------------------------------- /test/stacks/phantom-2.0/phantom-2.0.worker: -------------------------------------------------------------------------------- 1 | runtime 'binary' 2 | stack 'phantom-2.0' 3 | exec 'phantom-2.0.sh' -------------------------------------------------------------------------------- /test/stacks/ruby-1.8/ruby-1.8.worker: -------------------------------------------------------------------------------- 1 | runtime 'binary' 2 | stack 'ruby-1.8' 3 | 4 | exec "ruby-1.8.sh" 5 | -------------------------------------------------------------------------------- /test/hello.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | int main() 4 | { 5 | printf("hello\n"); 6 | return 0; 7 | } 8 | -------------------------------------------------------------------------------- /test/workers/merge_gem_worker/use_iron_core.rb: -------------------------------------------------------------------------------- 1 | require 'iron_core' 2 | IronCore::Logger.logger.info "hello" 3 | -------------------------------------------------------------------------------- /test/workers/sleepy.worker: -------------------------------------------------------------------------------- 1 | runtime 'ruby' 2 | name 'sleepy' 3 | exec 'sleepy_worker.rb' 4 | gem 'stathat' 5 | -------------------------------------------------------------------------------- /test/stacks/go-1.4/go-1.4.worker: -------------------------------------------------------------------------------- 1 | runtime 'binary' 2 | stack 'go-1.4' 3 | file 'go-1.4.sh' 4 | exec 'go-1.4.sh' 5 | -------------------------------------------------------------------------------- /test/stacks/phantom-1.9-b/phantom-1.9-b.worker: -------------------------------------------------------------------------------- 1 | runtime 'binary' 2 | stack 'phantom-1.9-b' 3 | exec 'phantom-1.9-b.sh' -------------------------------------------------------------------------------- /test/stacks/phantom-2.0-b/phantom-2.0-b.worker: -------------------------------------------------------------------------------- 1 | runtime 'binary' 2 | stack 'phantom-2.0-b' 3 | exec 'phantom-2.0-b.sh' -------------------------------------------------------------------------------- /test/stacks/php-5.6/php-5.6.worker: -------------------------------------------------------------------------------- 1 | runtime 'binary' 2 | stack 'php-5.6' 3 | file 'php-5.6.php' 4 | exec 'php-5.6.sh' -------------------------------------------------------------------------------- /test/cli_runner.rb: -------------------------------------------------------------------------------- 1 | require_relative '../lib/iron_worker_ng.rb' 2 | load(File.dirname(__FILE__) + '/../bin/iron_worker') 3 | -------------------------------------------------------------------------------- /test/stacks/java-1.8/java-1.8.worker: -------------------------------------------------------------------------------- 1 | runtime 'binary' 2 | stack 'java-1.8' 3 | file 'java-1.8.sh' 4 | exec 'java-1.8.sh' 5 | -------------------------------------------------------------------------------- /test/workers/config_worker.rb: -------------------------------------------------------------------------------- 1 | puts "I am config_worker" 2 | 3 | puts "params:" 4 | p params 5 | 6 | puts "config:" 7 | p config 8 | -------------------------------------------------------------------------------- /test/workers/merge_gem_worker/merge_gem_worker.worker: -------------------------------------------------------------------------------- 1 | gem 'iron_core' 2 | exec 'use_iron_core.rb' 3 | name 'merge_gem_worker' 4 | -------------------------------------------------------------------------------- /test/Hello.java: -------------------------------------------------------------------------------- 1 | public class Hello { 2 | public static void main(String[] args) { 3 | System.out.println("hello"); 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /test/ng_tests_worker.rb: -------------------------------------------------------------------------------- 1 | Dir.chdir 'iwng' 2 | cmd = 'rake -f test/Rakefile test ' + params[:args] 3 | puts cmd 4 | STDOUT.flush 5 | exec cmd 6 | -------------------------------------------------------------------------------- /test/stacks/java-1.7/HelloWorldApp.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iron-io/iron_worker_ruby_ng/HEAD/test/stacks/java-1.7/HelloWorldApp.jar -------------------------------------------------------------------------------- /test/stacks/python-2.7/python2.py: -------------------------------------------------------------------------------- 1 | import sys 2 | if sys.version_info[0] == 2 & sys.version_info[1] == 7: 3 | raise "Must be using Python 2" -------------------------------------------------------------------------------- /test/stacks/python-3.2/python3.py: -------------------------------------------------------------------------------- 1 | import sys 2 | if sys.version_info[0] == 3 & sys.version_info[1] == 2: 3 | raise "Must be using Python 3" -------------------------------------------------------------------------------- /test/worker-test/mem-kill/mem-kill.rb: -------------------------------------------------------------------------------- 1 | max_mem = params['max_mem'] 2 | a = "x" * (max_mem * 1000 * 1000 + 20_000_000) 3 | #a = "x" * 500_000_000 -------------------------------------------------------------------------------- /test/workers/locale_csv_worker.rb: -------------------------------------------------------------------------------- 1 | require 'csv' 2 | t = "" 3 | CSV.foreach( "file.csv") do |row| 4 | t+=row.join('-') 5 | end 6 | puts "All good" 7 | -------------------------------------------------------------------------------- /test/rake_test_ng/webhook_uri: -------------------------------------------------------------------------------- 1 | https://worker-aws-us-east-1.iron.io/2/projects/5007f0b90948517baf003e29/tasks/webhook?code_name=RakeTestNg&oauth=zZjQ.... 2 | -------------------------------------------------------------------------------- /test/workers/with_build_command/build_command.worker: -------------------------------------------------------------------------------- 1 | name 'BuildCommand' 2 | remote_build_command 'echo "puts \'hello\'" > hello.rb' 3 | exec 'hello.rb' 4 | -------------------------------------------------------------------------------- /test/stacks/node-0.10/node.js: -------------------------------------------------------------------------------- 1 | var version = (process.version).split("."); 2 | if (version[0]!='v0' || version[1]!='10') { 3 | throw new Error('Wrong node version'); 4 | } 5 | -------------------------------------------------------------------------------- /test/stacks/node-0.11/node.js: -------------------------------------------------------------------------------- 1 | var version = (process.version).split("."); 2 | if (version[0]!='v0' || version[1]!='11') { 3 | throw new Error('Wrong node version'); 4 | } 5 | -------------------------------------------------------------------------------- /test/rake_test_ng/Workerfile: -------------------------------------------------------------------------------- 1 | require '../iron_io_config.rb' 2 | 3 | exec 'rake_test_ng.rb' 4 | gemfile 'Gemfile' 5 | iron_io_config 6 | file File.expand_path('~/.abt-ng-config') 7 | -------------------------------------------------------------------------------- /test/workers/ruby_runner_test_class.rb: -------------------------------------------------------------------------------- 1 | class RubyWorker 2 | 3 | attr_accessor :a, :b 4 | 5 | def run 6 | puts( { :a => a, :b => b }.to_json ) 7 | end 8 | 9 | end 10 | -------------------------------------------------------------------------------- /test/stacks/python-3.2/python-3.2.worker: -------------------------------------------------------------------------------- 1 | runtime 'python' 2 | stack 'python-3.2' 3 | exec 'python3.py' 4 | pip "requests" 5 | pip "dj-database-url" 6 | pip "Django" 7 | pip "requests" 8 | remote -------------------------------------------------------------------------------- /test/workers/ruby_runner_test_script.rb: -------------------------------------------------------------------------------- 1 | puts({ 2 | :payload => payload, 3 | :params => params, 4 | :iron_task_id => iron_task_id, 5 | :indifferent_access => params[:a].equal?(params["a"]) 6 | }.to_json) 7 | -------------------------------------------------------------------------------- /test/worker-test/stat/stat.rb: -------------------------------------------------------------------------------- 1 | puts '--------------CPU Info--------------' 2 | puts `lscpu` 3 | 4 | puts "\n\n--------------MEM Info--------------" 5 | puts `free -m` 6 | 7 | puts "\n\n--------------HDD Info--------------" 8 | puts `df -m` 9 | puts "\n\n" -------------------------------------------------------------------------------- /test/workers/full_remote_build/pdftk.worker: -------------------------------------------------------------------------------- 1 | runtime 'binary' 2 | full_remote_build true 3 | 4 | exec 'pdftk.sh' 5 | deb 'http://mirror.pnl.gov/ubuntu/pool/universe/p/pdftk/pdftk_1.44-3_amd64.deb' 6 | deb 'http://mirror.pnl.gov/ubuntu/pool/main/g/gcj-4.6/libgcj12_4.6.1-4ubuntu2_amd64.deb' 7 | -------------------------------------------------------------------------------- /lib/iron_worker_ng/version.rb: -------------------------------------------------------------------------------- 1 | module IronWorkerNG 2 | VERSION = '1.6.9' 3 | 4 | def self.version 5 | VERSION 6 | end 7 | 8 | def self.full_version 9 | 'iron_worker_ruby_ng-' + IronWorkerNG.version + ' (iron_core_ruby-' + IronCore.version + ')' 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /lib/iron_worker_ng/code/go.rb: -------------------------------------------------------------------------------- 1 | require 'iron_worker_ng/code/runtime/go' 2 | 3 | module IronWorkerNG 4 | module Code 5 | class Go < IronWorkerNG::Code::Base 6 | def initialize(*args, &block) 7 | runtime(:go) 8 | 9 | super(*args, &block) 10 | end 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /lib/iron_worker_ng/code/php.rb: -------------------------------------------------------------------------------- 1 | require 'iron_worker_ng/code/runtime/php' 2 | 3 | module IronWorkerNG 4 | module Code 5 | class PHP < IronWorkerNG::Code::Base 6 | def initialize(*args, &block) 7 | runtime(:php) 8 | 9 | super(*args, &block) 10 | end 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /lib/iron_worker_ng/code/java.rb: -------------------------------------------------------------------------------- 1 | require 'iron_worker_ng/code/runtime/java' 2 | 3 | module IronWorkerNG 4 | module Code 5 | class Java < IronWorkerNG::Code::Base 6 | def initialize(*args, &block) 7 | runtime(:java) 8 | 9 | super(*args, &block) 10 | end 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /lib/iron_worker_ng/code/mono.rb: -------------------------------------------------------------------------------- 1 | require 'iron_worker_ng/code/runtime/mono' 2 | 3 | module IronWorkerNG 4 | module Code 5 | class Mono < IronWorkerNG::Code::Base 6 | def initialize(*args, &block) 7 | runtime(:mono) 8 | 9 | super(*args, &block) 10 | end 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /lib/iron_worker_ng/code/node.rb: -------------------------------------------------------------------------------- 1 | require 'iron_worker_ng/code/runtime/node' 2 | 3 | module IronWorkerNG 4 | module Code 5 | class Node < IronWorkerNG::Code::Base 6 | def initialize(*args, &block) 7 | runtime(:node) 8 | 9 | super(*args, &block) 10 | end 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /lib/iron_worker_ng/code/perl.rb: -------------------------------------------------------------------------------- 1 | require 'iron_worker_ng/code/runtime/perl' 2 | 3 | module IronWorkerNG 4 | module Code 5 | class Perl < IronWorkerNG::Code::Base 6 | def initialize(*args, &block) 7 | runtime(:perl) 8 | 9 | super(*args, &block) 10 | end 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /lib/iron_worker_ng/code/ruby.rb: -------------------------------------------------------------------------------- 1 | require 'iron_worker_ng/code/runtime/ruby' 2 | 3 | module IronWorkerNG 4 | module Code 5 | class Ruby < IronWorkerNG::Code::Base 6 | def initialize(*args, &block) 7 | runtime(:ruby) 8 | 9 | super(*args, &block) 10 | end 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /lib/iron_worker_ng/code/binary.rb: -------------------------------------------------------------------------------- 1 | require 'iron_worker_ng/code/runtime/binary' 2 | 3 | module IronWorkerNG 4 | module Code 5 | class Binary < IronWorkerNG::Code::Base 6 | def initialize(*args, &block) 7 | runtime(:binary) 8 | 9 | super(*args, &block) 10 | end 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /lib/iron_worker_ng/code/python.rb: -------------------------------------------------------------------------------- 1 | require 'iron_worker_ng/code/runtime/python' 2 | 3 | module IronWorkerNG 4 | module Code 5 | class Python < IronWorkerNG::Code::Base 6 | def initialize(*args, &block) 7 | runtime(:python) 8 | 9 | super(*args, &block) 10 | end 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /test/stacks/java-1.8/java-1.8.sh: -------------------------------------------------------------------------------- 1 | correct_version="1.8" 2 | current_version=`java -version 2>&1 | head -n 1 | awk -F '"' '{print $2}' | head -c3` 3 | 4 | if [ "$current_version" = "$correct_version" ]; then 5 | echo "correct version $correct_version" 6 | exit 0 7 | else 8 | echo "incorrect version $current_version" 9 | exit 1 10 | fi 11 | -------------------------------------------------------------------------------- /lib/iron_worker_ng/compat.rb: -------------------------------------------------------------------------------- 1 | require 'tmpdir' 2 | require 'fileutils' 3 | 4 | unless Dir.const_defined?(:Tmpname) 5 | class Dir 6 | class Tmpname 7 | def self.make_tmpname(x, y) 8 | n = ::Dir.mktmpdir(x + y) 9 | FileUtils.rm_rf(n) 10 | 11 | n.sub(::Dir.tmpdir, '') 12 | end 13 | end 14 | end 15 | end 16 | 17 | -------------------------------------------------------------------------------- /test/stacks/go-1.4/go-1.4.sh: -------------------------------------------------------------------------------- 1 | correct_version="1.4.1" 2 | current_version=`go version 2>&1 | head -n 1 | awk -F ' ' '{print $3}' | tail -c6` 3 | 4 | if [ "$current_version" = "$correct_version" ]; then 5 | echo "correct version $correct_version" 6 | exit 0 7 | else 8 | echo "incorrect version $current_version" 9 | exit 1 10 | fi 11 | -------------------------------------------------------------------------------- /test/stacks/mono-3.6/mono3.sh: -------------------------------------------------------------------------------- 1 | correct_version="3.6" 2 | current_version=`mono --version 2>&1 | head -n 1 | awk -F 'version ' '{print $2}' | head -c3` 3 | 4 | if [ "$current_version" = "$correct_version" ]; then 5 | echo "correct version $correct_version" 6 | exit 0 7 | else 8 | echo "incorrect version $current_version" 9 | exit 1 10 | fi 11 | 12 | -------------------------------------------------------------------------------- /test/stacks/phantom-2.0-b/phantom-2.0-b.sh: -------------------------------------------------------------------------------- 1 | correct_version="2.0.0" 2 | current_version=`phantomjs -v 2>&1 | head -n 1 | awk -F ' ' '{print $1}'` 3 | 4 | if [ "$current_version" = "$correct_version" ]; then 5 | echo "correct version $correct_version" 6 | exit 0 7 | else 8 | echo "incorrect version $current_version" 9 | exit 1 10 | fi -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | .bundle 3 | .rvmrc 4 | pkg 5 | iron.json 6 | examples/github_to_hipchat_webhook_worker/webhook_config.yml 7 | examples/worker101/twitter_config.json 8 | examples/actionmailer_standalone/actionmailer_config.json 9 | examples/chargify_to_campfire_webhook_worker/campfire_config.json 10 | examples/email_worker/email_config.json 11 | *.gem 12 | -------------------------------------------------------------------------------- /test/examples/example_worker101.rb: -------------------------------------------------------------------------------- 1 | require_relative '../helpers' 2 | 3 | class ExampleWorker101Test < IWNGTest 4 | def test_example 5 | Dir.chdir 'examples/worker101' do 6 | assert cli('upload', 'worker101') =~ /Upload successful/ 7 | log = `ruby enqueue.rb` 8 | assert log =~ /Worker101 completed/ 9 | end 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /test/stacks/ruby-1.8/ruby-1.8.sh: -------------------------------------------------------------------------------- 1 | correct_version="1.8.7" 2 | current_version=`ruby -v 2>&1 | awk -F 'ruby ' '{print $2}' | head -c 5` 3 | 4 | if [ "$current_version" = "$correct_version" ]; then 5 | echo "OK! Correct version --> $correct_version" 6 | exit 0 7 | else 8 | echo "FAIL! Incorrect version --> $current_version" 9 | exit 1 10 | fi 11 | -------------------------------------------------------------------------------- /test/workers/sleepy_worker.rb: -------------------------------------------------------------------------------- 1 | require 'stathat' 2 | 3 | sleep_i = @params['sleep'] || 60 4 | puts "worker #{@params['i']}" 5 | puts "Going to sleep at #{Time.now} for #{sleep_i}..." 6 | puts "posting 1 to stathat" 7 | StatHat::API.ez_post_count("Max Concurrency Test", @params['stathat']['email'], 1) 8 | sleep sleep_i 9 | puts "Woke up at #{Time.now}" 10 | 11 | -------------------------------------------------------------------------------- /test/test_full_remote_build.rb: -------------------------------------------------------------------------------- 1 | require_relative 'helpers' 2 | 3 | class FullRemoteBuildTest < IWNGTest 4 | def test_basic 5 | client.codes_create code_bundle('test/workers/full_remote_build/pdftk') 6 | task_id = client.tasks.create('pdftk').id 7 | client.tasks.wait_for(task_id) 8 | assert client.tasks.log(task_id) =~ /pdftk 1\.44/ 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /lib/iron_worker_ng/code/runtime/go.rb: -------------------------------------------------------------------------------- 1 | module IronWorkerNG 2 | module Code 3 | module Runtime 4 | module Go 5 | include IronWorkerNG::Feature::Common::MergeExec::InstanceMethods 6 | 7 | def runtime_run_code(local, params) 8 | <&1 | head -n 1 | awk -F ' ' '{print $1}' | head -c6` 3 | 4 | if [ "$current_version" = "$correct_version" ]; then 5 | echo "Version of Phantomjs is correct. $correct_version" 6 | exit 0 7 | else 8 | echo "Version of Phantomjs is incorrect. $current_version=$correct_version" 9 | exit 1 10 | fi -------------------------------------------------------------------------------- /lib/iron_worker_ng/code/runtime/mono.rb: -------------------------------------------------------------------------------- 1 | module IronWorkerNG 2 | module Code 3 | module Runtime 4 | module Mono 5 | include IronWorkerNG::Feature::Common::MergeExec::InstanceMethods 6 | 7 | def runtime_run_code(local, params) 8 | <&1 | head -n 1 | awk -F ' ' '{print $1}' | head -c6` 3 | 4 | if [ "$current_version" = "$correct_version" ]; then 5 | echo "Version of Phantomjs is correct. $correct_version" 6 | exit 0 7 | else 8 | echo "Version of Phantomjs is incorrect. $current_version=$correct_version" 9 | exit 1 10 | fi 11 | -------------------------------------------------------------------------------- /test/stacks/ruby-1.9/ruby-1.9.rb: -------------------------------------------------------------------------------- 1 | require "nokogiri" 2 | require "mechanize" 3 | @browser = ['Linux Mozilla', 'Mac Mozilla', 'Mac Safari', 'Windows Mozilla', 'Windows IE 8', 'Windows IE 9'].sample 4 | @mecha = Mechanize.new { |agent| agent.user_agent_alias = @browser} 5 | page = @mecha.get("http://www.google.com") 6 | raise "Not found" unless page.inspect.include? "Google" 7 | raise "Wrong ruby version" unless RUBY_VERSION.start_with?('1.9') 8 | -------------------------------------------------------------------------------- /test/stacks/ruby-2.1/ruby-2.1.rb: -------------------------------------------------------------------------------- 1 | require "nokogiri" 2 | require "mechanize" 3 | @browser = ['Linux Mozilla', 'Mac Mozilla', 'Mac Safari', 'Windows Mozilla', 'Windows IE 8', 'Windows IE 9'].sample 4 | @mecha = Mechanize.new { |agent| agent.user_agent_alias = @browser} 5 | page = @mecha.get("http://www.google.com") 6 | raise "Not found" unless page.inspect.include? "Google" 7 | raise "Wrong ruby version" unless RUBY_VERSION.start_with?('2.1') 8 | -------------------------------------------------------------------------------- /test/examples/example_hello.rb: -------------------------------------------------------------------------------- 1 | require_relative '../helpers' 2 | 3 | class ExampleHelloTest < IWNGTest 4 | def test_basic 5 | Dir.chdir 'examples/hello_worker' do 6 | client.codes_create(code_bundle('hello')) 7 | 8 | log = `ruby -I../../../lib enqueue.rb` 9 | 10 | assert(log =~ /Starting RubyHelloWorker/, 'started') 11 | assert(log =~ /RubyHelloWorker completed/, 'completed') 12 | end 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /test/test_wfile_paths.rb: -------------------------------------------------------------------------------- 1 | require_relative 'helpers' 2 | 3 | class WfilePathsTest < IWNGTest 4 | 5 | def test_dir 6 | wf = 'test/workers/wfile_paths/wfile_paths.worker' 7 | code = IronWorkerNG::Code::Base.new(:workerfile => wf) 8 | 9 | client.codes.create code 10 | id = client.tasks.create('WfilePaths').id 11 | client.tasks.wait_for(id) 12 | 13 | assert_equal "hello\n", client.tasks.log(id) 14 | end 15 | 16 | end 17 | -------------------------------------------------------------------------------- /lib/iron_worker_ng/code/runtime/binary.rb: -------------------------------------------------------------------------------- 1 | module IronWorkerNG 2 | module Code 3 | module Runtime 4 | module Binary 5 | include IronWorkerNG::Feature::Common::MergeExec::InstanceMethods 6 | 7 | def runtime_run_code(local, params) 8 | < -------------------------------------------------------------------------------- /test/stacks/python-2.7/python-2.7.worker: -------------------------------------------------------------------------------- 1 | runtime 'python' 2 | stack 'python-2.7' 3 | exec 'python2.py' 4 | pip 'simples3' 5 | pip "requests" 6 | pip "dj-database-url" 7 | pip "Django" 8 | pip "boto" 9 | pip "requests" 10 | pip "xlwt" 11 | pip "xlrd" 12 | pip "xlutils" 13 | pip "boto" 14 | pip "html5lib" 15 | pip "tinycss" 16 | pip "cssselect" 17 | pip "pyphen" 18 | pip "cffi" 19 | pip "WeasyPrint" 20 | pip "iso8601" 21 | pip "six" 22 | pip "python-dateutil" 23 | remote -------------------------------------------------------------------------------- /test/rake_test_ng/Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: http://rubygems.org/ 3 | specs: 4 | aws-s3 (0.6.2) 5 | builder 6 | mime-types 7 | xml-simple 8 | builder (3.0.0) 9 | hipchat (0.4.1) 10 | httparty 11 | httparty (0.8.3) 12 | multi_json (~> 1.0) 13 | multi_xml 14 | mime-types (1.18) 15 | multi_json (1.3.5) 16 | multi_xml (0.5.1) 17 | xml-simple (1.1.1) 18 | 19 | PLATFORMS 20 | ruby 21 | 22 | DEPENDENCIES 23 | aws-s3 24 | hipchat 25 | -------------------------------------------------------------------------------- /test/test_java_worker.rb: -------------------------------------------------------------------------------- 1 | require_relative 'helpers' 2 | 3 | class JavaWorkerTest < IWNGTest 4 | 5 | def test_hello 6 | code = IronWorkerNG::Code::Base.new do 7 | runtime 'java' 8 | name 'JavaHello' 9 | exec 'test/hello.jar' 10 | end 11 | client.codes_create(code) 12 | 13 | task = client.tasks_create('JavaHello') 14 | 15 | client.tasks_wait_for(task.id) 16 | 17 | assert_equal "hello\n", client.tasks.log(task.id), 18 | 'correct output' 19 | end 20 | 21 | end 22 | -------------------------------------------------------------------------------- /test/test_local_run_with_config.rb: -------------------------------------------------------------------------------- 1 | require_relative 'helpers' 2 | 3 | class LocalRunTest < IWNGTest 4 | CONFIG_FILE = File.dirname(__FILE__) + "/local_run_with_config.json" 5 | WORKER_FILE = File.dirname(__FILE__) + "/local_run_with_config.worker" 6 | def test_with_worker_config_file 7 | test = /#{config_file_contents}/ 8 | assert cli('run', "#{WORKER_FILE} --worker-config #{CONFIG_FILE}") =~ test 9 | end 10 | def test_with_no_config 11 | test = /#{nil.to_json}/ 12 | assert cli('run', "#{WORKER_FILE}") =~ test 13 | end 14 | private 15 | def config_file_contents 16 | File.read(CONFIG_FILE) 17 | end 18 | end -------------------------------------------------------------------------------- /test/test_workerfile.rb: -------------------------------------------------------------------------------- 1 | require_relative 'helpers' 2 | 3 | class WorkerfileTest < IWNGTest 4 | 5 | def workerfile(str) 6 | Tempfile.open('workerfile_test', '.'){ |f| f << str } 7 | end 8 | 9 | def test_basic 10 | wf = workerfile < wf.path) 17 | 18 | client.codes.create(code) 19 | 20 | id = client.tasks.create('ShHello').id 21 | 22 | client.tasks.wait_for(id) 23 | 24 | assert_equal "hello\n", client.tasks.log(id) 25 | end 26 | 27 | end 28 | -------------------------------------------------------------------------------- /test/examples/README.md: -------------------------------------------------------------------------------- 1 | To sync examples with repo: 2 | 1. install git@github.com:apenwarr/git-subtree.git 3 | 2. split ng examples into separate branch 4 | ``` 5 | git clone git@github.com:iron-io/iron_worker_examples.git iw_examples 6 | cd iw_examples 7 | git subtree split --prefix ruby_ng --annotate="(split)" -b ruby_ng_split 8 | cd - 9 | ``` 10 | 3. merge them into iron_worker_ng subtree 11 | ``` 12 | git clone git@github.com:iron-io/iron_worker_ruby_ng.git iw 13 | cd iw 14 | git remote add iw_examples ../iw_examples 15 | git fetch iw_examples 16 | git subtree merge -P examples --squash iw_examples/ruby_ng_split -m 'Updating examples subtree' 17 | git push origin master 18 | cd - 19 | ``` -------------------------------------------------------------------------------- /test/examples/example_master_slave.rb: -------------------------------------------------------------------------------- 1 | require_relative '../helpers' 2 | 3 | class ExampleMasterSlaveTest < IWNGTest 4 | def test_basic 5 | Dir.chdir 'examples/master_slave' do 6 | client.codes_create(code_bundle('master')) 7 | client.codes_create(code_bundle('slave')) 8 | 9 | log = `ruby -I../../../lib enqueue.rb` 10 | 11 | assert(log =~ /task id =/, 'task created') 12 | assert(log =~ /Queueing slave one with params \{"foo"=>"bar"\}/, 13 | 'first slave enqueued') 14 | assert(log =~ /Queueing slave two with params \{"hello"=>"world"\}/, 15 | 'second slave enqueued') 16 | assert(log =~ /Done/, 'done') 17 | end 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /test/test_tasks_list.rb: -------------------------------------------------------------------------------- 1 | require_relative 'helpers' 2 | 3 | class TestTaskList < IWNGTest 4 | 5 | def test_task_list 6 | 7 | tasks = client.tasks.list() 8 | code_names = {} 9 | tasks.each do |t| 10 | puts "#{t.code_name} - #{t.status}" 11 | code_names[t.code_name] ||= 0 12 | code_names[t.code_name] += 1 13 | end 14 | puts "num codes: #{code_names.size}" 15 | 16 | assert code_names.size > 0 17 | 18 | tasks = client.tasks.list(:code_name=>"hello") 19 | code_names = {} 20 | tasks.each do |t| 21 | p t.code_name 22 | code_names[t.code_name] ||= 0 23 | code_names[t.code_name] += 1 24 | end 25 | puts "num codes: #{code_names.size}" 26 | 27 | end 28 | 29 | end 30 | -------------------------------------------------------------------------------- /test/examples/example_github_webhook.rb: -------------------------------------------------------------------------------- 1 | require_relative '../helpers' 2 | require 'rest' 3 | 4 | class ExampleGithubWebhookTest < IWNGTest 5 | 6 | def test_example 7 | Dir.chdir 'examples/github_to_hipchat_webhook_worker' do 8 | assert(cli('upload', 'github_webhook') =~ /Upload successful/) 9 | 10 | assert(cli('webhook', 'github_webhook') =~ /^\s*(https.*)$/) 11 | webhook = $1 12 | 13 | rest = Rest::Client::new 14 | resp = rest.post(webhook, body: File.read('sample_github_payload')) 15 | assert_equal 200, resp.code 16 | 17 | task_id = JSON.parse(resp.body)['id'] 18 | client.tasks.wait_for(task_id) 19 | 20 | assert client.tasks.log(task_id) =~ /^Done$/ 21 | end 22 | end 23 | 24 | end 25 | -------------------------------------------------------------------------------- /test/test_concurrency.rb: -------------------------------------------------------------------------------- 1 | require_relative 'helpers' 2 | 3 | class TestTmp < IWNGTest 4 | 5 | N_TASKS = 1 6 | puts 'here' 7 | 8 | def setup 9 | puts 'setup2' 10 | super 11 | 12 | end 13 | 14 | 15 | def test_max_concurrency 16 | name = 'text_max_concurrency' 17 | puts name 18 | 19 | worker_name = "sleepy" 20 | 21 | code = IronWorkerNG::Code::Base.new(:workerfile => "test/workers/#{worker_name}.worker") 22 | @client.codes.create(code) 23 | 24 | 1.times do |i| 25 | puts "Queuing #{i}..." 26 | @client.tasks.create(worker_name, 27 | {:sleep => 1*60, :i => i, :stathat => {:email => "travis@iron.io"}}, 28 | {:delay=>60} 29 | ) 30 | end 31 | 32 | 33 | end 34 | 35 | 36 | end 37 | -------------------------------------------------------------------------------- /test/test_config.rb: -------------------------------------------------------------------------------- 1 | require_relative 'helpers' 2 | 3 | class TestConfig < IWNGTest 4 | 5 | def setup 6 | super 7 | 8 | end 9 | 10 | 11 | def test_config 12 | 13 | client.codes.create(IronWorkerNG::Code::Base.new('test/workers/config_worker'), {config: {c1: "some config var"}}) 14 | 15 | task_ids = [] 16 | 1.times do |i| 17 | puts "#{i}" 18 | task_ids << client.tasks.create('config_worker', {:foo=>"bar"}).id 19 | end 20 | 21 | task_ids.each_with_index do |id, i| 22 | puts "#{i}" 23 | task = client.tasks.wait_for(id) 24 | p task 25 | assert_equal 'complete', task.status 26 | log = client.tasks.log(id) 27 | puts log 28 | assert log.include?("some config var") 29 | end 30 | 31 | end 32 | 33 | 34 | end 35 | -------------------------------------------------------------------------------- /test/examples/example_chargify_to_campfire_webhook.rb: -------------------------------------------------------------------------------- 1 | require_relative '../helpers' 2 | require 'rest' 3 | 4 | class ExampleChargifyToCampfireWebhookTest < IWNGTest 5 | 6 | def test_example 7 | Dir.chdir 'examples/chargify_to_campfire_webhook_worker' do 8 | assert(cli('upload', 'chargify_to_campfire') =~ /Upload successful/) 9 | 10 | assert(cli('webhook', 'chargify_to_campfire') =~ /^\s*(https.*)$/) 11 | webhook = $1 12 | 13 | rest = Rest::Client::new 14 | resp = rest.post(webhook, body: "payload[chargify]=testing&id=6825503&event=test") 15 | assert_equal 200, resp.code 16 | 17 | task_id = JSON.parse(resp.body)['id'] 18 | client.tasks.wait_for(task_id) 19 | 20 | assert client.tasks.log(task_id) =~ /^Done$/ 21 | end 22 | end 23 | 24 | end 25 | -------------------------------------------------------------------------------- /lib/iron_worker_ng/code/container/zip.rb: -------------------------------------------------------------------------------- 1 | require 'zip' 2 | 3 | Zip.continue_on_exists_proc = true 4 | 5 | module IronWorkerNG 6 | module Code 7 | module Container 8 | class Zip < IronWorkerNG::Code::Container::Base 9 | def initialize 10 | super 11 | 12 | @name = @name + '.zip' 13 | @zip = ::Zip::File.open(@name, ::Zip::File::CREATE) 14 | end 15 | 16 | def add(dest, src) 17 | @zip.add(clear_dest(dest), src) 18 | end 19 | 20 | def commit 21 | @zip.commit 22 | end 23 | 24 | def get_output_stream(dest, &block) 25 | @zip.get_output_stream(clear_dest(dest), &block) 26 | end 27 | 28 | def close 29 | @zip.close 30 | end 31 | end 32 | end 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /lib/iron_worker_ng/code/runtime/python.rb: -------------------------------------------------------------------------------- 1 | require 'iron_worker_ng/feature/python/merge_pip_dependency' 2 | require 'iron_worker_ng/feature/python/merge_pip' 3 | require 'iron_worker_ng/feature/python/merge_requirements' 4 | 5 | module IronWorkerNG 6 | module Code 7 | module Runtime 8 | module Python 9 | include IronWorkerNG::Feature::Common::MergeExec::InstanceMethods 10 | include IronWorkerNG::Feature::Python::MergePipDependency::InstanceMethods 11 | include IronWorkerNG::Feature::Python::MergeRequirements::InstanceMethods 12 | 13 | def runtime_run_code(local, params) 14 | < 'test', 10 | :exec => 'test/hello.rb') 11 | 12 | task_ids = [] 13 | mutex = Mutex.new 14 | (N_TASKS.times.map do 15 | Thread.new do 16 | task_id = client.tasks.create('test').id 17 | mutex.synchronize do 18 | task_ids << task_id 19 | end 20 | end 21 | end).each{|t| t.join} 22 | 23 | assert_equal N_TASKS, task_ids.size, 24 | "All tasks started" 25 | 26 | task_ids.each do |id| 27 | client.tasks.wait_for(id) 28 | assert_equal "hello\n", client.tasks.log(id), 29 | "correct output" 30 | end 31 | end 32 | 33 | end 34 | -------------------------------------------------------------------------------- /lib/iron_worker_ng/code/container/dir.rb: -------------------------------------------------------------------------------- 1 | require 'fileutils' 2 | 3 | module IronWorkerNG 4 | module Code 5 | module Container 6 | class Dir < IronWorkerNG::Code::Container::Base 7 | def full_dest(dest) 8 | @name + '/' + clear_dest(dest) 9 | end 10 | 11 | def add(dest, src) 12 | FileUtils.mkdir_p(File.dirname(full_dest(dest))) 13 | 14 | if File.directory?(src) 15 | FileUtils.mkdir(full_dest(dest)) 16 | else 17 | FileUtils.cp(src, full_dest(dest)) 18 | end 19 | end 20 | 21 | def get_output_stream(dest, &block) 22 | FileUtils.mkdir_p(File.dirname(full_dest(dest))) 23 | 24 | file = File.open(full_dest(dest), 'wb') 25 | block.call(file) 26 | file.close 27 | end 28 | end 29 | end 30 | end 31 | end 32 | -------------------------------------------------------------------------------- /test/stacks/ruby-1.9/ruby-1.9.worker: -------------------------------------------------------------------------------- 1 | @use_local_iron_worker_ng = true 2 | runtime "ruby" 3 | stack 'ruby-1.9' 4 | 5 | exec "ruby-1.9.rb" 6 | 7 | gem 'net-http-persistent', '2.9.4' 8 | gem 'rest' 9 | gem 'rubyzip', '0.9.9' 10 | gem 'iron_mq' 11 | gem 'iron_worker_ng' 12 | gem 'iron_cache', '>= 1.4.2' 13 | gem 'typhoeus' 14 | gem 'mime-types', '~> 1.25.1' 15 | gem 'retryable' 16 | gem 'stamp' 17 | gem 'actionpack' 18 | gem 'spliner' 19 | 20 | gem 'aws-s3' 21 | gem 'geocoder' 22 | gem 'rgeo' 23 | gem 'rgeo-geojson' 24 | gem 'google_distance_matrix' 25 | 26 | gem 'nokogiri', '~> 1.5.10' 27 | gem 'mechanize' 28 | gem 'bson_ext' 29 | gem 'mongoid', '~> 3.1.5' 30 | gem 'carrierwave', '~> 0.9.0' 31 | gem 'carrierwave-mongoid', '~> 0.6.3' 32 | gem 'carrierwave-processing' 33 | gem 'rmagick' 34 | gem 'fog', '~> 1.18.0' 35 | gem 'mongoid-app_settings' 36 | gem 'aasm', '~> 3.0.26' 37 | 38 | full_remote_build true 39 | -------------------------------------------------------------------------------- /test/stacks/ruby-2.1/ruby-2.1.worker: -------------------------------------------------------------------------------- 1 | @use_local_iron_worker_ng = true 2 | runtime "ruby" 3 | stack 'ruby-2.1' 4 | 5 | exec "ruby-2.1.rb" 6 | 7 | gem 'net-http-persistent', '2.9.4' 8 | gem 'rest' 9 | gem 'rubyzip', '0.9.9' 10 | gem 'iron_mq' 11 | gem 'iron_worker_ng' 12 | gem 'iron_cache', '>= 1.4.2' 13 | gem 'typhoeus' 14 | gem 'mime-types', '~> 1.25.1' 15 | gem 'retryable' 16 | gem 'stamp' 17 | gem 'actionpack' 18 | gem 'spliner' 19 | 20 | gem 'aws-s3' 21 | gem 'geocoder' 22 | gem 'rgeo' 23 | gem 'rgeo-geojson' 24 | gem 'google_distance_matrix' 25 | 26 | gem 'nokogiri', '~> 1.5.10' 27 | gem 'mechanize' 28 | gem 'bson_ext' 29 | gem 'mongoid', '~> 3.1.5' 30 | gem 'carrierwave', '~> 0.9.0' 31 | gem 'carrierwave-mongoid', '~> 0.6.3' 32 | gem 'carrierwave-processing' 33 | gem 'rmagick' 34 | gem 'fog', '~> 1.18.0' 35 | gem 'mongoid-app_settings' 36 | gem 'aasm', '~> 3.0.26' 37 | 38 | full_remote_build true 39 | -------------------------------------------------------------------------------- /test/stacks/php-5.4/php.php: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/remote_test.rb: -------------------------------------------------------------------------------- 1 | out = `gem build iron_worker_ng.gemspec` 2 | unless out =~ /File: (.*)$/ 3 | puts "Failed to build gem: #{out}" 4 | exit(1) 5 | end 6 | puts `gem install #{$1}` 7 | 8 | require_relative '../lib/iron_worker_ng.rb' 9 | require_relative 'iron_io_config.rb' 10 | 11 | client = IronWorkerNG::Client.new 12 | 13 | code = IronWorkerNG::Code::Base.new do 14 | name 'NgTestsWorker' 15 | runtime 'ruby' 16 | 17 | exec 'test/ng_tests_worker.rb' 18 | gemfile 'Gemfile', :default, :development, :test 19 | 20 | Dir.glob('*').each do |p| 21 | dir p, 'iwng' if File.directory? p 22 | file p, 'iwng' if File.file? p 23 | end 24 | 25 | iron_io_config 'iwng' 26 | end 27 | 28 | puts client.codes.create(code) 29 | 30 | task = client.tasks.create 'NgTestsWorker', args: $*.join(' ') 31 | 32 | client.tasks.wait_for task.id 33 | 34 | puts '-' * 80 35 | 36 | puts client.tasks.log(task.id) 37 | -------------------------------------------------------------------------------- /test/test_fetcher.rb: -------------------------------------------------------------------------------- 1 | require_relative 'helpers' 2 | 3 | class FetcherTest < IWNGTest 4 | 5 | HELLO = 'https://raw.github.com/iron-io/iron_worker_ruby_ng/master/test/hello.rb' 6 | 7 | def test_basic 8 | code = code_bundle { exec HELLO } 9 | client.codes.create(code) 10 | task_id = client.tasks.create(code.name).id 11 | client.tasks.wait_for(task_id) 12 | assert_equal "hello\n", client.tasks.log(task_id) 13 | end 14 | 15 | def test_with_workerfile 16 | Tempfile.open(['', '.worker']) do |f| 17 | f << "file '#{File.basename(f.path)}'\n" 18 | f << "exec '#{HELLO}'\n" 19 | f.close 20 | 21 | code = IronWorkerNG::Code::Base.new(workerfile: f.path) 22 | 23 | client.codes.create code 24 | task_id = client.tasks.create(code.name).id 25 | client.tasks.wait_for(task_id) 26 | assert_equal "hello\n", client.tasks.log(task_id) 27 | end 28 | end 29 | 30 | end 31 | -------------------------------------------------------------------------------- /tools/gems.json: -------------------------------------------------------------------------------- 1 | { 2 | "gems": [ 3 | ["bson_ext", ">= 1.8"], 4 | ["bson", ">= 2.3.0"], 5 | ["json", ">= 1.4"], 6 | ["nokogiri", ">= 1.5.0"], 7 | ["nokogiri", "= 1.6.2.1"], 8 | ["mini_magick", ">= 3.5"], 9 | ["rmagick", ">= 2.1"], 10 | ["mysql2", ">= 0.3"], 11 | ["sqlite3", ">= 1.3.9"], 12 | ["pg", ">= 0.17.1"], 13 | ["atomic", ">= 1.1.14"], 14 | ["eventmachine", ">= 1.0"], 15 | ["patron", ">= 0.4"], 16 | ["phashion", ">= 1.0.5"], 17 | ["ffi", ">= 1.9.0"], 18 | ["zipruby", ">= 1.1.2"], 19 | ["gsl", ">= 1.15"], 20 | ["hpricot", ">= 0.8"], 21 | ["curb", ">= 0.8"], 22 | ["bcrypt-ruby", ">= 3.0"], 23 | ["yajl-ruby", ">= 1.1.0"], 24 | ["patron", ">= 3.0.0"], 25 | ["http_parser", ">= 0.6.0"], 26 | ["bcrypt", ">= 3.0.0"], 27 | ["digest-sha3", ">= 1.0.0"], 28 | ["unf_ext", ">= 0"], 29 | ["twitter", ">= 5.8.0"], 30 | ["nokogumbo", ">= 1.1.9"] 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /lib/3rdparty/hashie/merge_initializer.rb: -------------------------------------------------------------------------------- 1 | # source: https://github.com/intridea/hashie/blob/6d21c6868512603e77a340827ec91ecd3bcef078/lib/hashie/extensions/merge_initializer.rb 2 | module Hashie 3 | module Extensions 4 | # The MergeInitializer is a super-simple mixin that allows 5 | # you to initialize a subclass of Hash with another Hash 6 | # to give you faster startup time for Hash subclasses. Note 7 | # that you can still provide a default value as a second 8 | # argument to the initializer. 9 | # 10 | # @example 11 | # class MyHash < Hash 12 | # include Hashie::Extensions::MergeInitializer 13 | # end 14 | # 15 | # h = MyHash.new(:abc => 'def') 16 | # h[:abc] # => 'def' 17 | # 18 | module MergeInitializer 19 | def initialize(hash = {}, default = nil, &block) 20 | default ? super(default) : super(&block) 21 | update(hash) 22 | end 23 | end 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /lib/iron_worker_ng/code/container/base.rb: -------------------------------------------------------------------------------- 1 | require 'tmpdir' 2 | require 'pathname' 3 | 4 | module IronWorkerNG 5 | module Code 6 | module Container 7 | class Base 8 | attr_reader :name 9 | attr_reader :runner_additions 10 | 11 | def initialize 12 | @name = ::Dir.tmpdir + '/' + ::Dir::Tmpname.make_tmpname('iron-worker-ng-', 'container') 13 | @runner_additions = '' 14 | end 15 | 16 | def clear_dest(dest) 17 | dest = Pathname.new(dest).cleanpath.to_s unless dest.empty? 18 | 19 | dest 20 | end 21 | 22 | def add(dest, src) 23 | end 24 | 25 | def get_output_stream(dest, &block) 26 | end 27 | 28 | def commit 29 | end 30 | 31 | def close 32 | end 33 | 34 | def runner_add(runner_code) 35 | @runner_additions << runner_code << "\n" 36 | end 37 | end 38 | end 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /test/Rakefile: -------------------------------------------------------------------------------- 1 | require 'tmpdir' 2 | require 'rake/testtask' 3 | 4 | Dir.chdir(File.dirname(__FILE__) + '/..') 5 | 6 | Rake::TestTask.new do |t| 7 | if ENV['NEW_PROJECT'] 8 | require_relative '../lib/iron_worker_ng.rb' 9 | 10 | client = IronWorkerNG::Client.new 11 | name = 'IWtest ' + Time.now.strftime('%b %-d %T') 12 | resp = client.api.post('projects', name: name) 13 | res = JSON.parse(resp.body) 14 | raise "Failed to create new project: #{res}" unless 15 | res['msg'].start_with? 'Project Created' 16 | 17 | ENV['IRON_PROJECT_ID'] = res['id'] 18 | end 19 | if ENV['IRON_PROJECT_ID'] 20 | t.options = "-- --project-id=#{ENV['IRON_PROJECT_ID']}" 21 | end 22 | 23 | t.libs << "lib" 24 | 25 | files = FileList['test/**/test_*.rb'] 26 | t.test_files = files.keep_if do |f| 27 | f =~ Regexp.new(ENV['TESTP'] || '') and 28 | not ( r = ENV['EXCLP'] and 29 | f =~ Regexp.new(r) ) 30 | end 31 | 32 | t.verbose = true 33 | end 34 | -------------------------------------------------------------------------------- /test/quick_run.rb: -------------------------------------------------------------------------------- 1 | require_relative 'helpers' 2 | 3 | require 'go' 4 | 5 | class QuickRun < IWNGTest 6 | 7 | N_TASKS = 100 8 | 9 | def setup 10 | super 11 | 12 | end 13 | 14 | def test_quick_run2 15 | 16 | client.codes.create(IronWorkerNG::Code::Base.new('hello')) 17 | 18 | task_ids = [] 19 | ch = Go::Channel.new 20 | N_TASKS.times do |i| 21 | go do 22 | puts "#{i}" 23 | ch << client.tasks.create('hello', {:foo => "bar"}, {:priority => 2}).id 24 | end 25 | end 26 | 27 | ch.each do |x| 28 | task_ids << x 29 | break if task_ids.size == N_TASKS 30 | end 31 | 32 | task_ids.each_with_index do |id, i| 33 | puts "#{i}" 34 | task = client.tasks.wait_for(id) 35 | p task 36 | assert_equal 'complete', task.status 37 | sleep 1 38 | log = client.tasks.log(id) 39 | puts log 40 | assert_equal "hello\n", log, "for #{i}th task, task_id: ##{id}" 41 | end 42 | end 43 | 44 | end 45 | -------------------------------------------------------------------------------- /test/test_cli.rb: -------------------------------------------------------------------------------- 1 | require_relative 'helpers' 2 | 3 | class CLITest < IWNGTest 4 | 5 | def test_basic 6 | assert cli('upload', 'test/hello.worker') =~ 7 | /Upload successful/ 8 | 9 | assert cli('queue', 'hello') =~ 10 | /Queued up.*"id":"(.{24})"/ 11 | 12 | assert cli('log', '--wait', $1) =~ 13 | /\nhello\n/ 14 | 15 | assert cli('schedule', 'hello') =~ 16 | /Scheduled/ 17 | end 18 | 19 | def test_stacks 20 | assert cli('stacks') =~ 21 | /mono/ 22 | end 23 | 24 | def test_argument 25 | assert cli('upload', 'test/workers/wfile_paths/wfile_paths.worker') =~ 26 | /Upload successful/ 27 | 28 | tmp = File.open('krumplumpl.worker', 'w') { |f| f << 'exec "test/hello.rb"' } 29 | assert cli('upload', 'krumplumpl') =~ 30 | /Upload successful/ 31 | File.unlink tmp 32 | end 33 | 34 | def test_forced_name 35 | assert cli('upload', 'test/hello.worker', name: 'Frobnicator') =~ 36 | /Upload successful.*Frobnicator/m 37 | end 38 | 39 | end 40 | -------------------------------------------------------------------------------- /lib/iron_worker_ng/code/runtime/java.rb: -------------------------------------------------------------------------------- 1 | require 'iron_worker_ng/feature/java/merge_jar' 2 | 3 | module IronWorkerNG 4 | module Code 5 | module Runtime 6 | module Java 7 | include IronWorkerNG::Feature::Common::MergeExec::InstanceMethods 8 | include IronWorkerNG::Feature::Java::MergeJar::InstanceMethods 9 | 10 | def runtime_run_code(local, params) 11 | classpath_array = [] 12 | 13 | classpath_array << @exec.path 14 | 15 | @features.each do |f| 16 | if f.respond_to?(:code_for_classpath) 17 | classpath_array << f.send(:code_for_classpath) 18 | end 19 | end 20 | 21 | classpath = classpath_array.join(':') 22 | 23 | IronCore::Logger.info 'IronWorkerNG', "Collected '#{classpath}' classpath" 24 | 25 | < -------------------------------------------------------------------------------- /test/test_code_delete.rb: -------------------------------------------------------------------------------- 1 | require_relative 'helpers' 2 | 3 | class CodeDeleteTest < IWNGTest 4 | def test_code_delete 5 | r = code = client.codes.create(code_bundle do 6 | name 'code_delete_test' 7 | worker_code 'puts "hello"' 8 | end) 9 | p r 10 | code_id1 = r.id 11 | task_id = client.tasks.create('code_delete_test').id 12 | client.tasks.wait_for(task_id) 13 | assert_equal "hello\n", client.tasks.log(task_id) 14 | 15 | client.codes.delete(code.id) 16 | 17 | assert_raises Rest::HttpError do 18 | r = client.codes.get(code.id) 19 | p r 20 | end 21 | 22 | assert_raises Rest::HttpError do 23 | task_id = client.tasks.create('code_delete_test').id 24 | end 25 | 26 | r = client.codes.create(code_bundle do 27 | name 'code_delete_test' 28 | worker_code 'puts "bye"' 29 | end) 30 | assert_not_equal(code_id1, r.id) 31 | task_id = client.tasks.create('code_delete_test').id 32 | client.tasks.wait_for(task_id) 33 | assert_equal "bye\n", client.tasks.log(task_id) 34 | end 35 | end 36 | -------------------------------------------------------------------------------- /test/test_stacks.rb: -------------------------------------------------------------------------------- 1 | require_relative 'helpers' 2 | 3 | class CodeCreateTest < IWNGTest 4 | 5 | def self.startup 6 | puts 'Starting stack tests' 7 | puts 'Please, input cluster:' 8 | @@cluster = gets.chomp 9 | @@cluster = 'default' if @@cluster == '' 10 | end 11 | 12 | def test_create 13 | stacks_list = client.stacks_list 14 | assert stacks_list.is_a? Array 15 | assert stacks_list.count > 0 16 | assert stacks_list.include? 'python-2.7' 17 | end 18 | 19 | def test_wrong_stack 20 | code = code_bundle(:exec => 'test/hello.rb',:name => 'sample') 21 | code.stack('none') 22 | assert_raise Rest::Wrappers::RestClientExceptionWrapper do 23 | client.codes_create code 24 | end 25 | end 26 | 27 | def test_stacks 28 | puts "Starting stack tests for \"#{@@cluster}\" cluster..." 29 | client.stacks_list.each do |stack| 30 | client.codes.create(IronWorkerNG::Code::Base.new("test/stacks/#{stack}/#{stack}")) 31 | id = client.tasks.create(stack, {}, {cluster: @@cluster}).id 32 | task = client.tasks.wait_for(id) 33 | assert_equal 'complete', task.status 34 | end 35 | end 36 | 37 | end 38 | -------------------------------------------------------------------------------- /test/test_builder.rb: -------------------------------------------------------------------------------- 1 | require_relative 'helpers' 2 | 3 | class BuilderTest < IWNGTest 4 | 5 | def test_basic 6 | code = code_bundle do 7 | runtime 'ruby' 8 | name 'CHello' 9 | file 'test/hello.c' 10 | remote_build_command 'gcc hello.c' 11 | worker_code 'exec("./a.out")' 12 | end 13 | 14 | start = Time.now 15 | client.codes.create(code) 16 | puts "uploading finished in #{(Time.now - start).to_i} seconds" 17 | 18 | task = client.tasks.create('CHello') 19 | client.tasks.wait_for(task.id) 20 | 21 | assert_equal "hello\n", client.tasks.log(task.id) 22 | end 23 | 24 | def test_async 25 | code = code_bundle 'test/workers/with_build_command/build_command.worker' 26 | 27 | builder_task_id = client.codes.create_async(code) 28 | puts builder_task_id 29 | 30 | builder_task = client.tasks.wait_for(builder_task_id) 31 | puts builder_task 32 | 33 | assert_equal 'complete', builder_task.status 34 | 35 | task = client.tasks.create(code.name) 36 | puts task 37 | client.tasks.wait_for(task.id) 38 | 39 | log = client.tasks.log(task.id) 40 | assert_equal "hello\n", log 41 | end 42 | 43 | end 44 | -------------------------------------------------------------------------------- /test/test_clusters.rb: -------------------------------------------------------------------------------- 1 | require_relative 'helpers' 2 | 3 | class ClustersTest < IWNGTest 4 | def test_create 5 | count_before = client.clusters_list.count 6 | id = client.clusters_create({name: 'test_name'}) 7 | count_after = client.clusters_list.count 8 | 9 | assert_true id.length == 24 10 | assert_equal 1, count_after - count_before 11 | end 12 | 13 | def test_get 14 | name = "test#{Time.now.to_i}" 15 | id = client.clusters_create({name: name}) 16 | 17 | cluster = client.clusters_get(id) 18 | 19 | assert_equal id, cluster.id 20 | assert_equal name, cluster.name 21 | end 22 | 23 | def test_update 24 | id = client.clusters_create({name: 'test_name'}) 25 | 26 | name = "test#{Time.now.to_i}" 27 | client.clusters_update(id, {name: name}) 28 | cluster = client.clusters_get(id) 29 | 30 | assert_equal name, cluster.name 31 | end 32 | 33 | def test_delete 34 | id = client.clusters_create({name: 'test'}) 35 | count_before = client.clusters_list.count 36 | 37 | client.clusters_delete(id) 38 | count_after = client.clusters_list.count 39 | 40 | assert_equal -1, count_after - count_before 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /lib/iron_worker_ng/feature/python/merge_requirements.rb: -------------------------------------------------------------------------------- 1 | module IronWorkerNG 2 | module Feature 3 | module Python 4 | module MergeRequirements 5 | module InstanceMethods 6 | 7 | def merge_requirements(path) 8 | feature = IronWorkerNG::Feature::Base.new(self) 9 | IronWorkerNG::Fetcher.fetch_to_file(feature.rebase(path)) do |requirements| 10 | specs = parse_requirements(requirements) 11 | specs.each {|spec| merge_pip(spec[:name], spec[:version])} 12 | end 13 | @features << feature 14 | end 15 | 16 | alias :requirements :merge_requirements 17 | 18 | private 19 | def parse_requirements(file) 20 | specs = [] 21 | File.readlines(file).each do |line| 22 | line = line.strip 23 | next if line.to_s.empty? || line.start_with?('#') 24 | spec = line.split(/(==|>=|<=|<|>)/, 2) 25 | version = spec[1]? spec[1]+spec[2]: '' 26 | specs << {name: spec[0], version:version} 27 | end 28 | specs 29 | end 30 | end 31 | end 32 | end 33 | end 34 | end 35 | 36 | -------------------------------------------------------------------------------- /lib/iron_worker_ng.rb: -------------------------------------------------------------------------------- 1 | puts 'This gem is deprecated. Please see this page https://github.com/iron-io/iron_worker_ruby_ng/ for more information about new tools.' 2 | 3 | begin 4 | require 'json' 5 | rescue LoadError 6 | raise "Please install json gem" 7 | end 8 | 9 | if (not ''.respond_to?(:start_with?)) or (not ''.respond_to?(:end_with?)) 10 | class ::String 11 | def start_with?(prefix) 12 | prefix = prefix.to_s 13 | self[0, prefix.length] == prefix 14 | end 15 | 16 | def end_with?(suffix) 17 | suffix = suffix.to_s 18 | self[-suffix.length, suffix.length] == suffix 19 | end 20 | end 21 | end 22 | 23 | require 'iron_worker_ng/version' 24 | require 'iron_worker_ng/compat' 25 | require 'iron_worker_ng/fetcher' 26 | require 'iron_worker_ng/client' 27 | require 'iron_worker_ng/code/base' 28 | require 'iron_worker_ng/code/ruby' 29 | require 'iron_worker_ng/code/binary' 30 | require 'iron_worker_ng/code/java' 31 | require 'iron_worker_ng/code/node' 32 | require 'iron_worker_ng/code/mono' 33 | require 'iron_worker_ng/code/python' 34 | require 'iron_worker_ng/code/php' 35 | require 'iron_worker_ng/code/go' 36 | require 'iron_worker_ng/code/perl' 37 | require 'iron_worker_ng/code/builder' 38 | -------------------------------------------------------------------------------- /lib/iron_worker_ng/feature/common/set_env.rb: -------------------------------------------------------------------------------- 1 | module IronWorkerNG 2 | module Feature 3 | module Common 4 | module SetEnv 5 | class Feature < IronWorkerNG::Feature::Base 6 | attr_reader :key 7 | attr_reader :value 8 | 9 | def initialize(code, key, value) 10 | super(code) 11 | 12 | @key = key 13 | @value = value 14 | end 15 | 16 | def bundle(container) 17 | container.runner_add "export #{@key}=\"#{@value.to_s.gsub('"','\\"')}\"" 18 | end 19 | 20 | def build_command 21 | if @code.remote_build_command || @code.full_remote_build 22 | "set_env \"#{@key}\", \"#{@value.to_s.gsub('"','\\"')}\"" 23 | else 24 | nil 25 | end 26 | end 27 | end 28 | 29 | module InstanceMethods 30 | def set_env(key, value) 31 | IronCore::Logger.info 'IronWorkerNG', "Setting ENV variable with name='#{key}' and value='#{value}'" 32 | 33 | @features << IronWorkerNG::Feature::Common::SetEnv::Feature.new(self, key, value) 34 | end 35 | end 36 | end 37 | end 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /test/examples/example_actionmailer_standalone.rb: -------------------------------------------------------------------------------- 1 | require_relative '../helpers' 2 | require 'rest' 3 | require 'net/imap' 4 | 5 | class ExampleActionMailerStandaloneTest < IWNGTest 6 | 7 | def test_example 8 | Dir.chdir 'examples/actionmailer_standalone' do 9 | assert(cli('upload', 'actionmailer_standalone') =~ /Upload successful/) 10 | assert(cli('queue', 'actionmailer_standalone', 11 | payload_file: 'actionmailer_config.json') =~ 12 | /queued with id='(.*)'/) 13 | 14 | # wait for delivery 15 | client.tasks.wait_for($1) 16 | sleep 10 17 | 18 | config = JSON.parse(File.read('actionmailer_config.json'))['gmail'] 19 | begin 20 | imap = Net::IMAP.new('imap.gmail.com',993,true) 21 | imap.login(config['username'], config['password']) 22 | imap.select('INBOX') 23 | ids = imap.search(['SUBJECT', 'Sample subject']) 24 | assert ids.size >= 1 25 | id = ids.last 26 | msg = imap.fetch(id, 'RFC822') 27 | assert msg[0].attr['RFC822'] =~ /message sent using IronWorker/ 28 | imap.store(id, "+FLAGS", [:Deleted]) 29 | ensure 30 | imap.logout 31 | imap.disconnect 32 | end 33 | end 34 | end 35 | 36 | end 37 | -------------------------------------------------------------------------------- /lib/iron_worker_ng/feature/base.rb: -------------------------------------------------------------------------------- 1 | module IronWorkerNG 2 | module Feature 3 | class Base 4 | def initialize(code) 5 | @code = code 6 | end 7 | 8 | def rebase(path) 9 | if IronWorkerNG::Fetcher.remote?(path) || path.start_with?('/') 10 | return path 11 | end 12 | 13 | @code.base_dir + path 14 | end 15 | 16 | def container_add(container, dest, src, commit = false) 17 | IronWorkerNG::Fetcher.fetch_to_file(src) do |local_src| 18 | if local_src.nil? || (not File.exists?(local_src)) 19 | IronCore::Logger.error 'IronWorkerNG', "Can't find src with path='#{src}'", IronCore::Error 20 | end 21 | 22 | if File.directory?(local_src) 23 | ::Dir.glob(local_src + '/**/**') do |path| 24 | container.add(@code.dest_dir + dest + path[local_src.length .. -1], path) 25 | end 26 | else 27 | container.add(@code.dest_dir + dest, local_src) 28 | end 29 | 30 | if IronWorkerNG::Fetcher.remote?(src) || commit 31 | container.commit 32 | end 33 | end 34 | end 35 | 36 | def bundle(container) 37 | end 38 | 39 | def build_command 40 | nil 41 | end 42 | end 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /test/test_change_runtime.rb: -------------------------------------------------------------------------------- 1 | require_relative 'helpers' 2 | 3 | class ChangeRuntimeTest < IWNGTest 4 | 5 | # Alexander S. 6 | # btw looks like its a bug: 7 | # 1) upload worker with any name 8 | # 2) upload worker with same name and different runtime 9 | # 3) launch it and gaze on error 10 | 11 | def ruby_worker(code_name) 12 | client.codes_create( code_bundle do 13 | name code_name 14 | worker_code 'puts "hello"' 15 | end ) 16 | 17 | task = client.tasks.create(code_name) 18 | client.tasks.wait_for task.id 19 | 20 | assert_equal "hello\n", client.tasks.log(task.id) 21 | end 22 | 23 | def sh_worker(code_name) 24 | code = IronWorkerNG::Code::Base.new do 25 | runtime 'binary' 26 | name code_name 27 | exec(Tempfile.open('sh') do |f| 28 | f << 'echo "hello"' 29 | end.path) 30 | end 31 | 32 | client.codes.create code 33 | 34 | task = client.tasks.create(code_name) 35 | client.tasks.wait_for task.id 36 | 37 | assert_equal "hello\n", client.tasks.log(task.id) 38 | end 39 | 40 | def test_sh_to_ruby 41 | sh_worker 'sh_to_ruby' 42 | ruby_worker 'sh_to_ruby' 43 | end 44 | 45 | def test_ruby_to_sh 46 | ruby_worker 'ruby_to_sh' 47 | sh_worker 'ruby_to_sh' 48 | end 49 | 50 | end 51 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012 Iron.io, Inc 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 2. Redistributions in binary form must reproduce the above copyright notice, 10 | this list of conditions and the following disclaimer in the documentation 11 | and/or other materials provided with the distribution. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 14 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 15 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 16 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 17 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 18 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 19 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 20 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 21 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 22 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | -------------------------------------------------------------------------------- /test/iron_io_config.rb: -------------------------------------------------------------------------------- 1 | require 'pathname' 2 | 3 | module IronWorkerNG 4 | module Feature 5 | class IronIOConfig < IronWorkerNG::Feature::Base 6 | attr_reader :path 7 | attr_reader :dest 8 | 9 | def initialize(api, dest) 10 | @path = File.open(Dir.mktmpdir + '/iron.json', 'w') do |f| 11 | f << { 12 | token: api.token, 13 | project_id: api.project_id 14 | }.to_json 15 | end.path 16 | @dest = dest 17 | @dest = Pathname.new(dest).cleanpath.to_s + '/' unless @dest.empty? 18 | end 19 | 20 | def bundle(zip) 21 | IronCore::Logger.debug 'IronWorkerNG', 'Bundling iron.io config' 22 | zip.add(@dest + File.basename(@path), @path) 23 | end 24 | end 25 | end 26 | 27 | module Code 28 | module Runtime 29 | module Ruby 30 | def iron_io_config(*args) 31 | dest = '' 32 | api = IronWorkerNG::Client.new.api 33 | args.each do |arg| 34 | dest = arg if arg.is_a? String 35 | api = arg.api if arg.is_a? IronWorkerNG::Client 36 | end 37 | IronCore::Logger.info 'IronWorkerNG', 38 | "Merging iron.io config (dest=#{dest})" 39 | @features << Feature::IronIOConfig.new(api, dest) 40 | end 41 | end 42 | end 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /iron_worker_ng.gemspec: -------------------------------------------------------------------------------- 1 | require File.expand_path('../lib/iron_worker_ng/version', __FILE__) 2 | 3 | Gem::Specification.new do |gem| 4 | gem.authors = ['Andrew Kirilenko', 'Iron.io, Inc'] 5 | gem.email = ['info@iron.io'] 6 | gem.description = 'New generation ruby client for IronWorker' 7 | gem.summary = 'New generation ruby client for IronWorker' 8 | gem.homepage = 'https://github.com/iron-io/iron_worker_ruby_ng' 9 | 10 | gem.files = Dir.glob('lib/**/**') + ['README.md', 'LICENSE'] 11 | gem.executables = 'iron_worker' 12 | gem.name = 'iron_worker_ng' 13 | gem.require_paths = ['lib'] 14 | gem.version = IronWorkerNG::VERSION 15 | 16 | gem.required_rubygems_version = '>= 1.3.6' 17 | gem.required_ruby_version = Gem::Requirement.new('>= 1.9') 18 | 19 | gem.add_runtime_dependency 'iron_core', '>= 1.0.6' 20 | gem.add_runtime_dependency 'bundler' 21 | gem.add_runtime_dependency 'rubyzip', '>= 1.0.0' 22 | 23 | gem.add_development_dependency 'test-unit' 24 | gem.add_development_dependency 'minitest' 25 | gem.add_development_dependency 'minitest-reporters', '>= 0.5.0' 26 | gem.add_development_dependency 'rake' 27 | gem.add_development_dependency 'twitter' 28 | gem.add_development_dependency 'actionmailer' 29 | gem.add_development_dependency 'iron_cache', '>= 1.4.0' 30 | gem.add_development_dependency 'go' 31 | end 32 | -------------------------------------------------------------------------------- /test/stacks/ffmpeg-2.3/ffmpeg-2.3.sh: -------------------------------------------------------------------------------- 1 | correct_version_mp4box="0.5.1" 2 | correct_version_ffmpeg="2.3" 3 | correct_version_node="v0.10.29" 4 | correct_version_ruby="1.9.3" 5 | correct_version_java="1.7" 6 | 7 | current_version_mp4box=`MP4Box -version 2>&1 | head -n 1 | awk -F 'version ' '{print $2}' | head -c5` 8 | current_version_ffmpeg=`ffmpeg -version 2>&1 | head -n 1 | awk -F 'version ' '{print $2}' | head -c3` 9 | current_version_node=`node -v` 10 | current_version_ruby=`ruby -v 2>&1 | awk -F 'ruby ' '{print $2}' | head -c5` 11 | current_version_java=`java -version 2>&1 | head -n 1 | awk -F '"' '{print $2}' | head -c3` 12 | 13 | check_version_consistency() { 14 | name=$1 15 | current_version=$2 16 | correct_version=$3 17 | if [ "$current_version" = "$correct_version" ]; then 18 | echo "Version of $name is correct. $correct_version" 19 | else 20 | echo "Version of $name is incorrect. $current_version" 21 | exit 1 22 | fi 23 | } 24 | check_version_consistency "mp4box" $current_version_mp4box $correct_version_mp4box 25 | check_version_consistency "ffmpeg" $current_version_ffmpeg $correct_version_ffmpeg 26 | check_version_consistency "node" $current_version_node $correct_version_node 27 | check_version_consistency "ruby" $current_version_ruby $correct_version_ruby 28 | check_version_consistency "java" $current_version_java $correct_version_java 29 | 30 | exit 0 31 | -------------------------------------------------------------------------------- /test/test_ruby_merges.rb: -------------------------------------------------------------------------------- 1 | require_relative 'helpers' 2 | 3 | class RubyMergesTest < IWNGTest 4 | 5 | def test_merge_file 6 | client.codes.create( code_bundle(:name => 'test_merge_file') do 7 | merge_file 'test/hello.rb' 8 | worker_code "require 'hello'" 9 | end ) 10 | 11 | id = client.tasks.create('test_merge_file').id 12 | 13 | client.tasks.wait_for(id) 14 | 15 | assert_equal "hello\n", client.tasks.log(id), 16 | 'correct output' 17 | end 18 | 19 | def test_merge_dir 20 | client.codes.create( code_bundle(:name => 'test_merge_dir') do 21 | merge_dir 'test/data/dir2' 22 | worker_code "puts File.read('dir2/test')" 23 | end ) 24 | 25 | id = client.tasks.create('test_merge_dir').id 26 | 27 | client.tasks.wait_for(id) 28 | 29 | assert_equal "test\n", client.tasks.log(id), 30 | 'correct output' 31 | end 32 | 33 | def test_merge_gem 34 | wf = 'test/workers/merge_gem_worker/merge_gem_worker.worker' 35 | client.codes.create IronWorkerNG::Code::Base.new wf 36 | 37 | task = client.tasks.create('merge_gem_worker') 38 | client.tasks.wait_for(task.id) 39 | 40 | assert client.tasks.log(task.id) =~ /hello\n/, 41 | 'correct output' 42 | end 43 | 44 | end 45 | -------------------------------------------------------------------------------- /test/test_code_create.rb: -------------------------------------------------------------------------------- 1 | require_relative 'helpers' 2 | 3 | class CodeCreateTest < IWNGTest 4 | 5 | def test_create 6 | code = code_bundle(:name => 'asdfasdf') 7 | assert_equal nil, code.exec 8 | assert_equal 'asdfasdf', code.name 9 | 10 | code = code_bundle(:exec => 'test/hello.rb', :name => 'dfdfd') 11 | assert_equal 'test/hello.rb', code.exec.path 12 | assert_equal 'dfdfd', code.name 13 | 14 | code = code_bundle 15 | assert_equal nil, code.exec 16 | assert_equal nil, code.name 17 | end 18 | 19 | def test_big_file_upload 20 | system "dd if=/dev/urandom of=test/big_file.txt count=10240 bs=1024" 21 | code = code_bundle(:exec => 'test/big_file.txt', :name => 'big_file') 22 | resp = client.codes_create code 23 | assert_equal "Upload successful.", resp.msg 24 | assert resp.id =~ /[0-9a-f]{24}/, "has id" 25 | end 26 | 27 | def test_name 28 | resp = client.codes_create code_bundle('test/hello.worker') 29 | assert_equal "Upload successful.", resp.msg 30 | assert resp.id =~ /[0-9a-f]{24}/, "has id" 31 | end 32 | 33 | def test_block_init 34 | i = 0 35 | IronWorkerNG::Code::Base.new do 36 | exec 'test/hello.rb' 37 | i += 1 38 | end 39 | assert_equal 1, i, 40 | "block should be executed once" 41 | end 42 | 43 | def test_invalid 44 | assert_raise do 45 | code_bundle(asdf: 1) 46 | end 47 | end 48 | 49 | end 50 | -------------------------------------------------------------------------------- /lib/iron_worker_ng/feature/ruby/merge_gemfile.rb: -------------------------------------------------------------------------------- 1 | require 'bundler' 2 | 3 | module IronWorkerNG 4 | module Feature 5 | module Ruby 6 | module MergeGemfile 7 | class Feature < IronWorkerNG::Feature::Base 8 | attr_reader :path 9 | attr_reader :groups 10 | 11 | def initialize(code, path, groups) 12 | super(code) 13 | 14 | @path = path 15 | @groups = groups 16 | end 17 | end 18 | 19 | module InstanceMethods 20 | def merge_gemfile(path, *groups) 21 | groups = groups.map { |g| g.to_sym } 22 | groups << :default if groups.length == 0 23 | 24 | IronCore::Logger.info 'IronWorkerNG', "Adding ruby gems dependencies from #{groups.join(', ')} group#{groups.length > 1 ? 's' : ''} of #{path}" 25 | 26 | feature = IronWorkerNG::Feature::Ruby::MergeGemfile::Feature.new(self, path, groups) 27 | 28 | IronWorkerNG::Fetcher.fetch_to_file(feature.rebase(path)) do |gemfile| 29 | specs = Bundler::Definition.build(gemfile, path + '.lock', nil).specs_for(groups) 30 | 31 | specs.each do |spec| 32 | merge_gem(spec.name, spec.version.to_s) 33 | end 34 | end 35 | 36 | @features << feature 37 | end 38 | 39 | alias :gemfile :merge_gemfile 40 | end 41 | end 42 | end 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /test/stacks/newrelic/newrelic.sh: -------------------------------------------------------------------------------- 1 | correct_version_node="v0.11.15" 2 | correct_version_python="2.7.3" 3 | correct_version_ruby="1.9.3" 4 | correct_version_java="1.7" 5 | 6 | current_version_node=`node -v` 7 | current_version_python=`python -V 2>&1 | awk -F 'Python ' '{print $2}' | head -c5` 8 | current_version_ruby=`ruby -v 2>&1 | awk -F 'ruby ' '{print $2}' | head -c5` 9 | current_version_java=`java -version 2>&1 | head -n 1 | awk -F '"' '{print $2}' | head -c3` 10 | 11 | check_version_consistency() { 12 | name=$1 13 | current_version=$2 14 | correct_version=$3 15 | 16 | if [ "$current_version" = "$correct_version" ]; then 17 | echo "Version of $name is correct. $correct_version" 18 | else 19 | echo "Version of $name is incorrect. $current_version=$correct_version" 20 | exit 1 21 | fi 22 | } 23 | 24 | check_version_consistency "node" $current_version_node $correct_version_node 25 | check_version_consistency "python" $current_version_python $correct_version_python 26 | check_version_consistency "ruby" $current_version_ruby $correct_version_ruby 27 | check_version_consistency "java" $current_version_java $correct_version_java 28 | 29 | check_newrelic_installation() { 30 | language=$1 31 | output_text=$2 32 | 33 | case "$output_text" in 34 | *newrelic*) echo "Newrelic installed for $language" ;; 35 | * ) echo "Newrelic not installed for $language"; exit 1 ;; 36 | esac 37 | } 38 | 39 | check_newrelic_installation "ruby" `gem list | grep newrelic` 40 | check_newrelic_installation "python" `pip freeze | grep newrelic` 41 | 42 | exit 0 43 | -------------------------------------------------------------------------------- /lib/iron_worker_ng/feature/java/merge_jar.rb: -------------------------------------------------------------------------------- 1 | module IronWorkerNG 2 | module Feature 3 | module Java 4 | module MergeJar 5 | class Feature < IronWorkerNG::Feature::Base 6 | attr_reader :path 7 | 8 | def initialize(code, path) 9 | super(code) 10 | 11 | @path = path 12 | end 13 | 14 | def bundle(container) 15 | IronCore::Logger.debug 'IronWorkerNG', "Bundling java jar with path='#{@path}'" 16 | 17 | if (not @code.full_remote_build) || (not IronWorkerNG::Fetcher.remote?(rebase(@path))) 18 | container_add(container, File.basename(@path), rebase(@path)) 19 | end 20 | end 21 | 22 | def build_command 23 | if @code.remote_build_command || @code.full_remote_build 24 | if @code.full_remote_build && IronWorkerNG::Fetcher.remote?(rebase(@path)) 25 | "jar '#{rebase(@path)}'" 26 | else 27 | "jar '#{@code.dest_dir}#{File.basename(@path)}'" 28 | end 29 | else 30 | nil 31 | end 32 | end 33 | 34 | def code_for_classpath 35 | File.basename(@path) 36 | end 37 | end 38 | 39 | module InstanceMethods 40 | def merge_jar(path) 41 | IronCore::Logger.info 'IronWorkerNG', "Merging java jar with path='#{path}'" 42 | 43 | @features << IronWorkerNG::Feature::Java::MergeJar::Feature.new(self, path) 44 | end 45 | 46 | alias :jar :merge_jar 47 | end 48 | end 49 | end 50 | end 51 | end 52 | -------------------------------------------------------------------------------- /test/examples/example_simple.rb: -------------------------------------------------------------------------------- 1 | require_relative '../helpers' 2 | 3 | class ExampleSimpleTest < IWNGTest 4 | 5 | def test_example 6 | Dir.chdir 'examples/simple' do 7 | log = `ruby -I../../../lib simple.rb` 8 | 9 | assert(log =~ /default name is sample_worker/, 10 | 'Default worker name is executable without extension') 11 | 12 | assert(log =~ /hash: name is transmogrify, exec is sample_worker\.rb/, 13 | 'Constructor from hash works ok') 14 | 15 | assert(log =~ /block: name is transmogrify, exec is sample_worker\.rb/, 16 | 'Constructor from block works ok') 17 | 18 | assert(log =~ /WARN -- IronWorkerNG: Ignoring attempt to merge exec/, 19 | 'Warning about secondary exec') 20 | 21 | assert(log =~ /^exec is sample_worker\.rb$/, 22 | 'Exec unchanged') 23 | 24 | assert(log =~ /code id is .*, message is Upload successful/, 25 | 'Upload successful') 26 | 27 | assert(log =~ /\d+ codes created last hour/, 28 | 'codes created') 29 | 30 | assert(log =~ /transmogrify code info (.*)$/, 31 | 'transmogrify code info') 32 | info = $1 33 | assert_equal('transmogrify', eval(info)[:name], 'code name matches') 34 | 35 | assert(log =~ /another transmogrify code info (.*)$/, 36 | 'another transmogrify code info') 37 | assert_equal info, $1 38 | 39 | assert(log =~ /task created:/, 'task created') 40 | assert(log =~ /task finished with status complete/, 'task finished') 41 | 42 | assert(log =~ /^hello$/, 'correct log') 43 | end 44 | end 45 | 46 | end 47 | -------------------------------------------------------------------------------- /test/examples/example_email_worker.rb: -------------------------------------------------------------------------------- 1 | require_relative '../helpers' 2 | require 'rest' 3 | require 'iron_cache' 4 | require 'net/imap' 5 | require 'tempfile' 6 | 7 | class ExampleEmailWorkerTest < IWNGTest 8 | 9 | def test_example 10 | Dir.chdir 'examples/email_worker' do 11 | assert(cli('upload', 'email_worker') =~ /Upload successful/) 12 | 13 | cfg = JSON.parse(File.read('email_config.json')) 14 | 15 | cache = IronCache::Client.new 16 | cache.items.delete(cfg['to']) if cache.items.get(cfg['to']) 17 | 18 | cfg['iron'] = { token: client.api.token, 19 | project_id: client.api.project_id } 20 | tmp = Tempfile.new(['email_config', '.json']) 21 | tmp.write cfg.to_json 22 | tmp.close 23 | 24 | assert(cli('queue', 'email_worker', 25 | payload_file: tmp.path) =~ 26 | /queued with id='(.*)'/) 27 | 28 | # wait for delivery 29 | client.tasks.wait_for($1) 30 | sleep 10 31 | 32 | begin 33 | imap = Net::IMAP.new('imap.gmail.com',993,true) 34 | imap.login(cfg['smtp']['user_name'], cfg['smtp']['password']) 35 | imap.select('INBOX') 36 | ids = imap.search(['SUBJECT', 'hello from IronWorker']) 37 | assert ids.size >= 1 38 | id = ids.last 39 | msg = imap.fetch(id, 'RFC822') 40 | assert msg[0].attr['RFC822'] =~ /hello me/ 41 | imap.store(id, "+FLAGS", [:Deleted]) 42 | ensure 43 | imap.logout 44 | imap.disconnect 45 | end 46 | 47 | cache = IronCache::Client.new 48 | assert cache.items.get(cfg['to']).value =~ /hello me/ 49 | 50 | tmp.unlink 51 | end 52 | end 53 | 54 | end 55 | -------------------------------------------------------------------------------- /lib/iron_worker_ng/feature/common/merge_dir.rb: -------------------------------------------------------------------------------- 1 | module IronWorkerNG 2 | module Feature 3 | module Common 4 | module MergeDir 5 | class Feature < IronWorkerNG::Feature::Base 6 | attr_reader :path 7 | attr_reader :dest 8 | 9 | def initialize(code, path, dest) 10 | super(code) 11 | 12 | @path = path 13 | @dest = dest + (dest.empty? || dest.end_with?('/') ? '' : '/') 14 | end 15 | 16 | def bundle(container) 17 | IronCore::Logger.debug 'IronWorkerNG', "Bundling dir with path='#{@path}' and dest='#{@dest}'" 18 | 19 | if (not @code.full_remote_build) || (not IronWorkerNG::Fetcher.remote?(rebase(@path))) 20 | container_add(container, @dest + File.basename(@path), rebase(@path)) 21 | end 22 | end 23 | 24 | def build_command 25 | if @code.remote_build_command || @code.full_remote_build 26 | if @code.full_remote_build && IronWorkerNG::Fetcher.remote?(rebase(@path)) 27 | "dir '#{rebase(@path)}', '#{@dest}'" 28 | else 29 | "dir '#{@code.dest_dir}#{@dest}#{File.basename(@path)}', '#{@dest}'" 30 | end 31 | else 32 | nil 33 | end 34 | end 35 | end 36 | 37 | module InstanceMethods 38 | def merge_dir(path, dest = '') 39 | IronCore::Logger.info 'IronWorkerNG', "Merging dir with path='#{path}' and dest='#{dest}'" 40 | 41 | @features << IronWorkerNG::Feature::Common::MergeDir::Feature.new(self, path, dest) 42 | end 43 | 44 | alias :dir :merge_dir 45 | end 46 | end 47 | end 48 | end 49 | end 50 | -------------------------------------------------------------------------------- /lib/iron_worker_ng/feature/common/merge_file.rb: -------------------------------------------------------------------------------- 1 | module IronWorkerNG 2 | module Feature 3 | module Common 4 | module MergeFile 5 | class Feature < IronWorkerNG::Feature::Base 6 | attr_reader :path 7 | attr_reader :dest 8 | 9 | def initialize(code, path, dest) 10 | super(code) 11 | 12 | @path = path 13 | @dest = dest + (dest.empty? || dest.end_with?('/') ? '' : '/') 14 | end 15 | 16 | def bundle(container) 17 | IronCore::Logger.debug 'IronWorkerNG', "Bundling file with path='#{@path}' and dest='#{@dest}'" 18 | 19 | if (not @code.full_remote_build) || (not IronWorkerNG::Fetcher.remote?(rebase(@path))) 20 | container_add(container, @dest + File.basename(@path), rebase(@path)) 21 | end 22 | end 23 | 24 | def build_command 25 | if @code.remote_build_command || @code.full_remote_build 26 | if @code.full_remote_build && IronWorkerNG::Fetcher.remote?(rebase(@path)) 27 | "file '#{rebase(@path)}', '#{@dest}'" 28 | else 29 | "file '#{@code.dest_dir}#{@dest}#{File.basename(@path)}', '#{@dest}'" 30 | end 31 | else 32 | nil 33 | end 34 | end 35 | end 36 | 37 | module InstanceMethods 38 | def merge_file(path, dest = '') 39 | IronCore::Logger.info 'IronWorkerNG', "Merging file with path='#{path}' and dest='#{dest}'" 40 | 41 | @features << IronWorkerNG::Feature::Common::MergeFile::Feature.new(self, path, dest) 42 | end 43 | 44 | alias :file :merge_file 45 | end 46 | end 47 | end 48 | end 49 | end 50 | -------------------------------------------------------------------------------- /test/test_retry.rb: -------------------------------------------------------------------------------- 1 | require_relative 'helpers' 2 | 3 | class RetryTest < IWNGTest 4 | 5 | def test_retry 6 | code = IronWorkerNG::Code::Base.new do 7 | name 'hello' 8 | exec 'test/hello.rb' 9 | end 10 | client.codes.create(code) 11 | 12 | # queue 13 | task_id = client.tasks.create('hello').id 14 | client.tasks.wait_for(task_id) 15 | 16 | # retry 17 | task_id = client.tasks.retry(task_id).id 18 | client.tasks.wait_for(task_id) 19 | 20 | assert_equal "hello\n", client.tasks.log(task_id) 21 | end 22 | 23 | 24 | def test_auto_retry 25 | name = 'test_auto_retry' 26 | 27 | tasks = 1 28 | retries = 3 29 | retries_delay = 5 30 | 31 | client.codes.create(code_bundle(:name => name, 32 | :exec => 'test/workers/fail_worker.rb'), 33 | :retries => retries, 34 | :retries_delay => retries_delay) 35 | 36 | task_ids = [] 37 | tasks.times do 38 | task_ids << client.tasks.create(name, {}, {:priority => 2}).id 39 | end 40 | 41 | task_ids.each do |id| 42 | j = 0 43 | tid = id 44 | while tid != nil 45 | task = client.tasks.wait_for(tid) 46 | p task 47 | puts "retry_num: #{task.retry}" 48 | puts "original_task_id: #{task.original_task_id}" 49 | puts "retry_task_id: #{task.retry_task_id}" 50 | assert_equal 'error', task.status 51 | log = client.tasks.log(id) 52 | assert log.include?("Fail Whale") 53 | assert task.retry_task_id if j < retries 54 | tid = task.retry_task_id 55 | if j > 0 56 | assert_equal retries_delay, task.delay 57 | end 58 | j += 1 59 | end 60 | assert_equal retries+1, j 61 | end 62 | end 63 | 64 | end 65 | -------------------------------------------------------------------------------- /lib/iron_worker_ng/code/runtime/node.rb: -------------------------------------------------------------------------------- 1 | module IronWorkerNG 2 | module Code 3 | module Runtime 4 | module Node 5 | include IronWorkerNG::Feature::Common::MergeExec::InstanceMethods 6 | 7 | def runtime_bundle(container, local = false) 8 | container.get_output_stream(@dest_dir + 'node_modules/node_helper.js') do |runner| 9 | runner.write < 'test/workers/ruby_runner_test_script.rb' 7 | client.codes_create(code) 8 | task_id = client.tasks_create('ruby_runner_test_script', 9 | :a => 1, :b => 2).id 10 | client.tasks_wait_for(task_id) 11 | resp = JSON.parse client.tasks_log(task_id) 12 | 13 | puts resp.to_s 14 | 15 | assert_equal( { "a" => 1, "b" => 2 }, resp['params'], 16 | "correct params" ) 17 | 18 | assert_equal resp['params'], JSON.parse(resp['payload']), 19 | "params are parsed payload" 20 | 21 | assert resp['iron_task_id'] =~ /[0-9a-f]{24}/, 22 | "iron_task_id available" 23 | 24 | assert resp['indifferent_access'], "indifferent access works" 25 | end 26 | 27 | def test_class 28 | code = code_bundle do 29 | exec 'test/workers/ruby_runner_test_class.rb', 'RubyWorker' 30 | end 31 | client.codes_create(code) 32 | task_id = client.tasks_create('ruby_runner_test_class', 33 | :a => 1, :b => 2).id 34 | client.tasks_wait_for(task_id) 35 | 36 | assert_equal( { 'a' => 1, 'b' => 2 }, 37 | JSON.parse( client.tasks_log(task_id) ), 38 | "correct output" ) 39 | end 40 | 41 | def test_locale_bug 42 | code = code_bundle do 43 | exec 'test/workers/locale_csv_worker.rb' 44 | file 'test/workers/file.csv' 45 | end 46 | client.codes_create(code) 47 | task = client.tasks_create('locale_csv_worker') 48 | client.tasks_wait_for(task.id) 49 | task = client.tasks_get(task.id) 50 | assert_equal "complete", task.status 51 | assert_equal( "All good\n", 52 | client.tasks_log(task.id), 53 | "correct output" ) 54 | end 55 | 56 | end 57 | -------------------------------------------------------------------------------- /lib/iron_worker_ng/feature/python/merge_pip_dependency.rb: -------------------------------------------------------------------------------- 1 | module IronWorkerNG 2 | module Feature 3 | module Python 4 | module MergePipDependency 5 | class Feature < IronWorkerNG::Feature::Base 6 | attr_reader :name 7 | attr_reader :version 8 | 9 | def initialize(code, name, version) 10 | super(code) 11 | 12 | @name = name 13 | @version = version 14 | end 15 | 16 | def build_command 17 | if @code.full_remote_build 18 | "pip '#{@name}', '#{@version}'" 19 | elsif @code.remote_build_command 20 | "dir '__build__/__pips__'" 21 | else 22 | nil 23 | end 24 | end 25 | end 26 | 27 | module InstanceMethods 28 | def merge_pip(name, version = '') 29 | IronCore::Logger.info 'IronWorkerNG', "Adding python pip dependency with name='#{name}' and version='#{version}'" 30 | 31 | @features << IronWorkerNG::Feature::Python::MergePipDependency::Feature.new(self, name, version) 32 | 33 | unless @fixators.include?(:merge_pip_dependency_fixate) 34 | @fixators << :merge_pip_dependency_fixate 35 | end 36 | end 37 | 38 | alias :pip :merge_pip 39 | 40 | def merge_pip_dependency_fixate 41 | if not full_remote_build 42 | IronCore::Logger.info 'IronWorkerNG', 'Fixating pip dependencies' 43 | 44 | @features.reject! { |f| f.class == IronWorkerNG::Feature::Python::MergePip::Feature } 45 | 46 | deps = @features.reject { |f| f.class != IronWorkerNG::Feature::Python::MergePipDependency::Feature } 47 | 48 | @features << IronWorkerNG::Feature::Python::MergePip::Feature.new(self, deps) 49 | end 50 | end 51 | end 52 | end 53 | end 54 | end 55 | end 56 | -------------------------------------------------------------------------------- /test/test_quick_run.rb: -------------------------------------------------------------------------------- 1 | require_relative 'helpers' 2 | 3 | class QuickRunTest < IWNGTest 4 | 5 | N_TASKS = 50 6 | 7 | def setup 8 | super 9 | client.codes.create code_bundle(:name => 'test', 10 | :exec => 'test/hello.rb') 11 | end 12 | 13 | def test_quick_run 14 | task_ids = [] 15 | N_TASKS.times do 16 | task_ids << client.tasks.create('test').id 17 | end 18 | 19 | task_ids.each do |id| 20 | task = client.tasks.wait_for(id) 21 | assert_equal 'complete', task.status 22 | assert_equal "hello\n", client.tasks.log(id) 23 | end 24 | end 25 | 26 | MISTIMING_E = 10 27 | 28 | def test_scheduler_quick 29 | client.codes.create( code_bundle(:name => 'test_schedule') do 30 | worker_code 'sleep 10 and puts "hello"' 31 | end ) 32 | 33 | start = (Time.now + 10).utc 34 | 35 | id = client.schedules.create('test_schedule', :start_at => start).id 36 | sleep 5 while client.schedules.get(id).status == 'scheduled' 37 | assert_equal 'complete', client.schedules.get(id).status 38 | 39 | task = get_all_tasks. 40 | keep_if{ |t| t.code_name == 'test_schedule' }. 41 | max_by{ |t| Time.parse t.start_time } 42 | 43 | task = client.tasks.wait_for task.id 44 | 45 | actual_start = Time.parse(task.start_time) 46 | 47 | # if fails, ensure local time is correct, try ntpdate 48 | assert actual_start + MISTIMING_E >= start, 49 | ("actual start(#{actual_start}) " + 50 | "is earlier than expected(#{start})") 51 | 52 | assert Time.parse(task.start_time) + 10 <= Time.parse(task.end_time), 53 | ("worker complete in less than 10 seconds, "+ 54 | "start at #{task.start_time}, finish at #{task.end_time}") 55 | assert_equal 'complete', task.status 56 | assert_equal "hello\n", client.tasks.log(task.id) 57 | end 58 | 59 | end 60 | -------------------------------------------------------------------------------- /lib/iron_worker_ng/code/runtime/php.rb: -------------------------------------------------------------------------------- 1 | module IronWorkerNG 2 | module Code 3 | module Runtime 4 | module PHP 5 | include IronWorkerNG::Feature::Common::MergeExec::InstanceMethods 6 | 7 | def runtime_bundle(container, local = false) 8 | container.get_output_stream(@dest_dir + '__runner__.php') do |runner| 9 | runner.write < null, 'dir' => null, 'payload' => array(), 'config' => null, 'schedule_id' => null); 17 | 18 | foreach ($argv as $k => $v) { 19 | if (empty($argv[$k + 1])) continue; 20 | 21 | if ($v == '-id') $args['task_id'] = $argv[$k + 1]; 22 | if ($v == '-d') $args['dir'] = $argv[$k + 1]; 23 | if ($v == '-schedule_id') $args['schedule_id'] = $argv[$k + 1]; 24 | 25 | if ($v == '-payload' && file_exists($argv[$k + 1])) { 26 | $args['payload'] = file_get_contents($argv[$k + 1]); 27 | 28 | $parsed_payload = json_decode($args['payload'], $assoc); 29 | 30 | if ($parsed_payload != null) { 31 | $args['payload'] = $parsed_payload; 32 | } 33 | } 34 | 35 | if ($v == '-config' && file_exists($argv[$k + 1])) { 36 | $args['config'] = file_get_contents($argv[$k + 1]); 37 | 38 | $parsed_config = json_decode($args['config'], $assoc); 39 | 40 | if ($parsed_config != null) { 41 | $args['config'] = $parsed_config; 42 | } 43 | } 44 | } 45 | 46 | return $args; 47 | } 48 | 49 | function getPayload($assoc = false) { 50 | $args = getArgs($assoc); 51 | 52 | return $args['payload']; 53 | } 54 | 55 | function getConfig($assoc = true){ 56 | $args = getArgs($assoc); 57 | 58 | return $args['config']; 59 | } 60 | 61 | require '#{File.basename(@exec.path)}'; 62 | PHP_RUNNER 63 | end 64 | end 65 | 66 | def runtime_run_code(local, params) 67 | < expected_seconds 54 | errors.push "CPU test: Current execution time (#{count}) is not as expected" 55 | end 56 | #execution time ~ < 5sec 57 | 58 | 59 | 60 | #######---HDD CHECK---####### 61 | puts 'HDD test' 62 | available = `df -m 2>&1 | head -n 2 | tail -n 1 | awk '{print $2}'` 63 | 64 | if available.to_i < max_hdd - 500 65 | errors.push "HDD test: Current available size (#{available}) is not as expected" 66 | end 67 | #should be >9500 && <10000 68 | 69 | 70 | 71 | #######---NETWORK CHECK---####### 72 | require 'open-uri' 73 | puts 'Network test' 74 | url = 'https://s3.amazonaws.com/iron-examples/video/iron_man_2_trailer_official.flv' 75 | start = Time.now 76 | filename = 'video.flv' 77 | 78 | open(filename, 'wb') do |file| 79 | file << open(url).read 80 | end 81 | 82 | file_size = (File.size(filename).to_f / 2**20).round(2) 83 | 84 | elapsed_time = Time.now - start 85 | if elapsed_time > 20 86 | errors.push "Network test: Elapsed time is greater than 20 sec #{elapsed_time.to_s}" 87 | end 88 | if file_size < 20 89 | errors.push "Network test: Current downloaded file size(#{file_size}) is not as expected" 90 | end 91 | #elapsed time < 20 sec 92 | #file size >20mb && <25mb 93 | 94 | 95 | unless errors.empty? 96 | puts errors 97 | abort 98 | end 99 | 100 | 101 | #######---MEM CHECK---####### 102 | puts 'MEM test' 103 | a = "x" * (max_mem * 1000 * 1000 - 20_000_000) 104 | -------------------------------------------------------------------------------- /lib/iron_worker_ng/feature/common/merge_exec.rb: -------------------------------------------------------------------------------- 1 | module IronWorkerNG 2 | module Feature 3 | module Common 4 | module MergeExec 5 | class Feature < IronWorkerNG::Feature::Base 6 | attr_reader :path 7 | attr_reader :args 8 | 9 | def initialize(code, path, args) 10 | super(code) 11 | 12 | @path = path 13 | @args = args 14 | end 15 | 16 | def arg(name, i = nil) 17 | if @args.is_a?(Hash) 18 | return @args[name.to_sym] || @args[name.to_s] 19 | elsif @args.is_a?(Array) && (not i.nil?) 20 | return @args[i] 21 | end 22 | 23 | nil 24 | end 25 | 26 | def bundle(container) 27 | IronCore::Logger.debug 'IronWorkerNG', "Bundling exec with path='#{@path}' and args='#{@args.inspect}'" 28 | 29 | if (not @code.full_remote_build) || (not IronWorkerNG::Fetcher.remote?(rebase(@path))) 30 | container_add(container, File.basename(@path), rebase(@path)) 31 | end 32 | end 33 | 34 | def build_command 35 | if @code.remote_build_command || @code.full_remote_build 36 | if @code.full_remote_build && IronWorkerNG::Fetcher.remote?(rebase(@path)) 37 | "exec '#{rebase(@path)}', #{@args.inspect}" 38 | else 39 | "exec '#{@code.dest_dir}#{File.basename(@path)}', #{@args.inspect}" 40 | end 41 | else 42 | nil 43 | end 44 | end 45 | end 46 | 47 | module InstanceMethods 48 | def merge_exec(path = nil, args = {}) 49 | @exec ||= nil 50 | 51 | return @exec unless path 52 | 53 | if (not args.is_a?(Hash)) && (not args.is_a?(Array)) 54 | args = [args] 55 | end 56 | 57 | unless @exec.nil? 58 | IronCore::Logger.warn 'IronWorkerNG', "Ignoring attempt to merge exec with path='#{path}' and args='#{args.inspect}'" 59 | return 60 | end 61 | 62 | @exec = IronWorkerNG::Feature::Common::MergeExec::Feature.new(self, path, args) 63 | 64 | IronCore::Logger.info 'IronWorkerNG', "Detected exec with path='#{path}' and args='#{args.inspect}'" 65 | 66 | @features << @exec 67 | end 68 | 69 | alias :exec :merge_exec 70 | 71 | alias :merge_worker :merge_exec 72 | alias :worker :merge_worker 73 | end 74 | end 75 | end 76 | end 77 | end 78 | -------------------------------------------------------------------------------- /lib/iron_worker_ng/feature/common/merge_zip.rb: -------------------------------------------------------------------------------- 1 | module IronWorkerNG 2 | module Feature 3 | module Common 4 | module MergeZip 5 | class Feature < IronWorkerNG::Feature::Base 6 | attr_reader :path 7 | attr_reader :dest 8 | 9 | def initialize(code, path, dest) 10 | super(code) 11 | 12 | @path = path 13 | @dest = dest + (dest.empty? || dest.end_with?('/') ? '' : '/') 14 | end 15 | 16 | def bundle(container) 17 | IronCore::Logger.debug 'IronWorkerNG', "Bundling zip with path='#{@path}' and dest='#{@dest}'" 18 | 19 | if (not @code.full_remote_build) || (not IronWorkerNG::Fetcher.remote?(rebase(@path))) 20 | tmp_dir_name = ::Dir.tmpdir + '/' + ::Dir::Tmpname.make_tmpname('iron-worker-ng-', 'zip') 21 | 22 | ::Dir.mkdir(tmp_dir_name) 23 | 24 | IronWorkerNG::Fetcher.fetch_to_file(rebase(@path)) do |zip| 25 | zipf = ::Zip::File.open(zip) 26 | zipf.restore_permissions = true 27 | 28 | zipf.each do |f| 29 | next if zipf.get_entry(f).ftype == :directory 30 | 31 | FileUtils::mkdir_p(tmp_dir_name + '/' + File.dirname(f.name)) 32 | zipf.get_entry(f).extract(tmp_dir_name + '/' + f.name) 33 | end 34 | 35 | zipf.each do |f| 36 | next if zipf.get_entry(f).ftype == :directory 37 | 38 | container_add(container, @dest + f.name, tmp_dir_name + '/' + f.name) 39 | end 40 | end 41 | 42 | container.commit 43 | 44 | FileUtils.rm_rf(tmp_dir_name) 45 | end 46 | end 47 | 48 | def build_command 49 | if @code.remote_build_command || @code.full_remote_build 50 | if @code.full_remote_build && IronWorkerNG::Fetcher.remote?(rebase(@path)) 51 | "zip '#{rebase(@path)}', '#{@dest}'" 52 | else 53 | "zip '#{@code.dest_dir}#{@dest}#{File.basename(@path)}', '#{@dest}'" 54 | end 55 | else 56 | nil 57 | end 58 | end 59 | end 60 | 61 | module InstanceMethods 62 | def merge_zip(path, dest = '') 63 | IronCore::Logger.info 'IronWorkerNG', "Merging zip with path='#{path}' and dest='#{dest}'" 64 | 65 | @features << IronWorkerNG::Feature::Common::MergeZip::Feature.new(self, path, dest) 66 | end 67 | 68 | alias :zip :merge_zip 69 | end 70 | end 71 | end 72 | end 73 | end 74 | -------------------------------------------------------------------------------- /test/test_worker_consistency.rb: -------------------------------------------------------------------------------- 1 | require_relative 'helpers' 2 | 3 | class WorkerTest < IWNGTest 4 | 5 | def self.startup 6 | puts 'Please, input cluster:' 7 | @@cluster = gets.chomp 8 | @@cluster = 'default' if @@cluster == '' 9 | end 10 | 11 | def test_concurrency 12 | puts "Starting concurrency tests for \"#{@@cluster}\" cluster..." 13 | max_concurrency = gets_in('Please, input maximum concurrency (5 for default cluster):', 5) 14 | code_id = client.codes.create(IronWorkerNG::Code::Base.new('test/sleep'), {max_concurrency: max_concurrency}).id 15 | tasks = [] 16 | (max_concurrency * 2).times do 17 | tasks.push client.tasks.create('sleep', {:sleep => 1200}, {cluster: @@cluster}).id 18 | end 19 | 20 | sleep 120 21 | running = 0 22 | queued = 0 23 | tasks.each do |task| 24 | task_status = client.tasks.get(task).status 25 | running +=1 if task_status == 'running' 26 | queued +=1 if task_status == 'queued' 27 | end 28 | client.tasks.cancel_all(code_id) 29 | 30 | assert_equal max_concurrency, running 31 | assert_equal max_concurrency, queued 32 | 33 | end 34 | 35 | def test_workers 36 | stats = '' 37 | test_workers = Dir.entries('test/worker-test/') - %w(. ..) 38 | puts "Starting MEM, CPU, HDD, Network tests for \"#{@@cluster}\"..." 39 | mem_mb = gets_in('Please, input maximum available memory size in MB (320 for default cluster):', 320) 40 | hdd_mb = gets_in('Please, input maximum available HDD size in MB (10000 for default cluster):', 10000) 41 | cpu = gets_in('Please, input CPU performance: "high" - 1, "medium" - 2, "low" - 3 (2 - for default cluster):', 2) 42 | test_workers.each do |test_worker| 43 | client.codes.create(IronWorkerNG::Code::Base.new("test/worker-test/#{test_worker}/#{test_worker}")) 44 | id = client.tasks.create(test_worker, {max_mem: mem_mb, max_hdd: hdd_mb, cpu: cpu}, {cluster: @@cluster}).id 45 | task = client.tasks.wait_for(id) 46 | 47 | if test_worker == 'mem-kill' 48 | assert_equal 'error', task.status 49 | assert_equal "ERROR: WORKER RAM LIMIT EXCEEDED (#{mem_mb}M)", task.msg 50 | else 51 | assert_equal 'complete', task.status 52 | end 53 | stats = client.tasks.log(id) if test_worker == 'stat' 54 | end 55 | puts "\n\n======================WORKER INFO====================" 56 | puts stats 57 | end 58 | 59 | def gets_in(msg, default) 60 | puts msg 61 | input = gets 62 | return input.to_i if !!Integer(input) rescue false 63 | return default if input.chomp == '' 64 | puts 'That is not an integer' 65 | gets_in(msg, default) 66 | end 67 | end 68 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | PATH 2 | remote: . 3 | specs: 4 | iron_worker_ng (1.6.7) 5 | bundler 6 | iron_core (>= 1.0.6) 7 | rubyzip (>= 1.0.0) 8 | 9 | GEM 10 | remote: https://rubygems.org/ 11 | specs: 12 | actionmailer (4.1.8) 13 | actionpack (= 4.1.8) 14 | actionview (= 4.1.8) 15 | mail (~> 2.5, >= 2.5.4) 16 | actionpack (4.1.8) 17 | actionview (= 4.1.8) 18 | activesupport (= 4.1.8) 19 | rack (~> 1.5.2) 20 | rack-test (~> 0.6.2) 21 | actionview (4.1.8) 22 | activesupport (= 4.1.8) 23 | builder (~> 3.1) 24 | erubis (~> 2.7.0) 25 | activesupport (4.1.8) 26 | i18n (~> 0.6, >= 0.6.9) 27 | json (~> 1.7, >= 1.7.7) 28 | minitest (~> 5.1) 29 | thread_safe (~> 0.1) 30 | tzinfo (~> 1.1) 31 | addressable (2.3.6) 32 | ansi (1.4.3) 33 | buftok (0.2.0) 34 | builder (3.2.2) 35 | concur (2.1.1) 36 | equalizer (0.0.9) 37 | erubis (2.7.0) 38 | faraday (0.9.0) 39 | multipart-post (>= 1.2, < 3) 40 | go (1.1.0) 41 | concur 42 | http (0.6.3) 43 | http_parser.rb (~> 0.6.0) 44 | http_parser.rb (0.6.0) 45 | i18n (0.6.11) 46 | iron_cache (1.4.2) 47 | iron_core (>= 0.5.1) 48 | iron_core (1.0.7) 49 | rest (>= 3.0.4) 50 | json (1.8.1) 51 | mail (2.6.3) 52 | mime-types (>= 1.16, < 3) 53 | memoizable (0.4.2) 54 | thread_safe (~> 0.3, >= 0.3.1) 55 | mime-types (2.4.3) 56 | minitest (5.4.3) 57 | minitest-reporters (1.0.8) 58 | ansi 59 | builder 60 | minitest (>= 5.0) 61 | ruby-progressbar 62 | multipart-post (2.0.0) 63 | naught (1.0.0) 64 | net-http-persistent (2.9.4) 65 | netrc (0.10.3) 66 | power_assert (0.2.2) 67 | rack (1.5.2) 68 | rack-test (0.6.2) 69 | rack (>= 1.0) 70 | rake (10.4.0) 71 | rest (3.0.6) 72 | net-http-persistent (>= 2.9.1) 73 | netrc 74 | ruby-progressbar (1.7.0) 75 | rubyzip (1.1.7) 76 | simple_oauth (0.3.0) 77 | test-unit (3.0.7) 78 | power_assert 79 | thread_safe (0.3.4) 80 | twitter (5.13.0) 81 | addressable (~> 2.3) 82 | buftok (~> 0.2.0) 83 | equalizer (~> 0.0.9) 84 | faraday (~> 0.9.0) 85 | http (~> 0.6.0) 86 | http_parser.rb (~> 0.6.0) 87 | json (~> 1.8) 88 | memoizable (~> 0.4.0) 89 | naught (~> 1.0) 90 | simple_oauth (~> 0.3.0) 91 | tzinfo (1.2.2) 92 | thread_safe (~> 0.1) 93 | 94 | PLATFORMS 95 | ruby 96 | 97 | DEPENDENCIES 98 | actionmailer 99 | go 100 | iron_cache (>= 1.4.0) 101 | iron_worker_ng! 102 | minitest 103 | minitest-reporters (>= 0.5.0) 104 | rake 105 | test-unit 106 | twitter 107 | -------------------------------------------------------------------------------- /test/helpers.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'test/unit/ui/console/testrunner' 3 | require 'minitest/unit' 4 | require 'minitest/reporters' 5 | require 'tempfile' 6 | require 'optparse' 7 | 8 | require_relative '../lib/iron_worker_ng.rb' 9 | require_relative 'iron_io_config.rb' 10 | 11 | def code_bundle(*args,&block) 12 | code = IronWorkerNG::Code::Base.new(*args) 13 | 14 | class << code 15 | def worker_code(str) 16 | tmpdir = Dir.tmpdir + '/' + Digest::MD5.hexdigest(str) 17 | Dir.mkdir tmpdir unless Dir.exist? tmpdir 18 | 19 | tmpfname = tmpdir + '/worker.rb' 20 | File.open(tmpfname, "w") { |f| f << str } 21 | 22 | puts "created #{tmpfname}" 23 | merge_exec(tmpfname) 24 | end 25 | end 26 | 27 | code.instance_eval(&block) if block_given? 28 | 29 | code 30 | end 31 | 32 | def inspect_zip(code) 33 | zip_file = code.create_container 34 | yield Zip::File.open(zip_file) 35 | File.unlink zip_file 36 | end 37 | 38 | IronCore::Logger.logger.level = ::Logger::DEBUG 39 | 40 | class IWNGTest < Test::Unit::TestCase 41 | attr_accessor :client 42 | 43 | def setup 44 | options = { env: 'test' } 45 | OptionParser.new do |opts| 46 | opts.on('--project-id PROJECT_ID', String) do |p| 47 | options[:project_id] = p 48 | end 49 | end.parse! 50 | 51 | @client = IronWorkerNG::Client.new options 52 | end 53 | 54 | def get_all_tasks(options = { :from_time => (Time.now - 60 * 60).to_i }) 55 | prev_level = IronCore::Logger.logger.level 56 | IronCore::Logger.logger.level = ::Logger::INFO 57 | 58 | result = [] 59 | page = -1 60 | begin 61 | tasks = client.tasks.list({ :per_page => 100, 62 | :page => page += 1 63 | }.merge(options)) 64 | result += tasks 65 | end while tasks.size == 100 66 | 67 | IronCore::Logger.logger.level = prev_level 68 | 69 | result 70 | end 71 | 72 | def cli(*args) 73 | if args.last.is_a? Hash 74 | args.pop.each do |k,v| 75 | args << "--" + k.to_s.gsub(/_/,'-') + " " + v.to_s 76 | end 77 | end 78 | 79 | args << '--debug' 80 | 81 | out = Tempfile.new('cli_output').path 82 | args << "2>&1 >#{out}" 83 | 84 | test_dir = File.dirname(__FILE__) 85 | cmd = "ruby -I#{test_dir}/../lib #{test_dir}/cli_runner.rb " + args.join(' ') 86 | puts cmd 87 | 88 | exec(cmd) if fork.nil? 89 | Process.wait 90 | 91 | puts "--- cli output begin -------------------------------------" 92 | 93 | puts File.read(out) 94 | 95 | puts "--- cli output end ---------------------------------------" 96 | 97 | puts $? 98 | assert $?.success? 99 | 100 | File.read(out) 101 | end 102 | end 103 | 104 | class Test::Unit::UI::Console::TestRunner 105 | def guess_color_availability 106 | false 107 | end 108 | end 109 | -------------------------------------------------------------------------------- /tools/gb.rb: -------------------------------------------------------------------------------- 1 | require "rubygems" 2 | require "json" 3 | require "zip/zip" 4 | require "s3" 5 | require "open-uri" 6 | 7 | def create_gem(bucket, gem, version) 8 | begin 9 | puts "installing clean copy #{gem} #{version}" 10 | 11 | system("mkdir gem") 12 | system("gem uninstall #{gem} -q -I -x") 13 | res = system("cd gem && gem install #{gem} -v #{version} --no-ri --no-rdoc --ignore-dependencies -i __gems__") 14 | 15 | if not res 16 | puts "installation failed" 17 | raise 18 | end 19 | 20 | puts "packing" 21 | 22 | system("rm -rf gem/__gems__/build_info gem/__gems__/cache gem/__gems__/doc") 23 | 24 | Zip.options[:continue_on_exists_proc] = true 25 | 26 | Zip::File.open("#{gem}-#{version}.zip", Zip::ZipFile::CREATE) do |zip| 27 | Dir["gem/**/**"].each do |f| 28 | zip.add(f[4 .. -1], f) 29 | end 30 | end 31 | 32 | puts "creating deps file" 33 | 34 | deps = [Gem::Dependency.new(gem, version)] 35 | spec = Gem::Resolver.new(deps).resolve.first.spec 36 | dependencies = spec.dependencies.reject { |dep| dep.type != :runtime }.map { |dep| {gem: dep.name, version: dep.requirement.to_s} }.to_json 37 | 38 | File.open("#{gem}-#{version}.deps", "w") do |deps| 39 | deps.puts dependencies.to_json 40 | end 41 | 42 | puts "uploading to S3" 43 | 44 | zip = bucket.objects.build("#{gem}-#{version}.zip") 45 | zip.content = open("#{gem}-#{version}.zip") 46 | zip.save 47 | 48 | deps = bucket.objects.build("#{gem}-#{version}.deps") 49 | deps.content = open("#{gem}-#{version}.deps") 50 | deps.save 51 | rescue 52 | ensure 53 | system("rm -rf gem") 54 | system("rm #{gem}-#{version}.zip") 55 | system("rm #{gem}-#{version}.deps") 56 | end 57 | end 58 | 59 | stack = ARGV.length == 1 ? "-" + ARGV[0] : "" 60 | 61 | puts "Stack:#{stack}" 62 | 63 | config = JSON.parse(File.read("gb.json")) 64 | 65 | s3_access_key_id = config["s3_access_key_id"] 66 | s3_secret_access_key = config["s3_secret_access_key"] 67 | 68 | s3 = S3::Service.new(access_key_id: s3_access_key_id, secret_access_key: s3_secret_access_key) 69 | bucket = s3.buckets.find("iron_worker_ng_gems#{stack}") 70 | 71 | gems_data = open("https://raw.githubusercontent.com/iron-io/iron_worker_ruby_ng/master/tools/gems.json") { |f| f.read } 72 | gems = JSON.parse(gems_data)["gems"] 73 | 74 | gems.each do |gem| 75 | begin 76 | deps = [Gem::Dependency.new(gem[0], gem[1])] 77 | spec = Gem::Resolver.new(deps).resolve.first 78 | 79 | versions = spec.instance_variable_get("@others_possible").map { |o| o.version.to_s } 80 | versions << spec.spec.version.to_s 81 | 82 | versions.each do |version| 83 | begin 84 | bucket.objects.find("#{gem[0]}-#{version}.zip") 85 | bucket.objects.find("#{gem[0]}-#{version}.deps") 86 | rescue 87 | create_gem(bucket, gem[0], version) 88 | end 89 | end 90 | rescue 91 | puts "Unexpected error while building #{gem[0]} #{gem[1]}" 92 | end 93 | end 94 | -------------------------------------------------------------------------------- /test/rake_test_ng/rake_test_ng.rb: -------------------------------------------------------------------------------- 1 | require 'hipchat' 2 | require 'aws/s3' 3 | require 'tmpdir' 4 | require 'json' 5 | require 'cgi' 6 | 7 | push = JSON.parse(CGI::parse(payload)['payload'][0], :symbolize_names => true) 8 | 9 | if push[:ref] !~ /master/ 10 | puts 'Commit not to master branch -- ignoring.' 11 | exit(0) 12 | end 13 | 14 | config = JSON.parse( File.read('.abt-ng-config'), 15 | :symbolize_names => true ) 16 | 17 | msg = "#{push[:pusher][:name]} have pushed:\n" 18 | push[:commits].each do |c| 19 | msg << "#{ CGI::escapeHTML(c[:message]) }" 20 | msg << " by #{c[:author][:name]}\n" 21 | end 22 | 23 | root = Dir.pwd 24 | 25 | home = ENV['HOME'] = Dir.mktmpdir 26 | 27 | Dir.chdir home 28 | puts `git clone "http://github.com/iron-io/iron_worker_ruby_ng.git" iwng` 29 | Dir.chdir 'iwng' 30 | 31 | FileUtils.cp(root + '/iron.json', '.') 32 | 33 | bundler_path = home + '/bundler_gems' 34 | Dir.mkdir bundler_path 35 | 36 | puts `bundle install --path #{bundler_path}` 37 | puts `bundle install --gemfile test/Gemfile --path #{bundler_path}` 38 | 39 | gem_path = Dir.glob(bundler_path + '/*/*') 40 | ENV['GEM_PATH'] = (gem_path + (ENV['GEM_PATH'] || '').split(':')).join(':') 41 | 42 | run_tests = Proc.new do 43 | path = "test_log_#{ Time.now.strftime('%F_%H_%M') }.txt" 44 | 45 | fork { exec 'rake install' } 46 | Process.wait 47 | 48 | fork { exec "rake -f test/Rakefile test >#{path} 2>&1" } 49 | Process.wait 50 | 51 | AWS::S3::Base.establish_connection! config[:aws] 52 | 53 | AWS::S3::S3Object.store(path, open(path), 'abt-ng-logs', 54 | :access => :public_read) 55 | 56 | path 57 | end 58 | 59 | after = run_tests.call 60 | puts `git checkout #{push[:before]}` 61 | before = run_tests.call 62 | 63 | SUMMARY_R = %r/(\d+) tests, (\d+) assertions, (\d+) failures, (\d+) errors, (\d+) skips/ 64 | 65 | msg += "before: " 66 | msg += "full log " 67 | if log = File.read(before) and log =~ SUMMARY_R 68 | msg += $& 69 | end 70 | msg += "\n" 71 | 72 | msg += "after: " 73 | msg += "full log " 74 | if log = File.read(after) and pos = log =~ /^Finished( tests)? in/ 75 | res = log[pos .. -1] 76 | if pos = res =~ SUMMARY_R 77 | msg += ' ' + $& + "\n" 78 | res = res[0 .. pos - 1] 79 | end 80 | msg += ' 
' + CGI::escapeHTML(res) + '
' 81 | msg = msg.gsub( %r|#{Dir.pwd}/(.*):(\d+)|, 82 | '\0' ) 84 | end 85 | 86 | msg = msg.gsub(/\n+/,"
") 87 | 88 | hipchat = HipChat::Client.new(config[:hipchat]) 89 | color = if msg =~ /0 failures, 0 errors, 0 skips/ 90 | :green 91 | else 92 | :red 93 | end 94 | hipchat["IronWorker"].send('NG test', msg, 95 | :notify => false, 96 | :color => color) 97 | -------------------------------------------------------------------------------- /test/test_common_features.rb: -------------------------------------------------------------------------------- 1 | require_relative 'helpers' 2 | 3 | class CommonFeaturesTest < IWNGTest 4 | 5 | def test_merge_file 6 | code = code_bundle do 7 | merge_file('test/data/dir2/test', 'test/data/dir2') 8 | merge_exec('test/hello.rb') 9 | end 10 | 11 | inspect_zip(code) do |zip| 12 | assert zip.find_entry('test/data/dir2/test') 13 | end 14 | end 15 | 16 | def test_merge_file_no_dest 17 | code = code_bundle do 18 | merge_file('Gemfile') 19 | merge_exec('test/hello.rb') 20 | end 21 | 22 | inspect_zip(code) do |zip| 23 | assert zip.find_entry('Gemfile') 24 | end 25 | end 26 | 27 | def test_missing_file 28 | assert_raise IronCore::Error, "should check if merged file exists" do 29 | code_bundle do 30 | merge_file('krumplumpl', 'test/data') 31 | end.create_container 32 | end 33 | end 34 | 35 | def test_missing_dir 36 | assert_raise IronCore::Error, "should check if merged dir exists" do 37 | code_bundle do 38 | merge_dir('dir2', 'test/data') 39 | end.create_container 40 | end 41 | end 42 | 43 | def test_merge_dir 44 | code = code_bundle do 45 | merge_dir('test/data/dir2', 'test/data') 46 | merge_exec('test/hello.rb') 47 | end 48 | 49 | inspect_zip(code) do |zip| 50 | assert zip.find_entry('test/data/dir2/test') 51 | end 52 | end 53 | 54 | def test_merge_dir_no_dest 55 | code = code_bundle do 56 | merge_dir('test') 57 | merge_exec('test/hello.rb') 58 | end 59 | 60 | inspect_zip(code) do |zip| 61 | assert zip.find_entry('test/hello.rb') 62 | end 63 | end 64 | 65 | def test_wrong_merges 66 | def check(msg, &block) 67 | assert_raise IronCore::Error, msg do 68 | code_bundle(&block).create_container 69 | end 70 | end 71 | 72 | check "should check if merged file is a regular file" do 73 | file 'test' 74 | exec 'test/hello.rb' 75 | end 76 | 77 | check "should check if merged dir is a dir" do 78 | dir 'Gemfile' 79 | exec 'test/hello.rb' 80 | end 81 | 82 | check("should check if merged exec is a file"){ exec 'test' } 83 | end 84 | 85 | def test_symlinks 86 | File.unlink 'test/data/dir1/dir2' if 87 | File.symlink? 'test/data/dir1/dir2' 88 | 89 | Dir.chdir('test/data/dir1') do 90 | File.symlink('../dir2', 'dir2') 91 | end 92 | 93 | code = code_bundle :name => 'test_symlinks' do 94 | merge_dir('test/data/dir1', 'test/data') 95 | merge_dir('test/data/dir2', 'test/data') 96 | worker_code 'puts File.read("test/data/dir1/dir2/test")' 97 | end 98 | 99 | inspect_zip(code) do |zip| 100 | assert_equal '../dir2', zip.read('test/data/dir1/dir2') 101 | end 102 | 103 | client.codes_create(code) 104 | task_id = client.tasks_create('test_symlinks').id 105 | client.tasks_wait_for(task_id) 106 | log = client.tasks_log(task_id) 107 | 108 | assert_equal "test\n", log 109 | 110 | File.unlink 'test/data/dir1/dir2' 111 | end 112 | 113 | end 114 | -------------------------------------------------------------------------------- /lib/iron_worker_ng/code/builder.rb: -------------------------------------------------------------------------------- 1 | require 'iron_worker_ng/feature/ruby/merge_gem' 2 | 3 | module IronWorkerNG 4 | module Code 5 | class Builder < IronWorkerNG::Code::Base 6 | attr_accessor :builder_remote_build_command 7 | 8 | def initialize(*args, &block) 9 | @features = [] 10 | @fixators = [] 11 | 12 | @base_dir = '' 13 | @dest_dir = '' 14 | 15 | @remote_build_command = nil 16 | @full_remote_build = false 17 | 18 | @use_local_iron_worker_ng = false 19 | 20 | runtime(:ruby) 21 | end 22 | 23 | def bundle(container, local = false) 24 | @exec = IronWorkerNG::Feature::Common::MergeExec::Feature.new(self, '__builder__.rb', {}) 25 | 26 | super(container, local) 27 | 28 | if builder_remote_build_command 29 | container.get_output_stream(@dest_dir + '__builder__.sh') do |builder| 30 | builder.write < res.marshal_dump.to_json) 96 | BUILDER_RUBY 97 | end 98 | end 99 | end 100 | end 101 | end 102 | -------------------------------------------------------------------------------- /lib/iron_worker_ng/feature/ruby/merge_gem.rb: -------------------------------------------------------------------------------- 1 | module IronWorkerNG 2 | module Feature 3 | module Ruby 4 | module MergeGem 5 | def self.merge_binary? 6 | @merge_binary ||= false 7 | end 8 | 9 | def self.merge_binary=(merge_binary) 10 | @merge_binary = merge_binary 11 | end 12 | 13 | class Feature < IronWorkerNG::Feature::Base 14 | attr_reader :spec 15 | 16 | def initialize(code, spec) 17 | super(code) 18 | 19 | @spec = spec 20 | 21 | if @spec.name == 'nokogiri' && @spec.version.to_s.start_with?('1.6.') && (not code.use_build_cache) 22 | IronCore::Logger.warn 'IronWorkerNG', "WARNING: Building nokogiri version #{@spec.version} will take a lot of time. Switching to version '~> 1.5.9' should fix this" 23 | end 24 | end 25 | 26 | def gem_path 27 | path = @spec.full_gem_path 28 | 29 | # bundler fixes 30 | 31 | ['/gems/' + @spec.full_name, '/gems'].each do |bad_part| 32 | path.gsub!(bad_part + '/lib' + bad_part, bad_part) 33 | path.gsub!(bad_part + bad_part, bad_part) 34 | end 35 | 36 | path 37 | end 38 | 39 | def bundle(container) 40 | if not @code.full_remote_build 41 | if @spec.extensions.length == 0 || IronWorkerNG::Feature::Ruby::MergeGem.merge_binary? 42 | IronCore::Logger.debug 'IronWorkerNG', "Bundling ruby gem with name='#{@spec.name}' and version='#{@spec.version}'" 43 | 44 | loaded_from = @spec.loaded_from 45 | 46 | # yet another bundler fix 47 | 48 | if loaded_from.end_with?("/gems/bundler-#{@spec.version}/lib/bundler") 49 | loaded_from = loaded_from.gsub("/gems/bundler-#{@spec.version}/lib/bundler", "/specifications/bundler-#{@spec.version}.gemspec") 50 | end 51 | 52 | # and yet another one 53 | 54 | if loaded_from.end_with?("/gems/bundler-#{@spec.version}/lib/bundler/source") 55 | loaded_from = loaded_from.gsub("/gems/bundler-#{@spec.version}/lib/bundler/source", "/specifications/bundler-#{@spec.version}.gemspec") 56 | end 57 | 58 | if File.exists?(gem_path) 59 | container_add(container, '__gems__/gems/' + @spec.full_name, gem_path) 60 | else 61 | from_dir = File.dirname(loaded_from) 62 | 63 | @spec.files.each do |fname| 64 | container_add(container, '__gems__/gems/' + @spec.full_name + '/' + fname, from_dir + '/' + fname) if File.file?(from_dir + '/' + fname) 65 | end 66 | end 67 | 68 | container_add(container, "__gems__/specifications/#{@spec.full_name}.gemspec", loaded_from) 69 | else 70 | IronCore::Logger.warn 'IronWorkerNG', "Skipping ruby gem with name='#{@spec.name}' and version='#{@spec.version}' as it contains native extensions, switching to full remote build should fix this (add 'remote' to your .worker)" 71 | end 72 | end 73 | end 74 | end 75 | end 76 | end 77 | end 78 | end 79 | -------------------------------------------------------------------------------- /lib/iron_worker_ng/fetcher.rb: -------------------------------------------------------------------------------- 1 | require 'net/http' 2 | require 'net/https' 3 | require 'tmpdir' 4 | require 'fileutils' 5 | 6 | module IronWorkerNG 7 | class Fetcher 8 | def self.remote?(url) 9 | url.start_with?('http://') || url.start_with?('https://') 10 | end 11 | 12 | def self.exists?(url) 13 | if IronWorkerNG::Fetcher.remote?(url) 14 | url = IronWorkerNG::Fetcher.fix_github_url(url) 15 | 16 | uri = URI.parse(url) 17 | 18 | http = Net::HTTP.new(uri.host, uri.port) 19 | 20 | if uri.scheme == 'https' 21 | http.use_ssl = true 22 | http.verify_mode = OpenSSL::SSL::VERIFY_NONE 23 | end 24 | 25 | response = http.request(Net::HTTP::Head.new(uri.request_uri)) 26 | 27 | if response.kind_of?(Net::HTTPRedirection) 28 | return IronWorkerNG::Fetcher.exists?(response['location']) 29 | end 30 | 31 | return response.code.to_i == 200 32 | else 33 | File.exists?(url) 34 | end 35 | end 36 | 37 | def self.fix_github_url(url) 38 | if url.start_with?('http://github.com/') || url.start_with?('https://github.com/') 39 | fixed_url = url.sub('//github.com/', '//raw.github.com/').sub('/blob/', '/') 40 | 41 | IronCore::Logger.info 'IronWorkerNG', "Fixed github link with url='#{url}' to url='#{fixed_url}'" 42 | 43 | return fixed_url 44 | end 45 | 46 | url 47 | end 48 | 49 | def self.fetch(url, &block) 50 | if IronWorkerNG::Fetcher.remote?(url) 51 | url = IronWorkerNG::Fetcher.fix_github_url(url) 52 | 53 | uri = URI.parse(url) 54 | 55 | http = Net::HTTP.new(uri.host, uri.port) 56 | 57 | if uri.scheme == 'https' 58 | http.use_ssl = true 59 | http.verify_mode = OpenSSL::SSL::VERIFY_NONE 60 | end 61 | 62 | response = http.request(Net::HTTP::Get.new(uri.request_uri)) 63 | 64 | if response.kind_of?(Net::HTTPRedirection) 65 | IronWorkerNG::Fetcher.fetch(response['location'], &block) 66 | 67 | return 68 | end 69 | 70 | block.call(response.body) unless block.nil? 71 | else 72 | unless File.exists?(url) 73 | block.call(nil) unless block.nil? 74 | 75 | return 76 | end 77 | 78 | block.call(File.read(url)) unless block.nil? 79 | end 80 | end 81 | 82 | def self.fetch_to_file(url, &block) 83 | if IronWorkerNG::Fetcher.remote?(url) 84 | IronWorkerNG::Fetcher.fetch(url) do |data| 85 | unless data.nil? 86 | tmp_dir_name = ::Dir.tmpdir + '/' + ::Dir::Tmpname.make_tmpname('iron-worker-ng-', 'http') 87 | 88 | ::Dir.mkdir(tmp_dir_name) 89 | 90 | File.open(tmp_dir_name + '/' + File.basename(url), 'wb') do |f| 91 | f.write(data) 92 | end 93 | 94 | block.call(tmp_dir_name + '/' + File.basename(url)) unless block.nil? 95 | 96 | FileUtils.rm_rf(tmp_dir_name) 97 | end 98 | end 99 | else 100 | unless File.exists?(url) 101 | block.call(nil) unless block.nil? 102 | 103 | return 104 | end 105 | 106 | block.call(url) unless block.nil? 107 | end 108 | end 109 | end 110 | end 111 | -------------------------------------------------------------------------------- /lib/iron_worker_ng/feature/python/merge_pip.rb: -------------------------------------------------------------------------------- 1 | require 'tmpdir' 2 | require 'fileutils' 3 | 4 | module IronWorkerNG 5 | module Feature 6 | module Python 7 | module MergePip 8 | class Feature < IronWorkerNG::Feature::Base 9 | attr_reader :deps 10 | 11 | def initialize(code, deps) 12 | super(code) 13 | 14 | @deps = deps 15 | end 16 | 17 | def bundle(container) 18 | IronCore::Logger.debug 'IronWorkerNG', "Bundling pip dependencies" 19 | 20 | pip_dir_name = ::Dir.tmpdir + '/' + ::Dir::Tmpname.make_tmpname('iron-worker-ng-', 'pip') 21 | 22 | ::Dir.mkdir(pip_dir_name) 23 | 24 | p = 'pip-1.3.1' 25 | `cd #{pip_dir_name} && curl -O https://pypi.python.org/packages/source/p/pip/#{p}.tar.gz && tar xf #{p}.tar.gz && cd #{p} && python setup.py install --user --root #{pip_dir_name} && cd .. && rm -rf #{p} #{p}.tar.gz` 26 | 27 | pip_packages_dir = ::Dir.glob(pip_dir_name + ENV['HOME'] + '/.local/lib/python*').first 28 | pip_packages_dir ||= ::Dir.glob(pip_dir_name + ENV['HOME'] + '/Library/Python/*/lib/python*').first 29 | 30 | pip_binary = pip_dir_name + ENV['HOME'] + '/.local/bin/pip' 31 | 32 | unless File.exists?(pip_binary) 33 | pip_binary = Dir.glob(pip_dir_name + ENV['HOME'] + '/Library/Python/*/bin/pip').first 34 | 35 | if pip_binary.nil? 36 | IronCore::Logger.error 'IronWorkerNG', 'Unfamiliar Python environment, please switch to full remote build (add \'remote\' to your .worker)', IronCore::Error 37 | end 38 | end 39 | 40 | pips_dir_name = ::Dir.tmpdir + '/' + ::Dir::Tmpname.make_tmpname('iron-worker-ng-', 'pips') 41 | 42 | ::Dir.mkdir(pips_dir_name) 43 | 44 | deps_string = @deps.map { |dep| dep.name + dep.version.to_s.sub(/\A\d/, '==\0') }.join(' ') 45 | install_command = pip_binary + ' install -U -I --user --root ' + pips_dir_name + ' ' + deps_string 46 | 47 | fork do 48 | ENV['PYTHONPATH'] = pip_packages_dir + '/site-packages:' + pip_packages_dir + '/dist-packages' 49 | puts `#{install_command}` 50 | end 51 | 52 | Process.wait 53 | 54 | pips_packages_dir = ::Dir.glob(pips_dir_name + ENV['HOME'] + '/.local/lib/python*').first 55 | pips_packages_dir ||= ::Dir.glob(pips_dir_name + ENV['HOME'] + '/Library/Python/*/lib/python*').first 56 | 57 | pips_binary_dir = pips_dir_name + ENV['HOME'] + '/.local/bin' 58 | 59 | unless File.exists?(pips_binary_dir) 60 | pips_binary_dir = ::Dir.glob(pips_dir_name + ENV['HOME'] + '/Library/Python/*/bin').first 61 | end 62 | 63 | if (not pips_binary_dir.nil?) && File.exists?(pips_binary_dir) 64 | container_add(container, '__pips__/__bin__', pips_binary_dir, true) 65 | end 66 | 67 | if File.exists?(pips_packages_dir + '/site-packages') 68 | container_add(container, '__pips__', pips_packages_dir + '/site-packages' , true) 69 | end 70 | 71 | if File.exists?(pips_packages_dir + '/dist-packages') 72 | container_add(container, '__pips__', pips_packages_dir + '/dist-packages' , true) 73 | end 74 | 75 | FileUtils.rm_rf(pips_dir_name) 76 | FileUtils.rm_rf(pip_dir_name) 77 | end 78 | end 79 | end 80 | end 81 | end 82 | end 83 | -------------------------------------------------------------------------------- /lib/iron_worker_ng/feature/ruby/merge_gem_dependency.rb: -------------------------------------------------------------------------------- 1 | require 'bundler' 2 | 3 | module IronWorkerNG 4 | module Feature 5 | module Ruby 6 | module MergeGemDependency 7 | class Feature < IronWorkerNG::Feature::Base 8 | attr_reader :name 9 | attr_reader :version 10 | 11 | def initialize(code, name, version) 12 | super(code) 13 | 14 | @name = name 15 | @version = version 16 | end 17 | 18 | def build_command 19 | if @code.full_remote_build 20 | "gem '#{@name}', '#{@version}'" 21 | elsif @code.remote_build_command 22 | "dir '__build__/__gems__'" 23 | else 24 | nil 25 | end 26 | end 27 | end 28 | 29 | module InstanceMethods 30 | def merge_gem(name, version = '>= 0') 31 | IronCore::Logger.info 'IronWorkerNG', "Adding ruby gem dependency with name='#{name}' and version='#{version}'" 32 | 33 | @features << IronWorkerNG::Feature::Ruby::MergeGemDependency::Feature.new(self, name, version) 34 | 35 | unless @fixators.include?(:merge_gem_dependency_fixate) 36 | @fixators << :merge_gem_dependency_fixate 37 | end 38 | end 39 | 40 | alias :gem :merge_gem 41 | 42 | def merge_gem_dependency_fixate 43 | if not full_remote_build 44 | IronCore::Logger.info 'IronWorkerNG', 'Fixating gems dependencies' 45 | 46 | @features.reject! { |f| f.class == IronWorkerNG::Feature::Ruby::MergeGem::Feature } 47 | @features.reject! { |f| f.class == IronWorkerNG::Feature::Common::MergeZip::Feature && f.path.start_with?('http://s3.amazonaws.com/iron_worker_ng_gems') } 48 | 49 | deps = @features.reject { |f| f.class != IronWorkerNG::Feature::Ruby::MergeGemDependency::Feature } 50 | 51 | if deps.length > 0 52 | deps = deps.map { |dep| Bundler::DepProxy.new(Bundler::Dependency.new(dep.name, dep.version.split(', ')), Gem::Platform::RUBY) } 53 | 54 | source = nil 55 | 56 | begin 57 | source = Bundler::Source::Rubygems.new 58 | rescue Bundler::GemfileNotFound 59 | ENV['BUNDLE_GEMFILE'] = 'Gemfile' 60 | source = Bundler::Source::Rubygems.new 61 | end 62 | 63 | index = Bundler::Index.build { |index| index.use source.specs } 64 | 65 | spec_set = Bundler::Resolver.resolve(deps, index) 66 | 67 | spec_set.to_a.each do |spec| 68 | spec.__materialize__ 69 | 70 | if @use_build_cache 71 | cache_url = "http://s3.amazonaws.com/iron_worker_ng_gems#{@stack.nil? ? '' : '-' + @stack}/#{spec.name}-#{spec.version}.zip" 72 | 73 | if IronWorkerNG::Fetcher.exists?(cache_url) 74 | IronCore::Logger.info 'IronWorkerNG', "Merging cached ruby gem with name='#{spec.name}' and version='#{spec.version}'" 75 | 76 | @features << IronWorkerNG::Feature::Common::MergeZip::Feature.new(self, cache_url, '') 77 | 78 | next 79 | end 80 | end 81 | 82 | IronCore::Logger.info 'IronWorkerNG', "Merging ruby gem with name='#{spec.name}' and version='#{spec.version}'" 83 | 84 | @features << IronWorkerNG::Feature::Ruby::MergeGem::Feature.new(self, spec) 85 | end 86 | end 87 | end 88 | end 89 | end 90 | end 91 | end 92 | end 93 | end 94 | -------------------------------------------------------------------------------- /lib/3rdparty/hashie/indifferent_access.rb: -------------------------------------------------------------------------------- 1 | # source: https://github.com/intridea/hashie/blob/6d21c6868512603e77a340827ec91ecd3bcef078/lib/hashie/extensions/indifferent_access.rb 2 | module Hashie 3 | module Extensions 4 | # IndifferentAccess gives you the ability to not care 5 | # whether your hash has string or symbol keys. Made famous 6 | # in Rails for accessing query and POST parameters, this 7 | # is a handy tool for making sure your hash has maximum 8 | # utility. 9 | # 10 | # One unique feature of this mixin is that it will recursively 11 | # inject itself into sub-hash instances without modifying 12 | # the actual class of the sub-hash. 13 | # 14 | # @example 15 | # class MyHash < Hash 16 | # include Hashie::Extensions::MergeInitializer 17 | # include Hashie::Extensions::IndifferentAccess 18 | # end 19 | # 20 | # h = MyHash.new(:foo => 'bar', 'baz' => 'blip') 21 | # h['foo'] # => 'bar' 22 | # h[:foo] # => 'bar' 23 | # h[:baz] # => 'blip' 24 | # h['baz'] # => 'blip' 25 | # 26 | module IndifferentAccess 27 | def self.included(base) 28 | base.class_eval do 29 | alias_method :regular_writer, :[]= 30 | alias_method :[]=, :indifferent_writer 31 | %w(default update fetch delete key? values_at).each do |m| 32 | alias_method "regular_#{m}", m 33 | alias_method m, "indifferent_#{m}" 34 | end 35 | end 36 | end 37 | 38 | # This will inject indifferent access into an instance of 39 | # a hash without modifying the actual class. This is what 40 | # allows IndifferentAccess to spread to sub-hashes. 41 | def self.inject!(hash) 42 | (class << hash; self; end).send :include, Hashie::Extensions::IndifferentAccess 43 | hash.convert! 44 | end 45 | 46 | # Injects indifferent access into a duplicate of the hash 47 | # provided. See #inject! 48 | def self.inject(hash) 49 | inject!(hash.dup) 50 | end 51 | 52 | def convert_key(key) 53 | key.to_s 54 | end 55 | 56 | # Iterates through the keys and values, reconverting them to 57 | # their proper indifferent state. Used when IndifferentAccess 58 | # is injecting itself into member hashes. 59 | def convert! 60 | keys.each do |k| 61 | regular_writer convert_key(k), convert_value(self.regular_delete(k)) 62 | end 63 | self 64 | end 65 | 66 | def convert_value(value) 67 | if hash_lacking_indifference?(value) 68 | Hashie::Extensions::IndifferentAccess.inject(value.dup) 69 | elsif value.is_a?(::Array) 70 | value.dup.replace(value.map { |e| convert_value(e) }) 71 | else 72 | value 73 | end 74 | end 75 | 76 | def indifferent_default(key = nil) 77 | return self[convert_key(key)] if key?(key) 78 | regular_default(key) 79 | end 80 | 81 | def indifferent_update(other_hash) 82 | return regular_update(other_hash) if hash_with_indifference?(other_hash) 83 | other_hash.each_pair do |k,v| 84 | self[k] = v 85 | end 86 | end 87 | 88 | def indifferent_writer(key, value); regular_writer convert_key(key), convert_value(value) end 89 | def indifferent_fetch(key, *args); regular_fetch convert_key(key), *args end 90 | def indifferent_delete(key); regular_delete convert_key(key) end 91 | def indifferent_key?(key); regular_key? convert_key(key) end 92 | def indifferent_values_at(*indices); indices.map{|i| self[i] } end 93 | 94 | def indifferent_access?; true end 95 | 96 | protected 97 | 98 | def hash_lacking_indifference?(other) 99 | other.is_a?(::Hash) && 100 | !(other.respond_to?(:indifferent_access?) && 101 | other.indifferent_access?) 102 | end 103 | 104 | def hash_with_indifference?(other) 105 | other.is_a?(::Hash) && 106 | other.respond_to?(:indifferent_access?) && 107 | other.indifferent_access? 108 | end 109 | end 110 | end 111 | end 112 | -------------------------------------------------------------------------------- /lib/iron_worker_ng/code/runtime/ruby.rb: -------------------------------------------------------------------------------- 1 | require 'tmpdir' 2 | require 'fileutils' 3 | 4 | require 'iron_worker_ng/feature/ruby/merge_gem_dependency' 5 | require 'iron_worker_ng/feature/ruby/merge_gemfile' 6 | require 'iron_worker_ng/feature/ruby/merge_gem' 7 | 8 | module IronWorkerNG 9 | module Code 10 | module Runtime 11 | module Ruby 12 | include IronWorkerNG::Feature::Common::MergeExec::InstanceMethods 13 | include IronWorkerNG::Feature::Ruby::MergeGemDependency::InstanceMethods 14 | include IronWorkerNG::Feature::Ruby::MergeGemfile::InstanceMethods 15 | 16 | def runtime_bundle(container, local = false) 17 | container.get_output_stream(@dest_dir + '__runner__.rb') do |runner| 18 | runner.write < 'https', 12 | :host => IronWorkerNG::APIClient::AWS_US_EAST_HOST, 13 | :port => 443, 14 | :api_version => 2, 15 | :user_agent => IronWorkerNG.full_version 16 | } 17 | 18 | super('iron', 'worker', options, default_options, [:project_id, :token, :jwt, :api_version]) 19 | 20 | #puts "nhp.proxy yo #{rest.wrapper.http.proxy_uri}" if defined? Net::HTTP::Persistent 21 | #puts "RestClient.proxy yo #{RestClient.proxy}" if defined? RestClient 22 | #puts "InternalClient.proxy yo #{Rest::InternalClient.proxy}" if defined? Rest::InternalClient 23 | 24 | IronCore::Logger.error 'IronWorkerNG', "Token is not set", IronCore::Error if @token.nil? && @jwt.nil? 25 | 26 | check_id(@project_id, 'project_id') 27 | end 28 | 29 | def headers 30 | if !@jwt.nil? 31 | super.merge({'Authorization' => "JWT #{@jwt}"}) 32 | else 33 | super.merge({'Authorization' => "OAuth #{@token}"}) 34 | end 35 | end 36 | 37 | def base_url 38 | super + @api_version.to_s + '/' 39 | end 40 | 41 | def stacks_list 42 | parse_response(get("stacks")) 43 | end 44 | 45 | def codes_list(options = {}) 46 | parse_response(get("projects/#{@project_id}/codes", options)) 47 | end 48 | 49 | def codes_get(id) 50 | check_id(id) 51 | parse_response(get("projects/#{@project_id}/codes/#{id}")) 52 | end 53 | 54 | def codes_create(name, file, runtime, runner, options) 55 | file_instance = file.to_s.strip == '' ? '' : File.new(file, 'rb') 56 | options = {:name => name, :runtime => runtime, :file_name => runner}.merge(options) 57 | parse_response(post_file("projects/#{@project_id}/codes", :file, file_instance, :data, options)) 58 | end 59 | 60 | def codes_delete(id) 61 | check_id(id) 62 | parse_response(delete("projects/#{@project_id}/codes/#{id}")) 63 | end 64 | 65 | def codes_revisions(id, options = {}) 66 | check_id(id) 67 | parse_response(get("projects/#{@project_id}/codes/#{id}/revisions", options)) 68 | end 69 | 70 | def codes_download(id, options = {}) 71 | check_id(id) 72 | parse_response(get("projects/#{@project_id}/codes/#{id}/download", options), false) 73 | end 74 | 75 | def codes_pause_task_queue(id, options = {}) 76 | check_id(id) 77 | parse_response(post("projects/#{@project_id}/codes/#{id}/pause_task_queue", options)) 78 | end 79 | 80 | def codes_resume_task_queue(id, options = {}) 81 | check_id(id) 82 | parse_response(post("projects/#{@project_id}/codes/#{id}/resume_task_queue", options)) 83 | end 84 | 85 | def tasks_list(options = {}) 86 | parse_response(get("projects/#{@project_id}/tasks", options)) 87 | end 88 | 89 | def tasks_get(id) 90 | check_id(id) 91 | parse_response(get("projects/#{@project_id}/tasks/#{id}")) 92 | end 93 | 94 | def tasks_create(code_name, payload, options = {}) 95 | parse_response(post("projects/#{@project_id}/tasks", {:tasks => [{:code_name => code_name, :payload => payload}.merge(options)]})) 96 | end 97 | 98 | def tasks_cancel(id) 99 | check_id(id) 100 | parse_response(post("projects/#{@project_id}/tasks/#{id}/cancel")) 101 | end 102 | 103 | def tasks_cancel_all(id) 104 | check_id(id) 105 | parse_response(post("projects/#{@project_id}/codes/#{id}/cancel_all")) 106 | end 107 | 108 | def tasks_log(id) 109 | check_id(id) 110 | if block_given? 111 | stream_get("projects/#{@project_id}/tasks/#{id}/log") do |chunk| 112 | yield chunk 113 | end 114 | else 115 | parse_response(get("projects/#{@project_id}/tasks/#{id}/log"), false) 116 | end 117 | end 118 | 119 | def tasks_stdout(id) 120 | check_id(id) 121 | if block_given? 122 | stream_get("projects/#{@project_id}/tasks/#{id}/outlog") do |chunk| 123 | yield chunk 124 | end 125 | else 126 | parse_response(get("projects/#{@project_id}/tasks/#{id}/outlog"), false) 127 | end 128 | end 129 | 130 | def tasks_set_progress(id, options = {}) 131 | check_id(id) 132 | parse_response(post("projects/#{@project_id}/tasks/#{id}/progress", options)) 133 | end 134 | 135 | def tasks_retry(id, options = {}) 136 | check_id(id) 137 | parse_response(post("projects/#{@project_id}/tasks/#{id}/retry", options)) 138 | end 139 | 140 | def schedules_list(options = {}) 141 | parse_response(get("projects/#{@project_id}/schedules", options)) 142 | end 143 | 144 | def schedules_get(id) 145 | check_id(id) 146 | parse_response(get("projects/#{@project_id}/schedules/#{id}")) 147 | end 148 | 149 | def schedules_create(code_name, payload, options = {}) 150 | options[:start_at] = options[:start_at].iso8601 if (not options[:start_at].nil?) && options[:start_at].respond_to?(:iso8601) 151 | options[:end_at] = options[:end_at].iso8601 if (not options[:end_at].nil?) && options[:end_at].respond_to?(:iso8601) 152 | 153 | parse_response(post("projects/#{@project_id}/schedules", {:schedules => [{:code_name => code_name, :payload => payload}.merge(options)]})) 154 | end 155 | 156 | def schedules_update(id, options = {}) 157 | check_id(id) 158 | parse_response(put("projects/#{@project_id}/schedules/#{id}", options)) 159 | end 160 | 161 | def schedules_cancel(id, options = {}) 162 | check_id(id) 163 | parse_response(post("projects/#{@project_id}/schedules/#{id}/cancel", options)) 164 | end 165 | 166 | def projects_get 167 | parse_response(get("projects/#{@project_id}")) 168 | end 169 | 170 | def clusters_list(options = {}) 171 | parse_response(get("clusters", options)) 172 | end 173 | 174 | def clusters_get(cluster_id) 175 | parse_response(get("clusters/#{cluster_id}")) 176 | end 177 | 178 | def clusters_credentials(cluster_id) 179 | parse_response(get("clusters/#{cluster_id}/credentials")) 180 | end 181 | 182 | def clusters_create(options = {}) 183 | parse_response(post("clusters", options)) 184 | end 185 | 186 | def clusters_update(cluster_id, options = {}) 187 | parse_response(put("clusters/#{cluster_id}", options)) 188 | end 189 | 190 | def clusters_delete(cluster_id) 191 | parse_response(delete("clusters/#{cluster_id}")) 192 | end 193 | 194 | def clusters_share(cluster_id, options = {}) 195 | parse_response(post("clusters/#{cluster_id}/share", options)) 196 | end 197 | 198 | def clusters_unshare(cluster_id, user_id) 199 | parse_response(post("clusters/#{cluster_id}/unshare/#{user_id}")) 200 | end 201 | end 202 | end 203 | -------------------------------------------------------------------------------- /lib/iron_worker_ng/code/base.rb: -------------------------------------------------------------------------------- 1 | require 'fileutils' 2 | 3 | require 'iron_worker_ng/code/container/base' 4 | require 'iron_worker_ng/code/container/zip' 5 | require 'iron_worker_ng/code/container/dir' 6 | require 'iron_worker_ng/feature/base' 7 | require 'iron_worker_ng/feature/common/merge_exec' 8 | require 'iron_worker_ng/feature/common/merge_file' 9 | require 'iron_worker_ng/feature/common/merge_dir' 10 | require 'iron_worker_ng/feature/common/merge_deb' 11 | require 'iron_worker_ng/feature/common/merge_zip' 12 | require 'iron_worker_ng/feature/common/set_env' 13 | 14 | module IronWorkerNG 15 | module Code 16 | class Base 17 | attr_accessor :features 18 | attr_accessor :fixators 19 | 20 | attr_accessor :base_dir 21 | attr_accessor :dest_dir 22 | attr_accessor :env 23 | 24 | attr_accessor :zip_package 25 | 26 | attr_accessor :use_local_iron_worker_ng 27 | attr_accessor :use_build_cache 28 | attr_accessor :fix_params 29 | 30 | undef exec 31 | undef gem 32 | 33 | include IronWorkerNG::Feature::Common::MergeFile::InstanceMethods 34 | include IronWorkerNG::Feature::Common::MergeDir::InstanceMethods 35 | include IronWorkerNG::Feature::Common::MergeDeb::InstanceMethods 36 | include IronWorkerNG::Feature::Common::MergeZip::InstanceMethods 37 | include IronWorkerNG::Feature::Common::SetEnv::InstanceMethods 38 | 39 | def initialize(*args, &block) 40 | @features = [] 41 | @fixators = [] 42 | 43 | @base_dir = '' 44 | @dest_dir = '' 45 | 46 | @full_remote_build = false 47 | 48 | @runtime = nil 49 | 50 | @name = nil 51 | @exec = nil 52 | 53 | @zip_package = nil 54 | 55 | @use_local_iron_worker_ng = false 56 | 57 | worker_files = [] 58 | 59 | if args.length == 1 && args[0].is_a?(String) 60 | @name = args[0] 61 | elsif args.length == 1 && args[0].is_a?(Hash) 62 | @name = args[0][:name] || args[0]['name'] 63 | 64 | worker_file = args[0][:workerfile] || args[0]['workerfile'] 65 | worker_files << worker_file unless worker_file.nil? 66 | 67 | @env = args[0][:env] || args[0]['env'] 68 | 69 | exec = args[0][:exec] || args[0]['exec'] || args[0][:worker] || args[0]['worker'] 70 | unless exec.nil? 71 | merge_exec(exec) 72 | end 73 | end 74 | 75 | if @name.is_a?(String) && @name.end_with?('.worker') 76 | @name = @name.gsub(/\.worker$/, '') 77 | end 78 | 79 | if @name.is_a?(String) && @name.end_with?('.zip') 80 | @zip_package = @name 81 | @name = @name.gsub(/\.zip/, '') 82 | end 83 | 84 | if @name.nil? and (not @exec.nil?) 85 | @name = guess_name_for_path(@exec.path) 86 | end 87 | 88 | if (not @name.nil?) && @zip_package.nil? 89 | worker_files << @name + '.worker' 90 | end 91 | 92 | worker_files.each do |worker_file| 93 | IronWorkerNG::Fetcher.fetch(worker_file) do |content| 94 | unless content.nil? 95 | IronCore::Logger.info 'IronWorkerNG', "Found workerfile with path='#{worker_file}'" 96 | 97 | if IronWorkerNG::Fetcher.remote?(worker_file) 98 | @full_remote_build = true 99 | end 100 | 101 | @base_dir = File.dirname(worker_file) == '.' ? '' : File.dirname(worker_file) + '/' 102 | 103 | eval(content) 104 | 105 | break 106 | end 107 | end 108 | end 109 | 110 | unless block.nil? 111 | instance_eval(&block) 112 | end 113 | 114 | if @name.nil? and (not @exec.nil?) 115 | @name = guess_name_for_path(@exec.path) 116 | end 117 | 118 | unless @name.nil? 119 | @name = File.basename(@name) 120 | end 121 | end 122 | 123 | def method_missing(name, *args, &block) 124 | if @runtime.nil? 125 | runtime(:ruby) 126 | send(name, *args, &block) 127 | else 128 | super(name, *args, &block) 129 | end 130 | end 131 | 132 | def guess_name_for_path(path) 133 | File.basename(path).gsub(/#{File.extname(path)}$/, '') 134 | end 135 | 136 | def name(name = nil) 137 | @name = name if name 138 | 139 | @name 140 | end 141 | 142 | def name=(name) 143 | @name = name 144 | end 145 | 146 | def stack(stack_name = nil) 147 | @stack = stack_name if stack_name 148 | 149 | @use_local_iron_worker_ng = true if @stack == 'ruby-2.1' 150 | 151 | @stack 152 | end 153 | 154 | 155 | def remote_build_command(remote_build_command = nil) 156 | @remote_build_command = remote_build_command unless remote_build_command.nil? 157 | 158 | @remote_build_command 159 | end 160 | 161 | def remote_build_command=(remote_build_command) 162 | @remote_build_command = remote_build_command 163 | end 164 | 165 | alias :build :remote_build_command 166 | alias :build= :remote_build_command= 167 | 168 | def full_remote_build(full_remote_build = nil) 169 | @full_remote_build = full_remote_build unless full_remote_build.nil? 170 | 171 | @full_remote_build 172 | end 173 | 174 | def full_remote_build=(full_remote_build) 175 | @full_remote_build = full_remote_build 176 | end 177 | 178 | def remote 179 | @full_remote_build = true 180 | end 181 | 182 | def runtime(runtime = nil, version = nil) 183 | return @runtime unless runtime 184 | 185 | unless @runtime.nil? 186 | IronCore::Logger.error 'IronWorkerNG', "Runtime is already set to '#{@runtime}'", IronCore::Error 187 | end 188 | 189 | runtime_module = nil 190 | runtime = runtime.to_s 191 | 192 | if version 193 | s = "#{runtime}-#{version}" 194 | IronCore::Logger.info 'IronWorkerNG', "Trying to set stack to:#{s}'" 195 | stack(s) 196 | end 197 | 198 | begin 199 | runtime_module = IronWorkerNG::Code::Runtime.const_get(runtime.capitalize) 200 | rescue 201 | begin 202 | runtime_module = IronWorkerNG::Code::Runtime.const_get(runtime.upcase) 203 | rescue 204 | end 205 | end 206 | 207 | if runtime_module.nil? 208 | IronCore::Logger.error 'IronWorkerNG', "Can't find runtime '#{runtime}'" 209 | end 210 | 211 | self.extend(runtime_module) 212 | 213 | @runtime = runtime 214 | end 215 | 216 | def runtime=(runtime) 217 | runtime(runtime) 218 | end 219 | 220 | def fixate 221 | @fixators.each do |f| 222 | send(f) 223 | end 224 | end 225 | 226 | def runtime_bundle(container, local = false) 227 | end 228 | 229 | def runtime_run_code(local, params) 230 | '' 231 | end 232 | 233 | def bundle(container, local = false) 234 | @features.each do |feature| 235 | feature.bundle(container) 236 | end 237 | 238 | if local || ((not remote_build_command) && (not full_remote_build)) 239 | params = "\"$@\"" 240 | 241 | if @fix_params 242 | params="-`echo \"$@\" | sed \"s/ -/ --/g\"`" 243 | end 244 | 245 | container.get_output_stream(@dest_dir + '__runner__.sh') do |runner| 246 | runner.write < @config, :env => @env, :project_id => @project_id, 71 | :token => @token, 72 | :jwt => @jwt, 73 | :http_proxy => @http_proxy 74 | ) 75 | 76 | project = client.projects.get 77 | 78 | log "Project '#{project.name}' with id='#{project._id}'" 79 | end 80 | 81 | @client 82 | end 83 | 84 | def stacks_list 85 | client 86 | log_group "List of available stacks" 87 | puts client.stacks_list.map{ |stack| "* #{stack}" }.join("\n") 88 | end 89 | 90 | def upload(name, params, options) 91 | client 92 | 93 | log_group "Creating code package" 94 | 95 | code = IronWorkerNG::Code::Base.new(:name => name, :env => @env) 96 | 97 | code.name(params[:name]) unless params[:name].nil? 98 | 99 | code.full_remote_build = params[:full_remote_build] unless params[:full_remote_build].nil? 100 | 101 | log "Code package name is '#{code.name}'" 102 | log "Max concurrency set to '#{options[:max_concurrency]}'" unless options[:max_concurrency].nil? 103 | log "Default priority set to '#{options[:default_priority]}'" unless options[:default_priority].nil? 104 | log "Retries set to '#{options[:retries]}'" unless options[:retries].nil? 105 | log "Retries delay set to '#{options[:retries_delay]}'" unless options[:retries_delay].nil? 106 | log "Host set to '#{options[:host]}'" unless options[:host].nil? 107 | 108 | if options[:worker_config] 109 | log "Loading worker_config at #{options[:worker_config]}" 110 | c = IO.read(options[:worker_config]) 111 | options[:config] = c 112 | end 113 | 114 | if code.remote_build_command || code.full_remote_build 115 | log_group "Uploading and building code package '#{code.name}'" 116 | else 117 | log_group "Uploading code package '#{code.name}'" 118 | end 119 | 120 | if (params[:async] || params['async']) && code.remote_build_command 121 | builder_task_id = client.codes.create_async(code, options) 122 | 123 | log 'Code package is building' 124 | log "Check 'https://hud.iron.io/tq/projects/#{client.api.project_id}/tasks/#{builder_task_id}' for more info" 125 | else 126 | code_id = client.codes.create(code, options)._id 127 | code_info = client.codes.get(code_id) 128 | 129 | log "Code package uploaded with id='#{code_id}' and revision='#{code_info.rev}'" 130 | log "Check 'https://hud.iron.io/tq/projects/#{client.api.project_id}/code/#{code_id}' for more info" 131 | end 132 | end 133 | 134 | def patch(name, params, options) 135 | client 136 | 137 | log_group "Patching code package '#{name}'" 138 | 139 | code_id = client.codes.patch(name, options)._id 140 | code_info = client.codes.get(code_id) 141 | 142 | log "Patched code package uploaded with id='#{code_id}' and revision='#{code_info.rev}'" 143 | log "Check 'https://hud.iron.io/tq/projects/#{client.api.project_id}/code/#{code_id}' for more info" 144 | end 145 | 146 | def queue(name, params, options) 147 | client 148 | 149 | log_group "Queueing task" 150 | 151 | id = client.tasks.create(name, params[:payload] || params['payload'], options)._id 152 | 153 | log "Code package '#{name}' queued with id='#{id}'" 154 | log "Check 'https://hud.iron.io/tq/projects/#{client.api.project_id}/jobs/#{id}' for more info" 155 | 156 | if options[:wait] == true 157 | getlog(id, {}, {:wait => true}) 158 | end 159 | end 160 | 161 | def schedule(name, params, options) 162 | client 163 | 164 | log_group "Scheduling task" 165 | 166 | id = client.schedules.create(name, params[:payload] || params['payload'], options)._id 167 | 168 | log "Code package '#{name}' scheduled with id='#{id}'" 169 | log "Check 'https://hud.iron.io/tq/projects/#{client.api.project_id}/scheduled_jobs/#{id}' for more info" 170 | end 171 | 172 | def retry(task_id, params, options) 173 | client 174 | 175 | log_group "Retrying task with id='#{task_id}'" 176 | 177 | retry_task_id = client.tasks.retry(task_id, options)._id 178 | 179 | log "Task retried with id='#{retry_task_id}'" 180 | log "Check 'https://hud.iron.io/tq/projects/#{client.api.project_id}/jobs/#{retry_task_id}' for more info" 181 | 182 | if options[:wait] == true 183 | getlog(retry_task_id, {}, {:wait => true}) 184 | end 185 | end 186 | 187 | def update_schedule(schedule_id, params) 188 | client 189 | 190 | log_group "Updating scheduled task with id=#{schedule_id}" 191 | 192 | response = client.schedules.update(schedule_id, JSON.parse(params[:schedule], :symbolize_names => true)) 193 | 194 | if response.msg == 'Updated' 195 | log 'Scheduled task updated successfully' 196 | log "Check 'https://hud.iron.io/tq/projects/#{client.api.project_id}/scheduled_jobs/#{schedule_id}' for more info" 197 | else 198 | log 'Something went wrong' 199 | end 200 | end 201 | 202 | def getlog(task_id, params, options) 203 | client 204 | 205 | wait = options[:wait] || options['wait'] 206 | 207 | log_group "Getting log for task with id='#{task_id}'" 208 | 209 | log = '' 210 | 211 | if wait 212 | begin 213 | log = client.tasks.log(task_id) 214 | rescue 215 | end 216 | else 217 | log = client.tasks.log(task_id) 218 | end 219 | 220 | print log 221 | 222 | if wait 223 | client.tasks.wait_for(task_id) do |task| 224 | if task.status == 'running' 225 | begin 226 | next_log = client.tasks.log(task_id) 227 | print next_log[log.length .. - 1] 228 | log = next_log 229 | rescue 230 | end 231 | end 232 | end 233 | 234 | begin 235 | next_log = client.tasks.log(task_id) 236 | print next_log[log.length .. - 1] 237 | rescue 238 | end 239 | end 240 | end 241 | 242 | def run(name, params, options) 243 | log_group "Creating code package" 244 | 245 | code = IronWorkerNG::Code::Base.new(name) 246 | 247 | log "Code package name is '#{code.name}'" 248 | log_group "Running '#{code.name}'" 249 | 250 | if options[:worker_config] 251 | log "Loading worker_config at #{options[:worker_config]}" 252 | c = IO.read(options[:worker_config]) 253 | options[:config] = c 254 | end 255 | code.run(params[:payload] || params['payload'], options[:config] || options['config']) 256 | end 257 | 258 | def install(name, params, options) 259 | log_group "Installing dependencies for code package with name='#{name}'" 260 | 261 | code = IronWorkerNG::Code::Base.new(name) 262 | 263 | code.install 264 | end 265 | 266 | def webhook(name, params, options) 267 | client 268 | 269 | log_group 'Generating code package webhook' 270 | 271 | log 'You can invoke your worker by POSTing to the following URL' 272 | log client.api.url("projects/#{client.api.project_id}/tasks/webhook?code_name=#{name}&oauth=#{client.api.token}") 273 | end 274 | 275 | def info_code(name, params, options) 276 | client 277 | 278 | log_group 'Getting code package info' 279 | 280 | codes = client.codes.list({:all => true}) 281 | code = codes.find { |code| code.name == name } 282 | 283 | unless code 284 | log "Code package with name='#{name}' not found" 285 | exit 1 286 | end 287 | 288 | data = [] 289 | data << ['id', code._id] 290 | data << ['name', code.name] 291 | data << ['revision', code.rev] 292 | data << ['uploaded', parse_time(code.latest_change) || '-'] 293 | data << ['max concurrency', code.max_concurrency || '-'] 294 | data << ['retries', code.retries || '-'] 295 | data << ['retries delay', code.retries_delay || '-'] 296 | data << ['info', "https://hud.iron.io/tq/projects/#{client.api.project_id}/code/#{code._id}"] 297 | data << ['tasks info', "https://hud.iron.io/tq/projects/#{client.api.project_id}/jobs/#{code._id}/activity"] 298 | 299 | display_table(data) 300 | end 301 | 302 | def info_task(task_id, params, options) 303 | client 304 | 305 | log_group 'Getting task info' 306 | 307 | task = client.tasks.get(task_id) 308 | 309 | data = [] 310 | data << ['id', task._id] 311 | data << ['code package', task.code_name] 312 | data << ['code revision', task.code_rev] 313 | data << ['status', task.status] 314 | data << ['label', task.label] if task.label 315 | data << ['priority', task.priority || 2] 316 | data << ['queued', parse_time(task.created_at) || '-'] 317 | data << ['started', parse_time(task.start_time) || '-'] 318 | data << ['finished', parse_time(task.end_time) || '-'] 319 | data << ['payload', task.payload] 320 | data << ['info', "https://hud.iron.io/tq/projects/#{client.api.project_id}/jobs/#{task._id}"] 321 | 322 | display_table(data) 323 | end 324 | 325 | def info_schedule(schedule_id, params, options) 326 | client 327 | 328 | log_group 'Getting schedule info' 329 | 330 | schedule = client.schedules.get(schedule_id) 331 | 332 | data = [] 333 | data << ['id', schedule._id] 334 | data << ['code package', schedule.code_name] 335 | data << ['status', schedule.status] 336 | data << ['label', schedule.label] if schedule.label 337 | data << ['created', parse_time(schedule.created_at) || '-'] 338 | data << ['next start', parse_time(schedule.next_start) || '-'] 339 | data << ['run count', schedule.run_count || '-'] 340 | data << ['payload', schedule.payload] 341 | data << ['info', "https://hud.iron.io/tq/projects/#{client.api.project_id}/scheduled_jobs/#{schedule._id}"] 342 | 343 | display_table(data) 344 | end 345 | 346 | def pause(name, params, options) 347 | code = get_code_by_name(name) 348 | log "Pausing task queue for code '#{code._id}'" 349 | 350 | response = client.codes.pause_task_queue(code._id, options) 351 | 352 | if response.msg == 'Paused' 353 | log "Task queue and schedules for #{name} paused successfully" 354 | elsif response.msg == 'Already paused' 355 | log "Task queue and schedules for #{name} are already paused" 356 | else 357 | log 'Something went wrong' 358 | end 359 | log "Check 'https://hud.iron.io/tq/projects/#{client.api.project_id}/code/#{code._id}' for more info" 360 | end 361 | 362 | def resume(name, params, options) 363 | code = get_code_by_name(name) 364 | log "Resuming task queue for code '#{code._id}'" 365 | 366 | response = client.codes.resume_task_queue(code._id, options) 367 | 368 | if response.msg == 'Resumed' 369 | log "Task queue and schedules for #{name} resumed successfully" 370 | elsif response.msg == 'Already resumed' 371 | log "Task queue and schedules for #{name} are already resumed" 372 | else 373 | log 'Something went wrong' 374 | end 375 | log "Check 'https://hud.iron.io/tq/projects/#{client.api.project_id}/code/#{code._id}' for more info" 376 | end 377 | 378 | def get_code_by_name(name) 379 | client 380 | 381 | codes = client.codes.list({:all => true}) 382 | code = codes.find { |code| code.name == name } 383 | 384 | unless code 385 | log "Code package with name='#{name}' not found" 386 | exit 1 387 | end 388 | code 389 | end 390 | 391 | 392 | def parse_time(s) 393 | t = Time.parse(s) 394 | 395 | return nil if t == Time.utc(1) 396 | 397 | t 398 | end 399 | 400 | def display_table(t) 401 | t.each do |r| 402 | log sprintf('%-16s %s', r[0], r[1]) 403 | end 404 | end 405 | end 406 | end 407 | -------------------------------------------------------------------------------- /bin/iron_worker: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'optparse' 4 | require 'time' 5 | 6 | require 'iron_worker_ng' 7 | require 'iron_worker_ng/cli' 8 | 9 | class IronWorkerCLILoggerFormatter < ::Logger::Formatter 10 | def call(severity, time, proname, msg) 11 | msg = IronWorkerNG::CLI::LOG_ENTRY + msg unless msg[0, IronWorkerNG::CLI::LOG_GROUP.length] == IronWorkerNG::CLI::LOG_GROUP 12 | 13 | msg + "\n" 14 | end 15 | end 16 | 17 | IronCore::Logger.logger.formatter = IronWorkerCLILoggerFormatter.new 18 | 19 | @cli = IronWorkerNG::CLI.new 20 | 21 | def common_opts(opts) 22 | opts.on('--config CONFIG', 'config file') do |v| 23 | @cli.config = v 24 | end 25 | 26 | opts.on('-e', '--env ENV', 'environment') do |v| 27 | @cli.env = v 28 | end 29 | 30 | opts.on('--project-id PROJECT_ID', 'project id') do |v| 31 | @cli.project_id = v 32 | end 33 | 34 | opts.on('--token TOKEN', 'token') do |v| 35 | @cli.token = v 36 | end 37 | 38 | opts.on('--http-proxy HTTP_PROXY', 'http proxy') do |v| 39 | @cli.http_proxy = v 40 | end 41 | end 42 | 43 | if $*.size == 1 && ($*[0] == '-v' || $*[0] == '--version') 44 | puts IronWorkerNG.full_version 45 | exit 0 46 | end 47 | 48 | commands = ['upload', 'patch', 'queue', 'retry', 'schedule', 'update', 'log', 'run', 'install', 'webhook', 'info', 'stacks', 'pause', 'resume'] 49 | 50 | if $*.size == 0 || (not commands.include?($*[0])) 51 | puts 'usage: iron_worker COMMAND [OPTIONS]' 52 | puts " COMMAND: #{commands.join(', ')}" 53 | puts ' run iron_worker COMMAND --help to get more information about each command' 54 | exit 1 55 | end 56 | 57 | command = $*.shift 58 | 59 | if $*.include?('--debug') 60 | IronCore::Logger.logger.level = ::Logger::DEBUG 61 | 62 | $*.reject! { |p| p == '--debug' } 63 | end 64 | 65 | if command == 'upload' 66 | params = {} 67 | options = {} 68 | 69 | opts = OptionParser.new do |opts| 70 | opts.banner = "usage: iron_worker upload CODE_PACKAGE_NAME_OR_PATH_TO_WORKERFILE [OPTIONS]" 71 | 72 | opts.on('-n', '--name NAME', 'override code name') do |v| 73 | params[:name] = v 74 | end 75 | 76 | opts.on('-c', '--max-concurrency CONCURRENCY', Integer, 'max number of concurrent workers for this code package') do |v| 77 | options[:max_concurrency] = v 78 | end 79 | 80 | opts.on('-p', '--default-priority DEFAULT_PRIORITY', Integer, 'default priority of the tasks') do |v| 81 | options[:default_priority] = v 82 | end 83 | 84 | opts.on('-r', '--retries NUM_RETRIES', Integer, 'max number of automatic retries on task fail') do |v| 85 | options[:retries] = v 86 | end 87 | 88 | opts.on('-d', '--retries-delay RETRIES_DELAY', Integer, 'delay between each automatic retry') do |v| 89 | options[:retries_delay] = v 90 | end 91 | 92 | opts.on('--worker-config CONFIG_FILE', 'config file for worker') do |v| 93 | options[:worker_config] = v 94 | end 95 | 96 | opts.on('-h', '--host HOST', String, 'host name, eg: www.mydomain.com') do |v| 97 | options[:host] = v 98 | end 99 | 100 | opts.on('-a', '--async', 'don\'t wait for package build') do |v| 101 | params[:async] = true 102 | end 103 | 104 | opts.on('--full-remote-build', 'activate full remote build') do |v| 105 | params[:full_remote_build] = true 106 | end 107 | 108 | common_opts(opts) 109 | end 110 | 111 | begin 112 | opts.parse! 113 | rescue OptionParser::ParseError 114 | puts $!.to_s 115 | exit 1 116 | end 117 | 118 | unless $*.size == 1 119 | puts 'Please specify code package name or path to workerfile' 120 | puts opts 121 | exit 1 122 | end 123 | 124 | @cli.upload($*[0], params, options) 125 | elsif command == 'patch' 126 | params = {} 127 | options = {} 128 | 129 | opts = OptionParser.new do |opts| 130 | opts.banner = "usage: iron_worker patch CODE_PACKAGE_NAME [OPTIONS]" 131 | 132 | opts.on('-p', '--patch PATCH', Array, 'target and patch file names: \'foo.rb=bar.rb,baz.rb=lib/qux.rb,foo1.rb,foo2.rb\'') do |v| 133 | options[:patch] = Hash[v.map{|s| s.include?('=') ? s.split('='): [s, s]}] 134 | end 135 | 136 | common_opts(opts) 137 | end 138 | 139 | begin 140 | opts.parse! 141 | rescue OptionParser::ParseError 142 | puts $!.to_s 143 | exit 1 144 | end 145 | 146 | unless $*.size == 1 147 | puts 'Please specify code package name' 148 | puts opts 149 | exit 1 150 | end 151 | 152 | if options[:patch].nil? 153 | puts 'Please specify patch' 154 | puts opts 155 | exit 1 156 | end 157 | 158 | @cli.patch($*[0], params, options) 159 | elsif command == 'queue' || command == 'schedule' 160 | params = {} 161 | options = {} 162 | 163 | opts = OptionParser.new do |opts| 164 | opts.banner = "usage: iron_worker #{command} CODE_PACKAGE_NAME [OPTIONS]" 165 | 166 | opts.on('-p', '--payload PAYLOAD', String, 'payload to pass') do |v| 167 | params[:payload] = v 168 | end 169 | 170 | opts.on('-f', '--payload-file PAYLOAD_FILE', String, 'payload file to pass') do |v| 171 | params[:payload] = File.read(v) 172 | end 173 | 174 | opts.on('--priority PRIORITY', Integer, '0 (default), 1, 2') do |v| 175 | options[:priority] = v 176 | end 177 | 178 | opts.on('--timeout TIMEOUT', Integer, 'maximum run time in seconds from 0 to 3600 (default)') do |v| 179 | options[:timeout] = v 180 | end 181 | 182 | opts.on('--cluster CLUSTER', String, 'Run task on selected cluster') do |v| 183 | options[:cluster] = v 184 | end 185 | 186 | desc = command == 'queue' ? 'task' : 'schedule' 187 | opts.on('--label LABEL', String, "Label #{desc} with given name") do |v| 188 | options[:label] = v 189 | end 190 | 191 | if command == 'queue' 192 | opts.on('--delay DELAY', Integer, 'delay before start in seconds') do |v| 193 | options[:delay] = v 194 | end 195 | 196 | opts.on('--wait', Integer, 'wait for task to complete and print log') do |v| 197 | options[:wait] = true 198 | end 199 | end 200 | 201 | if command == 'schedule' 202 | opts.on('--start-at TIME', 'start task at specified time') do |v| 203 | options[:start_at] = Time.parse(v) 204 | end 205 | 206 | opts.on('--end-at TIME', 'stop running task at specified time') do |v| 207 | options[:end_at] = Time.parse(v) 208 | end 209 | 210 | opts.on('--run-times RUN_TIMES', Integer, 'run task no more times than specified') do |v| 211 | options[:run_times] = v 212 | end 213 | 214 | opts.on('--run-every RUN_EVERY', Integer, 'run task every RUN_EVERY seconds') do |v| 215 | options[:run_every] = v 216 | end 217 | 218 | end 219 | 220 | common_opts(opts) 221 | end 222 | 223 | begin 224 | opts.parse! 225 | rescue OptionParser::ParseError 226 | puts $!.to_s 227 | exit 1 228 | end 229 | 230 | unless $*.size == 1 231 | puts 'Please specify code package name' 232 | puts opts 233 | exit 1 234 | end 235 | 236 | if command == 'queue' 237 | @cli.queue($*[0], params, options) 238 | else 239 | @cli.schedule($*[0], params, options) 240 | end 241 | elsif command == 'retry' 242 | params = {} 243 | options = {} 244 | 245 | opts = OptionParser.new do |opts| 246 | opts.banner = "usage: iron_worker retry TASK_ID [OPTIONS]" 247 | 248 | opts.on('--delay DELAY', Integer, 'delay before start in seconds') do |v| 249 | options[:delay] = v 250 | end 251 | 252 | opts.on('--wait', Integer, 'wait for task to complete and print log') do |v| 253 | options[:wait] = true 254 | end 255 | 256 | common_opts(opts) 257 | end 258 | 259 | begin 260 | opts.parse! 261 | rescue OptionParser::ParseError 262 | puts $!.to_s 263 | exit 1 264 | end 265 | 266 | unless $*.size == 1 267 | puts 'Please specify task id' 268 | puts opts 269 | exit 1 270 | end 271 | 272 | @cli.retry($*[0], params, options) 273 | elsif command == 'log' 274 | params = {} 275 | options = {} 276 | 277 | opts = OptionParser.new do |opts| 278 | opts.banner = "usage: iron_worker log TASK_ID [OPTIONS]" 279 | 280 | opts.on('-w', '--wait', 'wait for task to complete') do |v| 281 | options[:wait] = true 282 | end 283 | 284 | common_opts(opts) 285 | end 286 | 287 | begin 288 | opts.parse! 289 | rescue OptionParser::ParseError 290 | puts $!.to_s 291 | exit 1 292 | end 293 | 294 | unless $*.size == 1 295 | puts 'Please specify task id' 296 | puts opts 297 | exit 1 298 | end 299 | 300 | @cli.getlog($*[0], params, options) 301 | elsif command == 'run' 302 | params = {} 303 | options = {} 304 | 305 | opts = OptionParser.new do |opts| 306 | opts.banner = "usage: iron_worker run CODE_PACKAGE_NAME_OR_PATH_TO_WORKERFILE [OPTIONS]" 307 | 308 | opts.on('-p', '--payload PAYLOAD', String, 'payload to pass') do |v| 309 | params[:payload] = v 310 | end 311 | 312 | opts.on('-f', '--payload-file PAYLOAD_FILE', String, 'payload file to pass') do |v| 313 | params[:payload] = File.read(v) 314 | end 315 | 316 | opts.on('--worker-config CONFIG_FILE', 'config file for worker') do |v| 317 | options[:worker_config] = v 318 | end 319 | 320 | common_opts(opts) 321 | end 322 | 323 | begin 324 | opts.parse! 325 | rescue OptionParser::ParseError 326 | puts $!.to_s 327 | exit 1 328 | end 329 | 330 | unless $*.size == 1 331 | puts 'Please specify code package name or workerfile' 332 | puts opts 333 | exit 1 334 | end 335 | 336 | @cli.run($*[0], params, options) 337 | elsif command == 'install' 338 | params = {} 339 | options = {} 340 | 341 | opts = OptionParser.new do |opts| 342 | opts.banner = 'usage: iron_worker install CODE_PACKAGE_NAME_OR_PATH_TO_WORKERFILE [OPTIONS]' 343 | 344 | common_opts(opts) 345 | end 346 | 347 | begin 348 | opts.parse! 349 | rescue OptionParser::ParseError 350 | puts $!.to_s 351 | exit 1 352 | end 353 | 354 | unless $*.size == 1 355 | puts 'Please specify code package name or path to workerfile' 356 | puts opts 357 | exit 1 358 | end 359 | 360 | @cli.install($*[0], params, options) 361 | elsif command == 'webhook' 362 | params = {} 363 | options = {} 364 | 365 | opts = OptionParser.new do |opts| 366 | opts.banner = 'usage: iron_worker webhook CODE_PACKAGE_NAME [OPTIONS]' 367 | 368 | common_opts(opts) 369 | end 370 | 371 | begin 372 | opts.parse! 373 | rescue OptionParser::ParseError 374 | puts $!.to_s 375 | exit 1 376 | end 377 | 378 | unless $*.size == 1 379 | puts 'Please specify code package name' 380 | puts opts 381 | exit 1 382 | end 383 | 384 | @cli.webhook($*[0], params, options) 385 | elsif command == 'info' 386 | entities = ['code', 'task', 'schedule'] 387 | 388 | if $*.size == 0 || (not entities.include?($*[0])) 389 | puts 'usage: iron_worker info ENTITY [OPTIONS]' 390 | puts " ENTITY: #{entities.join(', ')}" 391 | puts ' run iron_worker info ENTITY --help to get more information about each entity' 392 | exit 1 393 | end 394 | 395 | entity = $*.shift 396 | 397 | if entity == 'code' 398 | params = {} 399 | options = {} 400 | 401 | opts = OptionParser.new do |opts| 402 | opts.banner = "usage: iron_worker info code CODE_PACAKGE_NAME [OPTIONS]" 403 | 404 | common_opts(opts) 405 | end 406 | 407 | begin 408 | opts.parse! 409 | rescue OptionParser::ParseError 410 | puts $!.to_s 411 | exit 1 412 | end 413 | 414 | unless $*.size == 1 415 | puts 'Please specify code package name' 416 | puts opts 417 | exit 1 418 | end 419 | 420 | @cli.info_code($*[0], params, options) 421 | elsif entity == 'task' 422 | params = {} 423 | options = {} 424 | 425 | opts = OptionParser.new do |opts| 426 | opts.banner = "usage: iron_worker info task TASK_ID [OPTIONS]" 427 | 428 | common_opts(opts) 429 | end 430 | 431 | begin 432 | opts.parse! 433 | rescue OptionParser::ParseError 434 | puts $!.to_s 435 | exit 1 436 | end 437 | 438 | unless $*.size == 1 439 | puts 'Please specify task id' 440 | puts opts 441 | exit 1 442 | end 443 | 444 | @cli.info_task($*[0], params, options) 445 | elsif entity == 'schedule' 446 | params = {} 447 | options = {} 448 | 449 | opts = OptionParser.new do |opts| 450 | opts.banner = "usage: iron_worker info schedule SCHEDULE_ID [OPTIONS]" 451 | 452 | common_opts(opts) 453 | end 454 | 455 | begin 456 | opts.parse! 457 | rescue OptionParser::ParseError 458 | puts $!.to_s 459 | exit 1 460 | end 461 | 462 | unless $*.size == 1 463 | puts 'Please specify schedule id' 464 | puts opts 465 | exit 1 466 | end 467 | 468 | @cli.info_schedule($*[0], params, options) 469 | end 470 | 471 | elsif command == 'stacks' 472 | @cli.stacks_list 473 | elsif command == 'update' 474 | entities = ['schedule'] 475 | 476 | if $*.size == 0 || (not entities.include?($*[0])) 477 | puts 'usage: iron_worker update ENTITY [OPTIONS]' 478 | puts " ENTITY: #{entities.join(', ')}" 479 | puts ' run iron_worker update ENTITY --help to get more information about each entity' 480 | exit 1 481 | end 482 | 483 | entity = $*.shift 484 | 485 | if entity == 'schedule' 486 | params = {} 487 | 488 | opts = OptionParser.new do |opts| 489 | opts.banner = "usage: iron_worker update schedule SCHEDULED_TASK_ID [OPTIONS]" 490 | 491 | opts.on('-s', '--schedule SCHEDULE', String, 'schedule params to pass, JSON string') do |v| 492 | params[:schedule] = v 493 | end 494 | 495 | common_opts(opts) 496 | end 497 | 498 | begin 499 | opts.parse! 500 | rescue OptionParser::ParseError 501 | puts $!.to_s 502 | exit 1 503 | end 504 | 505 | unless $*.size == 1 506 | puts 'Please specify scheduled task id' 507 | puts opts 508 | exit 1 509 | end 510 | 511 | @cli.update_schedule($*[0], params) 512 | end 513 | 514 | elsif command == 'pause' || command == 'resume' 515 | params = {} 516 | options = {} 517 | 518 | opts = OptionParser.new do |opts| 519 | opts.banner = "usage: iron_worker #{command} CODE_PACKAGE_NAME" 520 | 521 | common_opts(opts) 522 | end 523 | 524 | begin 525 | opts.parse! 526 | rescue OptionParser::ParseError 527 | puts $!.to_s 528 | exit 1 529 | end 530 | 531 | unless $*.size == 1 532 | puts 'Please specify code package name' 533 | puts opts 534 | exit 1 535 | end 536 | 537 | if command == 'resume' 538 | @cli.resume($*[0], params, options) 539 | else 540 | @cli.pause($*[0], params, options) 541 | end 542 | end 543 | --------------------------------------------------------------------------------