├── .gitignore ├── Rakefile ├── test ├── test_helper.rb └── batch_test.rb ├── batch.gemspec ├── batch.gemspec.erb ├── LICENSE ├── README.markdown └── lib └── batch.rb /.gitignore: -------------------------------------------------------------------------------- 1 | /pkg 2 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "rake/testtask" 2 | 3 | Rake::TestTask.new do |t| 4 | t.test_files = FileList["test/**/*_test.rb"] 5 | end 6 | 7 | task :default => :test 8 | -------------------------------------------------------------------------------- /test/test_helper.rb: -------------------------------------------------------------------------------- 1 | require "stringio" 2 | require "contest" 3 | 4 | require File.expand_path("./../lib/batch", File.dirname(__FILE__)) 5 | 6 | def capture 7 | stdout, $stdout = $stdout, StringIO.new 8 | stderr, $stderr = $stderr, StringIO.new 9 | yield 10 | [$stdout.string, $stderr.string] 11 | ensure 12 | $stdout = stdout 13 | $stderr = stderr 14 | end 15 | -------------------------------------------------------------------------------- /batch.gemspec: -------------------------------------------------------------------------------- 1 | Gem::Specification.new do |s| 2 | s.name = "batch" 3 | s.version = "0.0.3" 4 | s.summary = "Iterate Enumerables with progress reporting." 5 | s.authors = ["Damian Janowski", "Michel Martens"] 6 | s.email = ["djanowski@dimaion.com", "michel@soveran.com"] 7 | s.homepage = "http://github.com/djanowski/batch" 8 | s.files = ["LICENSE", "README.markdown", "Rakefile", "lib/batch.rb", "batch.gemspec", "test/batch_test.rb", "test/test_helper.rb"] 9 | end 10 | -------------------------------------------------------------------------------- /batch.gemspec.erb: -------------------------------------------------------------------------------- 1 | <% require "./lib/batch" -%> 2 | Gem::Specification.new do |s| 3 | s.name = "batch" 4 | s.version = "<%= Batch::VERSION %>" 5 | s.summary = "Iterate Enumerables with progress reporting." 6 | s.authors = ["Damian Janowski", "Michel Martens"] 7 | s.email = ["djanowski@dimaion.com", "michel@soveran.com"] 8 | s.homepage = "http://github.com/djanowski/batch" 9 | s.files = <%= Dir[ 10 | "LICENSE", 11 | "README.markdown", 12 | "Rakefile", 13 | "lib/**/*.rb", 14 | "*.gemspec", 15 | "test/*.*" 16 | ].inspect %> 17 | end 18 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2010 Michel Martens & Damian Janowski 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /test/batch_test.rb: -------------------------------------------------------------------------------- 1 | # encoding: UTF-8 2 | 3 | require File.expand_path("./test_helper", File.dirname(__FILE__)) 4 | 5 | class BatchTest < Test::Unit::TestCase 6 | should "report" do 7 | stdout, _ = capture do 8 | Batch.each((1..80).to_a) do |item| 9 | item + 1 10 | end 11 | end 12 | 13 | expected = <<-EOS 14 | 0% ........................................................................... 15 | 93% ..... 16 | 100% 17 | EOS 18 | 19 | assert_equal expected.rstrip, stdout.rstrip 20 | end 21 | 22 | should "not halt on errors" do 23 | stdout, stderr = capture do 24 | Batch.each((1..80).to_a) do |item| 25 | raise ArgumentError, "Oops" if item == 3 26 | end 27 | end 28 | 29 | expected_stdout = <<-EOS 30 | 0% ..E........................................................................ 31 | 93% ..... 32 | 100% 33 | EOS 34 | 35 | 36 | expected_stderr = <<-EOS 37 | 38 | 39 | Completed. 40 | 41 | Elapsed: 0 seconds 42 | Errors: 1 43 | 44 | Some errors occured: 45 | 46 | 3: Oops 47 | EOS 48 | 49 | assert_equal expected_stdout.rstrip, stdout.rstrip 50 | assert_equal expected_stderr.rstrip, stderr.rstrip 51 | end 52 | 53 | should "use BATCH_WIDTH" do 54 | ENV["BATCH_WIDTH"] = "40" 55 | stdout, _ = capture do 56 | Batch.each((1..80).to_a) do |item| 57 | item + 1 58 | end 59 | end 60 | 61 | expected = <<-EOS 62 | 0% ........................................ 63 | 50% ........................................ 64 | 100% 65 | EOS 66 | 67 | assert_equal expected.rstrip, stdout.rstrip 68 | end 69 | end 70 | -------------------------------------------------------------------------------- /README.markdown: -------------------------------------------------------------------------------- 1 | Batch 2 | ===== 3 | 4 | Keep your batch jobs under control. 5 | 6 | Description 7 | ----------- 8 | 9 | Say you have a thousand images to process. You write a script, fire it and go 10 | to bed, only to realize the morning after that and exception was raised and the 11 | script was aborted. Well, no more frustration: now you can use Batch to make 12 | sure your script continues working despite those exceptions, and you can now 13 | get a nice report to read while you drink your morning coffee. 14 | 15 | Usage 16 | ----- 17 | 18 | require "batch" 19 | 20 | Batch.each(Model.all) do |model| 21 | # do something with model 22 | # and see the overall progress 23 | end 24 | 25 | Given that `Model.all` responds to `each` and `size`, you'll get a nice 26 | progress report: 27 | 28 | 0% ........................................................................... 29 | 25% ........................................................................... 30 | 50% ........................................................................... 31 | 75% ........................................................................... 32 | 100% 33 | 34 | If errors occur, they are handled so that your long-running scripts 35 | don't get interrupted right after you go to bed: 36 | 37 | 0% .......E................................................................... 38 | 25% ..........................................E................................ 39 | 50% ....................E...................................................... 40 | 75% ........................................................................... 41 | 100% 42 | 43 | Some errors occured: 44 | 45 | # ... detailed exceptions here 46 | 47 | You can determine the line width by setting the environment variable 48 | `BATCH_WIDTH`, which defaults to 75. 49 | 50 | Logging 51 | ------- 52 | 53 | You may specify `:log` in the options for `#each`: 54 | 55 | Batch.each(Model.all, :log => File.open('foo.log', 'w')) do { |model| 56 | # do something with model 57 | } 58 | 59 | Or you may specify default options using `Batch.options`: 60 | 61 | Batch.options[:log] = File.open('foo.log', 'w') 62 | 63 | Batch.each(Model.all) do { |model| 64 | # do something with model 65 | } 66 | 67 | This will put all errors into the given log file, and periodically update 68 | it with the status. 69 | 70 | Showing status 71 | -------------- 72 | 73 | Press Ctrl+C to show the status. 74 | 75 | 0% ........................................................................... 76 | 15% ............E.........................E...................^C 77 | 78 | 11% done (58 of 500) 79 | Elapsed: 0 seconds 80 | Remaining: 4 seconds 81 | Errors: 2 82 | Press Ctrl+C again to abort. 83 | 84 | ........................................................................... 85 | 30% ..........E......E......................................................... 86 | 87 | You can disable this behavior with: 88 | 89 | Batch.options[:status_on_interrupt] = false 90 | 91 | Installation 92 | ------------ 93 | 94 | $ gem install batch 95 | 96 | License 97 | ------- 98 | 99 | Copyright (c) 2010 Damian Janowski and Michel Martens 100 | 101 | Permission is hereby granted, free of charge, to any person 102 | obtaining a copy of this software and associated documentation 103 | files (the "Software"), to deal in the Software without 104 | restriction, including without limitation the rights to use, 105 | copy, modify, merge, publish, distribute, sublicense, and/or sell 106 | copies of the Software, and to permit persons to whom the 107 | Software is furnished to do so, subject to the following 108 | conditions: 109 | 110 | The above copyright notice and this permission notice shall be 111 | included in all copies or substantial portions of the Software. 112 | 113 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 114 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 115 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 116 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 117 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 118 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 119 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 120 | OTHER DEALINGS IN THE SOFTWARE. 121 | -------------------------------------------------------------------------------- /lib/batch.rb: -------------------------------------------------------------------------------- 1 | class Batch 2 | VERSION = "0.0.3" 3 | 4 | attr :enumerable 5 | attr :processed 6 | 7 | def self.options 8 | @options ||= { 9 | :status_on_interrupt => true, 10 | :log => nil, 11 | :mark_interval => 300 12 | } 13 | end 14 | 15 | def initialize(enumerable, options={}) 16 | @enumerable = enumerable 17 | @width = (ENV["BATCH_WIDTH"] || 75).to_i 18 | @options = self.class.options.merge(options) 19 | @log = @options[:log] 20 | end 21 | 22 | def each(&block) 23 | @errors = [] 24 | @current = 0 25 | 26 | @start = Time.now 27 | last_interrupt = 0 28 | last_mark = Time.now 29 | 30 | print " 0% " 31 | log_start 32 | 33 | enumerable.each do |item| 34 | begin 35 | if (Time.now - last_mark).to_i >= @options[:mark_interval] 36 | last_mark = Time.now 37 | log_status 38 | end 39 | 40 | yield(item) 41 | print "." 42 | 43 | rescue Interrupt 44 | if @options[:status_on_interrupt] 45 | if (Time.now - last_interrupt).to_i <= 1 46 | break 47 | else 48 | report_interrupt_status 49 | last_interrupt = Time.now 50 | end 51 | else 52 | break 53 | end 54 | 55 | rescue Exception => e 56 | print "E" 57 | @errors << [item, e] 58 | log_error item, e 59 | 60 | ensure 61 | @current += 1 62 | 63 | report_progress if eol? 64 | end 65 | end 66 | 67 | log_done 68 | report_completed 69 | report_errors 70 | 71 | nil 72 | end 73 | 74 | def report_interrupt_status 75 | $stderr.print "\n" 76 | $stderr.puts 77 | $stderr.puts " %i%% done (%i of %i)" % [ progress, @current, total ] 78 | $stderr.puts " Elapsed: %s" % [ time_format(elapsed) ] 79 | $stderr.puts " Remaining: %s" % [ time_format(remaining) ] 80 | $stderr.puts " Errors: #{@errors.size}" if @errors.any? 81 | $stderr.puts " Press Ctrl+C again to abort." 82 | $stderr.puts 83 | $stderr.print " " 84 | end 85 | 86 | def report_completed 87 | $stderr.print "\n\n" 88 | $stderr.puts "Completed." 89 | $stderr.puts 90 | $stderr.puts " Elapsed: %s" % [ time_format(elapsed) ] 91 | $stderr.puts " Errors: #{@errors.size}" if @errors.any? 92 | end 93 | 94 | def log(type, message) 95 | return if @log.nil? 96 | 97 | if @log.respond_to?(type.to_sym) 98 | # Logger.info "xx" 99 | @log.send type.to_sym, message 100 | 101 | elsif @log.respond_to?(:write) 102 | # $stderr.write 103 | stamp = Time.now.strftime("%Y-%m-%d %H:%M:%S") 104 | msg = "[%s] %5s: %s\n" % [ stamp, type.to_s.upcase, message ] 105 | @log.write msg 106 | end 107 | end 108 | 109 | def log_start 110 | log :info, "Started" 111 | end 112 | 113 | def log_done 114 | log :info, "Completed in %s" % [ time_format(elapsed) ] 115 | log :info, "Errors: #{@errors.size}" if @errors.any? 116 | end 117 | 118 | def log_status 119 | log :info, "[status] %i%% done (%i of %i)" % [ progress, @current, total ] 120 | log :info, "[status] Elapsed: %s" % [ time_format(elapsed) ] 121 | log :info, "[status] Remaining: %s" % [ time_format(remaining) ] 122 | log :info, "[status] Errors: #{@errors.size}" if @errors.any? 123 | end 124 | 125 | def log_error(item, err) 126 | log :error, "#{item}: #{err.class}: #{err.message}" 127 | err.backtrace.each { |line| log :error, " #{line}" } 128 | end 129 | 130 | def time_format_segments(secs) 131 | hours = secs / 3600 132 | secs %= 3600 133 | 134 | mins = secs / 60 135 | secs %= 60 136 | 137 | [ hours, mins, secs ] 138 | end 139 | 140 | def time_format(secs) 141 | return "Unknown" if secs.nil? 142 | 143 | hours, mins, secs = time_format_segments(secs) 144 | 145 | words = Array.new 146 | 147 | words << "#{hours.to_i} hours" if hours > 1 148 | words << "1 hour" if hours == 1 149 | words << "#{mins.to_i} minutes" if mins > 1 150 | words << "1 minute" if mins == 1 151 | words << "#{secs.to_i} seconds" if secs > 1 152 | words << "1 second" if secs == 1 153 | words << "0 seconds" if words.empty? 154 | 155 | words.join(', ') 156 | end 157 | 158 | def elapsed 159 | Time.now - @start 160 | end 161 | 162 | def remaining 163 | return nil if @current == 0 164 | elapsed * total / @current - elapsed 165 | end 166 | 167 | def report_errors 168 | return if @errors.empty? 169 | 170 | $stderr.puts "\nSome errors occured:\n\n" 171 | 172 | @errors.each do |item, error| 173 | report_error(item, error) 174 | end 175 | end 176 | 177 | def report_error(item, error) 178 | $stderr.puts "#{item.inspect}: #{error}\n" 179 | end 180 | 181 | def total 182 | @enumerable.size 183 | end 184 | 185 | def progress 186 | @current * 100 / total 187 | end 188 | 189 | def report_progress 190 | print "\n#{progress.to_s.rjust 3, " "}% " 191 | end 192 | 193 | def eol? 194 | @current % @width == 0 || @current == total 195 | end 196 | 197 | def self.each(enumerable, options={}, &block) 198 | new(enumerable, options).each(&block) 199 | end 200 | end 201 | --------------------------------------------------------------------------------