├── .ruby-version ├── lib ├── adrian │ ├── version.rb │ ├── queue_item.rb │ ├── composite_queue.rb │ ├── worker.rb │ ├── rotating_directory_queue.rb │ ├── array_queue.rb │ ├── failure_handler.rb │ ├── girl_friday_dispatcher.rb │ ├── queue.rb │ ├── filters.rb │ ├── file_item.rb │ ├── dispatcher.rb │ └── directory_queue.rb └── adrian.rb ├── Gemfile ├── .gitignore ├── CHANGELOG.md ├── test ├── test_helper.rb ├── girl_friday_dispatcher_test.rb ├── array_queue_test.rb ├── worker_test.rb ├── queue_test.rb ├── composite_queue_test.rb ├── failure_handler_test.rb ├── rotating_directory_queue_test.rb ├── dispatcher_lifecycle_test.rb ├── dispatcher_test.rb ├── filters_test.rb ├── file_item_test.rb └── directory_queue_test.rb ├── Rakefile ├── .github ├── PULL_REQUEST_TEMPLATE.md └── workflows │ ├── ci.yml │ └── publish.yml ├── examples └── basic.rb ├── adrian.gemspec ├── Gemfile.lock ├── contributing.md ├── README.md └── LICENSE /.ruby-version: -------------------------------------------------------------------------------- 1 | 3.2.9 2 | -------------------------------------------------------------------------------- /lib/adrian/version.rb: -------------------------------------------------------------------------------- 1 | module Adrian 2 | VERSION = '2.0.2' 3 | end 4 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Specify your gem's dependencies in adrian.gemspec 4 | gemspec 5 | gem 'debug', '>= 1.0.0' 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | *.rbc 3 | .bundle 4 | .config 5 | .yardoc 6 | InstalledFiles 7 | _yardoc 8 | coverage 9 | doc/ 10 | lib/bundler/man 11 | pkg 12 | rdoc 13 | spec/reports 14 | test/tmp 15 | test/version_tmp 16 | tmp 17 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | ## [Unreleased] 6 | 7 | ### Changed 8 | - Drop support for Ruby 3.1 and below. Minimum required Ruby version is now 3.2. 9 | -------------------------------------------------------------------------------- /test/test_helper.rb: -------------------------------------------------------------------------------- 1 | require 'bundler/setup' 2 | 3 | begin 4 | require 'debugger' 5 | rescue LoadError => e 6 | end 7 | 8 | $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib')) 9 | 10 | require 'adrian' 11 | 12 | require 'minitest/autorun' 13 | require 'timecop' 14 | -------------------------------------------------------------------------------- /lib/adrian/queue_item.rb: -------------------------------------------------------------------------------- 1 | module Adrian 2 | class QueueItem 3 | attr_reader :value, :created_at 4 | attr_accessor :queue 5 | 6 | def initialize(value, created_at = Time.now) 7 | @value = value 8 | @created_at = created_at 9 | end 10 | 11 | def age 12 | Time.now - created_at 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env rake 2 | require 'bundler/setup' 3 | require 'bundler/gem_tasks' 4 | 5 | require 'rake/testtask' 6 | 7 | Rake::TestTask.new(:test) do |test| 8 | test.libs << 'lib' 9 | test.libs << 'test' 10 | test.pattern = 'test/**/*_test.rb' 11 | test.verbose = true 12 | test.warning = false 13 | end 14 | 15 | task :default => [:test] 16 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ### Description 2 | Describe the original problem and the changes made on this PR. 3 | 4 | - [ ] This task needs to be done. 5 | - [x] This task has been completed. 6 | 7 | ### Steps to reproduce 8 | 9 | 1. Include steps to reproduce bugs. 10 | 2. Include logs, screenshoots or original files. 11 | 12 | ### References 13 | - Links: 14 | 15 | ### Risks 16 | - ["High"|"Medium"|"Low"]: Describe overall Risk and Complexity. 17 | -------------------------------------------------------------------------------- /lib/adrian/composite_queue.rb: -------------------------------------------------------------------------------- 1 | require 'adrian/queue' 2 | 3 | module Adrian 4 | class CompositeQueue < Queue 5 | def initialize(*queues) 6 | super() 7 | @queues = queues.flatten 8 | end 9 | 10 | def pop 11 | @queues.each do |q| 12 | item = q.pop 13 | return item if item 14 | end 15 | 16 | nil 17 | end 18 | 19 | def push(item) 20 | raise "You can not push item to a composite queue" 21 | end 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: push 4 | 5 | jobs: 6 | tests: 7 | runs-on: ubuntu-latest 8 | 9 | strategy: 10 | fail-fast: false 11 | matrix: 12 | ruby-version: 13 | - "3.2" 14 | steps: 15 | - uses: zendesk/checkout@v3 16 | - name: Set up Ruby 17 | uses: zendesk/setup-ruby@v1 18 | with: 19 | ruby-version: ${{ matrix.ruby-version }} 20 | bundler-cache: true 21 | - run: bundle exec rake test 22 | -------------------------------------------------------------------------------- /lib/adrian/worker.rb: -------------------------------------------------------------------------------- 1 | module Adrian 2 | class Worker 3 | attr_reader :item 4 | 5 | def initialize(item) 6 | @item = item 7 | end 8 | 9 | def report_to(boss) 10 | @boss = boss 11 | end 12 | 13 | def perform 14 | exception = nil 15 | 16 | begin 17 | work 18 | rescue Exception => e 19 | exception = e 20 | end 21 | 22 | @boss.work_done(item, self, exception) if @boss 23 | end 24 | 25 | def work 26 | raise "You need to implement #{self.class.name}#work" 27 | end 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /lib/adrian/rotating_directory_queue.rb: -------------------------------------------------------------------------------- 1 | require 'adrian/directory_queue' 2 | require 'fileutils' 3 | 4 | module Adrian 5 | 6 | class RotatingDirectoryQueue < DirectoryQueue 7 | attr_reader :time_format 8 | 9 | def initialize(options = {}) 10 | super 11 | @time_format = options.fetch(:time_format, '%Y-%m-%d') 12 | end 13 | 14 | def available_path 15 | path = "#{super}/#{Time.now.strftime(time_format)}" 16 | 17 | if path != @previous_avaliable_path 18 | FileUtils.mkdir_p(path) 19 | @previous_avaliable_path = path 20 | end 21 | 22 | path 23 | end 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish to RubyGems.org 2 | 3 | on: 4 | push: 5 | branches: main 6 | paths: lib/adrian/version.rb 7 | workflow_dispatch: 8 | 9 | jobs: 10 | publish: 11 | runs-on: ubuntu-latest 12 | environment: rubygems-publish 13 | if: github.repository_owner == 'zendesk' 14 | permissions: 15 | id-token: write 16 | contents: write 17 | steps: 18 | - uses: actions/checkout@v4 19 | - name: Set up Ruby 20 | uses: ruby/setup-ruby@v1 21 | with: 22 | bundler-cache: false 23 | ruby-version: "3.4" 24 | - name: Install dependencies 25 | run: bundle install 26 | - uses: rubygems/release-gem@v1 27 | -------------------------------------------------------------------------------- /lib/adrian.rb: -------------------------------------------------------------------------------- 1 | require 'adrian/version' 2 | 3 | module Adrian 4 | autoload :ArrayQueue, 'adrian/array_queue' 5 | autoload :CompositeQueue, 'adrian/composite_queue' 6 | autoload :DirectoryQueue, 'adrian/directory_queue' 7 | autoload :RotatingDirectoryQueue, 'adrian/rotating_directory_queue' 8 | autoload :Dispatcher, 'adrian/dispatcher' 9 | autoload :FileItem, 'adrian/file_item' 10 | autoload :Filters, 'adrian/filters' 11 | autoload :GirlFridayDispatcher, 'adrian/girl_friday_dispatcher' 12 | autoload :Queue, 'adrian/queue' 13 | autoload :QueueItem, 'adrian/queue_item' 14 | autoload :Worker, 'adrian/worker' 15 | end 16 | -------------------------------------------------------------------------------- /lib/adrian/array_queue.rb: -------------------------------------------------------------------------------- 1 | require 'adrian/queue' 2 | 3 | module Adrian 4 | class ArrayQueue < Adrian::Queue 5 | def initialize(array = [], options = {}) 6 | super(options) 7 | @array = array.map { |item| wrap_item(item) } 8 | @mutex = Mutex.new 9 | end 10 | 11 | def pop_item 12 | @mutex.synchronize { @array.shift } 13 | end 14 | 15 | def push_item(item) 16 | item = wrap_item(item) 17 | @mutex.synchronize { @array << item } 18 | self 19 | end 20 | 21 | def length 22 | @mutex.synchronize { @array.size } 23 | end 24 | 25 | protected 26 | 27 | def wrap_item(item) 28 | item.is_a?(QueueItem) ? item : QueueItem.new(item) 29 | end 30 | end 31 | end 32 | -------------------------------------------------------------------------------- /test/girl_friday_dispatcher_test.rb: -------------------------------------------------------------------------------- 1 | require_relative 'test_helper' 2 | 3 | describe Adrian::GirlFridayDispatcher do 4 | before do 5 | $done_items = [] 6 | @q = Adrian::ArrayQueue.new 7 | @dispatcher = Adrian::GirlFridayDispatcher.new(:stop_when_done => true) 8 | end 9 | 10 | describe "work delegation" do 11 | it "should instantiate an instance of the worker for each item and ask it to perform" do 12 | worker = Class.new(Adrian::Worker) do 13 | def work 14 | sleep(rand) 15 | $done_items << @item.value 16 | end 17 | end 18 | 19 | @q.push(1) 20 | @q.push(2) 21 | @q.push(3) 22 | 23 | @dispatcher.start(@q, worker) 24 | 25 | _($done_items.sort).must_equal([1, 2, 3]) 26 | end 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /examples/basic.rb: -------------------------------------------------------------------------------- 1 | $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib')) 2 | 3 | $stdout.sync = true 4 | 5 | require 'adrian' 6 | 7 | class ExampleError < RuntimeError; end 8 | 9 | class Worker < Adrian::Worker 10 | def work 11 | raise ExampleError if item.value > 0.5 12 | end 13 | end 14 | 15 | 16 | 17 | 18 | q = Adrian::ArrayQueue.new 19 | q2 = Adrian::ArrayQueue.new 20 | 21 | dispatcher = Adrian::Dispatcher.new 22 | 23 | dispatcher.on_failure(ExampleError) do |item, worker, exception| 24 | puts "FAILURE!!! #{item.value}" 25 | q2.push(item) 26 | end 27 | 28 | dispatcher.on_done do |item, worker| 29 | puts "DONE!!! #{item.value}" 30 | end 31 | 32 | 33 | 34 | 35 | Thread.new do 36 | while true 37 | sleep(0.1) 38 | q.push(rand) 39 | end 40 | end 41 | 42 | dispatcher.start(q, Worker) 43 | -------------------------------------------------------------------------------- /test/array_queue_test.rb: -------------------------------------------------------------------------------- 1 | require_relative 'test_helper' 2 | 3 | describe Adrian::ArrayQueue do 4 | it 'should allow construction with an array' do 5 | q = Adrian::ArrayQueue.new([1,2,3]) 6 | _(q.pop.value).must_equal 1 7 | _(q.pop.value).must_equal 2 8 | _(q.pop.value).must_equal 3 9 | _(q.pop).must_be_nil 10 | end 11 | 12 | it 'should allow construction without an array' do 13 | q = Adrian::ArrayQueue.new 14 | _(q.pop).must_be_nil 15 | end 16 | 17 | it 'should act as a queue' do 18 | q = Adrian::ArrayQueue.new 19 | 20 | q.push(1) 21 | q.push(2) 22 | q.push(3) 23 | 24 | _(q.length).must_equal 3 25 | 26 | _(q.pop.value).must_equal 1 27 | _(q.pop.value).must_equal 2 28 | _(q.pop.value).must_equal 3 29 | _(q.pop).must_be_nil 30 | 31 | _(q.length).must_equal 0 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /adrian.gemspec: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | require File.expand_path('../lib/adrian/version', __FILE__) 3 | 4 | Gem::Specification.new 'adrian', Adrian::VERSION do |gem| 5 | gem.authors = ['Mick Staugaard', 'Eric Chapweske'] 6 | gem.description = 'A work dispatcher and some queue implementations' 7 | gem.summary = 'Adrian does not do any real work, but is really good at delegating it' 8 | gem.homepage = 'https://github.com/zendesk/adrian' 9 | gem.license = 'Apache License Version 2.0' 10 | gem.files = `git ls-files lib`.split("\n") 11 | gem.required_ruby_version = '>= 3.2' 12 | 13 | gem.add_development_dependency 'rake', '~> 13' 14 | gem.add_development_dependency 'minitest', '~> 5' 15 | gem.add_development_dependency 'girl_friday', '~> 0.11.0' 16 | gem.add_development_dependency 'timecop', '~> 0.9' 17 | end 18 | -------------------------------------------------------------------------------- /lib/adrian/failure_handler.rb: -------------------------------------------------------------------------------- 1 | module Adrian 2 | class FailureHandler 3 | def initialize 4 | @rules = [] 5 | end 6 | 7 | def add_rule(*exceptions, &block) 8 | exceptions.each do |exception_class| 9 | @rules << Rule.new(exception_class, block) 10 | end 11 | end 12 | 13 | def handle(exception) 14 | if rule = @rules.find { |r| r.match(exception) } 15 | rule.block 16 | end 17 | end 18 | 19 | class Rule 20 | attr_reader :block 21 | 22 | def initialize(exception_class, block) 23 | @exception_class = exception_class 24 | @block = block 25 | end 26 | 27 | def match(exception) 28 | return @exception_class.nil? if exception.nil? 29 | 30 | return false if @exception_class.nil? 31 | 32 | exception.is_a?(@exception_class) 33 | end 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | PATH 2 | remote: . 3 | specs: 4 | adrian (2.0.2) 5 | 6 | GEM 7 | remote: https://rubygems.org/ 8 | specs: 9 | connection_pool (1.2.0) 10 | debug (1.7.2) 11 | irb (>= 1.5.0) 12 | reline (>= 0.3.1) 13 | girl_friday (0.11.2) 14 | connection_pool (~> 1.0) 15 | rubinius-actor 16 | io-console (0.6.0) 17 | irb (1.6.3) 18 | reline (>= 0.3.0) 19 | minitest (5.18.0) 20 | rake (13.0.6) 21 | reline (0.3.3) 22 | io-console (~> 0.5) 23 | rubinius-actor (0.0.2) 24 | rubinius-core-api 25 | rubinius-core-api (0.0.1) 26 | timecop (0.9.6) 27 | 28 | PLATFORMS 29 | arm64-darwin-21 30 | arm64-darwin-22 31 | arm64-linux 32 | ruby 33 | x86_64-darwin-21 34 | x86_64-linux 35 | 36 | DEPENDENCIES 37 | adrian! 38 | debug (>= 1.0.0) 39 | girl_friday (~> 0.11.0) 40 | minitest (~> 5) 41 | rake (~> 13) 42 | timecop (~> 0.9) 43 | 44 | BUNDLED WITH 45 | 2.4.10 46 | -------------------------------------------------------------------------------- /lib/adrian/girl_friday_dispatcher.rb: -------------------------------------------------------------------------------- 1 | require 'girl_friday' 2 | 3 | module Adrian 4 | class GirlFridayDispatcher < Dispatcher 5 | def gf_queue_name 6 | @options[:name] || 'adrian_queue' 7 | end 8 | 9 | def gf_queue_size 10 | @options[:size] 11 | end 12 | 13 | def gf_queue 14 | @gf_queue ||= GirlFriday::WorkQueue.new(gf_queue_name, :size => gf_queue_size) do |item, worker_class| 15 | worker = worker_class.new(item) 16 | worker.report_to(self) 17 | worker.perform 18 | end 19 | end 20 | 21 | def delegate_work(item, worker_class) 22 | gf_queue.push([item, worker_class]) 23 | end 24 | 25 | def wait_for_empty 26 | gf_queue.wait_for_empty 27 | 28 | sleep(0.5) 29 | 30 | while gf_queue.status[gf_queue_name][:busy] != 0 31 | sleep(0.5) 32 | end 33 | end 34 | 35 | def stop 36 | super 37 | wait_for_empty 38 | end 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /test/worker_test.rb: -------------------------------------------------------------------------------- 1 | require_relative 'test_helper' 2 | 3 | describe Adrian::Worker do 4 | describe "#perform" do 5 | before { @item = 2} 6 | 7 | it "should report back to the boss" do 8 | worker_class = Class.new(Adrian::Worker) do 9 | def work; item + 2; end 10 | end 11 | 12 | worker = worker_class.new(@item) 13 | boss = MiniTest::Mock.new 14 | worker.report_to(boss) 15 | 16 | boss.expect(:work_done, nil, [@item, worker, nil]) 17 | worker.perform 18 | 19 | boss.verify 20 | end 21 | 22 | it "should NEVER raise an exception" do 23 | worker_class = Class.new(Adrian::Worker) do 24 | def work; raise "STRIKE!"; end 25 | end 26 | 27 | worker = worker_class.new(@item) 28 | boss = MiniTest::Mock.new 29 | worker.report_to(boss) 30 | 31 | boss.expect(:work_done, nil, [@item, worker, RuntimeError]) 32 | 33 | worker.perform 34 | 35 | boss.verify 36 | end 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /contributing.md: -------------------------------------------------------------------------------- 1 | ## Contributing 2 | 3 | - Fork it 4 | - Create your feature branch (`git checkout -b my-new-feature`) 5 | - Commit your changes (`git commit -am 'Added some feature'`) 6 | - Push to the branch (`git push origin my-new-feature`) 7 | - Include specs, make sure are green. 8 | 9 | ### Releasing a new version 10 | A new version is published to RubyGems.org every time a change to `version.rb` is pushed to the `main` branch. 11 | In short, follow these steps: 12 | 1. Update `version.rb`, 13 | 2. run `bundle lock` to update `Gemfile.lock`, 14 | 3. merge this change into `main`, and 15 | 4. look at [the action](https://github.com/zendesk/adrian/actions/workflows/publish.yml) for output. 16 | 17 | To create a pre-release from a non-main branch: 18 | 1. change the version in `version.rb` to something like `1.2.0.pre.1` or `2.0.0.beta.2`, 19 | 2. push this change to your branch, 20 | 3. go to [Actions → “Publish to RubyGems.org” on GitHub](https://github.com/zendesk/adrian/actions/workflows/publish.yml), 21 | 4. click the “Run workflow” button, 22 | 5. pick your branch from a dropdown. 23 | -------------------------------------------------------------------------------- /test/queue_test.rb: -------------------------------------------------------------------------------- 1 | require_relative 'test_helper' 2 | 3 | describe Adrian::Queue do 4 | class TestQueue < Adrian::Queue 5 | attr_accessor :item 6 | 7 | def pop_item 8 | @item 9 | end 10 | 11 | def push_item(item) 12 | @item = item 13 | end 14 | end 15 | 16 | describe 'when a max age is defined' do 17 | before { @q = TestQueue.new(:max_age => 60) } 18 | 19 | it 'validates the age of items' do 20 | item = Adrian::QueueItem.new('value', Time.now) 21 | @q.push(item) 22 | _(@q.pop).must_equal item 23 | 24 | item = Adrian::QueueItem.new('value', Time.now - 120) 25 | @q.push(item) 26 | _(lambda { @q.pop }).must_raise(Adrian::Queue::ItemTooOldError) 27 | end 28 | 29 | end 30 | 31 | it 'sets the queue on the items' do 32 | q = TestQueue.new 33 | 34 | item = Adrian::QueueItem.new('value', Time.now) 35 | 36 | _(item.queue).must_be_nil 37 | 38 | q.push(item) 39 | 40 | _(item.queue).must_be_nil 41 | 42 | popped_item = q.pop 43 | 44 | _(popped_item).must_equal item 45 | _(item.queue).must_equal q 46 | end 47 | 48 | end 49 | -------------------------------------------------------------------------------- /lib/adrian/queue.rb: -------------------------------------------------------------------------------- 1 | module Adrian 2 | class Queue 3 | class ItemTooOldError < RuntimeError 4 | attr_reader :item, :queue 5 | 6 | def initialize(item, queue) 7 | super() 8 | @item = item 9 | @queue = queue 10 | end 11 | end 12 | 13 | def initialize(options = {}) 14 | @options = options 15 | end 16 | 17 | def pop 18 | item = pop_item 19 | item.queue = self if item 20 | verify_age!(item) 21 | end 22 | 23 | def push(item) 24 | push_item(item) 25 | end 26 | 27 | def verify_age!(item) 28 | if item && max_age && item.age > max_age 29 | raise ItemTooOldError.new(item, self) 30 | end 31 | 32 | item 33 | end 34 | 35 | def max_age 36 | @max_age ||= @options[:max_age] 37 | end 38 | 39 | def pop_item 40 | raise "#{self.class.name}#pop_item is not defined" 41 | end 42 | 43 | def push_item(item) 44 | raise "#{self.class.name}#push_item is not defined" 45 | end 46 | 47 | def length 48 | raise "#{self.class.name}#length is not defined" 49 | end 50 | end 51 | end 52 | -------------------------------------------------------------------------------- /test/composite_queue_test.rb: -------------------------------------------------------------------------------- 1 | require_relative 'test_helper' 2 | 3 | describe Adrian::CompositeQueue do 4 | before do 5 | @q1 = Adrian::ArrayQueue.new 6 | @q2 = Adrian::ArrayQueue.new 7 | @q = Adrian::CompositeQueue.new(@q1, @q2) 8 | end 9 | 10 | describe "popping" do 11 | it 'should return nil when all queues are empty' do 12 | _(@q.pop).must_be_nil 13 | end 14 | 15 | it 'should return an item from the first queue that has items' do 16 | @q1.push(1) 17 | @q1.push(2) 18 | @q2.push(3) 19 | @q2.push(4) 20 | 21 | _(@q.pop.value).must_equal(1) 22 | _(@q.pop.value).must_equal(2) 23 | _(@q.pop.value).must_equal(3) 24 | _(@q.pop.value).must_equal(4) 25 | _(@q.pop).must_be_nil 26 | _(@q1.pop).must_be_nil 27 | _(@q2.pop).must_be_nil 28 | end 29 | 30 | it 'sets the original queue on the item' do 31 | @q1.push(1) 32 | @q2.push(2) 33 | 34 | _(@q.pop.queue).must_equal @q1 35 | _(@q.pop.queue).must_equal @q2 36 | end 37 | end 38 | 39 | describe "pushing" do 40 | it "should not be allowed" do 41 | _(lambda { @q.push(1) }).must_raise(RuntimeError) 42 | end 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Adrian 2 | 3 | [![Build Status](https://travis-ci.org/zendesk/adrian.svg?branch=master)](https://travis-ci.org/zendesk/adrian) 4 | 5 | Adrian is a work dispatcher and some queue implementations. 6 | Adrian does not do any real work, but is really good at delegating it. 7 | Adrian **does not care how** real work get's done, but **makes damn sure that it does get done**. 8 | 9 | ## Installation 10 | 11 | Add this line to your application's Gemfile: 12 | 13 | gem 'adrian' 14 | 15 | And then execute: 16 | 17 | $ bundle 18 | 19 | Or install it yourself as: 20 | 21 | $ gem install adrian 22 | 23 | ## Copyright and license 24 | 25 | Copyright 2013-2016 Zendesk 26 | 27 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. 28 | You may obtain a copy of the License at 29 | 30 | http://www.apache.org/licenses/LICENSE-2.0 31 | 32 | Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 33 | -------------------------------------------------------------------------------- /test/failure_handler_test.rb: -------------------------------------------------------------------------------- 1 | require_relative 'test_helper' 2 | 3 | require 'adrian/failure_handler' 4 | 5 | describe Adrian::FailureHandler do 6 | before do 7 | @handler = Adrian::FailureHandler.new 8 | 9 | $failure = nil 10 | 11 | @handler.add_rule(RuntimeError) { :runtime } 12 | @handler.add_rule(StandardError) { :standard } 13 | end 14 | 15 | it "should match rules in the order they were added" do 16 | block = @handler.handle(RuntimeError.new) 17 | assert block 18 | _(block.call).must_equal :runtime 19 | 20 | block = @handler.handle(StandardError.new) 21 | assert block 22 | _(block.call).must_equal :standard 23 | end 24 | 25 | it "should do nothing when no rules match" do 26 | _(@handler.handle(Exception.new)).must_be_nil 27 | end 28 | 29 | describe "the success rule" do 30 | before do 31 | @handler = Adrian::FailureHandler.new 32 | @handler.add_rule(nil) { :success } 33 | end 34 | 35 | it "should match when there is no exception" do 36 | _(@handler.handle(RuntimeError.new)).must_be_nil 37 | 38 | block = @handler.handle(nil) 39 | assert block 40 | _(block.call).must_equal :success 41 | end 42 | end 43 | 44 | end 45 | -------------------------------------------------------------------------------- /lib/adrian/filters.rb: -------------------------------------------------------------------------------- 1 | module Adrian 2 | module Filters 3 | 4 | def filters 5 | @filters ||= [] 6 | end 7 | 8 | def filter?(item) 9 | !filters.all? { |filter| filter.allow?(item) } 10 | end 11 | 12 | class Delay 13 | FIFTEEN_MINUTES = 900 14 | 15 | def initialize(options = {}) 16 | @options = options 17 | end 18 | 19 | def allow?(item) 20 | item.updated_at <= (Time.now - duration) 21 | end 22 | 23 | def duration 24 | @options[:duration] ||= FIFTEEN_MINUTES 25 | end 26 | 27 | end 28 | 29 | class FileLock 30 | ONE_HOUR = 3_600 31 | 32 | def initialize(options = {}) 33 | @options = options 34 | @reserved_path = @options.fetch(:reserved_path) 35 | end 36 | 37 | def allow?(item) 38 | !locked?(item) || lock_expired?(item) 39 | end 40 | 41 | def lock_expired?(item) 42 | item.updated_at <= (Time.now - duration) 43 | end 44 | 45 | def locked?(item) 46 | @reserved_path == File.dirname(item.path) 47 | end 48 | 49 | def duration 50 | @options[:duration] ||= ONE_HOUR 51 | end 52 | 53 | end 54 | 55 | end 56 | end 57 | -------------------------------------------------------------------------------- /lib/adrian/file_item.rb: -------------------------------------------------------------------------------- 1 | module Adrian 2 | class FileItem < QueueItem 3 | attr_accessor :logger 4 | 5 | def initialize(value) 6 | @value = value 7 | stat 8 | end 9 | 10 | def path 11 | value 12 | end 13 | 14 | def name 15 | File.basename(path) 16 | end 17 | 18 | def ==(other) 19 | other.respond_to?(:name) && 20 | name == other.name 21 | end 22 | 23 | def move(destination) 24 | destination_path = File.join(destination, File.basename(path)) 25 | logger.info("Moving #{path} to #{destination_path}") if logger 26 | File.rename(path, destination_path) 27 | @value = destination_path 28 | end 29 | 30 | def stat 31 | @stat ||= File.stat(path) 32 | rescue Errno::ENOENT 33 | nil 34 | end 35 | 36 | def atime 37 | stat ? stat.atime.utc : nil 38 | end 39 | 40 | def mtime 41 | stat ? stat.mtime.utc : nil 42 | end 43 | 44 | def updated_at 45 | @updated_at ||= atime 46 | end 47 | 48 | def created_at 49 | @created_at ||= mtime 50 | end 51 | 52 | def touch(updated_at = Time.now) 53 | @updated_at = updated_at.utc 54 | File.utime(updated_at, created_at, path) 55 | end 56 | 57 | def exist? 58 | File.exist?(path) 59 | end 60 | 61 | end 62 | end 63 | -------------------------------------------------------------------------------- /test/rotating_directory_queue_test.rb: -------------------------------------------------------------------------------- 1 | require_relative 'test_helper' 2 | require 'tempfile' 3 | require 'tmpdir' 4 | require 'fileutils' 5 | 6 | describe Adrian::RotatingDirectoryQueue do 7 | before do 8 | @root_path = Dir.mktmpdir('dir_queue_test') 9 | @q = Adrian::RotatingDirectoryQueue.create(:path => @root_path) 10 | Timecop.freeze 11 | end 12 | 13 | after do 14 | Timecop.return 15 | FileUtils.rm_r(@root_path, :force => true) 16 | end 17 | 18 | describe 'pop' do 19 | it 'only provides files available in the current time-stamped directory' do 20 | @item1 = Adrian::FileItem.new(Tempfile.new('item1').path) 21 | @item2 = Adrian::FileItem.new(Tempfile.new('item2').path) 22 | @item3 = Adrian::FileItem.new(Tempfile.new('item3').path) 23 | 24 | todays_directory = File.join(@root_path, Time.now.strftime('%Y-%m-%d')) 25 | tomorrows_directory = File.join(@root_path, (Time.now + 60 * 60 * 24).strftime('%Y-%m-%d')) 26 | 27 | FileUtils.mkdir_p(todays_directory) 28 | FileUtils.mkdir_p(tomorrows_directory) 29 | 30 | @item1.move(todays_directory) 31 | @item2.move(tomorrows_directory) 32 | @item3.move(@root_path) 33 | 34 | _(@q.pop).must_equal @item1 35 | _(@q.pop).must_be_nil 36 | end 37 | end 38 | 39 | describe 'push' do 40 | before do 41 | @item = Adrian::FileItem.new(Tempfile.new('item').path) 42 | end 43 | 44 | it 'moves the file to the time-stamped available directory' do 45 | original_path = @item.path 46 | @q.push(@item) 47 | 48 | assert_equal false, File.exist?(original_path) 49 | assert_equal true, File.exist?(File.join(@q.available_path, @item.name)) 50 | 51 | _(@item.path).must_equal File.join(@root_path, Time.now.strftime('%Y-%m-%d'), @item.name) 52 | end 53 | end 54 | 55 | end 56 | -------------------------------------------------------------------------------- /test/dispatcher_lifecycle_test.rb: -------------------------------------------------------------------------------- 1 | require_relative 'test_helper' 2 | 3 | describe "Adrian::Dispatcher lifecycle" do 4 | class Worker < Adrian::Worker 5 | def work 6 | $done_items << item.value 7 | end 8 | end 9 | 10 | before do 11 | $done_items = [] 12 | @q = Adrian::ArrayQueue.new([1,2,3]) 13 | end 14 | 15 | describe "stop_when_done" do 16 | describe "set to true" do 17 | before do 18 | @dispatcher = Adrian::Dispatcher.new(:stop_when_done => true) 19 | end 20 | 21 | it "should have all the work done and stop" do 22 | t = Thread.new do 23 | @dispatcher.start(@q, Worker) 24 | end 25 | 26 | sleep(0.5) 27 | 28 | _(@q.pop).must_be_nil 29 | 30 | _($done_items).must_equal([1,2,3]) 31 | 32 | _(@dispatcher.running).must_equal false 33 | end 34 | end 35 | 36 | describe "set to false" do 37 | before do 38 | @dispatcher = Adrian::Dispatcher.new(:stop_when_done => false) 39 | end 40 | 41 | it "should have all the work done and continue" do 42 | t = Thread.new do 43 | @dispatcher.start(@q, Worker) 44 | end 45 | 46 | sleep(0.5) 47 | 48 | _(@q.pop).must_be_nil 49 | 50 | _($done_items).must_equal([1,2,3]) 51 | 52 | _(@dispatcher.running).must_equal true 53 | t.kill 54 | end 55 | end 56 | end 57 | 58 | describe "#stop" do 59 | before do 60 | @dispatcher = Adrian::Dispatcher.new(:sleep => 0.1) 61 | end 62 | 63 | it "should stop a running dispatcher" do 64 | t = Thread.new do 65 | @dispatcher.start(@q, Worker) 66 | end 67 | 68 | sleep(0.5) 69 | 70 | _(@dispatcher.running).must_equal true 71 | _(t.status).wont_equal false 72 | 73 | @dispatcher.stop 74 | 75 | sleep(0.5) 76 | 77 | _(@dispatcher.running).must_equal false 78 | _(t.status).must_equal false 79 | end 80 | end 81 | 82 | end 83 | -------------------------------------------------------------------------------- /lib/adrian/dispatcher.rb: -------------------------------------------------------------------------------- 1 | require 'adrian/failure_handler' 2 | 3 | module Adrian 4 | class Dispatcher 5 | attr_reader :running 6 | 7 | def initialize(options = {}) 8 | @failure_handler = FailureHandler.new 9 | @stop_when_done = !!options[:stop_when_done] 10 | @stop_when_signalled = options.fetch(:stop_when_signalled, true) 11 | @sleep = options[:sleep] || 0.5 12 | @options = options 13 | end 14 | 15 | def on_failure(*exceptions, &blk) 16 | @failure_handler.add_rule(*exceptions, &blk) 17 | end 18 | 19 | def on_done(&blk) 20 | @failure_handler.add_rule(nil, &blk) 21 | end 22 | 23 | def start(queue, worker_class) 24 | trap_stop_signals if @stop_when_signalled 25 | @running = true 26 | 27 | while @running do 28 | begin 29 | item = queue.pop 30 | rescue Adrian::Queue::ItemTooOldError => e 31 | if handler = @failure_handler.handle(e) 32 | handler.call(e.item, nil, e) 33 | end 34 | item = nil 35 | next 36 | end 37 | 38 | if item 39 | delegate_work(item, worker_class) 40 | else 41 | if @stop_when_done 42 | stop 43 | else 44 | sleep(@sleep) if @sleep 45 | end 46 | end 47 | end 48 | end 49 | 50 | def stop 51 | @running = false 52 | end 53 | 54 | def trap_stop_signals 55 | Signal.trap('TERM') { stop } 56 | Signal.trap('INT') { stop } 57 | end 58 | 59 | def delegate_work(item, worker_class) 60 | worker = worker_class.new(item) 61 | worker.report_to(self) 62 | worker.perform 63 | end 64 | 65 | def work_done(item, worker, exception = nil) 66 | if handler = @failure_handler.handle(exception) 67 | handler.call(item, worker, exception) 68 | else 69 | raise exception if exception 70 | end 71 | end 72 | 73 | end 74 | end 75 | -------------------------------------------------------------------------------- /lib/adrian/directory_queue.rb: -------------------------------------------------------------------------------- 1 | require 'adrian/queue' 2 | require 'fileutils' 3 | 4 | module Adrian 5 | class DirectoryQueue < Adrian::Queue 6 | include Filters 7 | 8 | def self.create(options = {}) 9 | queue = new(options) 10 | FileUtils.mkdir_p(queue.available_path) 11 | FileUtils.mkdir_p(queue.reserved_path) 12 | queue 13 | end 14 | 15 | attr_reader :available_path, :reserved_path, :logger 16 | 17 | # Note: 18 | # There is the possibility of an item being consumed by multiple processes when its still in the queue after its lock expires. 19 | # The reason for allowing this is: 20 | # 1. It's much simpler than introducing a seperate monitoring process to handle lock expiry. 21 | # 2. This is an acceptable and rare event. e.g. it only happens when the process working on the item crashes without being able to release the lock 22 | def initialize(options = {}) 23 | super 24 | @available_path = options.fetch(:path) 25 | @reserved_path = options.fetch(:reserved_path, default_reserved_path) 26 | @logger = options[:logger] 27 | filters << Filters::FileLock.new(:duration => options[:lock_duration], :reserved_path => reserved_path) 28 | filters << Filters::Delay.new(:duration => options[:delay]) if options[:delay] 29 | end 30 | 31 | def pop_item 32 | while item = items.shift 33 | return item if reserve(item) 34 | end 35 | end 36 | 37 | def push_item(value) 38 | item = wrap_item(value) 39 | item.move(available_path) 40 | item.touch 41 | self 42 | end 43 | 44 | def length 45 | available_files.count { |file| File.file?(file) } 46 | end 47 | 48 | def include?(value) 49 | item = wrap_item(value) 50 | items.include?(item) 51 | end 52 | 53 | protected 54 | 55 | def wrap_item(value) 56 | item = value.is_a?(FileItem) ? value : FileItem.new(value) 57 | item.logger ||= logger 58 | item 59 | end 60 | 61 | def reserve(item) 62 | item.move(reserved_path) 63 | item.touch 64 | true 65 | rescue Errno::ENOENT => e 66 | false 67 | end 68 | 69 | def items 70 | return @items if @items && @items.length > 0 71 | @items = files.map! { |file| wrap_item(file) } 72 | @items.reject! { |item| !item.exist? || filter?(item) } 73 | @items.sort_by!(&:updated_at) 74 | end 75 | 76 | def files 77 | (available_files + reserved_files).select { |file| File.file?(file) } 78 | end 79 | 80 | def available_files 81 | Dir.glob("#{available_path}/*") 82 | end 83 | 84 | def reserved_files 85 | Dir.glob("#{reserved_path}/*") 86 | end 87 | 88 | def default_reserved_path 89 | File.join(@available_path, 'cur') 90 | end 91 | 92 | end 93 | end 94 | -------------------------------------------------------------------------------- /test/dispatcher_test.rb: -------------------------------------------------------------------------------- 1 | require_relative 'test_helper' 2 | 3 | describe Adrian::Dispatcher do 4 | before do 5 | $done_items = [] 6 | @q = Adrian::ArrayQueue.new([], :max_age => 1000) 7 | @dispatcher = Adrian::Dispatcher.new(:stop_when_done => true) 8 | @worker = TestWorker 9 | end 10 | 11 | class TestWorker 12 | 13 | def initialize(item) 14 | @item = item 15 | @done_items = [] 16 | end 17 | 18 | def perform 19 | $done_items << @item.value 20 | end 21 | 22 | def report_to(boss) 23 | @boss = boss 24 | end 25 | 26 | end 27 | 28 | describe "work delegation" do 29 | it "should instantiate an instance of the worker for each item and ask it to perform" do 30 | @q.push(1) 31 | @q.push(2) 32 | @q.push(3) 33 | 34 | @dispatcher.start(@q, @worker) 35 | 36 | _($done_items).must_equal([1, 2, 3]) 37 | end 38 | end 39 | 40 | describe "a queue with old items" do 41 | before do 42 | @q.push(Adrian::QueueItem.new(1, Time.now)) 43 | @old_item = Adrian::QueueItem.new(2, Time.now - 2000) 44 | @q.push(@old_item) 45 | @q.push(Adrian::QueueItem.new(3, Time.now)) 46 | end 47 | 48 | it 'skips the old items' do 49 | @dispatcher.start(@q, @worker) 50 | 51 | _($done_items).must_equal([1, 3]) 52 | end 53 | 54 | it 'calls the handler for Adrian::Queue::ItemTooOldError' do 55 | handled_items = [] 56 | handled_exceptions = [] 57 | 58 | @dispatcher.on_failure(Adrian::Queue::ItemTooOldError) do |item, worker, exception| 59 | handled_items << item 60 | handled_exceptions << exception 61 | end 62 | 63 | @dispatcher.start(@q, @worker) 64 | 65 | _(handled_items).must_equal [@old_item] 66 | _(handled_exceptions.size).must_equal 1 67 | _(handled_exceptions.first).must_be_instance_of Adrian::Queue::ItemTooOldError 68 | end 69 | end 70 | 71 | describe "work evaluation" do 72 | 73 | it "stops when receiving a termination signal" do 74 | @dispatcher = Adrian::Dispatcher.new(:stop_when_done => false) 75 | dispatch_thread = Thread.new { @dispatcher.start(@q, @worker) } 76 | sleep(0.1) 77 | assert_equal true, @dispatcher.running 78 | 79 | Process.kill('TERM', Process.pid) 80 | assert_equal false, @dispatcher.running 81 | dispatch_thread.exit 82 | end 83 | 84 | it "should use the failure handler to handle the result" do 85 | @dispatcher.on_failure(RuntimeError) do |item, worker, exception| 86 | @q.push(item) 87 | end 88 | 89 | @dispatcher.work_done(1, nil) 90 | _(@q.pop).must_be_nil 91 | 92 | @dispatcher.work_done(1, nil, nil) 93 | _(@q.pop).must_be_nil 94 | 95 | @dispatcher.work_done(1, nil, RuntimeError.new) 96 | _(@q.pop.value).must_equal 1 97 | end 98 | end 99 | end 100 | -------------------------------------------------------------------------------- /test/filters_test.rb: -------------------------------------------------------------------------------- 1 | require_relative 'test_helper' 2 | require 'tempfile' 3 | require 'tmpdir' 4 | 5 | 6 | describe Adrian::Filters do 7 | before do 8 | @q = Object.new.extend(Adrian::Filters) 9 | @item = Adrian::QueueItem.new("hello") 10 | end 11 | 12 | class FakeFilter 13 | 14 | def initialize(options = {}) 15 | @allow = options.fetch(:allow) 16 | end 17 | 18 | def allow?(item) 19 | @allow == true 20 | end 21 | 22 | end 23 | 24 | describe "#filter?" do 25 | 26 | it "is true when any filter denies the item" do 27 | @q.filters << FakeFilter.new(:allow => true) 28 | @q.filters << FakeFilter.new(:allow => false) 29 | 30 | assert_equal true, @q.filter?(@item) 31 | end 32 | 33 | it "is false when all filters allow the item" do 34 | @q.filters << FakeFilter.new(:allow => true) 35 | assert_equal false, @q.filter?(@item) 36 | end 37 | 38 | end 39 | 40 | module Updatable 41 | attr_accessor :updated_at 42 | end 43 | 44 | describe Adrian::Filters::Delay do 45 | before do 46 | @filter = Adrian::Filters::Delay.new 47 | @updatable_item = Adrian::QueueItem.new("hello") 48 | @updatable_item.extend(Updatable) 49 | @updatable_item.updated_at = Time.now 50 | @fifteen_minutes = 900 51 | end 52 | 53 | it "allows items that have not been recently updated" do 54 | Time.stub(:now, @updatable_item.updated_at + @fifteen_minutes) do 55 | assert_equal true, @filter.allow?(@updatable_item) 56 | end 57 | end 58 | 59 | it "denies items that have been recently updated" do 60 | assert_equal false, @filter.allow?(@updatable_item) 61 | end 62 | 63 | it "has a configurable recently updated duration that defaults to 15 minutes" do 64 | assert_equal @fifteen_minutes, @filter.duration 65 | configured_filter = Adrian::Filters::Delay.new(:duration => 1) 66 | 67 | assert_equal 1, configured_filter.duration 68 | end 69 | 70 | end 71 | 72 | describe Adrian::Filters::FileLock do 73 | before do 74 | @filter = Adrian::Filters::FileLock.new(:reserved_path => 'path/to/locked') 75 | @available_item = Adrian::FileItem.new("path/to/file") 76 | @locked_item = Adrian::FileItem.new("path/to/locked/file") 77 | @one_hour = 3_600 78 | end 79 | 80 | it "allows items that are not locked" do 81 | assert_equal true, @filter.allow?(@available_item) 82 | end 83 | 84 | it "allows items with an expired lock" do 85 | @locked_item.stub(:updated_at, Time.now - @one_hour) do 86 | assert_equal true, @filter.allow?(@locked_item) 87 | end 88 | end 89 | 90 | it "does not allow items with a fresh lock" do 91 | @locked_item.stub(:updated_at, Time.now) do 92 | assert_equal false, @filter.allow?(@locked_item) 93 | end 94 | end 95 | 96 | it "has a configurable lock expiry duration that defaults to one hour" do 97 | assert_equal @one_hour, @filter.duration 98 | configured_filter = Adrian::Filters::FileLock.new(:duration => 1, :reserved_path => 'path/to/locked') 99 | 100 | assert_equal 1, configured_filter.duration 101 | end 102 | 103 | end 104 | 105 | end 106 | -------------------------------------------------------------------------------- /test/file_item_test.rb: -------------------------------------------------------------------------------- 1 | require_relative 'test_helper' 2 | require 'tempfile' 3 | 4 | describe Adrian::FileItem do 5 | before do 6 | @item = Adrian::FileItem.new(Tempfile.new('file_item_test').path) 7 | end 8 | 9 | it 'aliases value as path' do 10 | item = Adrian::FileItem.new('path/a') 11 | assert_equal 'path/a', item.value 12 | end 13 | 14 | it 'has a name from the path' do 15 | item = Adrian::FileItem.new('path/name.ext') 16 | assert_equal 'name.ext', item.name 17 | end 18 | 19 | it 'is equal to another item when they have the same name' do 20 | item1 = Adrian::FileItem.new('path/a') 21 | item2 = Adrian::FileItem.new('path/b') 22 | assert(item1 != item2) 23 | 24 | item3 = Adrian::FileItem.new('path/a') 25 | assert_equal item1, item3 26 | end 27 | 28 | describe 'updated_at' do 29 | 30 | it 'is the atime of the file' do 31 | _(@item.updated_at).must_equal File.atime(@item.path) 32 | end 33 | 34 | it 'is nil when moved by another process' do 35 | item = Adrian::FileItem.new('moved/during/initialize') 36 | assert_equal false, item.exist? 37 | assert_nil item.updated_at 38 | end 39 | 40 | it 'is cached' do 41 | updated_at = @item.updated_at 42 | assert @item.updated_at 43 | File.unlink(@item.path) 44 | assert_equal false, @item.exist? 45 | 46 | assert_equal updated_at, @item.updated_at 47 | end 48 | 49 | end 50 | 51 | describe 'created_at' do 52 | 53 | it 'is the mtime of the file' do 54 | _(@item.created_at).must_equal File.mtime(@item.path) 55 | end 56 | 57 | it 'is nil when moved by another process' do 58 | item = Adrian::FileItem.new('moved/during/initialize') 59 | assert_equal false, item.exist? 60 | assert_nil item.created_at 61 | end 62 | 63 | it 'is cached' do 64 | created_at = @item.created_at 65 | assert @item.created_at 66 | File.unlink(@item.path) 67 | assert_equal false, @item.exist? 68 | 69 | assert_equal created_at, @item.created_at 70 | end 71 | 72 | end 73 | 74 | describe 'move' do 75 | before do 76 | @destination = Dir.mktmpdir('file_item_move_test') 77 | end 78 | 79 | it 'moves the file to the given directory' do 80 | @item.move(@destination) 81 | assert_equal true, File.exist?(File.join(@destination, @item.name)) 82 | end 83 | 84 | it 'updates the path to its new location' do 85 | @item.move(@destination) 86 | assert_equal @destination, File.dirname(@item.path) 87 | end 88 | 89 | it 'logs the move on the logger' do 90 | destination_file_name = File.join(@destination, File.basename(@item.path)) 91 | logger = MiniTest::Mock.new 92 | logger.expect(:info, nil, ["Moving #{@item.path} to #{destination_file_name}"]) 93 | @item.logger = logger 94 | 95 | @item.move(@destination) 96 | 97 | logger.verify 98 | end 99 | 100 | it 'does not change the atime' do 101 | atime = File.atime(@item.path) 102 | @item.move(@destination) 103 | _(File.atime(@item.path)).must_equal atime 104 | end 105 | 106 | it 'does not change the mtime' do 107 | mtime = File.mtime(@item.path) 108 | @item.move(@destination) 109 | _(File.mtime(@item.path)).must_equal mtime 110 | end 111 | 112 | end 113 | 114 | describe 'touch' do 115 | 116 | it 'changes the update timestamp to the current time' do 117 | now = Time.now - 100 118 | Time.stub(:now, now) { @item.touch } 119 | 120 | assert_equal now.to_i, @item.updated_at.to_i 121 | end 122 | 123 | it 'changes the atime' do 124 | atime = File.atime(@item.path).to_i 125 | 126 | now = (Time.now - 100) 127 | Time.stub(:now, now) { @item.touch } 128 | 129 | _(now.to_i).wont_equal atime 130 | _(File.atime(@item.path).to_i).must_equal now.to_i 131 | end 132 | 133 | it 'does not change the mtime' do 134 | mtime = File.mtime(@item.path).to_i 135 | 136 | now = (Time.now - 100) 137 | Time.stub(:new, now) { @item.touch } 138 | 139 | _(now.to_i).wont_equal mtime 140 | _(File.mtime(@item.path).to_i).must_equal mtime 141 | end 142 | 143 | end 144 | 145 | it 'exists when the file at the given path exists' do 146 | assert_equal true, @item.exist? 147 | File.unlink(@item.path) 148 | 149 | assert_equal false, @item.exist? 150 | end 151 | 152 | end 153 | -------------------------------------------------------------------------------- /test/directory_queue_test.rb: -------------------------------------------------------------------------------- 1 | require_relative 'test_helper' 2 | require 'tempfile' 3 | require 'tmpdir' 4 | require 'fileutils' 5 | require 'logger' 6 | 7 | describe Adrian::DirectoryQueue do 8 | before do 9 | @logger = Logger.new('/dev/null') 10 | @q = Adrian::DirectoryQueue.create(:path => Dir.mktmpdir('dir_queue_test'), :delay => 0, :logger => @logger) 11 | end 12 | 13 | after do 14 | FileUtils.rm_r(@q.available_path, :force => true) 15 | FileUtils.rm_r(@q.reserved_path, :force => true) 16 | end 17 | 18 | it 'should act as a queue for files' do 19 | item1 = Tempfile.new('item1-').path 20 | item2 = Tempfile.new('item2-').path 21 | item3 = Tempfile.new('item3-').path 22 | 23 | @q.push(item1) 24 | @q.push(item2) 25 | @q.push(item3) 26 | 27 | _(@q.length).must_equal 3 28 | 29 | _(@q.pop).must_equal Adrian::FileItem.new(item1) 30 | _(@q.pop).must_equal Adrian::FileItem.new(item2) 31 | _(@q.pop).must_equal Adrian::FileItem.new(item3) 32 | _(@q.pop).must_be_nil 33 | 34 | _(@q.length).must_equal 0 35 | end 36 | 37 | describe 'file backend' do 38 | 39 | describe 'pop' do 40 | before do 41 | @item = Adrian::FileItem.new(Tempfile.new('item').path) 42 | end 43 | 44 | it 'provides an available file' do 45 | @q.push(@item) 46 | assert_equal @item, @q.pop 47 | end 48 | 49 | it 'moves the file to the reserved directory' do 50 | @q.push(@item) 51 | original_path = @item.path 52 | item = @q.pop 53 | assert_equal @item, item 54 | 55 | assert_equal false, File.exist?(original_path) 56 | assert_equal true, File.exist?(File.join(@q.reserved_path, @item.name)) 57 | end 58 | 59 | it 'reserves the file for an hour by default' do 60 | @q.push(@item) 61 | reserved_item = @q.pop 62 | assert reserved_item 63 | one_hour = 3_600 64 | 65 | Time.stub(:now, reserved_item.updated_at + one_hour - 1) do 66 | assert_nil @q.pop 67 | end 68 | 69 | Time.stub(:now, reserved_item.updated_at + one_hour) do 70 | assert_equal @item, @q.pop 71 | end 72 | 73 | end 74 | 75 | it 'touches the item' do 76 | @q.push(@item) 77 | now = Time.now + 100 78 | item = nil 79 | Time.stub(:now, now) { item = @q.pop } 80 | 81 | assert_equal now.to_i, item.updated_at.to_i 82 | end 83 | 84 | it 'skips the file when moved by another process' do 85 | def @q.files 86 | [ 'no/longer/exists' ] 87 | end 88 | assert_nil @q.pop 89 | end 90 | 91 | it "only provides normal files" do 92 | not_file = Dir.mktmpdir('directory_queue_x', @q.available_path) 93 | assert_nil @q.pop 94 | end 95 | 96 | it "sets the logger on the item" do 97 | _(@item.logger).must_be_nil 98 | @q.push(@item) 99 | _(@q.pop.logger).must_equal @logger 100 | end 101 | 102 | describe "items list" do 103 | before do 104 | @item1 = Tempfile.new('item1-').path 105 | @item2 = Tempfile.new('item2-').path 106 | @item3 = Tempfile.new('item3-').path 107 | @item4 = Tempfile.new('item4-').path 108 | end 109 | 110 | it "populates items list on first pop" do 111 | _(items_count).must_equal 0 112 | @q.push(@item1) 113 | @q.push(@item2) 114 | _(items_count).must_equal 0 115 | 116 | @q.pop 117 | _(items_count).must_equal 1 118 | end 119 | 120 | it "populates items list when #include? is used" do 121 | @q.push(@item1) 122 | _(items_count).must_equal 0 123 | assert @q.include?(@item1) 124 | end 125 | 126 | describe "only repopulates items list from directory after its current contents are emptied" do 127 | before do 128 | @q.push(@item1) 129 | @q.push(@item2) 130 | @q.pop 131 | _(items_count).must_equal 1 132 | 133 | @q.push(@item3) 134 | @q.push(@item4) 135 | refute @q.include?(@item4) 136 | _(items_count).must_equal 1 137 | 138 | @q.pop 139 | _(items_count).must_equal 0 140 | end 141 | 142 | it "and #pop is called" do 143 | @q.pop 144 | assert @q.include?(@item4) 145 | _(items_count).must_equal 1 146 | end 147 | 148 | it "and #include? is called" do 149 | assert @q.include?(@item3) 150 | _(items_count).must_equal 2 151 | end 152 | end 153 | end 154 | end 155 | 156 | describe 'push' do 157 | before do 158 | @item = Adrian::FileItem.new(Tempfile.new('item').path) 159 | end 160 | 161 | it 'moves the file to the available directory' do 162 | original_path = @item.path 163 | @q.push(@item) 164 | 165 | assert_equal false, File.exist?(original_path) 166 | assert_equal true, File.exist?(File.join(@q.available_path, @item.name)) 167 | end 168 | 169 | it 'touches the item' do 170 | now = Time.now - 100 171 | Time.stub(:now, now) { @q.push(@item) } 172 | 173 | assert_equal now.to_i, @item.updated_at.to_i 174 | end 175 | 176 | end 177 | 178 | describe 'filters' do 179 | it 'should add a delay filter if the :delay option is given' do 180 | q = Adrian::DirectoryQueue.create(:path => Dir.mktmpdir('dir_queue_test')) 181 | filter = q.filters.find {|filter| filter.is_a?(Adrian::Filters::Delay)} 182 | _(filter).must_be_nil 183 | 184 | q = Adrian::DirectoryQueue.create(:path => Dir.mktmpdir('dir_queue_test'), :delay => 300) 185 | filter = q.filters.find {|filter| filter.is_a?(Adrian::Filters::Delay)} 186 | _(filter).wont_equal nil 187 | _(filter.duration).must_equal 300 188 | end 189 | 190 | it 'should add a lock filter that can be configured with the :lock_duration option' do 191 | q = Adrian::DirectoryQueue.create(:path => Dir.mktmpdir('dir_queue_test')) 192 | filter = q.filters.find {|filter| filter.is_a?(Adrian::Filters::FileLock)} 193 | _(filter).wont_equal nil 194 | _(filter.duration).must_equal 3600 # default value 195 | 196 | q = Adrian::DirectoryQueue.create(:path => Dir.mktmpdir('dir_queue_test'), :lock_duration => 300) 197 | filter = q.filters.find {|filter| filter.is_a?(Adrian::Filters::FileLock)} 198 | _(filter).wont_equal nil 199 | _(filter.duration).must_equal 300 200 | end 201 | end 202 | end 203 | 204 | def items_count 205 | (@q.instance_variable_get(:@items) || []).size 206 | end 207 | 208 | end 209 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS --------------------------------------------------------------------------------