├── .travis.yml ├── lib ├── localjob │ ├── version.rb │ ├── mock_adapter.rb │ ├── sysv_adapter.rb │ ├── cli.rb │ └── worker.rb └── localjob.rb ├── bin └── localjob ├── Gemfile ├── .editorconfig ├── Rakefile ├── test ├── integration.rb ├── jobs.rb ├── mock_adapter.rb ├── sysv_adapter_test.rb ├── test_helper.rb ├── localjob_test.rb └── worker_test.rb ├── .gitignore ├── LICENSE.txt ├── localjob.gemspec └── README.md /.travis.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | rvm: 3 | - 2.0.0 4 | - 2.1.1 5 | -------------------------------------------------------------------------------- /lib/localjob/version.rb: -------------------------------------------------------------------------------- 1 | class Localjob 2 | VERSION = "0.4.1" 3 | end 4 | -------------------------------------------------------------------------------- /bin/localjob: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | require 'localjob' 3 | require 'localjob/cli' 4 | 5 | Localjob::CLI.start(ARGV) 6 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Specify your gem's dependencies in localjob.gemspec 4 | gemspec 5 | 6 | gem 'coveralls', require: false 7 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://EditorConfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | charset = utf-8 7 | indent_style = space 8 | indent_size = 2 9 | insert_final_newline = true 10 | 11 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | 3 | task :default => :test 4 | 5 | require 'rake/testtask' 6 | Rake::TestTask.new do |t| 7 | t.libs << "test" 8 | t.pattern = "test/**/*_test.rb" 9 | end 10 | -------------------------------------------------------------------------------- /test/integration.rb: -------------------------------------------------------------------------------- 1 | require 'localjob' 2 | 3 | class PrintJob 4 | def perform 5 | puts "hello world" 6 | end 7 | end 8 | 9 | if ARGV[0] == "produce" 10 | queue = Localjob.new 11 | queue << PrintJob.new 12 | end 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | *.rbc 3 | .bundle 4 | .config 5 | .yardoc 6 | .idea 7 | Gemfile.lock 8 | InstalledFiles 9 | _yardoc 10 | coverage 11 | doc/ 12 | lib/bundler/man 13 | pkg 14 | rdoc 15 | spec/reports 16 | test/tmp 17 | test/version_tmp 18 | tmp 19 | -------------------------------------------------------------------------------- /test/jobs.rb: -------------------------------------------------------------------------------- 1 | class WalrusJob < Struct.new(:action) 2 | def perform 3 | "Nice try, but there's no way I'll #{action}.." 4 | end 5 | end 6 | 7 | class AngryWalrusJob < Struct.new(:angryness) 8 | def perform 9 | raise "I am this angry: #{angryness}" 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /test/mock_adapter.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | require 'localjob/mock_adapter' 3 | 4 | class MockAdapterTest < LocaljobTestCase 5 | def setup 6 | @localjob = queue 7 | @localjob.queue = Localjob::MockAdapter.new("localjob") 8 | end 9 | 10 | def test_push_to_queue 11 | @localjob << "hello world" 12 | assert_equal 1, @localjob.size 13 | end 14 | 15 | def test_push_and_pop_from_queue 16 | @localjob << "hello world" 17 | assert_equal "hello world", @localjob.shift 18 | end 19 | 20 | def test_destroy_queue 21 | @localjob << "hello world" 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /lib/localjob/mock_adapter.rb: -------------------------------------------------------------------------------- 1 | class Localjob 2 | class Channel 3 | def shift 4 | queue = @queues.find { |q| q.size > 0 } 5 | return queue.shift 6 | end 7 | end 8 | 9 | class MockAdapter 10 | def initialize(name = 'default') 11 | @@queues ||= {} 12 | @name = name 13 | @@queues[@name] ||= [] 14 | end 15 | 16 | def receive 17 | @@queues[@name].shift 18 | end 19 | 20 | def send(message) 21 | @@queues[@name] << message 22 | end 23 | 24 | def size 25 | @@queues[@name].size 26 | end 27 | 28 | def destroy 29 | @@queues[@name] = nil 30 | end 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /lib/localjob.rb: -------------------------------------------------------------------------------- 1 | require 'yaml' 2 | require 'logger' 3 | require 'forwardable' 4 | 5 | require "localjob/version" 6 | require 'localjob/worker' 7 | require 'localjob/sysv_adapter' 8 | 9 | class Localjob 10 | extend Forwardable 11 | 12 | attr_reader :name 13 | attr_writer :queue 14 | 15 | def_delegators :queue, :stats, :destroy, :size 16 | 17 | # LOCALJOB in 1337speak 18 | def initialize(name = 0x10CA110B) 19 | @name = name 20 | end 21 | 22 | def serializer 23 | YAML 24 | end 25 | 26 | def queue 27 | @queue ||= SysvAdapter.new(@name) 28 | end 29 | 30 | def <<(object) 31 | queue.send serializer.dump(object) 32 | end 33 | 34 | def shift 35 | serializer.load queue.receive 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /lib/localjob/sysv_adapter.rb: -------------------------------------------------------------------------------- 1 | require 'sysvmq' 2 | 3 | class Localjob 4 | class SysvAdapter 5 | RECEIVE_ALL_TYPES = 0 6 | 7 | attr_reader :queue 8 | 9 | def initialize(key, size: 8192, flags: SysVMQ::IPC_CREAT | 0660) 10 | @key = key 11 | @queue = SysVMQ.new(key, size, flags) 12 | end 13 | 14 | def receive 15 | queue.receive(RECEIVE_ALL_TYPES) 16 | end 17 | 18 | def send(message) 19 | queue.send(message, 1) 20 | end 21 | 22 | def size 23 | queue.stats[:count] 24 | end 25 | 26 | def stats 27 | queue.stats 28 | end 29 | 30 | def destroy 31 | queue.destroy 32 | rescue Errno::EINVAL 33 | # Queue likely doesn't exist or was destroyed. 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /test/sysv_adapter_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | 3 | class SysvAdapterTest < LocaljobTestCase 4 | def setup 5 | @localjob = queue 6 | @localjob.queue = Localjob::SysvAdapter.new(0xDEADC0DE) 7 | end 8 | 9 | def teardown 10 | @localjob.destroy 11 | end 12 | 13 | def test_send_and_receive 14 | msg = "Hello World" 15 | @localjob << msg 16 | assert_equal msg, @localjob.shift 17 | end 18 | 19 | def test_size 20 | @localjob << "Hello World" 21 | assert_equal 1, @localjob.size 22 | end 23 | 24 | def test_stats 25 | @localjob << "Hello World" 26 | assert_equal 20, @localjob.stats[:size] 27 | assert_equal 1, @localjob.stats[:count] 28 | end 29 | 30 | def test_multiple_destroys_do_not_raise_exception 31 | @localjob.destroy 32 | @localjob.destroy 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /test/test_helper.rb: -------------------------------------------------------------------------------- 1 | require 'coveralls' 2 | Coveralls.wear! 3 | 4 | require 'minitest/autorun' 5 | $:<< File.dirname(__FILE__) + "/../lib" 6 | require 'localjob' 7 | require "mocha/setup" 8 | require 'jobs' 9 | 10 | class LocaljobTestCase < MiniTest::Test 11 | protected 12 | # This is a method to make sure the logger is set right. 13 | def worker(queue) 14 | Localjob::Worker.new(queue, logger: logger) 15 | end 16 | 17 | # This is a method to make sure all queues are registred and destroyed after 18 | # each teach run. 19 | def queue(name = 0x10CA110B) 20 | @queues ||= [] 21 | queue = Localjob.new(name) 22 | @queues << queue 23 | queue 24 | end 25 | 26 | def teardown 27 | clear_queues 28 | end 29 | 30 | def logger 31 | @logger ||= Logger.new(ENV["DEBUG"] ? STDOUT : "/dev/null") 32 | end 33 | 34 | def clear_queues 35 | @queues.each(&:destroy) if @queues 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /test/localjob_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | 3 | class LocaljobTest < LocaljobTestCase 4 | def setup 5 | @localjob = queue 6 | end 7 | 8 | def test_push_should_put_a_job_in_queue 9 | @localjob << WalrusJob.new("move") 10 | assert_equal 1, @localjob.size 11 | end 12 | 13 | def test_pop_from_queue 14 | @localjob << WalrusJob.new("move") 15 | 16 | job = @localjob.shift 17 | assert_instance_of WalrusJob, job 18 | assert_equal "move", job.action 19 | end 20 | 21 | def test_handles_multiple_queues 22 | @localjob << WalrusJob.new("move") 23 | 24 | other = queue(0xDEADCAFE) 25 | other << WalrusJob.new("dance") 26 | 27 | assert_equal 1, @localjob.size 28 | assert_equal 1, other.size 29 | end 30 | 31 | def test_send_raises_error_if_queue_does_not_exist 32 | assert_raises Errno::EINVAL do 33 | @localjob.destroy 34 | @localjob << WalrusJob.new("move") 35 | end 36 | end 37 | 38 | def test_shift_raises_error_if_queue_does_not_exist 39 | assert_raises Errno::EINVAL do 40 | @localjob.destroy 41 | @localjob.shift 42 | end 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013 Simon Eskildsen 2 | 3 | MIT License 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /localjob.gemspec: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | lib = File.expand_path('../lib', __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | require 'localjob/version' 5 | 6 | Gem::Specification.new do |spec| 7 | spec.name = "localjob" 8 | spec.version = Localjob::VERSION 9 | spec.authors = ["Simon Eskildsen"] 10 | spec.email = ["sirup@sirupsen.com"] 11 | spec.description = %q{Simple, self-contained background queue built on top of POSIX message queues.} 12 | spec.summary = %q{Simple, self-contained background queue built on top of POSIX message queues. It only works on Linux.} 13 | spec.homepage = "" 14 | spec.license = "MIT" 15 | 16 | spec.files = `git ls-files`.split($/) 17 | spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) } 18 | spec.test_files = spec.files.grep(%r{^(test|spec|features)/}) 19 | spec.require_paths = ["lib"] 20 | 21 | spec.add_dependency "thor", "0.18.1" 22 | spec.add_dependency "sysvmq", "~> 0.1" 23 | 24 | spec.add_development_dependency "bundler", "~> 1.3" 25 | spec.add_development_dependency "rake" 26 | spec.add_development_dependency "mocha" 27 | spec.add_development_dependency "minitest", "~> 5.8.4" 28 | end 29 | -------------------------------------------------------------------------------- /lib/localjob/cli.rb: -------------------------------------------------------------------------------- 1 | require 'thor' 2 | 3 | class Localjob 4 | class CLI < Thor 5 | option :queue, aliases: ["-q"], type: :string, default: "0x10CA110B" 6 | option :require, aliases: ["-r"], type: :string, default: "." 7 | option :pid_file, aliases: ["-p"], type: :string 8 | desc "work", "Start worker to process jobs" 9 | def work 10 | load_environment options[:require] 11 | Localjob::Worker.new(options[:queue].to_i(16), *options.slice(:pid_file)).work 12 | end 13 | 14 | private 15 | def load_environment(file) 16 | if rails?(file) 17 | require 'rails' 18 | require File.expand_path("#{file}/config/environment.rb") 19 | ::Rails.application.eager_load! 20 | elsif File.file?(file) 21 | require File.expand_path(file) 22 | else 23 | puts < SysV MQ for short). The SysV 5 | message queue API is implemented by the kernel. It's very stable and performant. 6 | 7 | This means workers and the app pushing to the queue must reside on the same 8 | machine. It's the sqlite of background queues. Here's a post about [how it 9 | works][blog]. You can run Localjob either as a seperate process, or as a thread 10 | in your app. 11 | 12 | Localjob is for early-development situations where you don't need a 13 | full-featured background queue, but just want to get started with something 14 | simple that does not rely on any external services. The advantage of the SysV 15 | queues is that your Rails app or worker can restart at any time, without loosing 16 | any events. 17 | 18 | Localjob works on Ruby >= 2.0.0 on Linux and OS X. 19 | 20 | Add it to your Gemfile: 21 | 22 | ```ruby 23 | gem 'localjob' 24 | ``` 25 | 26 | ## Usage 27 | 28 | Localjobs have the following format: 29 | 30 | ```ruby 31 | class EmailJob 32 | def initialize(user_id, email) 33 | @user, @email = User.find(user_id), email 34 | end 35 | 36 | def perform 37 | @email.deliver_to(@user) 38 | end 39 | end 40 | ``` 41 | 42 | To queue a job, create an instance of it and push it to the queue: 43 | 44 | ```ruby 45 | queue = Localjob.new 46 | queue << EmailJob.new(current_user.id, welcome_email) 47 | ``` 48 | 49 | A job is serialized with YAML and pushed onto a persistent SysV Message Queue. 50 | This means a worker does not have to listen on the queue to push things to it. 51 | Pops off the message queue are atomic, so only one will receive the queue. This 52 | means you can run multiple workers on the same machine if you wish. The workers 53 | will deserialize the message to create an instance of your object, and call 54 | `#perform` on the object. 55 | 56 | ### Rails initializer 57 | 58 | For easy access to your queues in Rails, you can add an initializer to set up a 59 | constant referencing each of your queues. This allows easy access anywhere in 60 | your app. In `config/initializers/localjob.rb`: 61 | 62 | ```ruby 63 | BackgroundQueue = Localjob.new 64 | ``` 65 | 66 | Then in your app you can simply reference the constant to push to the queue: 67 | 68 | ```ruby 69 | BackgroundQueue << EmailJob.new(current_user.id, welcome_email) 70 | ``` 71 | 72 | ### Managing workers 73 | 74 | There are two ways to spawn workers, either a thread inside the process emitting 75 | events, or as a separate process managed with the `localjob` command-line 76 | utility. 77 | 78 | #### Thread 79 | 80 | Spawn the worker thread in an initializer where you are initializing Localjob as 81 | well: 82 | 83 | ```ruby 84 | BackgroundQueue = Localjob.new 85 | 86 | worker = Localjob::Worker.new(BackgroundQueue, logger: Rails.logger) 87 | worker.work(thread: true) 88 | ``` 89 | 90 | #### Process 91 | 92 | Spawning workers can be done with `localjob`. Run `localjob work` to spawn a 93 | single worker. It takes a few arguments. The most important being `--require` 94 | which takes a path the worker will require before processing jobs. For Rails, 95 | you can run `localjob work` without any arguments. `localjob(2)` has a few other 96 | commands such as `list` to list all queues and `size` to list the size of all 97 | queues. `localjob help` to list all commands. 98 | 99 | Gracefully shut down workers by sending `SIGQUIT` to them. This will make sure 100 | the worker completes its current job before shutting down. Jobs can be sent to 101 | the queue meanwhile, and the worker will process them once it starts again. 102 | 103 | ### Testing 104 | 105 | Create your instance of the queue as normal in your setup: 106 | 107 | ```ruby 108 | def setup 109 | @queue = Localjob.new 110 | @worker = Localjob::Worker.new(@queue) 111 | end 112 | ``` 113 | 114 | In your `teardown` you'll want to destroy your queue: 115 | 116 | ```ruby 117 | def teardown 118 | @queue.destroy 119 | end 120 | ``` 121 | 122 | You can get the size of your queue by calling `@queue.size`. You pop off the 123 | queue with `@queue.shift`. Other than that, just use the normal API. You can 124 | also read the tests for Localjob to get an idea of how to test. Sample test: 125 | 126 | ```ruby 127 | def test_pop_and_send_to_worker 128 | WalrusJob.any_instance.expects(:perform) 129 | 130 | @localjob << WalrusJob.new("move") 131 | 132 | job = @localjob.shift 133 | @worker.process(job) 134 | 135 | assert_equal 0, @localjob.size 136 | end 137 | ``` 138 | 139 | ### Multiple queues 140 | 141 | If you wish to have multiple queues you can have multiple Localjob objects 142 | referencing different queues. A worker can only work off a single queue at a 143 | time, so you will have to spawn multiple thread or process workers. Example: 144 | 145 | ```ruby 146 | MailQueue = Localjob.new(0xDEADC0DE) 147 | DefaultQueue = Localjob.new(0xDEADCAFE) 148 | ``` 149 | 150 | Note that Localjob takes a hex value as the queue name. This is how the SysV 151 | adapter identifies different queues. 152 | 153 | [sysv]: http://man7.org/linux/man-pages/man7/svipc.7.html 154 | [blog]: http://sirupsen.com/unix-background-queue/ 155 | --------------------------------------------------------------------------------