├── .github └── workflows │ └── tests.yml ├── .gitignore ├── .ruby-version ├── CHANGELOG.md ├── CONTRIBUTING.md ├── Gemfile ├── LICENCE.md ├── README.md ├── Rakefile ├── docs └── upgrading.md ├── lib ├── propono.rb └── propono │ ├── components │ ├── aws_client.rb │ ├── aws_config.rb │ ├── client.rb │ ├── queue.rb │ ├── queue_subscription.rb │ ├── sqs_message.rb │ └── topic.rb │ ├── configuration.rb │ ├── logger.rb │ ├── propono_error.rb │ ├── services │ ├── publisher.rb │ └── queue_listener.rb │ ├── utils.rb │ └── version.rb ├── propono.gemspec └── test ├── components ├── aws_client_test.rb ├── aws_config_test.rb ├── client_test.rb ├── queue_subscription_test.rb ├── queue_test.rb └── topic_test.rb ├── config.yml.example ├── configuration_test.rb ├── integration ├── integration_test.rb ├── slow_queue_test.rb └── sns_to_sqs_test.rb ├── logger_test.rb ├── services ├── publisher_test.rb └── queue_listener_test.rb ├── test_helper.rb └── utils └── hash_test.rb /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | pull_request: 7 | branches: [main] 8 | 9 | jobs: 10 | test: 11 | name: Ruby ${{ matrix.ruby-version }} - ${{ matrix.os }} - ${{ github.event_name }} 12 | runs-on: ${{ matrix.os }} 13 | strategy: 14 | fail-fast: false 15 | matrix: 16 | os: 17 | - ubuntu-latest 18 | ruby-version: [2.6, 2.7] 19 | 20 | steps: 21 | - uses: actions/checkout@v2 22 | 23 | - name: Set up Ruby 24 | uses: ruby/setup-ruby@a699edbce608a2c128dedad88e3b6a0e28687b3c 25 | with: 26 | ruby-version: ${{ matrix.ruby-version }} 27 | bundler-cache: true 28 | 29 | - name: Test 30 | run: bundle exec rake test:local 31 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | *.rbc 3 | .bundle 4 | .config 5 | .yardoc 6 | Gemfile.lock 7 | InstalledFiles 8 | _yardoc 9 | coverage 10 | doc/ 11 | lib/bundler/man 12 | pkg 13 | rdoc 14 | spec/reports 15 | test/tmp 16 | test/version_tmp 17 | test/config.yml 18 | manual_test.rb 19 | tmp 20 | *.swp 21 | -------------------------------------------------------------------------------- /.ruby-version: -------------------------------------------------------------------------------- 1 | 2.6.6 2 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # 3.0.0 / 2021-01-10 2 | * [FEATURE] Improve AWS configuration. 3 | 4 | # 2.2.0 / 2019-09-21 5 | * [FEATURE] Add setting to disable slow queue 6 | 7 | # 2.1.0 / 2017-03-14 8 | * [FEATURE] Added visibility_timeout to listen 9 | 10 | # 2.0.0 / 2017-03-14 11 | * [FEATURE] Remove UDP and TCP support 12 | * [FEATURE] Change default publish behaviour from async to sync 13 | * [FEATURE] Propono.subscripe_by_post has been removed 14 | * [FEATURE] Propono.subscripe_by_queue has been renamed to subscribe 15 | * [FEATURE] Change to Propono::Client interface 16 | * [FEATURE] Switch fog out for aws gems 17 | * [FEATURE] Use long polling 18 | 19 | # 1.7.0 / 2017-01-17 20 | * [FEATURE] Added num_messages_per_poll config option to allow you to change how many messages you pull from AWS per poll cycle. 21 | 22 | # 1.6.0 / 2015-06-05 23 | * [FEATURE] Require fog-aws gem instead of fog (:blue_heart: @mhuggins) 24 | * [FEATURE] Change licence to MIT (:blue_heart: @BiggerNoise) 25 | 26 | # 1.5.0 / 2015-03-16 27 | * [BUGFIX] Fix inability to use queue if the message visibility timeout has changed. 28 | 29 | # 1.4.0 / 2014-07-12 30 | * [FEATURE] Move symbolize_keys to Propono namespace to avoid ActiveSupport conflict (:blue_heart: @tardate) 31 | * [BUGFIX] Drain integration tests drain queues before starting (:blue_heart: @tardate) 32 | * [BUGFIX] Fix typos in log messages (:blue_heart: @tardate) 33 | * [BUGFIX] Fix issue with tests failing when ran in a certain order 34 | 35 | # 1.3.0 / 2014-07-12 36 | * [FEATURE] Add {async: false} option to publisher 37 | 38 | # 1.2.0 / 2014-05-25 39 | * [BUGFIX] Restrict SQS policy to only allow SNS topic publishes. 40 | 41 | # 1.1.3 / 2014-05-14 42 | * [FEATURE] Added ability to drain queue. Also allow dot releases of Fog. 43 | 44 | # 1.1.2 / 2014-03-31 45 | * [BUGFIX] Move topic lookup into publishing thread. 46 | 47 | # 1.1.1 / 2014-02-22 48 | * [BUGFIX] Logger.error only takes (0..1) arguments. 49 | 50 | # 1.1.0 / 2014-02-18 51 | * [FEATURE] Added slow queue to allow processing of lower priority messages. 52 | 53 | # 1.0.0.rc3 / 2013-12-20 54 | * [FEATURE] Create failed and corrupt queues when subscribe. 55 | 56 | # 1.0.0.rc2 / 2013-12-15 57 | * [FEATURE] Make queue_suffix optional 58 | 59 | # 1.0.0.rc1 / 2013-12-15 60 | * [FEATURE] Improve transactional handling of messages. 61 | * [FEATURE] Add failed/corrupt queues. 62 | 63 | # 0.11.1 / 2013-12-09 64 | * [BUGFIX] Re raise 403 forbidden excetion instead of continuing. 65 | 66 | # 0.11.0 / 2013-12-03 67 | * [FEATURE] Add support for IAM profiles for AWS auth 68 | 69 | # 0.10.0 / 2013-12-03 70 | * [FEATURE] Add queue_suffix config variable 71 | 72 | # 0.9.1 / Unreleased 73 | * [FEATURE] Propono will raise exceptions if the message processing fails 74 | 75 | # 0.9.0 / Unreleased 76 | * [FEATURE] Add message ids that track throughout Propono 77 | 78 | # 0.8.2 / 2013-11-01 79 | * [BUGFIX] Replace thread library with standard ruby threads to fix Unicorn problems. 80 | 81 | # 0.8.1 / 2013-11-01 82 | * [FEATURE] Log all messages published from Propono. 83 | 84 | # 0.8.0 / 2013-11-01 85 | * [FEATURE] SNS publish now delegates to a thread pool. The SNS response can be accessed via a future. 86 | 87 | # 0.7.0 / 2013-10-23 88 | * [FEATURE] Add TCP publish and listen methods. 89 | 90 | # 0.6.3 / 2013-10-20 91 | * [FEATURE] Catch all StandardError exceptions for UDP publishes. 92 | 93 | # 0.6.2 / 2013-10-20 94 | * [BUGFIX] Fixed integration tests that sometimes failed due to shared UDP ports or slow SQS subscriptions. 95 | 96 | # 0.6.1 / 2013-10-20 97 | * [BUGFIX] Added `require 'json'` to udp_listener.rb 98 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How To Contribute 2 | 3 | First of all, **thank you** for contributing to this library! 4 | 5 | ## Issues 6 | Please file issues on the [GitHub issues list](https://github.com/meducation/propono/issues) and give as much detail as possible. 7 | 8 | ## Features / Pull Requests 9 | 10 | If you want a feature implemented, the best way to get it done is to submit a pull request that implements it. Please make sure it has tests. 11 | 12 | To get the implementation tests to run, you'll need to create a `test/config.yml` file. There's an example to get you started. 13 | 14 | You can run the tests with: 15 | 16 | ``` 17 | bundle exec rake test 18 | ``` 19 | 20 | If you've not contributed to a repository before - this is the accepted pattern to use: 21 | 22 | 1. Fork it (big button on Github at the top right) 23 | 2. Create your feature branch (`git checkout -b my-new-feature`) 24 | 3. Make your changes (please add tests!) 25 | 4. Commit your changes (`git commit -am 'Add some feature'`) 26 | 5. Push to the branch (`git push origin my-new-feature`) 27 | 6. Create new Pull Request on Github 28 | 29 | Thank you again! 30 | :heart: :sparkling_heart: :heart: 31 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Specify your gem's dependencies in propono.gemspec 4 | gemspec 5 | -------------------------------------------------------------------------------- /LICENCE.md: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 New Media Education Ltd 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | **Propono v3.0.0 has been released with new AWS configuration options. Check out the [upgrading doc](https://github.com/iHiD/propono/blob/main/docs/upgrading.md) for more information. Thanks to @dougal for this work!** 2 | 3 | # Propono 4 | 5 | ![Tests](https://github.com/iHiD/propono/workflows/Tests/badge.svg) 6 | [![Code Climate](https://codeclimate.com/github/iHiD/propono.png)](https://codeclimate.com/github/iHiD/propono) 7 | 8 | Propono is a [pub/sub](http://en.wikipedia.org/wiki/Publish-subscribe_pattern) gem built on top of Amazon Web Services (AWS). It uses Simple Notification Service (SNS) and Simple Queue Service (SQS) to seamlessly pass messages throughout your infrastructure. 9 | 10 | It's beautifully simple to use. [Watch an introduction](https://www.youtube.com/watch?v=ZM3-Gl5DVgM) 11 | 12 | ```ruby 13 | # On Machine A 14 | Propono::Client.new.listen('some-topic') do |message| 15 | puts "I just received: #{message}" 16 | end 17 | 18 | # On Machine B 19 | Propono::Client.new.publish('some-topic', "The Best Message Ever") 20 | 21 | # Output on Machine A a second later. 22 | # - "I just received The Best Message Ever" 23 | ``` 24 | 25 | ## Upgrading 26 | 27 | Upgrades from v1 to v2, and v2 to v3 are covered in the [upgrade documentation](docs/upgrading.md). 28 | 29 | ## Installation 30 | 31 | Add this line to your application's Gemfile: 32 | 33 | gem 'propono' 34 | 35 | And then execute: 36 | 37 | $ bundle install 38 | 39 | ## Usage 40 | 41 | The first thing to do is setup some configuration for Propono. 42 | It's best to do this in an initializer, or at the start of your application. 43 | If you need to setup AWS authentication, see the [AWS Configuration](#aws-configuration) section. 44 | 45 | ```ruby 46 | client = Propono::Client.new 47 | ``` 48 | 49 | You can then start publishing messages easily from anywhere in your codebase. 50 | 51 | ```ruby 52 | client = Propono::Client.new 53 | client.publish('some-topic', "Some string") 54 | client.publish('some-topic', {some: ['hash', 'or', 'array']}) 55 | ``` 56 | 57 | Listening for messages is easy too. Just tell Propono what your application is called and start listening. You'll get a block yielded for each message. 58 | 59 | ```ruby 60 | client = Propono::Client.new 61 | client.config.application_name = "application-name" # Something unique to this app. 62 | client.listen('some-topic') do |message| 63 | # ... Do something interesting with the message 64 | end 65 | ``` 66 | In the background, Propono is automatically setting up a queue using SQS, a notification system using SNS, and gluing them all together for you. But you don't have to worry about any of that. 67 | 68 | **Does it matter what I set my `application_name` to?** 69 | For a simple publisher and subscriber deployment, no. 70 | However, the `application_name` has a direct impact on subscriber behaviour when more than one is in play. 71 | This is because a queue is established for each application_name/topic combination. In practice: 72 | * subscribers that share the same `application_name` will act as multiple workers on the same queue. Only one will get to process each message. 73 | * subscribers that have a different `application_name` will each get a copy of a message to process independently i.e. acts as a one-to-many broadcast. 74 | 75 | ### AWS Configuration 76 | 77 | By default, Propono will create SQS and SNS clients with no options. 78 | In the absence of options, these clients will make use of the credentials on the current host. 79 | See the [AWS SDK For Ruby Configuration documentation](https://docs.aws.amazon.com/sdk-for-ruby/v3/developer-guide/setup-config.html) for more details. 80 | 81 | To manually configure options for use with AWS, use `aws_options`, which sets options to be passed to both clients. For example: 82 | 83 | client = Propono::Client.new do |config| 84 | config.aws_options = { 85 | region: 'aws_region', 86 | access_key_id: 'your_access_key_id', 87 | secret_access_key: 'your_secret_access_key' 88 | } 89 | end 90 | 91 | In addition to this, there are also `sqs_options` and `sns_options`, used to configure each client independently. 92 | See the [SQS Client](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/SQS/Client.html#initialize-instance_method) and [SNS Client](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/SNS/Client.html#initialize-instance_method) documentation for available options. 93 | These individual options are merged with `aws_options` with the per-client options taking precendence. 94 | 95 | ### General Configuration 96 | 97 | ``` 98 | Propono::Client.new do |config| 99 | # AWS Configuration, see above. 100 | config.aws_options = {...} 101 | config.sqs_options = {...} 102 | config.sns_options = {...} 103 | 104 | config.application_name = "A name unique in your network" 105 | config.logger = "A logger such as Log4r or Rails.logger" 106 | 107 | config.max_retries = "The number of retries if a message raises an exception before being placed on the failed queue" 108 | config.num_messages_per_poll = "The number of messages retrieved per poll to SQS" 109 | 110 | config.slow_queue_enabled = true 111 | end 112 | ``` 113 | 114 | ### Options 115 | 116 | #### Async 117 | 118 | By default messages are posted inline, blocking the main thread. The `async: true` option can be sent when posting a message, which will spawn a new thread for the message networking calls, and unblocking the main thread. 119 | 120 | #### Visiblity Timeout 121 | 122 | For certain tasks (e.g. video processing), being able to hold messages for longer is important. To achieve this, the [visibility timeout of a message](http://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html) can be changed on the call to listen. e.g. 123 | 124 | ``` 125 | client.listen('long-running-tasks', visiblity_timeout: 3600) do |message| 126 | puts "I just received: #{message}" 127 | end 128 | ``` 129 | 130 | ### Slow Queue 131 | 132 | The slow queue can be disabled by setting `slow_queue_enabled` to `false`. This will yield performance improvements if you do not make use of the "slow queue" functionality. 133 | 134 | ### Is it any good? 135 | 136 | [Yes.](http://news.ycombinator.com/item?id=3067434) 137 | 138 | ## Contributing 139 | 140 | Firstly, thank you!! :heart::sparkling_heart::heart: 141 | 142 | We'd love to have you involved. Please read our [contributing guide](https://github.com/iHiD/propono/tree/master/CONTRIBUTING.md) for information on how to get stuck in. 143 | 144 | ### Contributors 145 | 146 | This project is managed by the [Jeremy Walker](http://ihid.co.uk). 147 | 148 | These individuals have come up with the ideas and written the code that made this possible: 149 | 150 | - [Jeremy Walker](https://github.com/iHiD) 151 | - [Malcolm Landon](https://github.com/malcyL) 152 | - [Charles Care](https://github.com/ccare) 153 | - [Rob Styles](https://github.com/mmmmmrob) 154 | 155 | ## Licence 156 | 157 | Copyright (C) 2017 Jeremy Walker 158 | 159 | This program is free software: you can redistribute it and/or modify 160 | it under the terms of the MIT License. 161 | 162 | This program is distributed in the hope that it will be useful, 163 | but WITHOUT ANY WARRANTY; without even the implied warranty of 164 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 165 | MIT License for more details. 166 | 167 | A copy of the MIT License is available in [LICENCE.md](https://github.com/iHiD/propono/blob/master/LICENCE.md) 168 | along with this program. 169 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | require 'rake/testtask' 3 | require 'yard' 4 | 5 | YARD::Rake::YardocTask.new do |t| 6 | end 7 | 8 | Rake::TestTask.new do |t| 9 | t.pattern = "test/**/*_test.rb" 10 | end 11 | 12 | namespace :test do 13 | Rake::TestTask.new(:local) do |t| 14 | t.pattern = "test/{components/,services/,utils/,}*_test.rb" 15 | end 16 | end 17 | 18 | task default: :test 19 | -------------------------------------------------------------------------------- /docs/upgrading.md: -------------------------------------------------------------------------------- 1 | # Upgrading 2 | 3 | ## Changes from v2 to v3 4 | 5 | Version 3 changed the way configuration options for the two AWS services are 6 | passed to the client gems. Instead of Propono attempting to guess which 7 | configuration options you might want, it now accepts hashes for AWS 8 | configuration which are passed directly to the appropriate clients. 9 | 10 | If you are upgrading from v2 to v3, and using the configuration as previously 11 | given in the README, you need to change from: 12 | 13 | ```ruby 14 | client = Propono::Client.new 15 | client.config.queue_region = 'aws_region' 16 | client.config.access_key = 'your_access_key_id' 17 | client.config.secret_key = 'your_secret_access_key' 18 | ``` 19 | 20 | To: 21 | 22 | ```ruby 23 | client = Propono::Client.new do |config| 24 | config.aws_options = { 25 | region: 'aws_region', 26 | access_key_id: 'your_access_key_id', 27 | secret_access_key: 'your_secret_access_key' 28 | } 29 | end 30 | ``` 31 | 32 | For a full rundown, see the [AWS Configuration 33 | section](../README.md#aws-configuration) of the README. 34 | 35 | 36 | ## Changes from v1 to v2 37 | 38 | Version 2 of Propono changed a few things: 39 | - We moved from a global interface to a client interface. Rather than calling 40 | `publish` and equivalent on `Propono`, you should now initialize a 41 | `Propono::Client` and then call everything on that client. This fixes issues 42 | with thread safety and global config. 43 | - We have also removed the dependancy on Fog and instead switch to the `sns` 44 | and `sqs` mini-gems of `aws-sdk`. 45 | - UDP and TCP support have been removed, and `subscribe_by_post` has been 46 | removed. 47 | - We are now using long-polling. This makes Propono **significantly** faster 48 | (10-100x). 49 | -------------------------------------------------------------------------------- /lib/propono.rb: -------------------------------------------------------------------------------- 1 | # Propono 2 | # 3 | # Propono is a pub/sub gem built on top of Amazon Web Services (AWS). It uses Simple Notification Service (SNS) and Simple Queue Service (SQS) to seamlessly pass messages throughout your infrastructure. 4 | require "propono/version" 5 | require 'propono/propono_error' 6 | require 'propono/logger' 7 | require 'propono/configuration' 8 | require "propono/utils" 9 | 10 | require 'propono/components/client' 11 | 12 | require 'propono/components/aws_config' 13 | require 'propono/components/aws_client' 14 | 15 | require "propono/components/queue" 16 | require "propono/components/topic" 17 | require "propono/components/queue_subscription" 18 | require "propono/components/sqs_message" 19 | 20 | require "propono/services/publisher" 21 | require "propono/services/queue_listener" 22 | 23 | # Propono is a pub/sub gem built on top of Amazon Web Services (AWS). 24 | # It uses Simple Notification Service (SNS) and Simple Queue Service (SQS) 25 | # to seamlessly pass messages throughout your infrastructure. 26 | module Propono 27 | end 28 | -------------------------------------------------------------------------------- /lib/propono/components/aws_client.rb: -------------------------------------------------------------------------------- 1 | require 'aws-sdk-sns' 2 | require 'aws-sdk-sqs' 3 | 4 | module Propono 5 | class AwsClient 6 | attr_reader :aws_config 7 | def initialize(aws_config) 8 | @aws_config = aws_config 9 | end 10 | 11 | def publish_to_sns(topic, message) 12 | sns_client.publish( 13 | topic_arn: topic.arn, 14 | message: message.to_json 15 | ) 16 | end 17 | 18 | def send_to_sqs(queue, message) 19 | sqs_client.send_message( 20 | queue_url: queue.url, 21 | message_body: message 22 | ) 23 | end 24 | 25 | def create_topic(name) 26 | Topic.new(sns_client.create_topic(name: name)) 27 | end 28 | 29 | def create_queue(name) 30 | url = sqs_client.create_queue(queue_name: name).queue_url 31 | attributes = sqs_client.get_queue_attributes(queue_url: url, attribute_names: ["QueueArn"]).attributes 32 | Queue.new(url, attributes) 33 | end 34 | 35 | def subscribe_sqs_to_sns(queue, topic) 36 | sns_client.subscribe( 37 | topic_arn: topic.arn, 38 | protocol: 'sqs', 39 | endpoint: queue.arn 40 | ) 41 | end 42 | 43 | def set_sqs_policy(queue, policy) 44 | sqs_client.set_queue_attributes( 45 | queue_url: queue.url, 46 | attributes: { Policy: policy } 47 | ) 48 | end 49 | 50 | def read_from_sqs(queue, num_messages, long_poll: true, visibility_timeout: nil) 51 | wait_time_seconds = long_poll ? 20 : 0 52 | visibility_timeout ||= 30 53 | sqs_client.receive_message( 54 | queue_url: queue.url, 55 | wait_time_seconds: wait_time_seconds, 56 | max_number_of_messages: num_messages, 57 | visibility_timeout: visibility_timeout 58 | ).messages 59 | end 60 | 61 | def delete_from_sqs(queue, receipt_handle) 62 | sqs_client.delete_message( 63 | queue_url: queue.url, 64 | receipt_handle: receipt_handle 65 | ) 66 | end 67 | 68 | private 69 | 70 | def sns_client 71 | @sns_client ||= Aws::SNS::Client.new(aws_config.sns_options) 72 | end 73 | 74 | def sqs_client 75 | @sqs_client ||= Aws::SQS::Client.new(aws_config.sqs_options) 76 | end 77 | end 78 | end 79 | -------------------------------------------------------------------------------- /lib/propono/components/aws_config.rb: -------------------------------------------------------------------------------- 1 | module Propono 2 | class AwsConfig 3 | 4 | def initialize(config) 5 | @config = config 6 | end 7 | 8 | def sqs_options 9 | @config.aws_options.merge(@config.sqs_options) 10 | end 11 | 12 | def sns_options 13 | @config.aws_options.merge(@config.sns_options) 14 | end 15 | 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /lib/propono/components/client.rb: -------------------------------------------------------------------------------- 1 | module Propono 2 | class Client 3 | 4 | # Propono configuration. 5 | # 6 | # Settings should be set in an initializer or using some 7 | # other method that ensures they are set before any 8 | # Propono code is used. 9 | # 10 | # They can be set in one of the following ways: 11 | # 12 | # 1. As options passed to new as a hash. 13 | # 14 | # Propono::Client.new(application_name: 'my-application') 15 | # 16 | # 2. As options passed to new using a block. 17 | # 18 | # Propono::Client.new do |config" 19 | # config.application_name: 'my-application' 20 | # end 21 | # 22 | # 3. By calling the Propono::Client#configure. 23 | # client.configure do |config| 24 | # config.access_key = "my-access-key" 25 | # end 26 | 27 | attr_reader :config, :aws_client 28 | def initialize(settings = {}, &block) 29 | @config = Configuration.new 30 | if block_given? 31 | configure(&block) 32 | else 33 | settings.each do |key, value| 34 | config.send("#{key}=", value) 35 | end 36 | end 37 | 38 | @aws_client = AwsClient.new(AwsConfig.new(config)) 39 | end 40 | 41 | def configure 42 | yield config 43 | end 44 | 45 | # Publishes a new message into the Propono pub/sub network. 46 | # 47 | # This requires a topic and a message. By default this pushes 48 | # out AWS SNS. 49 | # 50 | # @param [String] topic The name of the topic to publish to. 51 | # @param [String] message The message to post. 52 | def publish(topic, message, options = {}) 53 | suffixed_topic = "#{topic}#{config.queue_suffix}" 54 | Publisher.publish(aws_client, config, suffixed_topic, message, options) 55 | end 56 | 57 | # Creates a new SNS-SQS subscription on the specified topic. 58 | # 59 | # This is implicitly called by {#listen}. 60 | # 61 | # @param [String] topic The name of the topic to subscribe to. 62 | def subscribe(topic) 63 | QueueSubscription.create(aws_client, config, topic) 64 | end 65 | 66 | # Listens on a queue and yields for each message 67 | # 68 | # Calling this will enter a queue-listening loop that 69 | # yields the message_processor for each messages. 70 | # 71 | # This method will automatically create a subscription if 72 | # one does not exist, so there is no need to call 73 | # subscribe in addition. 74 | # 75 | # @param [String] topic The topic to subscribe to. 76 | # @param &message_processor The block to yield for each message. 77 | def listen(topic_name, options = {}, &message_processor) 78 | QueueListener.listen(aws_client, config, topic_name, options, &message_processor) 79 | end 80 | 81 | # Listens on a queue and yields for each message 82 | # 83 | # Calling this will enter a queue-listening loop that 84 | # yields the message_processor for each messages. The 85 | # loop will end when all messages have been processed. 86 | # 87 | # This method will automatically create a subscription if 88 | # one does not exist, so there is no need to call 89 | # subscribe in addition. 90 | # 91 | # @param [String] topic The topic to subscribe to. 92 | # @param &message_processor The block to yield for each message. 93 | def drain_queue(topic, &message_processor) 94 | QueueListener.drain(aws_client, config, topic, &message_processor) 95 | end 96 | end 97 | end 98 | 99 | -------------------------------------------------------------------------------- /lib/propono/components/queue.rb: -------------------------------------------------------------------------------- 1 | module Propono 2 | class Queue 3 | 4 | attr_reader :url, :attributes 5 | def initialize(url, attributes) 6 | @url = url 7 | @attributes = attributes 8 | end 9 | 10 | def arn 11 | @arn ||= attributes["QueueArn"] 12 | end 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /lib/propono/components/queue_subscription.rb: -------------------------------------------------------------------------------- 1 | module Propono 2 | class QueueSubscription 3 | 4 | attr_reader :aws_client, :propono_config, :topic_arn, :queue_name, :queue, :failed_queue, :corrupt_queue, :slow_queue 5 | 6 | def self.create(*args) 7 | new(*args).tap do |subscription| 8 | subscription.create 9 | end 10 | end 11 | 12 | def initialize(aws_client, propono_config, topic_name) 13 | @aws_client = aws_client 14 | @propono_config = propono_config 15 | @topic_name = topic_name 16 | @suffixed_topic_name = "#{topic_name}#{propono_config.queue_suffix}" 17 | @suffixed_slow_topic_name = "#{topic_name}#{propono_config.queue_suffix}-slow" 18 | @queue_name = "#{propono_config.application_name.tr(" ", "_")}-#{@suffixed_topic_name}" 19 | end 20 | 21 | def create 22 | raise ProponoError.new("topic_name is nil") unless @topic_name 23 | create_and_subscribe_main_queue 24 | create_and_subscribe_slow_queue 25 | create_misc_queues 26 | end 27 | 28 | def create_and_subscribe_main_queue 29 | @queue = aws_client.create_queue(queue_name) 30 | topic = aws_client.create_topic(@suffixed_topic_name) 31 | aws_client.subscribe_sqs_to_sns(@queue, topic) 32 | aws_client.set_sqs_policy(@queue, generate_policy(@queue, topic)) 33 | end 34 | 35 | def create_misc_queues 36 | @failed_queue = aws_client.create_queue("#{queue_name}-failed") 37 | @corrupt_queue = aws_client.create_queue("#{queue_name}-corrupt") 38 | end 39 | 40 | def create_and_subscribe_slow_queue 41 | @slow_queue = aws_client.create_queue("#{queue_name}-slow") 42 | slow_topic = aws_client.create_topic(@suffixed_slow_topic_name) 43 | aws_client.subscribe_sqs_to_sns(@slow_queue, slow_topic) 44 | aws_client.set_sqs_policy(@slow_queue, generate_policy(@slow_queue, slow_topic)) 45 | end 46 | 47 | private 48 | 49 | def generate_policy(queue, topic) 50 | <<-EOS 51 | { 52 | "Version": "2008-10-17", 53 | "Id": "#{queue.arn}/SQSDefaultPolicy", 54 | "Statement": [ 55 | { 56 | "Sid": "#{queue.arn}-Sid", 57 | "Effect": "Allow", 58 | "Principal": { 59 | "AWS": "*" 60 | }, 61 | "Action": "SQS:*", 62 | "Resource": "#{queue.arn}", 63 | "Condition": { 64 | "StringEquals": { 65 | "aws:SourceArn": "#{topic.arn}" 66 | } 67 | } 68 | } 69 | ] 70 | } 71 | EOS 72 | end 73 | end 74 | end 75 | -------------------------------------------------------------------------------- /lib/propono/components/sqs_message.rb: -------------------------------------------------------------------------------- 1 | module Propono 2 | class SqsMessage 3 | attr_reader :context, :message, :receipt_handle, :failure_count 4 | def initialize(raw_message) 5 | raw_body = raw_message.body 6 | @raw_body_json = JSON.parse(raw_body) 7 | body = JSON.parse(@raw_body_json["Message"]) 8 | 9 | @context = Propono::Utils.symbolize_keys body 10 | @failure_count = context[:num_failures] || 0 11 | @message = context.delete(:message) 12 | @receipt_handle = raw_message.receipt_handle 13 | end 14 | 15 | def to_json_with_exception(exception) 16 | message = @raw_body_json.dup 17 | context = {} 18 | context[:id] = @context[:id] 19 | context[:message] = @message 20 | context[:last_exception_message] = exception.message 21 | context[:last_exception_stack_trace] = exception.backtrace 22 | context[:last_exception_time] = Time.now 23 | context[:num_failures] = failure_count + 1 24 | context[:last_context] = @context 25 | message['Message'] = context.to_json 26 | JSON.pretty_generate(message) 27 | end 28 | 29 | def ==(other) 30 | other.is_a?(SqsMessage) && other.receipt_handle == @receipt_handle 31 | end 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /lib/propono/components/topic.rb: -------------------------------------------------------------------------------- 1 | module Propono 2 | class Topic 3 | attr_reader :aws_topic 4 | def initialize(aws_topic) 5 | @aws_topic = aws_topic 6 | end 7 | 8 | def arn 9 | @arn ||= aws_topic.topic_arn 10 | end 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /lib/propono/configuration.rb: -------------------------------------------------------------------------------- 1 | module Propono 2 | 3 | class ProponoConfigurationError < ProponoError 4 | end 5 | 6 | class Configuration 7 | 8 | def self.add_setting(sym, required: true) 9 | define_method(sym) do 10 | required ? get_or_raise(sym) : @settings[sym] 11 | end 12 | 13 | define_method("#{sym}=") do |new_value| 14 | @settings[sym] = new_value 15 | end 16 | end 17 | 18 | add_setting :aws_options 19 | add_setting :sqs_options 20 | add_setting :sns_options 21 | add_setting :application_name 22 | add_setting :logger 23 | add_setting :max_retries 24 | add_setting :num_messages_per_poll 25 | add_setting :slow_queue_enabled, required: false 26 | add_setting :queue_suffix, required: false 27 | 28 | def initialize 29 | @settings = { 30 | aws_options: {}, 31 | sqs_options: {}, 32 | sns_options: {}, 33 | logger: Propono::Logger.new, 34 | queue_suffix: "", 35 | max_retries: 0, 36 | num_messages_per_poll: 1, 37 | slow_queue_enabled: true 38 | } 39 | end 40 | 41 | private 42 | 43 | def get_or_raise(setting) 44 | @settings[setting] || raise(ProponoConfigurationError.new("Configuration for #{setting} is not set")) 45 | end 46 | end 47 | end 48 | -------------------------------------------------------------------------------- /lib/propono/logger.rb: -------------------------------------------------------------------------------- 1 | module Propono 2 | class Logger 3 | 4 | StdLevels = %W{debug info warn} 5 | ErrorLevels = %W{error fatal} 6 | 7 | StdLevels.each do |level| 8 | define_method level do |*args| 9 | $stdout.puts(*args) 10 | end 11 | end 12 | 13 | ErrorLevels.each do |level| 14 | define_method level do |*args| 15 | $stderr.puts(*args) 16 | end 17 | end 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /lib/propono/propono_error.rb: -------------------------------------------------------------------------------- 1 | module Propono 2 | 3 | # The exception from which all other exceptions in this library derive. 4 | class ProponoError < StandardError 5 | end 6 | 7 | end 8 | -------------------------------------------------------------------------------- /lib/propono/services/publisher.rb: -------------------------------------------------------------------------------- 1 | require 'socket' 2 | 3 | module Propono 4 | class PublisherError < ProponoError 5 | end 6 | 7 | class Publisher 8 | def self.publish(*args) 9 | new(*args).publish 10 | end 11 | 12 | attr_reader :aws_client, :propono_config, :topic_name, :message, :id, :async 13 | 14 | def initialize(aws_client, propono_config, topic_name, message, async: false, id: nil) 15 | raise PublisherError.new("Topic is nil") if topic_name.nil? 16 | raise PublisherError.new("Message is nil") if message.nil? 17 | 18 | @aws_client = aws_client 19 | @propono_config = propono_config 20 | @topic_name = topic_name 21 | @message = message 22 | @async = async 23 | 24 | random_id = SecureRandom.hex(3) 25 | @id = id ? "#{id}-#{random_id}" : random_id 26 | end 27 | 28 | def publish 29 | propono_config.logger.info "Propono [#{id}]: Publishing #{message} to #{topic_name}" 30 | async ? publish_asyncronously : publish_syncronously 31 | end 32 | 33 | private 34 | 35 | def publish_asyncronously 36 | Thread.new { publish_syncronously } 37 | end 38 | 39 | def publish_syncronously 40 | begin 41 | topic = aws_client.create_topic(topic_name) 42 | rescue => e 43 | propono_config.logger.error "Propono [#{id}]: Failed to get or create topic #{topic_name}: #{e}" 44 | raise 45 | end 46 | 47 | begin 48 | aws_client.publish_to_sns(topic, body) 49 | rescue => e 50 | propono_config.logger.error "Propono [#{id}]: Failed to send via sns: #{e}" 51 | raise 52 | end 53 | end 54 | 55 | def body 56 | { 57 | id: id, 58 | message: message 59 | } 60 | end 61 | end 62 | end 63 | -------------------------------------------------------------------------------- /lib/propono/services/queue_listener.rb: -------------------------------------------------------------------------------- 1 | module Propono 2 | 3 | class QueueListener 4 | def self.listen(*args, &message_processor) 5 | new(*args, &message_processor).listen 6 | end 7 | 8 | def self.drain(*args, &message_processor) 9 | new(*args, &message_processor).drain 10 | end 11 | 12 | attr_reader :aws_client, :propono_config, :topic_name, :visibility_timeout, :message_processor 13 | def initialize(aws_client, propono_config, topic_name, options = {}, &message_processor) 14 | @aws_client = aws_client 15 | @propono_config = propono_config 16 | @topic_name = topic_name 17 | @message_processor = message_processor 18 | @visibility_timeout = options[:visibility_timeout] || nil 19 | end 20 | 21 | def listen 22 | raise ProponoError.new("topic_name is nil") unless topic_name 23 | loop do 24 | read_messages 25 | end 26 | end 27 | 28 | def drain 29 | raise ProponoError.new("topic_name is nil") unless topic_name 30 | true while read_messages_from_queue(main_queue, 10, long_poll: false) 31 | true while read_messages_from_queue(slow_queue, 10, long_poll: false) if propono_config.slow_queue_enabled 32 | end 33 | 34 | private 35 | 36 | def read_messages 37 | read_messages_from_queue(main_queue, propono_config.num_messages_per_poll) || 38 | (propono_config.slow_queue_enabled ? read_messages_from_queue(slow_queue, 1) : nil) 39 | end 40 | 41 | def read_messages_from_queue(queue, num_messages, long_poll: true) 42 | messages = aws_client.read_from_sqs(queue, num_messages, long_poll: long_poll, visibility_timeout: visibility_timeout) 43 | if messages.empty? 44 | false 45 | else 46 | messages.each { |msg| process_raw_message(msg, queue) } 47 | true 48 | end 49 | rescue => e #Aws::Errors => e 50 | propono_config.logger.error "Unexpected error reading from queue #{queue.url}" 51 | propono_config.logger.error e.class.name 52 | propono_config.logger.error e.message 53 | propono_config.logger.error e.backtrace 54 | false 55 | end 56 | 57 | # The calls to delete_message are deliberately duplicated so 58 | # as to ensure the message is only deleted if the preceeding line 59 | # has completed succesfully. We do *not* want to ensure that the 60 | # message is deleted regardless of what happens in this method. 61 | def process_raw_message(raw_sqs_message, queue) 62 | sqs_message = parse(raw_sqs_message, queue) 63 | return unless sqs_message 64 | 65 | propono_config.logger.info "Propono [#{sqs_message.context[:id]}]: Received from sqs." 66 | handle(sqs_message) 67 | aws_client.delete_from_sqs(queue, sqs_message.receipt_handle) 68 | end 69 | 70 | def parse(raw_sqs_message, queue) 71 | SqsMessage.new(raw_sqs_message) 72 | rescue 73 | propono_config.logger.error "Error parsing message, moving to corrupt queue", $!, $!.backtrace 74 | move_to_corrupt_queue(raw_sqs_message) 75 | aws_client.delete_from_sqs(queue, raw_sqs_message.receipt_handle) 76 | nil 77 | end 78 | 79 | def handle(sqs_message) 80 | process_message(sqs_message) 81 | rescue => e 82 | propono_config.logger.error("Failed to handle message #{e.message} #{e.backtrace}") 83 | requeue_message_on_failure(sqs_message, e) 84 | end 85 | 86 | def process_message(sqs_message) 87 | return false unless message_processor 88 | message_processor.call(sqs_message.message, sqs_message.context) 89 | end 90 | 91 | def move_to_corrupt_queue(raw_sqs_message) 92 | aws_client.send_to_sqs(corrupt_queue, raw_sqs_message.body) 93 | end 94 | 95 | def requeue_message_on_failure(sqs_message, exception) 96 | next_queue = (sqs_message.failure_count < propono_config.max_retries) ? main_queue : failed_queue 97 | propono_config.logger.error "Error processing message, moving to queue: #{next_queue}" 98 | aws_client.send_to_sqs(next_queue, sqs_message.to_json_with_exception(exception)) 99 | end 100 | 101 | def delete_message(raw_sqs_message, queue) 102 | aws_client.delete_from_sqs(queue, raw_sqs_message.receipt_handle) 103 | end 104 | 105 | def main_queue 106 | @main_queue ||= subscription.queue 107 | end 108 | 109 | def failed_queue 110 | @failed_queue ||= subscription.failed_queue 111 | end 112 | 113 | def corrupt_queue 114 | @corrupt_queue ||= subscription.corrupt_queue 115 | end 116 | 117 | def slow_queue 118 | @slow_queue ||= subscription.slow_queue 119 | end 120 | 121 | def subscription 122 | QueueSubscription.create(aws_client, propono_config, topic_name) 123 | end 124 | end 125 | end 126 | -------------------------------------------------------------------------------- /lib/propono/utils.rb: -------------------------------------------------------------------------------- 1 | module Propono 2 | module Utils 3 | 4 | # Returns +hash+ with all primary and nested keys to string values symbolised 5 | # To avoid conflicts with ActiveSupport and other libraries that provide Hash symbolisation, 6 | # this method is kept within the Propono namespace and not mixed into Hash 7 | def self.symbolize_keys(hash) 8 | hash.inject({}) do |result, (key, value)| 9 | new_key = key.is_a?(String) ? key.to_sym : key 10 | new_value = value.is_a?(Hash) ? symbolize_keys(value) : value 11 | result[new_key] = new_value 12 | result 13 | end 14 | end 15 | 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /lib/propono/version.rb: -------------------------------------------------------------------------------- 1 | module Propono 2 | VERSION = "3.0.0" 3 | end 4 | -------------------------------------------------------------------------------- /propono.gemspec: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | lib = File.expand_path('../lib', __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | require 'propono/version' 5 | 6 | Gem::Specification.new do |spec| 7 | spec.name = "propono" 8 | spec.version = Propono::VERSION 9 | spec.authors = ["iHiD", "dougal", "ccare", "MalcyL"] 10 | spec.email = ["jez.walker@gmail.com"] 11 | spec.description = %q{Pub / Sub Library using Amazon Web Services} 12 | spec.summary = %q{General purpose pub/sub library built on top of AWS SNS and SQS} 13 | spec.homepage = "https://github.com/iHiD/propono/" 14 | spec.license = "MIT" 15 | 16 | spec.files = `git ls-files`.split($/) 17 | spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) } 18 | spec.test_files = spec.files.grep(%r{^(test|spec|features)/}) 19 | spec.require_paths = ["lib"] 20 | 21 | spec.add_dependency "aws-sdk-sns" 22 | spec.add_dependency "aws-sdk-sqs" 23 | 24 | spec.add_development_dependency "bundler", "~> 2.1" 25 | spec.add_development_dependency "rake" 26 | spec.add_development_dependency "mocha" 27 | spec.add_development_dependency "yard" 28 | spec.add_development_dependency "minitest", "~> 5.0.8" 29 | end 30 | -------------------------------------------------------------------------------- /test/components/aws_client_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path('../../test_helper', __FILE__) 2 | 3 | module Propono 4 | class AwsClientTest < Minitest::Test 5 | 6 | def test_publish_to_sns_proxies 7 | client = AwsClient.new(nil) 8 | sns_client = mock 9 | message = {foo: 'bar'} 10 | topic_arn = "asd" 11 | topic = mock(arn: topic_arn) 12 | sns_client.expects(:publish).with( 13 | topic_arn: topic_arn, 14 | message: message.to_json 15 | ) 16 | client.stubs(sns_client: sns_client) 17 | client.publish_to_sns(topic, message) 18 | end 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /test/components/aws_config_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path('../../test_helper', __FILE__) 2 | 3 | module Propono 4 | class AwsConfigTest < Minitest::Test 5 | 6 | def setup 7 | super 8 | @config = Propono::Configuration.new 9 | 10 | @config.aws_options = { a: 'any', b: 'aws-specific' } 11 | @config.sqs_options = { a: 'sqs', c: 'sqs-specific' } 12 | @config.sns_options = { a: 'sns', c: 'sns-specific' } 13 | 14 | @aws_config = Propono::AwsConfig.new(@config) 15 | end 16 | 17 | def test_overwritten_keys_take_precendence 18 | assert_equal 'sqs', @aws_config.sqs_options[:a] 19 | assert_equal 'sns', @aws_config.sns_options[:a] 20 | end 21 | 22 | def test_common_keys_remain 23 | assert_equal 'aws-specific', @aws_config.sqs_options[:b] 24 | assert_equal 'aws-specific', @aws_config.sns_options[:b] 25 | end 26 | 27 | def test_specific_keys_remain 28 | assert_equal 'sqs-specific', @aws_config.sqs_options[:c] 29 | assert_equal 'sns-specific', @aws_config.sns_options[:c] 30 | end 31 | 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /test/components/client_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path('../../test_helper', __FILE__) 2 | 3 | module Propono 4 | class ClientTest < Minitest::Test 5 | 6 | def test_publish_calls_publisher_publish 7 | topic, message = "Foo", "Bar" 8 | client = Propono::Client.new 9 | Publisher.expects(:publish).with( 10 | client.aws_client, 11 | client.config, 12 | topic, 13 | message, 14 | {} 15 | ) 16 | client.publish(topic, message) 17 | end 18 | 19 | def test_publish_sets_suffix_publish 20 | queue_suffix = "-bar" 21 | topic = "foo" 22 | message = "asdasdasda" 23 | 24 | client = Propono::Client.new 25 | client.config.queue_suffix = queue_suffix 26 | Publisher.expects(:publish).with( 27 | client.aws_client, 28 | client.config, 29 | "#{topic}#{queue_suffix}", 30 | message, 31 | {} 32 | ) 33 | client.publish(topic, message) 34 | end 35 | 36 | def test_listen_calls_queue_listener 37 | topic = 'foobar' 38 | 39 | client = Propono::Client.new 40 | QueueListener.expects(:listen).with( 41 | client.aws_client, 42 | client.config, 43 | topic, 44 | {} 45 | ) 46 | client.listen(topic) 47 | end 48 | 49 | def test_listen_calls_queue_listener_with_options 50 | topic = 'foobar' 51 | options = {foo: 'bar'} 52 | 53 | client = Propono::Client.new 54 | QueueListener.expects(:listen).with( 55 | client.aws_client, 56 | client.config, 57 | topic, 58 | options 59 | ) 60 | client.listen(topic, options) 61 | end 62 | 63 | 64 | 65 | def test_drain_queue_calls_queue_listener 66 | topic = 'foobar' 67 | 68 | client = Propono::Client.new 69 | QueueListener.expects(:drain).with( 70 | client.aws_client, 71 | client.config, 72 | topic 73 | ) 74 | client.drain_queue(topic) 75 | end 76 | 77 | def test_block_configuration_syntax 78 | test_application_name = "my-application" 79 | client = Propono::Client.new do |config| 80 | config.application_name = test_application_name 81 | end 82 | assert_equal test_application_name, client.config.application_name 83 | end 84 | end 85 | end 86 | -------------------------------------------------------------------------------- /test/components/queue_subscription_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path('../../test_helper', __FILE__) 2 | 3 | module Propono 4 | class QueueSubscriptionTest < Minitest::Test 5 | def setup 6 | super 7 | @suffix = "-suf" 8 | propono_config.queue_suffix = @suffix 9 | end 10 | 11 | def teardown 12 | super 13 | propono_config.queue_suffix = "" 14 | end 15 | 16 | def test_create_calls_submethods 17 | subscription = QueueSubscription.new(aws_client, propono_config, "foobar") 18 | subscription.expects(:create_and_subscribe_main_queue) 19 | subscription.expects(:create_and_subscribe_slow_queue) 20 | subscription.expects(:create_misc_queues) 21 | subscription.create 22 | end 23 | 24 | def test_create_main_queue 25 | policy = "Some policy" 26 | topic_name = "SomeName" 27 | 28 | subscription = QueueSubscription.new(aws_client, propono_config, topic_name) 29 | subscription.stubs(:create_and_subscribe_slow_queue) 30 | subscription.stubs(:create_misc_queues) 31 | subscription.stubs(generate_policy: policy) 32 | queue_name = subscription.send(:queue_name) 33 | 34 | topic = mock 35 | queue = mock 36 | aws_client.expects(:create_topic).with("#{topic_name}#{@suffix}").returns(topic) 37 | aws_client.expects(:create_queue).with(queue_name).returns(queue) 38 | aws_client.expects(:subscribe_sqs_to_sns).with(queue, topic) 39 | aws_client.expects(:set_sqs_policy).with(queue, policy) 40 | 41 | subscription.create 42 | assert_equal queue, subscription.queue 43 | end 44 | 45 | def test_create_slow_queue 46 | policy = "Some policy" 47 | topic_name = "SomeName" 48 | slow_queue = mock 49 | 50 | subscription = QueueSubscription.new(aws_client, propono_config, topic_name) 51 | subscription.stubs(:create_and_subscribe_main_queue) 52 | subscription.stubs(:create_misc_queues) 53 | subscription.stubs(generate_policy: policy) 54 | queue_name = subscription.send(:queue_name) 55 | 56 | topic = mock 57 | slow_queue = mock 58 | aws_client.expects(:create_topic).with("#{topic_name}#{@suffix}-slow").returns(topic) 59 | aws_client.expects(:create_queue).with("#{queue_name}-slow").returns(slow_queue) 60 | aws_client.expects(:subscribe_sqs_to_sns).with(slow_queue, topic) 61 | aws_client.expects(:set_sqs_policy).with(slow_queue, policy) 62 | 63 | subscription.create 64 | assert_equal slow_queue, subscription.slow_queue 65 | end 66 | 67 | def test_create_misc_queues 68 | policy = "Some policy" 69 | topic_name = "SomeName" 70 | failed_queue = mock 71 | corrupt_queue = mock 72 | 73 | subscription = QueueSubscription.new(aws_client, propono_config, topic_name) 74 | subscription.stubs(:create_and_subscribe_main_queue) 75 | subscription.stubs(:create_and_subscribe_slow_queue) 76 | subscription.stubs(generate_policy: policy) 77 | queue_name = subscription.send(:queue_name) 78 | 79 | aws_client.expects(:create_queue).with("#{queue_name}-failed").returns(failed_queue) 80 | aws_client.expects(:create_queue).with("#{queue_name}-corrupt").returns(corrupt_queue) 81 | 82 | subscription.create 83 | 84 | assert_equal failed_queue, subscription.failed_queue 85 | assert_equal corrupt_queue, subscription.corrupt_queue 86 | end 87 | 88 | def test_subscription_queue_name 89 | propono_config.application_name = "MyApp" 90 | 91 | topic_name = "Foobar" 92 | subscription = QueueSubscription.new(aws_client, propono_config, topic_name) 93 | 94 | assert_equal "MyApp-Foobar#{@suffix}", subscription.send(:queue_name) 95 | end 96 | 97 | def test_subscription_queue_name_with_spaces 98 | propono_config.application_name = "My App" 99 | 100 | topic_name = "Foobar" 101 | subscription = QueueSubscription.new(aws_client, propono_config, topic_name) 102 | 103 | assert_equal "My_App-Foobar#{@suffix}", subscription.send(:queue_name) 104 | end 105 | 106 | def test_create_raises_with_nil_topic 107 | subscription = QueueSubscription.new(aws_client, propono_config, nil) 108 | assert_raises ProponoError do 109 | subscription.create 110 | end 111 | end 112 | 113 | def test_generate_policy 114 | queue_arn = "queue-arn" 115 | topic_arn = "topic-arn" 116 | queue = mock().tap {|m|m.stubs(arn: queue_arn)} 117 | topic = mock().tap {|m|m.stubs(arn: topic_arn)} 118 | 119 | policy = <<-EOS 120 | { 121 | "Version": "2008-10-17", 122 | "Id": "#{queue_arn}/SQSDefaultPolicy", 123 | "Statement": [ 124 | { 125 | "Sid": "#{queue_arn}-Sid", 126 | "Effect": "Allow", 127 | "Principal": { 128 | "AWS": "*" 129 | }, 130 | "Action": "SQS:*", 131 | "Resource": "#{queue_arn}", 132 | "Condition": { 133 | "StringEquals": { 134 | "aws:SourceArn": "#{topic_arn}" 135 | } 136 | } 137 | } 138 | ] 139 | } 140 | EOS 141 | 142 | assert_equal policy, QueueSubscription.new(aws_client, propono_config, nil).send(:generate_policy, queue, topic) 143 | end 144 | end 145 | end 146 | -------------------------------------------------------------------------------- /test/components/queue_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path('../../test_helper', __FILE__) 2 | 3 | module Propono 4 | class QueueTest < Minitest::Test 5 | def test_url 6 | url = 'foobar' 7 | queue = Queue.new(url, nil) 8 | assert url, queue.url 9 | end 10 | 11 | def test_arn 12 | arn = 'foobar' 13 | attributes = {"QueueArn" => arn} 14 | queue = Queue.new(nil, attributes) 15 | assert arn, queue.arn 16 | end 17 | end 18 | end 19 | 20 | -------------------------------------------------------------------------------- /test/components/topic_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path('../../test_helper', __FILE__) 2 | 3 | module Propono 4 | class TopicTest < Minitest::Test 5 | def test_arn 6 | arn = 'foobar' 7 | aws_topic = mock 8 | aws_topic.expects(:topic_arn).returns(arn) 9 | topic = Topic.new(aws_topic) 10 | assert arn, topic.arn 11 | end 12 | end 13 | end 14 | 15 | -------------------------------------------------------------------------------- /test/config.yml.example: -------------------------------------------------------------------------------- 1 | application_name: tests-yourinitials 2 | 3 | # Whatever keys are in aws_options are passed directly to the AWS clients. 4 | 5 | # Option 1 - Do nothing. 6 | # AWS clients will either use the default profile in ~/.aws/credentials, or use an IAM Role if on EC2. 7 | 8 | # Option 2 - Use environment variables 9 | # You can set a non-default local profile with the AWS_PROFILE environment variable. 10 | # You can also set keys directly: https://docs.aws.amazon.com/sdk-for-ruby/v3/developer-guide/setup-config.html#aws-ruby-sdk-credentials-environment 11 | 12 | # Option 3 - Set values directly. 13 | aws_options: 14 | # Required 15 | region: 'test-aws-region' 16 | 17 | # Set keys: 18 | # access_key_id: test-aws-access-key 19 | # secret_access_key: test-aws-secret-key 20 | 21 | # Or set a profile: 22 | # profile: profile-name 23 | 24 | -------------------------------------------------------------------------------- /test/configuration_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path('../test_helper', __FILE__) 2 | 3 | module Propono 4 | class ConfigurationTest < Minitest::Test 5 | 6 | def setup 7 | super 8 | Propono.instance_variable_set("@config", nil) 9 | end 10 | 11 | def test_obtaining_singletion 12 | refute propono_config.nil? 13 | end 14 | 15 | def test_application_name 16 | application_name = "test-application-name" 17 | propono_config.application_name = application_name 18 | assert_equal application_name, propono_config.application_name 19 | end 20 | 21 | def test_default_aws_options 22 | assert_equal({}, propono_config.aws_options) 23 | end 24 | 25 | def test_aws_options 26 | opts = { foo: 'bar' } 27 | propono_config.aws_options = opts 28 | assert_equal opts, propono_config.aws_options 29 | end 30 | 31 | def test_default_sqs_options 32 | assert_equal({}, propono_config.sqs_options) 33 | end 34 | 35 | def test_sqs_options 36 | opts = { foo: 'bar' } 37 | propono_config.sqs_options = opts 38 | assert_equal opts, propono_config.sqs_options 39 | end 40 | 41 | def test_default_sns_options 42 | assert_equal({}, propono_config.sns_options) 43 | end 44 | 45 | def test_sns_options 46 | opts = { foo: 'bar' } 47 | propono_config.sns_options = opts 48 | assert_equal opts, propono_config.sns_options 49 | end 50 | 51 | def test_default_logger 52 | assert propono_config.logger.is_a?(Propono::Logger) 53 | end 54 | 55 | def test_logger 56 | propono_config.logger = :my_logger 57 | assert_equal :my_logger, propono_config.logger 58 | end 59 | 60 | def test_default_queue_suffix 61 | assert_equal "", propono_config.queue_suffix 62 | end 63 | 64 | def test_queue_suffix 65 | queue_suffix = "test-application-name" 66 | propono_config.queue_suffix = queue_suffix 67 | assert_equal queue_suffix, propono_config.queue_suffix 68 | end 69 | 70 | def test_default_num_messages_per_poll 71 | assert_equal 1, propono_config.num_messages_per_poll 72 | end 73 | 74 | def test_num_messages_per_poll 75 | val = 3 76 | propono_config.num_messages_per_poll = val 77 | assert_equal val, propono_config.num_messages_per_poll 78 | end 79 | 80 | def test_missing_application_name_throws_exception 81 | assert_raises(ProponoConfigurationError) do 82 | propono_config.application_name 83 | end 84 | end 85 | 86 | def test_missing_logger_throws_exception 87 | propono_config.logger = nil 88 | assert_raises(ProponoConfigurationError) do 89 | propono_config.logger 90 | end 91 | end 92 | 93 | def test_missing_max_retries_throws_exception 94 | propono_config.max_retries = nil 95 | assert_raises(ProponoConfigurationError) do 96 | propono_config.max_retries 97 | end 98 | end 99 | 100 | def test_missing_num_messages_per_poll_throws_exception 101 | propono_config.num_messages_per_poll = nil 102 | assert_raises(ProponoConfigurationError) do 103 | propono_config.num_messages_per_poll 104 | end 105 | end 106 | 107 | def test_default_max_retries 108 | assert_equal 0, propono_config.max_retries 109 | end 110 | 111 | def test_max_retries 112 | val = 5 113 | propono_config.max_retries = val 114 | assert_equal 5, propono_config.max_retries 115 | end 116 | 117 | def propono_config 118 | @propono_config ||= Propono::Configuration.new 119 | end 120 | end 121 | end 122 | 123 | -------------------------------------------------------------------------------- /test/integration/integration_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path('../../test_helper', __FILE__) 2 | 3 | module Propono 4 | class IntegrationTest < Minitest::Test 5 | 6 | def propono_client 7 | config_file = YAML.load_file( File.expand_path('../../config.yml', __FILE__)) 8 | @propono_client ||= Propono::Client.new do |config| 9 | config.aws_options = config_file['aws_options'] 10 | config.application_name = config_file['application_name'] 11 | end 12 | end 13 | 14 | # Wait a max of 20secs before failing the test 15 | def wait_for_thread(thread, secs = 20) 16 | (secs * 10).times do |x| 17 | return true unless thread.alive? 18 | sleep(0.1) 19 | end 20 | false 21 | end 22 | end 23 | end 24 | 25 | 26 | -------------------------------------------------------------------------------- /test/integration/slow_queue_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path('../integration_test', __FILE__) 2 | 3 | module Propono 4 | class SlowQueueTest < IntegrationTest 5 | def test_slow_messages_are_received 6 | topic = "propono-tests-slow-queue-topic" 7 | slow_topic = "propono-tests-slow-queue-topic-slow" 8 | text = "This is my message #{DateTime.now} #{rand()}" 9 | slow_text = "This is my slow message #{DateTime.now} #{rand()}" 10 | flunks = [] 11 | message_received = false 12 | slow_message_received = false 13 | 14 | propono_client.drain_queue(topic) 15 | propono_client.drain_queue(slow_topic) 16 | 17 | propono_client.subscribe(topic) 18 | 19 | thread = Thread.new do 20 | begin 21 | propono_client.listen(topic) do |message, context| 22 | flunks << "Wrong message" unless (message == text || message == slow_text) 23 | message_received = true if message == text 24 | slow_message_received = true if message == slow_text 25 | thread.terminate if message_received && slow_message_received 26 | end 27 | rescue => e 28 | flunks << e.message 29 | ensure 30 | thread.terminate 31 | end 32 | end 33 | 34 | Thread.new do 35 | sleep(1) while !message_received 36 | sleep(1) while !slow_message_received 37 | sleep(5) # Make sure all the message deletion clear up in the thread has happened 38 | thread.terminate 39 | end 40 | 41 | sleep(1) # Make sure the listener has started 42 | 43 | propono_client.publish(slow_topic, slow_text) 44 | propono_client.publish(topic, text) 45 | 46 | flunks << "Test Timeout" unless wait_for_thread(thread, 60) 47 | flunk(flunks.join("\n")) unless flunks.empty? 48 | ensure 49 | thread.terminate if thread 50 | end 51 | end 52 | end 53 | -------------------------------------------------------------------------------- /test/integration/sns_to_sqs_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path('../integration_test', __FILE__) 2 | 3 | module Propono 4 | class SnsToSqsTest < IntegrationTest 5 | def test_the_message_gets_there 6 | topic = "propono-tests-sns-to-sqs-topic" 7 | text = "This is my message #{DateTime.now} #{rand()}" 8 | flunks = [] 9 | message_received = false 10 | 11 | propono_client.drain_queue(topic) 12 | propono_client.subscribe(topic) 13 | 14 | thread = Thread.new do 15 | begin 16 | propono_client.listen(topic) do |message, context| 17 | flunks << "Wrong message" unless message == text 18 | flunks << "Wrong id" unless context[:id] =~ Regexp.new("[a-z0-9]{6}") 19 | message_received = true 20 | thread.terminate 21 | end 22 | rescue => e 23 | flunks << e.message 24 | ensure 25 | thread.terminate 26 | end 27 | end 28 | 29 | Thread.new do 30 | sleep(1) while !message_received 31 | sleep(5) # Make sure all the message deletion clear up in the thread has happened 32 | thread.terminate 33 | end 34 | 35 | sleep(1) # Make sure the listener has started 36 | 37 | propono_client.publish(topic, text) 38 | flunks << "Test Timeout" unless wait_for_thread(thread) 39 | flunk(flunks.join("\n")) unless flunks.empty? 40 | ensure 41 | thread.terminate if thread 42 | end 43 | 44 | =begin 45 | def test_failed_messge_is_transferred_to_failed_channel 46 | topic = "propono-tests-sns-to-sqs-topic-failed" 47 | text = "This is my message #{DateTime.now} #{rand()}" 48 | flunks = [] 49 | message_received = false 50 | 51 | propono_client.drain_queue(topic) 52 | propono_client.subscribe(topic) 53 | 54 | thread = Thread.new do 55 | begin 56 | propono_client.listen(topic) do |message, context| 57 | raise StandardError.new 'BOOM' 58 | end 59 | rescue => e 60 | flunks << e.message 61 | ensure 62 | thread.terminate 63 | end 64 | end 65 | 66 | failure_thread = Thread.new do 67 | begin 68 | propono_client.listen(topic, channel: :failed) do |message, context| 69 | flunks << "Wrong message" unless message == text 70 | flunks << "Wrong id" unless context[:id] =~ Regexp.new("[a-z0-9]{6}") 71 | message_received = true 72 | failure_thread.terminate 73 | end 74 | rescue => e 75 | flunks << e.message 76 | ensure 77 | thread.terminate 78 | end 79 | end 80 | 81 | Thread.new do 82 | sleep(1) while !message_received 83 | p "Message received" 84 | sleep(5) # Make sure all the message deletion clear up in the thread has happened 85 | thread.terminate 86 | failure_thread.terminate 87 | end 88 | 89 | sleep(1) # Make sure the listener has started 90 | 91 | propono_client.publish(topic, text) 92 | flunks << "Test Timeout" unless wait_for_thread(thread) 93 | flunk(flunks.join("\n")) unless flunks.empty? 94 | ensure 95 | thread.terminate if thread 96 | end 97 | =end 98 | end 99 | end 100 | -------------------------------------------------------------------------------- /test/logger_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path('../test_helper', __FILE__) 2 | 3 | module Propono 4 | class LoggerTest < Minitest::Test 5 | def setup 6 | super 7 | @logger = Logger.new 8 | end 9 | 10 | def test_debug 11 | $stdout.expects(:puts).with("foobar") 12 | @logger.debug "foobar" 13 | end 14 | 15 | def test_info 16 | $stdout.expects(:puts).with("foobar") 17 | @logger.info "foobar" 18 | end 19 | 20 | def test_warn 21 | $stdout.expects(:puts).with("foobar") 22 | @logger.warn "foobar" 23 | end 24 | 25 | def test_error 26 | $stderr.expects(:puts).with("foobar") 27 | @logger.error "foobar" 28 | end 29 | 30 | def test_fatal 31 | $stderr.expects(:puts).with("foobar") 32 | @logger.fatal "foobar" 33 | end 34 | end 35 | end 36 | -------------------------------------------------------------------------------- /test/services/publisher_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path('../../test_helper', __FILE__) 2 | 3 | module Propono 4 | class PublisherTest < Minitest::Test 5 | 6 | def test_initialization 7 | publisher = Publisher.new(aws_client, propono_config, 'topic', 'message') 8 | refute publisher.nil? 9 | end 10 | 11 | def test_self_publish_calls_new 12 | topic = "topic123" 13 | message = "message123" 14 | Publisher.expects(:new).with(aws_client, topic, message).returns(mock(publish: nil)) 15 | Publisher.publish(aws_client, topic, message) 16 | end 17 | 18 | def test_initializer_generates_an_id 19 | publisher = Publisher.new(aws_client, propono_config, 'x','y') 20 | assert publisher.instance_variable_get(:@id) 21 | end 22 | 23 | def test_initializer_concats_an_id 24 | id = "q1w2e3" 25 | hex = "313abd" 26 | SecureRandom.expects(:hex).with(3).returns(hex) 27 | publisher = Publisher.new(aws_client, propono_config, 'x','y', id: id) 28 | assert_equal "#{id}-#{hex}", publisher.id 29 | end 30 | 31 | def test_self_publish_calls_publish 32 | Publisher.any_instance.expects(:publish) 33 | Publisher.publish(aws_client, propono_config, "topic", "message") 34 | end 35 | 36 | def test_publish_logs 37 | publisher = Publisher.new(aws_client, propono_config, "foo", "bar") 38 | publisher.instance_variable_set(:@id, 'abc') 39 | publisher.stubs(:publish_syncronously) 40 | propono_config.logger.expects(:info).with {|x| x =~ /^Propono \[abc\]: Publishing bar to foo.*/} 41 | publisher.publish 42 | end 43 | 44 | def test_publish_should_call_sns_on_correct_topic_and_message 45 | topic_name = "topic123" 46 | id = "f123" 47 | message = "message123" 48 | 49 | topic = mock 50 | topic_arn = "arn123" 51 | topic.stubs(arn: topic_arn) 52 | 53 | aws_client.expects(:create_topic).with(topic_name).returns(topic) 54 | aws_client.expects(:publish_to_sns).with( 55 | topic, 56 | {id: id, message: message} 57 | ) 58 | 59 | publisher = Publisher.new(aws_client, propono_config, topic_name, message) 60 | publisher.stubs(id: id) 61 | publisher.publish 62 | end 63 | 64 | def test_publish_should_accept_a_hash_for_message 65 | topic_name = "topic123" 66 | id = "foobar123" 67 | message = {something: ['some', 123, true]} 68 | body = {id: id, message: message} 69 | 70 | topic = mock 71 | topic_arn = "arn123" 72 | topic.stubs(topic_arn: topic_arn) 73 | 74 | topic = mock 75 | topic_arn = "arn123" 76 | topic.stubs(arn: topic_arn) 77 | 78 | aws_client.expects(:create_topic).with(topic_name).returns(topic) 79 | aws_client.expects(:publish_to_sns).with(topic, body) 80 | 81 | publisher = Publisher.new(aws_client, propono_config, topic_name, message) 82 | publisher.stubs(id: id) 83 | publisher.publish 84 | end 85 | 86 | def test_publish_async_should_return_future_of_the_sns_response 87 | skip "Rebuild this maybe" 88 | topic = "topic123" 89 | id = "foobar123" 90 | message = "message123" 91 | body = {id: id, message: message} 92 | 93 | topic_arn = "arn123" 94 | topic = Topic.new(topic_arn) 95 | 96 | sns = mock() 97 | sns.expects(:publish).with(topic_arn, body.to_json).returns(:response) 98 | publisher = Publisher.new(aws_client, propono_config, topic, message, async: true) 99 | publisher.stubs(id: id, sns: sns) 100 | assert_same :response, publisher.send(:publish_syncronously).value 101 | end 102 | 103 | def test_publish_should_propogate_exception_on_topic_creation_error 104 | aws_client.expects(:create_topic).raises(RuntimeError) 105 | publisher = Publisher.new(aws_client, propono_config, "topic", "message") 106 | 107 | assert_raises(RuntimeError) do 108 | publisher.publish 109 | end 110 | end 111 | 112 | def test_publish_should_raise_exception_if_topic_is_nil 113 | assert_raises(PublisherError, "Topic is nil") do 114 | Publisher.publish(aws_client, propono_config, nil, "foobar") 115 | end 116 | end 117 | 118 | def test_publish_should_raise_exception_if_message_is_nil 119 | assert_raises(PublisherError, "Message is nil") do 120 | Publisher.publish(aws_client, propono_config, "foobar", nil) 121 | end 122 | end 123 | 124 | def test_publish_can_be_called_syncronously 125 | publisher = Publisher.new(aws_client, propono_config, "topic_name", "message", async: true) 126 | publisher.expects(:publish_syncronously).never 127 | publisher.expects(:publish_asyncronously).once 128 | publisher.publish 129 | end 130 | 131 | def test_publish_is_normally_called_syncronously 132 | publisher = Publisher.new(aws_client, propono_config, "topic_name", "message") 133 | publisher.expects(:publish_syncronously) 134 | publisher.publish 135 | end 136 | end 137 | end 138 | -------------------------------------------------------------------------------- /test/services/queue_listener_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path('../../test_helper', __FILE__) 2 | 3 | module Propono 4 | class QueueListenerTest < Minitest::Test 5 | 6 | def setup 7 | super 8 | @topic_name = "some-topic" 9 | 10 | @receipt_handle1 = "test-receipt-handle1" 11 | @receipt_handle2 = "test-receipt-handle2" 12 | @message1 = {cat: "Foobar 123"} 13 | @message2 = "Barfoo 543" 14 | @message1_id = "abc123" 15 | @message2_id = "987whf" 16 | @body1 = {id: @message1_id, message: @message1} 17 | @body2 = {id: @message2_id, message: @message2} 18 | 19 | @sqs_message1 = mock 20 | @sqs_message1.stubs(receipt_handle: @receipt_handle1, body: {"Message" => @body1.to_json}.to_json) 21 | @sqs_message2 = mock 22 | @sqs_message2.stubs(receipt_handle: @receipt_handle2, body: {"Message" => @body2.to_json}.to_json) 23 | 24 | @queue = mock.tap {|q| q.stubs(url: "foobar", arn: "qarn") } 25 | @topic = mock.tap {|t| t.stubs(arn: "tarn") } 26 | aws_client.stubs( 27 | create_queue: @queue, 28 | create_topic: @topic 29 | ) 30 | aws_client.stubs(:subscribe_sqs_to_sns) 31 | aws_client.stubs(:set_sqs_policy) 32 | 33 | @messages = [@sqs_message1, @sqs_message2] 34 | aws_client.stubs(read_from_sqs: @messages) 35 | aws_client.stubs(:delete_from_sqs) 36 | 37 | @listener = QueueListener.new(aws_client, propono_config, @topic_name) {} 38 | 39 | @slow_queue = mock 40 | @slow_queue.stubs(url: "some_queue_url") 41 | @failed_queue = mock 42 | @corrupt_queue = mock 43 | @listener.stubs(slow_queue: @slow_queue, corrupt_queue: @corrupt_queue, failed_queue: @failed_queue) 44 | 45 | propono_config.num_messages_per_poll = 14 46 | propono_config.max_retries = 0 47 | end 48 | 49 | def test_listen_should_loop 50 | @listener.expects(:loop) 51 | @listener.listen 52 | end 53 | 54 | def test_listen_raises_with_nil_topic 55 | listener = QueueListener.new(aws_client, propono_config, nil) {} 56 | assert_raises ProponoError do 57 | listener.listen 58 | end 59 | end 60 | 61 | # Keep this test in sync with the one below, just with the config enabled 62 | def test_drain_should_continue_if_queue_empty 63 | @listener.expects(:read_messages_from_queue).with(@slow_queue, 10, long_poll: false).returns(false) 64 | @listener.expects(:read_messages_from_queue).with(@queue, 10, long_poll: false).returns(false) 65 | @listener.drain 66 | assert true 67 | end 68 | 69 | # Keep this test in sync with the one above, just with the config disabled 70 | def test_drain_ignores_slow_queue_if_disabled 71 | propono_config.slow_queue_enabled = false 72 | 73 | @listener.expects(:read_messages_from_queue).with(@slow_queue, 10, long_poll: false).never 74 | @listener.expects(:read_messages_from_queue).with(@queue, 10, long_poll: false).returns(false) 75 | @listener.drain 76 | assert true 77 | end 78 | 79 | def test_drain_raises_with_nil_topic 80 | listener = QueueListener.new(aws_client, propono_config, nil) {} 81 | assert_raises ProponoError do 82 | listener.drain 83 | end 84 | end 85 | 86 | def test_read_messages_should_subscribe 87 | queue = mock 88 | queue.stubs(:url) 89 | QueueSubscription.expects(:create).with(aws_client, propono_config, @topic_name).returns(mock(queue: queue)) 90 | @listener.send(:read_messages) 91 | end 92 | 93 | def test_read_message_from_sqs 94 | max_number_of_messages = 5 95 | aws_client.expects(:read_from_sqs).with(@queue, max_number_of_messages, long_poll: true, visibility_timeout: nil) 96 | @listener.send(:read_messages_from_queue, @queue, max_number_of_messages) 97 | end 98 | 99 | def test_log_message_from_sqs 100 | propono_config.logger.expects(:info).with() {|x| x == "Propono [#{@message1_id}]: Received from sqs."} 101 | @listener.send(:read_messages_from_queue, @queue, propono_config.num_messages_per_poll) 102 | end 103 | 104 | def test_read_messages_calls_process_message_for_each_msg 105 | @listener.expects(:process_raw_message).with(@sqs_message1, @queue) 106 | @listener.expects(:process_raw_message).with(@sqs_message2, @queue) 107 | @listener.send(:read_messages_from_queue, @queue, propono_config.num_messages_per_poll) 108 | end 109 | 110 | def test_read_messages_does_not_call_process_messages_if_there_are_none 111 | aws_client.stubs(read_from_sqs: []) 112 | @listener.expects(:process_message).never 113 | @listener.send(:read_messages_from_queue, @queue, propono_config.num_messages_per_poll) 114 | end 115 | 116 | def test_exception_from_sqs_is_logged 117 | aws_client.stubs(:read_from_sqs).raises(StandardError) 118 | propono_config.logger.expects(:error).with("Unexpected error reading from queue #{@queue.url}") 119 | @listener.send(:read_messages_from_queue, @queue, propono_config.num_messages_per_poll) 120 | end 121 | 122 | def test_exception_from_sqs_returns_false 123 | aws_client.stubs(:read_from_sqs).raises(StandardError) 124 | refute @listener.send(:read_messages) 125 | end 126 | 127 | def test_each_message_processor_is_yielded 128 | messages_yielded = [] 129 | @listener = QueueListener.new(aws_client, propono_config, @topic_name) { |m, _| messages_yielded.push(m) } 130 | @listener.send(:read_messages) 131 | 132 | assert_equal messages_yielded.size, 2 133 | assert messages_yielded.include?(@message1) 134 | assert messages_yielded.include?(@message2) 135 | end 136 | 137 | def test_ok_if_message_processor_is_nil 138 | messages_yielded = [] 139 | @listener = QueueListener.new(aws_client, propono_config, @topic_name) 140 | 141 | @listener.send(:process_message, "") 142 | assert_equal messages_yielded.size, 0 143 | end 144 | 145 | def test_each_message_processor_context 146 | ids = [] 147 | @listener = QueueListener.new(aws_client, propono_config, @topic_name) { |_, context| ids << context[:id] } 148 | @listener.send(:read_messages) 149 | 150 | assert_equal ids.size, 2 151 | assert ids.include?(@message1_id) 152 | assert ids.include?(@message2_id) 153 | end 154 | 155 | def test_each_message_is_deleted 156 | queue = "test-queue-url" 157 | 158 | aws_client.expects(:delete_from_sqs).with(queue, @receipt_handle1) 159 | aws_client.expects(:delete_from_sqs).with(queue, @receipt_handle2) 160 | 161 | @listener.stubs(queue: queue) 162 | @listener.send(:read_messages_from_queue, queue, propono_config.num_messages_per_poll) 163 | end 164 | 165 | def test_messages_are_deleted_if_there_is_an_exception_processing 166 | aws_client.expects(:delete_from_sqs).with(@queue, @receipt_handle1) 167 | aws_client.expects(:delete_from_sqs).with(@queue, @receipt_handle2) 168 | 169 | exception = StandardError.new("Test Error") 170 | @listener = QueueListener.new(aws_client, propono_config, @topic_name) { raise exception } 171 | @listener.stubs(:requeue_message_on_failure).with(SqsMessage.new(@sqs_message1), exception) 172 | @listener.stubs(:requeue_message_on_failure).with(SqsMessage.new(@sqs_message2), exception) 173 | @listener.send(:read_messages_from_queue, @queue, propono_config.num_messages_per_poll) 174 | end 175 | 176 | def test_messages_are_retried_or_abandoned_on_failure 177 | exception = StandardError.new("Test Error") 178 | @listener = QueueListener.new(aws_client, propono_config, @topic_name) { raise exception } 179 | @listener.expects(:requeue_message_on_failure).with(SqsMessage.new(@sqs_message1), exception) 180 | @listener.expects(:requeue_message_on_failure).with(SqsMessage.new(@sqs_message2), exception) 181 | @listener.send(:read_messages_from_queue, @queue, propono_config.num_messages_per_poll) 182 | end 183 | 184 | def test_failed_on_moving_to_failed_queue_does_not_delete 185 | exception = StandardError.new("Test Error") 186 | @listener = QueueListener.new(aws_client, propono_config, @topic_name) { raise exception } 187 | @listener.stubs(:requeue_message_on_failure).with(SqsMessage.new(@sqs_message1), exception).raises(StandardError.new("failed to move")) 188 | @listener.stubs(:requeue_message_on_failure).with(SqsMessage.new(@sqs_message2), exception).raises(StandardError.new("failed to move")) 189 | @listener.expects(:delete_message).with(@sqs_message1).never 190 | @listener.expects(:delete_message).with(@sqs_message2).never 191 | @listener.send(:read_messages_from_queue, @queue, propono_config.num_messages_per_poll) 192 | end 193 | 194 | def test_messages_are_moved_to_corrupt_queue_if_there_is_an_parsing_exception 195 | sqs_message1 = mock(body: "foobar", receipt_handle: "123") 196 | sqs_message2 = mock(body: "barfoo", receipt_handle: "321") 197 | @messages[0] = sqs_message1 198 | @messages[1] = sqs_message2 199 | 200 | @listener.expects(:move_to_corrupt_queue).with(sqs_message1) 201 | @listener.expects(:move_to_corrupt_queue).with(sqs_message2) 202 | @listener.send(:read_messages_from_queue, @queue, propono_config.num_messages_per_poll) 203 | end 204 | 205 | def test_message_moved_to_failed_queue_if_there_is_an_exception_and_retry_count_is_zero 206 | aws_client.expects(:send_to_sqs).with(@failed_queue, anything) 207 | @listener.send(:requeue_message_on_failure, SqsMessage.new(@sqs_message1), StandardError.new) 208 | end 209 | 210 | def test_message_requeued_if_there_is_an_exception_but_failure_count_less_than_retry_count 211 | propono_config.max_retries = propono_config.num_messages_per_poll 212 | message = SqsMessage.new(@sqs_message1) 213 | message.stubs(failure_count: 4) 214 | aws_client.expects(:send_to_sqs).with(@queue, anything) 215 | @listener.send(:requeue_message_on_failure, message, StandardError.new) 216 | end 217 | 218 | def test_message_requeued_if_there_is_an_exception_but_failure_count_exceeds_than_retry_count 219 | propono_config.max_retries = propono_config.num_messages_per_poll 220 | message = SqsMessage.new(@sqs_message1) 221 | message.stubs(failure_count: propono_config.num_messages_per_poll) 222 | aws_client.expects(:send_to_sqs).with(@failed_queue, anything) 223 | @listener.send(:requeue_message_on_failure, message, StandardError.new) 224 | end 225 | 226 | def test_move_to_corrupt_queue 227 | aws_client.expects(:send_to_sqs).with(@corrupt_queue, @sqs_message1.body) 228 | @listener.send(:move_to_corrupt_queue, @sqs_message1) 229 | end 230 | 231 | # Keep this test in sync with the one below, just with the config enabled 232 | def test_if_no_messages_read_from_normal_queue_read_from_slow_queue 233 | main_queue = mock 234 | @listener.stubs(main_queue: main_queue) 235 | slow_queue = mock 236 | @listener.stubs(slow_queue: slow_queue) 237 | 238 | @listener.expects(:read_messages_from_queue).with(main_queue, propono_config.num_messages_per_poll).returns(false) 239 | @listener.expects(:read_messages_from_queue).with(slow_queue, 1) 240 | @listener.send(:read_messages) 241 | end 242 | 243 | # Keep this test in sync with the one above, just with the config disabled 244 | def ignore_slow_queue_if_disabled 245 | propono_config.slow_queue_enabled = false 246 | 247 | main_queue = mock 248 | @listener.stubs(main_queue: main_queue) 249 | slow_queue = mock 250 | @listener.stubs(slow_queue: slow_queue) 251 | 252 | @listener.expects(:read_messages_from_queue).with(main_queue, propono_config.num_messages_per_poll).returns(false) 253 | @listener.expects(:read_messages_from_queue).with(slow_queue, 1).never 254 | @listener.send(:read_messages) 255 | end 256 | 257 | def test_if_read_messages_from_normal_do_not_read_from_slow_queue 258 | main_queue = mock 259 | @listener.stubs(main_queue: main_queue) 260 | 261 | @listener.expects(:read_messages_from_queue).with(main_queue, propono_config.num_messages_per_poll).returns(true) 262 | @listener.send(:read_messages) 263 | end 264 | end 265 | end 266 | -------------------------------------------------------------------------------- /test/test_helper.rb: -------------------------------------------------------------------------------- 1 | gem "minitest" 2 | require "minitest/autorun" 3 | require "minitest/pride" 4 | require "minitest/mock" 5 | require "mocha/setup" 6 | 7 | lib = File.expand_path('../../lib', __FILE__) 8 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 9 | 10 | require "propono" 11 | 12 | class Minitest::Test 13 | def setup 14 | end 15 | 16 | def propono_config 17 | @propono_config ||= Propono::Configuration.new.tap do |c| 18 | c.application_name = "MyApp" 19 | c.queue_suffix = "" 20 | 21 | c.logger.stubs(:debug) 22 | c.logger.stubs(:info) 23 | c.logger.stubs(:error) 24 | end 25 | end 26 | 27 | def aws_client 28 | @aws_client ||= Propono::AwsClient.new(mock).tap do |c| 29 | c.stubs(:sns_client) 30 | c.stubs(:sqs_client) 31 | end 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /test/utils/hash_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path('../../test_helper', __FILE__) 2 | 3 | module Propono 4 | class HashTest < Minitest::Test 5 | def test_symbolize_keys_works 6 | input = { 7 | "foo" => "bar", 8 | cat: 1, 9 | "nest" => { 10 | "dog" => [ 11 | {"mouse" => true} 12 | ] 13 | } 14 | } 15 | expected = { 16 | foo: 'bar', 17 | cat: 1, 18 | nest: { 19 | dog: [ 20 | {"mouse" => true} 21 | ] 22 | } 23 | } 24 | 25 | assert_equal expected, Propono::Utils.symbolize_keys(input) 26 | end 27 | end 28 | end 29 | 30 | 31 | --------------------------------------------------------------------------------