├── .gitignore ├── .ruby-gemset ├── .ruby-version ├── .travis.yml ├── CHANGELOG.md ├── Gemfile ├── LICENSE.txt ├── README.md ├── Rakefile ├── bin └── ryespy ├── lib ├── ryespy.rb └── ryespy │ ├── app.rb │ ├── listener │ ├── amzn_s3.rb │ ├── base.rb │ ├── fogable.rb │ ├── ftp.rb │ ├── goog_cs.rb │ ├── goog_drv.rb │ ├── imap.rb │ └── rax_cf.rb │ ├── notifier │ └── sidekiq.rb │ └── version.rb ├── ryespy.gemspec └── test ├── helper.rb └── ryespy ├── app_test.rb ├── listener ├── amzn_s3_test.rb ├── ftp_test.rb ├── goog_cs_test.rb ├── goog_drv_test.rb ├── imap_test.rb └── rax_cf_test.rb ├── notifier └── sidekiq_test.rb └── version_test.rb /.gitignore: -------------------------------------------------------------------------------- 1 | /*.gem 2 | /*.rbc 3 | /.bundle/ 4 | /.config 5 | /.yardoc/ 6 | /Gemfile.lock 7 | /InstalledFiles 8 | /_yardoc/ 9 | /coverage/ 10 | /doc/ 11 | /lib/bundler/man/ 12 | /pkg/ 13 | /rdoc/ 14 | /spec/reports/ 15 | /test/version_tmp/ 16 | /tmp/ 17 | -------------------------------------------------------------------------------- /.ruby-gemset: -------------------------------------------------------------------------------- 1 | ryespy 2 | -------------------------------------------------------------------------------- /.ruby-version: -------------------------------------------------------------------------------- 1 | 2.2.3 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | rvm: 3 | - 2.0.0 4 | - 2.1.7 5 | - 2.2.3 6 | services: 7 | - redis-server 8 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Ryespy Changelog 2 | 3 | This changelog documents the main changes between released versions. 4 | For a full list of changes, consult the commit history. 5 | 6 | 7 | ## 1.1.1 8 | 9 | - [#2] Google Drive listener fix changing max files from 100 to 1000; outstanding [#3] to remove limit entirely (thank you, @Lewis-Clayton) 10 | 11 | 12 | ## 1.1.0 13 | 14 | - start of support for Ruby 2.1.1 15 | - start of support for Ruby 2.1.2 16 | - improved `README` with workers examples 17 | - [#1] new Google Drive listener (`--listener goog-drv`) (thank you, @Lewis-Clayton) 18 | - extension of [#1] with changes detection and tests 19 | 20 | 21 | ## 1.0.0 22 | 23 | - first major release; Redis key structure frozen 24 | - Redis key structure backwards-incompatible with 0.x.x (sorry! :( ) 25 | - start of support for Ruby 2.1.0 26 | - end of support for Ruby 1.9.2 27 | - new Amazon S3 listener (`--listener amzn-s3`) 28 | - new Google Cloud Storage listener (`--listener goog-cs`) 29 | - new Rackspace Cloud Files listener (`--listener rax-cf`) 30 | - change of `--verbose` mode to `--debug` mode 31 | - broader error-catching, in case weird things happen when `--eternal` 32 | - missing FTP listener (`--listener ftp`) `--ftp-port` fix 33 | - dynamic requiring of listeners (some have their own dependencies) 34 | - comprehensive `README` with lots of examples 35 | - major refactoring and improvement of code throughout 36 | - a plethora of tests; most of the core is now covered 37 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Specify your gem's dependencies in ryespy.gemspec 4 | gemspec 5 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) tiredpixel and others 2013 - 2015 2 | 3 | MIT License 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Ryespy 2 | 3 | [![Gem Version](https://badge.fury.io/rb/ryespy.png)](http://badge.fury.io/rb/ryespy) 4 | [![Build Status](https://travis-ci.org/tiredpixel/ryespy.png?branch=master,stable)](https://travis-ci.org/tiredpixel/ryespy) 5 | [![Code Climate](https://codeclimate.com/github/tiredpixel/ryespy.png)](https://codeclimate.com/github/tiredpixel/ryespy) 6 | 7 | [Redis](http://redis.io) [Sidekiq](https://github.com/mperham/sidekiq)/ 8 | [Resque](https://github.com/resque/resque) 9 | IMAP, FTP, Amazon S3, Google Cloud Storage, Google Drive, Rackspace Cloud Files 10 | listener. 11 | 12 | Ryespy provides an executable for listening to IMAP mailboxes, FTP folders, 13 | Amazon S3 buckets, Google Cloud Storage buckets, Google Drive accounts, or 14 | Rackspace Cloud Files containers, keeps track of what it's seen using 15 | Redis, and writes Sidekiq/Resque-compatible payloads. 16 | 17 | Ryespy was inspired by [Redimap](https://github.com/tiredpixel/redimap). 18 | Yes, it's sometimes possible to inspire oneself. Ryespy with my little eye. 19 | 20 | More sleep lost by [tiredpixel](https://www.tiredpixel.com). 21 | 22 | 23 | ## Externals 24 | 25 | - [Redis](http://redis.io) 26 | 27 | 28 | ## Installation 29 | 30 | Install using: 31 | 32 | gem install ryespy 33 | 34 | Listener dependencies are required dynamically. That means that it may be 35 | necessary to manually install the indicated gems if you are using that listener. 36 | If you are not using that listener, the dependencies need not be installed. 37 | 38 | ### Amazon S3 Listener 39 | 40 | gem install fog -v '~> 1.19' 41 | 42 | ### Google Cloud Storage Listener 43 | 44 | gem install fog -v '~> 1.19' 45 | 46 | ### Google Drive Listener 47 | 48 | gem install google_drive -v '~> 0.3' 49 | 50 | ### Rackspace Cloud Files Listener 51 | 52 | gem install fog -v '~> 1.19' 53 | 54 | The default Ruby version supported is defined in `.ruby-version`. 55 | Any other versions supported are defined in `.travis.yml`. 56 | 57 | 58 | ## Usage 59 | 60 | View the available options: 61 | 62 | ryespy --help 63 | 64 | It is necessary to specify a listener and at least one notifier. 65 | Currently, the only notifier is `--notifier-sidekiq`. 66 | 67 | To run eternally, use `--eternal` (no need for Cron, but you can if you prefer). 68 | 69 | 70 | ### IMAP Listener 71 | 72 | Check IMAP, queue new email UIDs, and quit: 73 | 74 | ryespy --listener imap --imap-host mail.example.com --imap-username a@example.com --imap-password helpimacarrot --notifier-sidekiq 75 | 76 | For non-SSL, use `--no-imap-ssl`. 77 | For non-INBOX or multiple mailboxes, use `--imap-mailboxes INBOX,Sent`. 78 | 79 | #### Example Worker 80 | 81 | class RyespyIMAPJob 82 | include Sidekiq::Worker 83 | 84 | sidekiq_options :queue => :ryespy 85 | 86 | def perform(mailbox, uid) 87 | end 88 | end 89 | 90 | ### FTP Listener 91 | 92 | Check FTP, queue new file paths, and quit: 93 | 94 | ryespy --listener ftp --ftp-host ftp.example.com --ftp-username b@example.com --ftp-password helpimacucumber --notifier-sidekiq 95 | 96 | For PASSIVE mode, use `--ftp-passive`. 97 | For non-root or multiple directories, use `--ftp-dirs /DIR1,/DIR2`. 98 | 99 | #### Example Worker 100 | 101 | class RyespyFTPJob 102 | include Sidekiq::Worker 103 | 104 | sidekiq_options :queue => :ryespy 105 | 106 | def perform(dir, filename) 107 | end 108 | end 109 | 110 | ### Amazon S3 Listener 111 | 112 | Check Amazon S3, queue new file keys, and quit: 113 | 114 | ryespy --listener amzn-s3 --amzn-s3-access-key c/example/com --amzn-s3-secret-key helpimabroccoli --amzn-s3-bucket vegetable-box --notifier-sidekiq 115 | 116 | For non-* or multiple key prefix filters, 117 | use `--amzn-s3-prefixes virtual-dir1/,virtual-dir`. 118 | 119 | #### Example Worker 120 | 121 | class RyespyAmznS3Job 122 | include Sidekiq::Worker 123 | 124 | sidekiq_options :queue => :ryespy 125 | 126 | def perform(filename) 127 | end 128 | end 129 | 130 | ### Google Cloud Storage Listener 131 | 132 | Check Google Cloud Storage, queue new file keys, and quit: 133 | 134 | ryespy --listener goog-cs --goog-cs-access-key d/example/com --goog-cs-secret-key helpimanasparagus --goog-cs-bucket vegetable-box --notifier-sidekiq 135 | 136 | For non-* or multiple key prefix filters, 137 | use `--goog-cs-prefixes virtual-dir1/,virtual-dir`. 138 | 139 | #### Example Worker 140 | 141 | class RyespyGoogCSJob 142 | include Sidekiq::Worker 143 | 144 | sidekiq_options :queue => :ryespy 145 | 146 | def perform(filename) 147 | end 148 | end 149 | 150 | ### Google Drive Listener 151 | 152 | *Note that the Google Drive listener currently has a pre-filter maximum of 1000 153 | files. That means that having more files than this in an account is not (yet) 154 | supported; such files might be ignored entirely, or strange behaviour might be 155 | encountered depending on the order in which the files are returned.* 156 | ** 157 | ** 158 | 159 | Check Google Drive, queue new file resource ids, and quit: 160 | 161 | ryespy --listener goog-drv --goog-drv-username vegetable-box@gmail.com --goog-drv-password helpimgarlic --goog-drv-filters vegetable-box --notifier-sidekiq 162 | 163 | #### Example Worker 164 | 165 | class RyespyGoogDrvJob 166 | include Sidekiq::Worker 167 | 168 | sidekiq_options :queue => :ryespy 169 | 170 | def perform(resource_id) 171 | end 172 | end 173 | 174 | ### Rackspace Cloud Files Listener 175 | 176 | Check Rackspace Cloud Files, queue new file keys, and quit: 177 | 178 | ryespy --listener rax-cf --rax-cf-username vegetable --rax-cf-api-key helpimacelery --rax-cf-container vegetable-box --notifier-sidekiq 179 | 180 | For non-DFW region, use `--rax-cf-region lon`. 181 | For non-US auth endpoint, use `--rax-cf-endpoint uk`. 182 | Is your Rackspace account in London? Fret not; combine these and 183 | use `--rax-cf-endpoint uk --rax-cf-region lon`. 184 | For non-* or multiple key prefix filters, 185 | use `--rax-cf-prefixes virtual-dir1/,virtual-dir`. 186 | 187 | #### Example Worker 188 | 189 | class RyespyRaxCFJob 190 | include Sidekiq::Worker 191 | 192 | sidekiq_options :queue => :ryespy 193 | 194 | def perform(filename) 195 | end 196 | end 197 | 198 | 199 | ## Advanced Usage 200 | 201 | If you want to do something rather more magical such as checking multiple 202 | accounts for a listener or even multiple listeners, then you may wish to use the 203 | Ryespy library directly instead of the `ryespy` executable. 204 | 205 | Depend upon the `ryespy` gem in a `Gemfile`, remembering to add any manual 206 | dependencies for listeners as detailed in [Installation](#installation): 207 | 208 | # Gemfile 209 | 210 | gem 'ryespy' 211 | gem 'fog' # example manual dependency 212 | 213 | Configure Ryespy Redis and require Ryespy: 214 | 215 | require 'redis' 216 | require 'redis/namespace' 217 | 218 | Redis.current = Redis::Namespace.new('ryespy', 219 | :redis => Redis.connect(:url => nil) # Redis default 220 | ) 221 | 222 | require 'ryespy' 223 | 224 | Create the notifiers: 225 | 226 | require 'ryespy/notifier/sidekiq' 227 | 228 | notifiers = [] 229 | notifiers << Ryespy::Notifier::Sidekiq.new( 230 | :url => nil, # Redis default 231 | :namespace => 'resque' 232 | ) 233 | 234 | For each listener, configure like in `ryespy --help` but without the prefix and 235 | with `-` changed to `_` (e.g. `--amzn-s3-access-key` => `:access_key`). Pass in 236 | an array of notifiers. Note that the `check()` argument varies per listener, 237 | meaning IMAP mailbox, FTP directory, Google Drive filter, or storage key prefix. 238 | 239 | require 'ryespy/listener/amzn_s3' 240 | 241 | Ryespy::Listener::AmznS3.new( 242 | :access_key => 'ACCESS_KEY', 243 | :secret_key => 'SECRET_KEY', 244 | :bucket => 'BUCKET', 245 | :notifiers => notifiers 246 | ) do |listener| 247 | listener.check('prefix/') 248 | end 249 | 250 | That's about the size of it. 251 | 252 | 253 | ## Stay Tuned 254 | 255 | We have a [Librelist](http://librelist.com) mailing list! 256 | To subscribe, send an email to . 257 | To unsubscribe, send an email to . 258 | There be [archives](http://librelist.com/browser/ryespy/). 259 | That was easy. 260 | 261 | You can also become a [watcher](https://github.com/tiredpixel/ryespy/watchers) 262 | on GitHub. And don't forget you can become a [stargazer](https://github.com/tiredpixel/ryespy/stargazers) if you are so minded. :D 263 | 264 | 265 | ## Contributions 266 | 267 | Contributions are embraced with much love and affection! 268 | Please fork the repository and wizard your magic, preferably with plenty of 269 | fairy-dust sprinkled over the tests. ;) 270 | Then send me a pull request. Simples! 271 | If you'd like to discuss what you're doing or planning to do, or if you get 272 | stuck on something, then just wave. :) 273 | 274 | Do whatever makes you happy. We'll probably still like you. :) 275 | 276 | Tests are written using [minitest](https://github.com/seattlerb/minitest), 277 | which is included by default in Ruby 1.9 onwards. To run all tests: 278 | 279 | rake test 280 | 281 | When using the `ryespy` executable in development, you'll probably want to set 282 | `--debug` mode so debug-level messages are logged and stack traces raised. 283 | 284 | We render grateful thanks unto our 285 | [contributors](https://github.com/tiredpixel/ryespy/graphs/contributors). 286 | 287 | 288 | ## Blessing 289 | 290 | May you find peace, and help others to do likewise. 291 | 292 | 293 | ## Licence 294 | 295 | © [tiredpixel](https://www.tiredpixel.com) and [others](https://github.com/tiredpixel/ryespy/graphs/contributors) 2013 - 2015. 296 | It is free software, released under the MIT License, and may be redistributed under the terms specified in `LICENSE.txt`. 297 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'bundler/gem_tasks' 2 | require 'rake/testtask' 3 | 4 | 5 | Rake::TestTask.new do |t| 6 | t.libs.push 'lib' 7 | t.test_files = FileList['test/**/*_test.rb'] 8 | t.verbose = true 9 | end 10 | 11 | task :default => 'test' 12 | -------------------------------------------------------------------------------- /bin/ryespy: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | $stdout.sync = true 4 | 5 | require 'optparse' 6 | require 'ostruct' 7 | require 'logger' 8 | 9 | require File.expand_path(File.dirname(__FILE__) + '/../lib/ryespy') 10 | 11 | 12 | # = Parse opts 13 | 14 | options = OpenStruct.new( 15 | :notifiers => {} 16 | ) 17 | 18 | OptionParser.new do |opts| 19 | opts.version = "v#{Ryespy::VERSION}" 20 | 21 | opts.banner = opts.ver 22 | 23 | opts.separator "" 24 | opts.separator "Usage: ryespy [OPTIONS]" 25 | 26 | opts.separator "" 27 | opts.separator "Listener:" 28 | 29 | opts.on("-l", "--listener LISTENER", [ 30 | 'imap', 31 | 'ftp', 32 | 'amzn-s3', 33 | 'goog-cs', 34 | 'goog-drv', 35 | 'rax-cf', 36 | ], "Listener (imap|ftp|amzn-s3|goog-cs|goog-drv|rax-cf)") do |o| 37 | options[:listener] = o.tr('-', '_').to_sym 38 | end 39 | 40 | opts.separator "" 41 | opts.separator "Polling:" 42 | 43 | opts.on("-e", "--[no-]eternal", "Run eternally (default: no)") do |o| 44 | options[:eternal] = o 45 | end 46 | 47 | opts.on("--polling-interval [N]", Integer, "Poll every N seconds (default: 60)") do |o| 48 | options[:polling_interval] = o 49 | end 50 | 51 | opts.separator "" 52 | opts.separator "Redis:" 53 | 54 | opts.on("--redis-url [URL]", "Ryespy URL (default: Redis default)") do |o| 55 | options[:redis_url] = o 56 | end 57 | 58 | opts.on("--redis-ns-ryespy [NS]", "Ryespy namespace (default: ryespy)") do |o| 59 | options[:redis_ns_ryespy] = o 60 | end 61 | 62 | opts.separator "" 63 | opts.separator "--listener imap (IMAP) :" 64 | 65 | opts.on("--imap-host HOST", "Server HOST") do |o| 66 | options[:imap_host] = o 67 | end 68 | 69 | opts.on("--imap-port [PORT]", Integer, "Server PORT (default: 993)") do |o| 70 | options[:imap_port] = o 71 | end 72 | 73 | opts.on("--[no-]imap-ssl", "Use SSL (default: yes)") do |o| 74 | options[:imap_ssl] = o 75 | end 76 | 77 | opts.on("--imap-username USERNAME", "Auth USERNAME") do |o| 78 | options[:imap_username] = o 79 | end 80 | 81 | opts.on("--imap-password PASSWORD", "Auth PASSWORD") do |o| 82 | options[:imap_password] = o 83 | end 84 | 85 | opts.on("--imap-mailboxes [INBOX,Mu]", Array, "Mailboxes INBOX,Mu (default: INBOX)") do |o| 86 | options[:imap_filters] = o 87 | end 88 | 89 | opts.separator "" 90 | opts.separator "--listener ftp (FTP) :" 91 | 92 | opts.on("--ftp-host HOST", "Server HOST") do |o| 93 | options[:ftp_host] = o 94 | end 95 | 96 | opts.on("--ftp-port [PORT]", Integer, "Server PORT (default: 21)") do |o| 97 | options[:ftp_port] = o 98 | end 99 | 100 | opts.on("--[no-]ftp-passive", "Use PASSIVE mode (default: no)") do |o| 101 | options[:ftp_passive] = o 102 | end 103 | 104 | opts.on("--ftp-username USERNAME", "Auth USERNAME") do |o| 105 | options[:ftp_username] = o 106 | end 107 | 108 | opts.on("--ftp-password PASSWORD", "Auth PASSWORD") do |o| 109 | options[:ftp_password] = o 110 | end 111 | 112 | opts.on("--ftp-dirs [/D1,/D2]", Array, "Directories /D1,/D2 (default: /)") do |o| 113 | options[:ftp_filters] = o 114 | end 115 | 116 | opts.separator "" 117 | opts.separator "--listener amzn-s3 (Amazon S3) :" 118 | 119 | opts.on("--amzn-s3-access-key ACCESS", "Account ACCESS key") do |o| 120 | options[:amzn_s3_access_key] = o 121 | end 122 | 123 | opts.on("--amzn-s3-secret-key SECRET", "Account SECRET key") do |o| 124 | options[:amzn_s3_secret_key] = o 125 | end 126 | 127 | opts.on("--amzn-s3-bucket BUCKET", "Storage BUCKET") do |o| 128 | options[:amzn_s3_bucket] = o 129 | end 130 | 131 | opts.on("--amzn-s3-prefixes [P1,P2]", Array, "Prefixes P1,P2 (default: *)") do |o| 132 | options[:amzn_s3_filters] = o 133 | end 134 | 135 | opts.separator "" 136 | opts.separator "--listener goog-cs (Google Cloud Storage) :" 137 | 138 | opts.on("--goog-cs-access-key ACCESS", "Account ACCESS key") do |o| 139 | options[:goog_cs_access_key] = o 140 | end 141 | 142 | opts.on("--goog-cs-secret-key SECRET", "Account SECRET key") do |o| 143 | options[:goog_cs_secret_key] = o 144 | end 145 | 146 | opts.on("--goog-cs-bucket BUCKET", "Storage BUCKET") do |o| 147 | options[:goog_cs_bucket] = o 148 | end 149 | 150 | opts.on("--goog-cs-prefixes [P1,P2]", Array, "Prefixes P1,P2 (default: *)") do |o| 151 | options[:goog_cs_filters] = o 152 | end 153 | 154 | opts.separator "" 155 | opts.separator "--listener goog-drv (Google Drive) :" 156 | 157 | opts.on("--goog-drv-username USERNAME", "USERNAME") do |o| 158 | options[:goog_drv_username] = o 159 | end 160 | 161 | opts.on("--goog-drv-password PASSWORD", "PASSWORD") do |o| 162 | options[:goog_drv_password] = o 163 | end 164 | 165 | opts.on("--goog-drv-filters [F1,F2]", Array, "Filters F1,F2 (default: *)") do |o| 166 | options[:goog_drv_filters] = o 167 | end 168 | 169 | opts.separator "" 170 | opts.separator "--listener rax-cf (Rackspace Cloud Files) :" 171 | 172 | opts.on("--rax-cf-endpoint [ENDPOINT]", "Auth ENDPOINT (default: us)") do |o| 173 | options[:rax_cf_endpoint] = o 174 | end 175 | 176 | opts.on("--rax-cf-username USERNAME", "Account USERNAME") do |o| 177 | options[:rax_cf_username] = o 178 | end 179 | 180 | opts.on("--rax-cf-api-key API_KEY", "Account API_KEY") do |o| 181 | options[:rax_cf_api_key] = o 182 | end 183 | 184 | opts.on("--rax-cf-container CONTAINER", "Storage CONTAINER") do |o| 185 | options[:rax_cf_container] = o 186 | end 187 | 188 | opts.on("--rax-cf-region [REGION]", "Storage REGION (default: dfw)") do |o| 189 | options[:rax_cf_region] = o 190 | end 191 | 192 | opts.on("--rax-cf-prefixes [P1,P2]", Array, "Prefixes P1,P2 (default: *)") do |o| 193 | options[:rax_cf_filters] = o 194 | end 195 | 196 | opts.separator "" 197 | opts.separator "Notifiers:" 198 | 199 | opts.on("--redis-ns-notifiers [NS]", "Notifiers namespace (default: resque)") do |o| 200 | options[:redis_ns_notifiers] = o 201 | end 202 | 203 | opts.on("--notifier-sidekiq [URL]", "Sidekiq Redis URL (default: Redis default)") do |o| 204 | (options.notifiers[:sidekiq] ||= []) << o 205 | end 206 | 207 | opts.separator "" 208 | opts.separator "Other:" 209 | 210 | opts.on("-d", "--debug", "Set debug mode") do |o| 211 | options[:debug] = o 212 | end 213 | 214 | opts.on_tail("--help", 215 | "Output (this) help and exit" 216 | ) do 217 | puts opts 218 | exit 219 | end 220 | 221 | opts.on_tail("--version", 222 | "Output version and exit" 223 | ) do 224 | puts opts.ver 225 | exit 226 | end 227 | end.parse! 228 | 229 | [ 230 | :listener, 231 | ].each do |o| 232 | unless options[o] 233 | raise OptionParser::MissingArgument, "--#{o}" 234 | end 235 | end 236 | 237 | 238 | # = Create app 239 | 240 | @app = Ryespy::App.new(options[:eternal], 241 | :logger => Logger.new($stdout) 242 | ) 243 | 244 | 245 | # = Configure 246 | 247 | @app.configure do |c| 248 | c.log_level = :DEBUG if options[:debug] 249 | 250 | c.listener = options[:listener] 251 | 252 | [ 253 | :polling_interval, 254 | :redis_url, 255 | :redis_ns_ryespy, 256 | :redis_ns_notifiers, 257 | :notifiers, 258 | ].each { |k| c[k] = options[k] unless options[k].nil? } 259 | 260 | { 261 | :imap => [ 262 | :host, 263 | :port, 264 | :ssl, 265 | :username, 266 | :password, 267 | :filters, # mailboxes 268 | ], 269 | :ftp => [ 270 | :host, 271 | :port, 272 | :passive, 273 | :username, 274 | :password, 275 | :filters, # dirs 276 | ], 277 | :amzn_s3 => [ 278 | :access_key, 279 | :secret_key, 280 | :bucket, 281 | :filters, # prefixes 282 | ], 283 | :goog_cs => [ 284 | :access_key, 285 | :secret_key, 286 | :bucket, 287 | :filters, # prefixes 288 | ], 289 | :goog_drv => [ 290 | :username, 291 | :password, 292 | :filters, 293 | ], 294 | :rax_cf => [ 295 | :endpoint, 296 | :region, 297 | :username, 298 | :api_key, 299 | :container, 300 | :filters, # prefixes 301 | ], 302 | }[c.listener].each do |k| 303 | lk = "#{c.listener}_#{k}".to_sym 304 | 305 | c[c.listener][k] = options[lk] unless options[lk].nil? 306 | end 307 | end 308 | 309 | 310 | # = Run app 311 | 312 | trap('INT') { @app.stop } 313 | 314 | begin 315 | @app.start 316 | rescue => e 317 | warn "#{$0}: #{e}" 318 | 319 | if options[:debug] 320 | raise # reraise for stacktrace 321 | else 322 | exit 1 # same non-zero status as when --debug 323 | end 324 | end 325 | -------------------------------------------------------------------------------- /lib/ryespy.rb: -------------------------------------------------------------------------------- 1 | require_relative 'ryespy/version' 2 | 3 | require_relative 'ryespy/app' 4 | 5 | require_relative 'ryespy/listener/base' 6 | # ryespy/listener/X dynamically required in ryespy/app.rb 7 | 8 | require_relative 'ryespy/notifier/sidekiq' 9 | -------------------------------------------------------------------------------- /lib/ryespy/app.rb: -------------------------------------------------------------------------------- 1 | require 'logger' 2 | require 'ostruct' 3 | require 'redis' 4 | require 'redis-namespace' 5 | 6 | # listener dynamically required in App#setup 7 | 8 | 9 | module Ryespy 10 | class App 11 | 12 | def self.config_defaults 13 | { 14 | :log_level => :INFO, 15 | :polling_interval => 60, 16 | :redis_ns_ryespy => 'ryespy', 17 | :redis_ns_notifiers => 'resque', 18 | :imap => { 19 | :port => 993, 20 | :ssl => true, 21 | :filters => ['INBOX'], # mailboxes 22 | }, 23 | :ftp => { 24 | :port => 21, 25 | :passive => false, 26 | :filters => ['/'], # dirs 27 | }, 28 | :amzn_s3 => { 29 | :filters => [''], # prefixes 30 | }, 31 | :goog_cs => { 32 | :filters => [''], # prefixes 33 | }, 34 | :goog_drv => { 35 | :filters => [''], 36 | }, 37 | :rax_cf => { 38 | :endpoint => :us, 39 | :region => :dfw, 40 | :filters => [''], # prefixes 41 | }, 42 | } 43 | end 44 | 45 | attr_reader :config 46 | attr_reader :running 47 | 48 | def initialize(eternal = false, opts = {}) 49 | @eternal = eternal 50 | 51 | @logger = opts[:logger] || Logger.new(nil) 52 | 53 | @config = OpenStruct.new(self.class.config_defaults) 54 | 55 | @running = false 56 | @threads = {} 57 | end 58 | 59 | def configure 60 | yield @config 61 | 62 | @logger.level = Logger.const_get(@config.log_level) 63 | 64 | Redis.current = Redis::Namespace.new(@config.redis_ns_ryespy, 65 | :redis => Redis.connect(:url => @config.redis_url) 66 | ) 67 | 68 | @logger.debug { "Configured #{@config.to_s}" } 69 | end 70 | 71 | def notifiers 72 | unless @notifiers 73 | @notifiers = [] 74 | @config.notifiers[:sidekiq].each do |notifier_url| 75 | @notifiers << Notifier::Sidekiq.new( 76 | :url => notifier_url, 77 | :namespace => @config.redis_ns_notifiers, 78 | :logger => @logger 79 | ) 80 | end 81 | end 82 | 83 | @notifiers 84 | end 85 | 86 | def start 87 | begin 88 | @running = true 89 | 90 | setup 91 | 92 | @threads[:refresh] ||= Thread.new do 93 | refresh_loop # refresh frequently 94 | end 95 | 96 | @threads.values.each(&:join) 97 | ensure 98 | cleanup 99 | end 100 | end 101 | 102 | def stop 103 | @running = false 104 | 105 | @threads.values.each { |t| t.run if t.status == 'sleep' } 106 | end 107 | 108 | private 109 | 110 | def setup 111 | require_relative "listener/#{@config.listener}" 112 | end 113 | 114 | def cleanup 115 | end 116 | 117 | def refresh_loop 118 | while @running do 119 | begin 120 | check_all 121 | rescue StandardError => e 122 | @logger.error { e.to_s } 123 | 124 | raise if @config.log_level == :DEBUG 125 | end 126 | 127 | if !@eternal 128 | stop 129 | 130 | break 131 | end 132 | 133 | @logger.debug { "Snoring for #{@config.polling_interval} s" } 134 | 135 | sleep @config.polling_interval # sleep awhile (snore) 136 | end 137 | end 138 | 139 | def check_all 140 | listener_class_map = { 141 | :imap => :IMAP, 142 | :ftp => :FTP, 143 | :amzn_s3 => :AmznS3, 144 | :goog_cs => :GoogCS, 145 | :goog_drv => :GoogDrv, 146 | :rax_cf => :RaxCF, 147 | } 148 | 149 | listener_config = @config[@config.listener].merge({ 150 | :notifiers => notifiers, 151 | :logger => @logger, 152 | }) 153 | 154 | listener_class = Listener.const_get(listener_class_map[@config.listener]) 155 | 156 | listener_class.new(listener_config) do |listener| 157 | listener_config[:filters].each { |f| listener.check(f) } 158 | end 159 | end 160 | 161 | end 162 | end 163 | -------------------------------------------------------------------------------- /lib/ryespy/listener/amzn_s3.rb: -------------------------------------------------------------------------------- 1 | require 'fog' 2 | 3 | require_relative 'base' 4 | require_relative 'fogable' 5 | 6 | 7 | module Ryespy 8 | module Listener 9 | class AmznS3 < Base 10 | 11 | include Listener::Fogable 12 | 13 | REDIS_KEY_PREFIX = 'amzn_s3'.freeze 14 | SIDEKIQ_JOB_CLASS = 'RyespyAmznS3Job'.freeze 15 | 16 | def initialize(opts = {}) 17 | @config = { 18 | :access_key => opts[:access_key], 19 | :secret_key => opts[:secret_key], 20 | :directory => opts[:bucket], 21 | } 22 | 23 | super(opts) 24 | end 25 | 26 | private 27 | 28 | def connect_service 29 | @fog_storage = Fog::Storage.new({ 30 | :provider => 'AWS', 31 | :aws_access_key_id => @config[:access_key], 32 | :aws_secret_access_key => @config[:secret_key], 33 | }) 34 | end 35 | 36 | end 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /lib/ryespy/listener/base.rb: -------------------------------------------------------------------------------- 1 | require 'logger' 2 | require 'redis' 3 | 4 | 5 | module Ryespy 6 | module Listener 7 | class Base 8 | 9 | def initialize(opts = {}) 10 | @notifiers = opts[:notifiers] || [] 11 | @logger = opts[:logger] || Logger.new(nil) 12 | 13 | @redis = Redis.current 14 | 15 | connect_service 16 | 17 | if block_given? 18 | yield self 19 | 20 | close 21 | end 22 | end 23 | 24 | def close 25 | end 26 | 27 | private 28 | 29 | def connect_service 30 | end 31 | 32 | end 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /lib/ryespy/listener/fogable.rb: -------------------------------------------------------------------------------- 1 | module Ryespy 2 | module Listener 3 | module Fogable 4 | 5 | def check(prefix) 6 | @logger.debug { "prefix: #{prefix}" } 7 | 8 | @logger.debug { "redis_key: #{redis_key}" } 9 | 10 | seen_files = @redis.hgetall(redis_key) 11 | 12 | unseen_files = get_unseen_files(prefix, seen_files) 13 | 14 | @logger.debug { "unseen_files: #{unseen_files}" } 15 | 16 | unseen_files.each do |filename, checksum| 17 | @redis.hset(redis_key, filename, checksum) 18 | 19 | # prefix is not included as it is part of key, and list operations 20 | # return files (virtually) recursively. Constructing Redis key in this 21 | # way means a file matching multiple prefixes will only notify once. 22 | @notifiers.each do |notifier| 23 | notifier.notify(self.class::SIDEKIQ_JOB_CLASS, [filename]) 24 | end 25 | end 26 | 27 | @logger.info { "#{prefix}* has #{unseen_files.count} new files" } 28 | end 29 | 30 | private 31 | 32 | def redis_key 33 | [ 34 | self.class::REDIS_KEY_PREFIX, 35 | @config[:directory], 36 | ].join(':') 37 | end 38 | 39 | def get_unseen_files(prefix, seen_files) 40 | files = {} 41 | 42 | @fog_storage.directories.get(@config[:directory], 43 | :prefix => prefix 44 | ).files.each do |file| 45 | if file.content_type == 'application/directory' || file.content_length == 0 46 | next # virtual dirs or 0-length file 47 | end 48 | 49 | if seen_files[file.key] != file.etag # etag is server-side checksum 50 | files[file.key] = file.etag 51 | end 52 | end 53 | 54 | files 55 | end 56 | 57 | end 58 | end 59 | end 60 | -------------------------------------------------------------------------------- /lib/ryespy/listener/ftp.rb: -------------------------------------------------------------------------------- 1 | require 'net/ftp' 2 | 3 | require_relative 'base' 4 | 5 | 6 | module Ryespy 7 | module Listener 8 | class FTP < Base 9 | 10 | REDIS_KEY_PREFIX = 'ftp'.freeze 11 | SIDEKIQ_JOB_CLASS = 'RyespyFTPJob'.freeze 12 | 13 | def initialize(opts = {}) 14 | @ftp_config = { 15 | :host => opts[:host], 16 | :port => opts[:port], 17 | :passive => opts[:passive], 18 | :username => opts[:username], 19 | :password => opts[:password], 20 | } 21 | 22 | super(opts) 23 | end 24 | 25 | def close 26 | @ftp.close 27 | end 28 | 29 | def check(dir) 30 | @logger.debug { "dir: #{dir}" } 31 | 32 | @logger.debug { "redis_key: #{redis_key(dir)}" } 33 | 34 | seen_files = @redis.hgetall(redis_key(dir)) 35 | 36 | unseen_files = get_unseen_files(dir, seen_files) 37 | 38 | @logger.debug { "unseen_files: #{unseen_files}" } 39 | 40 | unseen_files.each do |filename, checksum| 41 | @redis.hset(redis_key(dir), filename, checksum) 42 | 43 | @notifiers.each { |n| n.notify(SIDEKIQ_JOB_CLASS, [dir, filename]) } 44 | end 45 | 46 | @logger.info { "#{dir} has #{unseen_files.count} new files" } 47 | end 48 | 49 | private 50 | 51 | def connect_service 52 | @ftp = Net::FTP.new 53 | 54 | @ftp.connect(@ftp_config[:host], @ftp_config[:port]) 55 | 56 | @ftp.passive = @ftp_config[:passive] 57 | 58 | @ftp.login(@ftp_config[:username], @ftp_config[:password]) 59 | end 60 | 61 | def redis_key(dir) 62 | [ 63 | REDIS_KEY_PREFIX, 64 | @ftp_config[:host], 65 | @ftp_config[:port], 66 | @ftp_config[:username], 67 | dir, 68 | ].join(':') 69 | end 70 | 71 | def get_unseen_files(dir, seen_files) 72 | @ftp.chdir(dir) 73 | 74 | files = {} 75 | 76 | @ftp.nlst.each do |file| 77 | mtime = @ftp.mtime(file).to_i 78 | size = @ftp.size(file) rescue nil # ignore non-file error 79 | 80 | if size # exclude directories 81 | checksum = "#{mtime},#{size}".freeze 82 | 83 | if seen_files[file] != checksum 84 | files[file] = checksum 85 | end 86 | end 87 | end 88 | 89 | files 90 | end 91 | 92 | end 93 | end 94 | end 95 | -------------------------------------------------------------------------------- /lib/ryespy/listener/goog_cs.rb: -------------------------------------------------------------------------------- 1 | require 'fog' 2 | 3 | require_relative 'base' 4 | require_relative 'fogable' 5 | 6 | 7 | module Ryespy 8 | module Listener 9 | class GoogCS < Base 10 | 11 | include Listener::Fogable 12 | 13 | REDIS_KEY_PREFIX = 'goog_cs'.freeze 14 | SIDEKIQ_JOB_CLASS = 'RyespyGoogCSJob'.freeze 15 | 16 | def initialize(opts = {}) 17 | @config = { 18 | :access_key => opts[:access_key], 19 | :secret_key => opts[:secret_key], 20 | :directory => opts[:bucket], 21 | } 22 | 23 | super(opts) 24 | end 25 | 26 | private 27 | 28 | def connect_service 29 | @fog_storage = Fog::Storage.new({ 30 | :provider => 'Google', 31 | :google_storage_access_key_id => @config[:access_key], 32 | :google_storage_secret_access_key => @config[:secret_key], 33 | }) 34 | end 35 | 36 | end 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /lib/ryespy/listener/goog_drv.rb: -------------------------------------------------------------------------------- 1 | require "google_drive" 2 | 3 | require_relative 'base' 4 | 5 | 6 | module Ryespy 7 | module Listener 8 | class GoogDrv < Base 9 | 10 | REDIS_KEY_PREFIX = 'goog_drv'.freeze 11 | SIDEKIQ_JOB_CLASS = 'RyespyGoogDrvJob'.freeze 12 | 13 | def initialize(opts = {}) 14 | @config = { 15 | :username => opts[:username], 16 | :password => opts[:password], 17 | } 18 | 19 | super(opts) 20 | end 21 | 22 | def check(filter) 23 | @logger.debug { "filter: #{filter}" } 24 | 25 | @logger.debug { "redis_key: #{redis_key}" } 26 | 27 | seen_files = @redis.hgetall(redis_key) 28 | 29 | unseen_files = get_unseen_files(filter, seen_files) 30 | 31 | @logger.debug { "unseen_files: #{unseen_files}" } 32 | 33 | unseen_files.each do |key, val| 34 | @redis.hset(redis_key, key, val) 35 | 36 | @notifiers.each { |n| n.notify(SIDEKIQ_JOB_CLASS, [key]) } 37 | end 38 | 39 | @logger.info { "#{filter} has #{unseen_files.count} new files" } 40 | end 41 | 42 | private 43 | 44 | def connect_service 45 | @google_drive = GoogleDrive.login(@config[:username], @config[:password]) 46 | end 47 | 48 | def redis_key 49 | [ 50 | REDIS_KEY_PREFIX, 51 | @config[:username] 52 | ].join(':') 53 | end 54 | 55 | def get_unseen_files(filter, seen_files) 56 | files = {} 57 | 58 | @google_drive.files('max-results' => 1000).each do |file| 59 | next unless file.title =~ /#{filter}/ && file.resource_id && file.resource_type != 'folder' 60 | 61 | # updated should be present for all resource_type , but there is often 62 | # a delay until it is set. 63 | updated = file.document_feed_entry.css('updated').first.text 64 | 65 | # etag is present for most resource_type , but is nil for spreadsheet. 66 | etag = file.document_feed_entry.attribute('etag') 67 | 68 | checksum = "#{updated},#{etag}" 69 | 70 | if seen_files[file.resource_id] != checksum 71 | files[file.resource_id] = checksum 72 | end 73 | end 74 | 75 | files 76 | end 77 | 78 | end 79 | end 80 | end 81 | -------------------------------------------------------------------------------- /lib/ryespy/listener/imap.rb: -------------------------------------------------------------------------------- 1 | require 'net/imap' 2 | 3 | require_relative 'base' 4 | 5 | 6 | module Ryespy 7 | module Listener 8 | class IMAP < Base 9 | 10 | REDIS_KEY_PREFIX = 'imap'.freeze 11 | SIDEKIQ_JOB_CLASS = 'RyespyIMAPJob'.freeze 12 | 13 | def initialize(opts = {}) 14 | @imap_config = { 15 | :host => opts[:host], 16 | :port => opts[:port], 17 | :ssl => opts[:ssl], 18 | :username => opts[:username], 19 | :password => opts[:password], 20 | } 21 | 22 | super(opts) 23 | end 24 | 25 | def close 26 | @imap.disconnect 27 | end 28 | 29 | def check(mailbox) 30 | @logger.debug { "mailbox: #{mailbox}" } 31 | 32 | @logger.debug { "redis_key: #{redis_key(mailbox)}" } 33 | 34 | last_seen_uid = @redis.get(redis_key(mailbox)).to_i 35 | 36 | unseen_uids = get_unseen_uids(mailbox, last_seen_uid) 37 | 38 | @logger.debug { "unseen_uids: #{unseen_uids}" } 39 | 40 | unseen_uids.each do |uid| 41 | @redis.set(redis_key(mailbox), uid) 42 | 43 | @notifiers.each { |n| n.notify(SIDEKIQ_JOB_CLASS, [mailbox, uid]) } 44 | end 45 | 46 | @logger.info { "#{mailbox} has #{unseen_uids.count} new emails" } 47 | end 48 | 49 | private 50 | 51 | def connect_service 52 | @imap = Net::IMAP.new(@imap_config[:host], { 53 | :port => @imap_config[:port], 54 | :ssl => @imap_config[:ssl], 55 | }) 56 | 57 | @imap.login(@imap_config[:username], @imap_config[:password]) 58 | end 59 | 60 | def redis_key(mailbox) 61 | [ 62 | REDIS_KEY_PREFIX, 63 | @imap_config[:host], 64 | @imap_config[:port], 65 | @imap_config[:username], 66 | mailbox, 67 | ].join(':') 68 | end 69 | 70 | def get_unseen_uids(mailbox, last_seen_uid = nil) 71 | @imap.select(mailbox) 72 | 73 | uids = @imap.uid_search("#{last_seen_uid + 1}:*") 74 | 75 | # filter as IMAP search gets fun with edge cases 76 | uids.find_all { |uid| uid > last_seen_uid } 77 | end 78 | 79 | end 80 | end 81 | end 82 | -------------------------------------------------------------------------------- /lib/ryespy/listener/rax_cf.rb: -------------------------------------------------------------------------------- 1 | require 'fog' 2 | 3 | require_relative 'base' 4 | require_relative 'fogable' 5 | 6 | 7 | module Ryespy 8 | module Listener 9 | class RaxCF < Base 10 | 11 | include Listener::Fogable 12 | 13 | REDIS_KEY_PREFIX = 'rax_cf'.freeze 14 | SIDEKIQ_JOB_CLASS = 'RyespyRaxCFJob'.freeze 15 | 16 | def initialize(opts = {}) 17 | @config = { 18 | :auth_url => Fog::Rackspace.const_get( 19 | "#{opts[:endpoint].upcase}_AUTH_ENDPOINT" 20 | ), 21 | :region => opts[:region].downcase.to_sym, 22 | :username => opts[:username], 23 | :api_key => opts[:api_key], 24 | :directory => opts[:container], 25 | } 26 | 27 | super(opts) 28 | end 29 | 30 | private 31 | 32 | def connect_service 33 | @fog_storage = Fog::Storage.new({ 34 | :provider => 'Rackspace', 35 | :rackspace_auth_url => @config[:auth_url], 36 | :rackspace_region => @config[:region], 37 | :rackspace_username => @config[:username], 38 | :rackspace_api_key => @config[:api_key], 39 | }) 40 | end 41 | 42 | def redis_key 43 | # CF container (directory) is unique across an account (region?). 44 | [ 45 | REDIS_KEY_PREFIX, 46 | @config[:username], 47 | @config[:directory], 48 | @config[:region], 49 | ].join(':') 50 | end 51 | 52 | end 53 | end 54 | end 55 | -------------------------------------------------------------------------------- /lib/ryespy/notifier/sidekiq.rb: -------------------------------------------------------------------------------- 1 | require 'logger' 2 | require 'redis' 3 | require 'redis-namespace' 4 | require 'json' 5 | require 'securerandom' 6 | 7 | 8 | module Ryespy 9 | module Notifier 10 | class Sidekiq 11 | 12 | SIDEKIQ_QUEUE = 'ryespy'.freeze 13 | SIDEKIQ_KEY_QUEUES = 'queues'.freeze 14 | SIDEKIQ_KEY_QUEUE_X = "queue:#{SIDEKIQ_QUEUE}".freeze 15 | 16 | def initialize(opts = {}) 17 | @redis_config = { 18 | :url => opts[:url], 19 | :namespace => opts[:namespace], 20 | } 21 | 22 | @logger = opts[:logger] || Logger.new(nil) 23 | 24 | connect_redis 25 | 26 | if block_given? 27 | yield self 28 | 29 | close 30 | end 31 | end 32 | 33 | def close 34 | @redis.quit 35 | end 36 | 37 | def notify(job_class, args) 38 | @redis.sadd(SIDEKIQ_KEY_QUEUES, SIDEKIQ_QUEUE) 39 | 40 | sidekiq_job_payload = sidekiq_job(job_class, args) 41 | 42 | @logger.debug { "Setting Redis Key #{SIDEKIQ_KEY_QUEUE_X} Payload #{sidekiq_job_payload.to_json}" } 43 | 44 | @redis.rpush(SIDEKIQ_KEY_QUEUE_X, sidekiq_job_payload.to_json) 45 | end 46 | 47 | private 48 | 49 | def connect_redis 50 | @redis = Redis::Namespace.new(@redis_config[:namespace], 51 | :redis => Redis.connect(:url => @redis_config[:url]) 52 | ) 53 | end 54 | 55 | def sidekiq_job(job_class, args) 56 | { 57 | # resque 58 | :class => job_class, 59 | :args => args, 60 | # sidekiq (extra) 61 | :queue => SIDEKIQ_QUEUE, 62 | :retry => true, 63 | :jid => SecureRandom.hex(12), 64 | } 65 | end 66 | 67 | end 68 | end 69 | end 70 | -------------------------------------------------------------------------------- /lib/ryespy/version.rb: -------------------------------------------------------------------------------- 1 | module Ryespy 2 | 3 | VERSION = '1.1.1'.freeze 4 | 5 | end 6 | -------------------------------------------------------------------------------- /ryespy.gemspec: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | lib = File.expand_path('../lib', __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | require 'ryespy/version' 5 | 6 | Gem::Specification.new do |spec| 7 | spec.name = "ryespy" 8 | spec.version = Ryespy::VERSION 9 | spec.authors = ["tiredpixel"] 10 | spec.email = ["tiredpixel@posteo.de"] 11 | spec.summary = %q{Redis Sidekiq/Resque IMAP, FTP, Amazon S3, Google Cloud Storage, Google Drive, Rackspace Cloud Files listener.} 12 | spec.description = %q{} 13 | spec.homepage = "https://github.com/tiredpixel/ryespy" 14 | spec.license = "MIT" 15 | 16 | spec.files = `git ls-files`.split($/) 17 | spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) } 18 | spec.test_files = spec.files.grep(%r{^(test|spec|features)/}) 19 | spec.require_paths = ["lib"] 20 | 21 | spec.add_dependency "redis", "~> 3.0" 22 | spec.add_dependency "redis-namespace", "~> 1.4" 23 | 24 | spec.add_development_dependency "bundler", "~> 1.3", "!= 1.5.0" 25 | spec.add_development_dependency "rake" 26 | spec.add_development_dependency "fog", "~> 1.19" # conditional dependency 27 | spec.add_development_dependency "mocha", "~> 0.14" 28 | spec.add_development_dependency "sidekiq-spy", ">= 0.3.2" 29 | spec.add_development_dependency "google_drive", '~> 0.3' 30 | end 31 | -------------------------------------------------------------------------------- /test/helper.rb: -------------------------------------------------------------------------------- 1 | require 'minitest/autorun' 2 | require 'mocha/setup' 3 | require 'securerandom' 4 | require 'json' 5 | require 'redis' 6 | require 'redis-namespace' 7 | 8 | 9 | module Ryespy 10 | module Test 11 | 12 | def self.config 13 | @config ||= { 14 | :redis => { 15 | :url => ENV['REDIS_URL'], # defaults 16 | :namespace => 'ryespy:test', 17 | }, 18 | } 19 | end 20 | 21 | module Redis 22 | 23 | def self.setup 24 | ::Redis.current = ::Redis::Namespace.new(self.namespace, 25 | :redis => ::Redis.connect(:url => Ryespy::Test.config[:redis][:url]) 26 | ) 27 | end 28 | 29 | def self.namespace 30 | "#{Ryespy::Test.config[:redis][:namespace]}:#{SecureRandom.hex}" 31 | end 32 | 33 | def self.flush_namespace(redis) 34 | # Redis::Namespace means only namespaced keys removed 35 | redis.keys('*').each { |k| redis.del(k) } 36 | end 37 | 38 | end 39 | 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /test/ryespy/app_test.rb: -------------------------------------------------------------------------------- 1 | require 'logger' 2 | require 'net/imap' 3 | 4 | 5 | require_relative '../helper' 6 | 7 | require_relative '../../lib/ryespy' 8 | 9 | 10 | def start_and_stop_app(app) 11 | app_thread = Thread.new { app.start } 12 | 13 | sleep 1 # patience, patience; give app time to start 14 | 15 | app.stop 16 | 17 | app_thread.join(2) 18 | 19 | Thread.kill(app_thread) 20 | end 21 | 22 | 23 | describe Ryespy::App do 24 | 25 | describe "#initialize" do 26 | before do 27 | @app = Ryespy::App.new 28 | end 29 | 30 | it "defaults running to false" do 31 | @app.running.must_equal false 32 | end 33 | end 34 | 35 | describe "#configure" do 36 | before do 37 | @app = Ryespy::App.new 38 | 39 | @config = @app.config 40 | end 41 | 42 | describe "main" do 43 | before do 44 | @app.configure do |c| 45 | c.log_level = 'ERROR' 46 | c.listener = 'imap' 47 | c.polling_interval = 13 48 | c.redis_url = 'redis://127.0.0.1:6379/1' 49 | c.redis_ns_ryespy = 'WithMyLittleEye!' 50 | c.redis_ns_notifiers = 'LaLaLiLi-' 51 | c.notifiers = [{ :sidekiq => 'redis://127.0.0.1:6379/2' }] 52 | end 53 | end 54 | 55 | it "configures log_level" do 56 | @config.log_level.must_equal 'ERROR' 57 | end 58 | 59 | it "sets logger level" do 60 | @app.instance_variable_get(:@logger).level.must_equal Logger::ERROR 61 | end 62 | 63 | it "configures listener" do 64 | @config.listener.must_equal 'imap' 65 | end 66 | 67 | it "configures polling_interval" do 68 | @config.polling_interval.must_equal 13 69 | end 70 | 71 | it "configures redis_url" do 72 | @config.redis_url.must_equal 'redis://127.0.0.1:6379/1' 73 | end 74 | 75 | it "configures redis_ns_ryespy" do 76 | @config.redis_ns_ryespy.must_equal 'WithMyLittleEye!' 77 | end 78 | 79 | it "configures redis_ns_notifiers" do 80 | @config.redis_ns_notifiers.must_equal 'LaLaLiLi-' 81 | end 82 | 83 | it "configures notifiers" do 84 | @config.notifiers.must_equal [{ :sidekiq => 'redis://127.0.0.1:6379/2' }] 85 | end 86 | end 87 | 88 | describe "listener IMAP" do 89 | before do 90 | @app.configure do |c| 91 | c.imap_host = 'imap.example.com' 92 | c.imap_port = 143 93 | c.imap_ssl = false 94 | c.imap_username = 'lucy.westenra@example.com' 95 | c.imap_password = 'white' 96 | c.imap_filters = 'BoxA,Sent Messages' 97 | end 98 | end 99 | 100 | it "configures imap_host" do 101 | @config.imap_host.must_equal 'imap.example.com' 102 | end 103 | 104 | it "configures imap_port" do 105 | @config.imap_port.must_equal 143 106 | end 107 | 108 | it "configures imap_ssl" do 109 | @config.imap_ssl.must_equal false 110 | end 111 | 112 | it "configures imap_username" do 113 | @config.imap_username.must_equal 'lucy.westenra@example.com' 114 | end 115 | 116 | it "configures imap_password" do 117 | @config.imap_password.must_equal 'white' 118 | end 119 | 120 | it "configures imap_filters" do 121 | @config.imap_filters.must_equal 'BoxA,Sent Messages' 122 | end 123 | end 124 | 125 | describe "listener FTP" do 126 | before do 127 | @app.configure do |c| 128 | c.ftp_host = 'ftp.example.org' 129 | c.ftp_port = 2121 130 | c.ftp_passive = true 131 | c.ftp_username = 'madam.mina@example.com' 132 | c.ftp_password = 'black' 133 | c.ftp_filters = ['BoxA', 'Sent Messages'] 134 | end 135 | end 136 | 137 | it "configures ftp_host" do 138 | @config.ftp_host.must_equal 'ftp.example.org' 139 | end 140 | 141 | it "configures ftp_port" do 142 | @config.ftp_port.must_equal 2121 143 | end 144 | 145 | it "configures ftp_passive" do 146 | @config.ftp_passive.must_equal true 147 | end 148 | 149 | it "configures ftp_username" do 150 | @config.ftp_username.must_equal 'madam.mina@example.com' 151 | end 152 | 153 | it "configures ftp_password" do 154 | @config.ftp_password.must_equal 'black' 155 | end 156 | 157 | it "configures ftp_filters" do 158 | @config.ftp_filters.must_equal ['BoxA', 'Sent Messages'] 159 | end 160 | end 161 | 162 | describe "listener amzn-s3" do 163 | before do 164 | @app.configure do |c| 165 | c.amzn_s3_access_key = 'r.m.renfield' 166 | c.amzn_s3_secret_key = 'master' 167 | c.amzn_s3_bucket = 'i-can-wait' 168 | c.amzn_s3_filters = ['flies/', 'spiders/'] 169 | end 170 | end 171 | 172 | it "configures amzn_s3_access_key" do 173 | @config.amzn_s3_access_key.must_equal 'r.m.renfield' 174 | end 175 | 176 | it "configures amzn_s3_secret_key" do 177 | @config.amzn_s3_secret_key.must_equal 'master' 178 | end 179 | 180 | it "configures amzn_s3_bucket" do 181 | @config.amzn_s3_bucket.must_equal 'i-can-wait' 182 | end 183 | 184 | it "configures amzn_s3_filters" do 185 | @config.amzn_s3_filters.must_equal ["flies/", "spiders/"] 186 | end 187 | end 188 | 189 | describe "listener goog-cs" do 190 | before do 191 | @app.configure do |c| 192 | c.goog_cs_access_key = 'r.m.renfield' 193 | c.goog_cs_secret_key = 'master' 194 | c.goog_cs_bucket = 'i-can-wait' 195 | c.goog_cs_filters = ['flies/', 'spiders/'] 196 | end 197 | end 198 | 199 | it "configures goog_cs_access_key" do 200 | @config.goog_cs_access_key.must_equal 'r.m.renfield' 201 | end 202 | 203 | it "configures goog_cs_secret_key" do 204 | @config.goog_cs_secret_key.must_equal 'master' 205 | end 206 | 207 | it "configures goog_cs_bucket" do 208 | @config.goog_cs_bucket.must_equal 'i-can-wait' 209 | end 210 | 211 | it "configures goog_cs_filters" do 212 | @config.goog_cs_filters.must_equal ["flies/", "spiders/"] 213 | end 214 | end 215 | 216 | describe "listener rax-cf" do 217 | before do 218 | @app.configure do |c| 219 | c.rax_cf_endpoint = 'uk' 220 | c.rax_cf_region = 'lon' 221 | c.rax_cf_username = 'van.helsing' 222 | c.rax_cf_api_key = 'M.D., D.Ph., D.Litt., etc.' 223 | c.rax_cf_container = 'the-milk-that-is-spilt-cries-not-out-afterwards' 224 | c.rax_cf_filters = ['abraham/', 'van/'] 225 | end 226 | end 227 | 228 | it "configures rax_cf_endpoint" do 229 | @config.rax_cf_endpoint.must_equal 'uk' 230 | end 231 | 232 | it "configures rax_cf_region" do 233 | @config.rax_cf_region.must_equal 'lon' 234 | end 235 | 236 | it "configures rax_cf_username" do 237 | @config.rax_cf_username.must_equal 'van.helsing' 238 | end 239 | 240 | it "configures rax_cf_api_key" do 241 | @config.rax_cf_api_key.must_equal 'M.D., D.Ph., D.Litt., etc.' 242 | end 243 | 244 | it "configures rax_cf_container" do 245 | @config.rax_cf_container.must_equal 'the-milk-that-is-spilt-cries-not-out-afterwards' 246 | end 247 | 248 | it "configures rax_cf_filters" do 249 | @config.rax_cf_filters.must_equal ["abraham/", "van/"] 250 | end 251 | end 252 | end 253 | 254 | describe "#notifiers" do 255 | before do 256 | @app = Ryespy::App.new 257 | 258 | @app.configure do |c| 259 | c.notifiers = { :sidekiq => ['redis://127.0.0.1:6379/11'] } 260 | end 261 | 262 | @app.instance_variable_set(:@notifiers, nil) 263 | end 264 | 265 | it "creates notifiers when empty" do 266 | @app.notifiers.map(&:class).must_equal [Ryespy::Notifier::Sidekiq] 267 | end 268 | 269 | it "returns notifiers when extant" do 270 | @notifiers = stub 271 | 272 | @app.instance_variable_set(:@notifiers, @notifiers) 273 | 274 | @app.notifiers.must_equal @notifiers 275 | end 276 | end 277 | 278 | describe "#start" do 279 | before do 280 | Net::IMAP.stubs(:new).returns(stub( 281 | :login => nil, 282 | :select => nil, 283 | :uid_search => [], 284 | :disconnect => nil 285 | )) 286 | 287 | @app = Ryespy::App.new(true) 288 | 289 | @app.instance_variable_set(:@logger, Logger.new(nil)) 290 | 291 | @app.configure do |c| 292 | c.listener = :imap 293 | c.polling_interval = 10 294 | end 295 | end 296 | 297 | it "sets status running within 1s" do 298 | thread_app = Thread.new { @app.start } 299 | 300 | sleep 1 # patience, patience; give app time to start 301 | 302 | @app.running.must_equal true 303 | 304 | Thread.kill(thread_app) 305 | end 306 | 307 | it "stops running within 1s" do 308 | thread_app = Thread.new { @app.start } 309 | 310 | sleep 1 # patience, patience; give app time to start 311 | 312 | @app.stop; t0 = Time.now 313 | 314 | thread_app.join(2) 315 | 316 | Thread.kill(thread_app) 317 | 318 | assert_operator (Time.now - t0), :<=, 1 319 | end 320 | 321 | it "calls #setup hook" do 322 | @app.expects(:setup) 323 | 324 | start_and_stop_app(@app) 325 | end 326 | 327 | it "calls #cleanup hook" do 328 | @app.expects(:cleanup) 329 | 330 | start_and_stop_app(@app) 331 | end 332 | end 333 | 334 | describe "#stop" do 335 | before do 336 | @app = Ryespy::App.new 337 | 338 | @app.instance_variable_set(:@running, true) 339 | end 340 | 341 | it "sets status not-running" do 342 | @app.stop 343 | 344 | @app.running.must_equal false 345 | end 346 | end 347 | 348 | end 349 | -------------------------------------------------------------------------------- /test/ryespy/listener/amzn_s3_test.rb: -------------------------------------------------------------------------------- 1 | require_relative '../../helper' 2 | 3 | require_relative '../../../lib/ryespy/listener/amzn_s3' 4 | 5 | 6 | describe Ryespy::Listener::AmznS3 do 7 | 8 | before do 9 | etag = 'QpD453wgum7qpJKUZaeHgcnHtGabP6CS' 10 | 11 | @files = [ 12 | stub(:content_length => 0, :content_type => '', :key => 'flies/', :etag => etag), 13 | stub(:content_length => 1, :content_type => '', :key => 'flies/a.txt', :etag => etag), 14 | stub(:content_length => 1, :content_type => '', :key => 'flies/b.txt', :etag => etag), 15 | stub(:content_length => 0, :content_type => '', :key => 'f/', :etag => etag), 16 | stub(:content_length => 1, :content_type => '', :key => 'f/flies_README.txt', :etag => etag), 17 | stub(:content_length => 0, :content_type => '', :key => 'spiders/', :etag => etag), 18 | stub(:content_length => 1, :content_type => '', :key => 'spiders/spider.txt', :etag => etag), 19 | ] 20 | 21 | @files_no_dirs = @files.select { |f| f.content_length != 0 } 22 | 23 | @fog_storage = stub 24 | 25 | @fog_storage_directories = stub 26 | 27 | @fog_storage_directories.stubs(:get).with('icw', { 28 | :prefix => 'flies/' 29 | }).returns(stub(:files => @files.select { |f| f.key =~ /^flies\// })) 30 | @fog_storage_directories.stubs(:get).with('icw', { 31 | :prefix => 'f' 32 | }).returns(stub(:files => @files.select { |f| f.key =~ /^f/ })) 33 | @fog_storage_directories.stubs(:get).with('icw', { 34 | :prefix => 'spiders/' 35 | }).returns(stub(:files => @files.select { |f| f.key =~ /^spiders\// })) 36 | @fog_storage_directories.stubs(:get).with('icw', { 37 | :prefix => '' 38 | }).returns(stub(:files => @files)) 39 | 40 | @fog_storage.stubs(:directories).returns(@fog_storage_directories) 41 | 42 | Fog::Storage.stubs(:new).with({ 43 | :provider => 'AWS', 44 | :aws_access_key_id => 'r.m.renfield', 45 | :aws_secret_access_key => 'master', 46 | }).returns(@fog_storage) 47 | end 48 | 49 | describe "#check" do 50 | before do 51 | Ryespy::Test::Redis::setup 52 | 53 | @notifier = mock() 54 | 55 | @amzn_s3 = Ryespy::Listener::AmznS3.new( 56 | :access_key => 'r.m.renfield', 57 | :secret_key => 'master', 58 | :bucket => 'icw', 59 | :notifiers => [@notifier], 60 | ) 61 | 62 | @redis = @amzn_s3.instance_variable_get(:@redis) 63 | end 64 | 65 | after do 66 | @amzn_s3.close 67 | 68 | Ryespy::Test::Redis::flush_namespace(@redis) 69 | end 70 | 71 | it "notifies when new files prefix *" do 72 | @files_no_dirs.each do |file| 73 | @notifier.expects(:notify).with('RyespyAmznS3Job', [file.key]).once 74 | end 75 | 76 | @amzn_s3.check('') 77 | end 78 | 79 | it "notifies when new files prefix spiders/" do 80 | @files_no_dirs.select { |f| f.key =~ /^spiders\// }.each do |file| 81 | @notifier.expects(:notify).with('RyespyAmznS3Job', [file.key]).once 82 | end 83 | 84 | @amzn_s3.check('spiders/') 85 | end 86 | 87 | it "notifies when new files prefix f" do 88 | @files_no_dirs.select { |f| f.key =~ /^f/ }.each do |file| 89 | @notifier.expects(:notify).with('RyespyAmznS3Job', [file.key]).once 90 | end 91 | 92 | @amzn_s3.check('f') 93 | end 94 | 95 | it "doesn't notify when no new files" do 96 | @notifier.expects(:notify).times(2) 97 | 98 | @amzn_s3.check('flies/') 99 | 100 | @notifier.expects(:notify).never 101 | 102 | @amzn_s3.check('flies/') 103 | end 104 | 105 | it "doesn't notify when no new files prefix subset" do 106 | @notifier.expects(:notify).times(3) 107 | 108 | @amzn_s3.check('f') 109 | 110 | @notifier.expects(:notify).never 111 | 112 | @amzn_s3.check('flies/') 113 | end 114 | 115 | it "notifies when new files prefix distinct" do 116 | @notifier.expects(:notify).times(3) 117 | 118 | @amzn_s3.check('f') 119 | 120 | @notifier.expects(:notify).times(1) 121 | 122 | @amzn_s3.check('spiders/') 123 | end 124 | 125 | it "notifies when changed etag" do 126 | @notifier.expects(:notify).times(2) 127 | 128 | @amzn_s3.check('flies/') 129 | 130 | @files[1].stubs(:etag).returns(-2303600400) 131 | 132 | @notifier.expects(:notify).with('RyespyAmznS3Job', ['flies/a.txt']).once 133 | 134 | @amzn_s3.check('flies/') 135 | end 136 | end 137 | 138 | end 139 | -------------------------------------------------------------------------------- /test/ryespy/listener/ftp_test.rb: -------------------------------------------------------------------------------- 1 | require_relative '../../helper' 2 | 3 | require_relative '../../../lib/ryespy/listener/ftp' 4 | 5 | 6 | describe Ryespy::Listener::FTP do 7 | 8 | before do 9 | @files = [ 10 | 'Bottlenose', 11 | 'Haeviside', 12 | 'Franciscana', 13 | ] 14 | 15 | @net_ftp = stub 16 | 17 | @net_ftp.stubs(:connect).with('ftp.example.com', 2121) 18 | @net_ftp.stubs(:passive=) 19 | @net_ftp.stubs(:login).with('d.adams', 'solongandthanksforallthefish') 20 | @net_ftp.stubs(:chdir).with('/dolphin') 21 | @net_ftp.stubs(:nlst).returns(@files) 22 | @net_ftp.stubs(:mtime).returns(-562032000) 23 | @net_ftp.stubs(:size).returns(42) 24 | @net_ftp.stubs(:close) 25 | 26 | Net::FTP.stubs(:new).returns(@net_ftp) 27 | end 28 | 29 | describe "#check" do 30 | before do 31 | Ryespy::Test::Redis::setup 32 | 33 | @notifier = mock() 34 | 35 | @ftp = Ryespy::Listener::FTP.new( 36 | :host => 'ftp.example.com', 37 | :port => 2121, 38 | :passive => true, 39 | :username => 'd.adams', 40 | :password => 'solongandthanksforallthefish', 41 | :notifiers => [@notifier], 42 | ) 43 | 44 | @redis = @ftp.instance_variable_get(:@redis) 45 | end 46 | 47 | after do 48 | @ftp.close 49 | 50 | Ryespy::Test::Redis::flush_namespace(@redis) 51 | end 52 | 53 | it "notifies when new files" do 54 | @files.each do |file| 55 | @notifier.expects(:notify).with('RyespyFTPJob', ['/dolphin', file]).once 56 | end 57 | 58 | @ftp.check('/dolphin') 59 | end 60 | 61 | it "doesn't notify when no new files" do 62 | @notifier.expects(:notify).times(3) 63 | 64 | @ftp.check('/dolphin') 65 | 66 | @notifier.expects(:notify).never 67 | 68 | @ftp.check('/dolphin') 69 | end 70 | 71 | it "notifies when changed mtime" do 72 | @notifier.expects(:notify).times(3) 73 | 74 | @ftp.check('/dolphin') 75 | 76 | @net_ftp.stubs(:mtime).with('Bottlenose').returns(-562031999) 77 | 78 | @notifier.expects(:notify).with('RyespyFTPJob', ['/dolphin', 'Bottlenose']).once 79 | 80 | @ftp.check('/dolphin') 81 | end 82 | 83 | it "notifies when changed size" do 84 | @notifier.expects(:notify).times(3) 85 | 86 | @ftp.check('/dolphin') 87 | 88 | @net_ftp.stubs(:size).with('Franciscana').returns(41) 89 | 90 | @notifier.expects(:notify).with('RyespyFTPJob', ['/dolphin', 'Franciscana']).once 91 | 92 | @ftp.check('/dolphin') 93 | end 94 | end 95 | 96 | end 97 | -------------------------------------------------------------------------------- /test/ryespy/listener/goog_cs_test.rb: -------------------------------------------------------------------------------- 1 | require_relative '../../helper' 2 | 3 | require_relative '../../../lib/ryespy/listener/goog_cs' 4 | 5 | 6 | describe Ryespy::Listener::GoogCS do 7 | 8 | before do 9 | etag = 'QpD453wgum7qpJKUZaeHgcnHtGabP6CS' 10 | 11 | @files = [ 12 | stub(:content_length => 0, :content_type => '', :key => 'flies/', :etag => etag), 13 | stub(:content_length => 1, :content_type => '', :key => 'flies/a.txt', :etag => etag), 14 | stub(:content_length => 1, :content_type => '', :key => 'flies/b.txt', :etag => etag), 15 | stub(:content_length => 0, :content_type => '', :key => 'f/', :etag => etag), 16 | stub(:content_length => 1, :content_type => '', :key => 'f/flies_README.txt', :etag => etag), 17 | stub(:content_length => 0, :content_type => '', :key => 'spiders/', :etag => etag), 18 | stub(:content_length => 1, :content_type => '', :key => 'spiders/spider.txt', :etag => etag), 19 | ] 20 | 21 | @files_no_dirs = @files.select { |f| f.content_length != 0 } 22 | 23 | @fog_storage = stub 24 | 25 | @fog_storage_directories = stub 26 | 27 | @fog_storage_directories.stubs(:get).with('icw', { 28 | :prefix => 'flies/' 29 | }).returns(stub(:files => @files.select { |f| f.key =~ /^flies\// })) 30 | @fog_storage_directories.stubs(:get).with('icw', { 31 | :prefix => 'f' 32 | }).returns(stub(:files => @files.select { |f| f.key =~ /^f/ })) 33 | @fog_storage_directories.stubs(:get).with('icw', { 34 | :prefix => 'spiders/' 35 | }).returns(stub(:files => @files.select { |f| f.key =~ /^spiders\// })) 36 | @fog_storage_directories.stubs(:get).with('icw', { 37 | :prefix => '' 38 | }).returns(stub(:files => @files)) 39 | 40 | @fog_storage.stubs(:directories).returns(@fog_storage_directories) 41 | 42 | Fog::Storage.stubs(:new).with({ 43 | :provider => 'Google', 44 | :google_storage_access_key_id => 'r.m.renfield', 45 | :google_storage_secret_access_key => 'master', 46 | }).returns(@fog_storage) 47 | end 48 | 49 | describe "#check" do 50 | before do 51 | Ryespy::Test::Redis::setup 52 | 53 | @notifier = mock() 54 | 55 | @goog_cs = Ryespy::Listener::GoogCS.new( 56 | :access_key => 'r.m.renfield', 57 | :secret_key => 'master', 58 | :bucket => 'icw', 59 | :notifiers => [@notifier], 60 | ) 61 | 62 | @redis = @goog_cs.instance_variable_get(:@redis) 63 | end 64 | 65 | after do 66 | @goog_cs.close 67 | 68 | Ryespy::Test::Redis::flush_namespace(@redis) 69 | end 70 | 71 | it "notifies when new files prefix *" do 72 | @files_no_dirs.each do |file| 73 | @notifier.expects(:notify).with('RyespyGoogCSJob', [file.key]).once 74 | end 75 | 76 | @goog_cs.check('') 77 | end 78 | 79 | it "notifies when new files prefix spiders/" do 80 | @files_no_dirs.select { |f| f.key =~ /^spiders\// }.each do |file| 81 | @notifier.expects(:notify).with('RyespyGoogCSJob', [file.key]).once 82 | end 83 | 84 | @goog_cs.check('spiders/') 85 | end 86 | 87 | it "notifies when new files prefix f" do 88 | @files_no_dirs.select { |f| f.key =~ /^f/ }.each do |file| 89 | @notifier.expects(:notify).with('RyespyGoogCSJob', [file.key]).once 90 | end 91 | 92 | @goog_cs.check('f') 93 | end 94 | 95 | it "doesn't notify when no new files" do 96 | @notifier.expects(:notify).times(2) 97 | 98 | @goog_cs.check('flies/') 99 | 100 | @notifier.expects(:notify).never 101 | 102 | @goog_cs.check('flies/') 103 | end 104 | 105 | it "doesn't notify when no new files prefix subset" do 106 | @notifier.expects(:notify).times(3) 107 | 108 | @goog_cs.check('f') 109 | 110 | @notifier.expects(:notify).never 111 | 112 | @goog_cs.check('flies/') 113 | end 114 | 115 | it "notifies when new files prefix distinct" do 116 | @notifier.expects(:notify).times(3) 117 | 118 | @goog_cs.check('f') 119 | 120 | @notifier.expects(:notify).times(1) 121 | 122 | @goog_cs.check('spiders/') 123 | end 124 | 125 | it "notifies when changed etag" do 126 | @notifier.expects(:notify).times(2) 127 | 128 | @goog_cs.check('flies/') 129 | 130 | @files[1].stubs(:etag).returns(-2303600400) 131 | 132 | @notifier.expects(:notify).with('RyespyGoogCSJob', ['flies/a.txt']).once 133 | 134 | @goog_cs.check('flies/') 135 | end 136 | end 137 | 138 | end 139 | -------------------------------------------------------------------------------- /test/ryespy/listener/goog_drv_test.rb: -------------------------------------------------------------------------------- 1 | require_relative '../../helper' 2 | 3 | require_relative '../../../lib/ryespy/listener/goog_drv' 4 | 5 | 6 | describe Ryespy::Listener::GoogDrv do 7 | 8 | before do 9 | updated = '1897-05-26 00:00:00 UTC' 10 | 11 | dfe = stub 12 | dfe.stubs(:css).with('updated').returns([stub(:text => updated)]) 13 | dfe.stubs(:attribute).with('etag').returns('E13l2izwZbgWKfS5') 14 | 15 | dfe2 = stub 16 | dfe2.stubs(:css).with('updated').returns([stub(:text => updated)]) 17 | dfe2.stubs(:attribute).with('etag').returns(nil) 18 | 19 | @files = [ 20 | stub( 21 | :resource_id => 'folder:FFTSKJtqo5v2TRrIADAmVOlsqe8h9TqM', 22 | :resource_type => 'folder', 23 | :title => 'Flies/', 24 | :document_feed_entry => dfe, 25 | ), 26 | stub( 27 | :resource_id => 'presentation:SuOQSdIwu3E58fq7HsbtfjG2aeuTDtOJ', 28 | :resource_type => 'presentation', 29 | :title => 'Flies Presentation', 30 | :document_feed_entry => dfe, 31 | ), 32 | stub( 33 | :resource_id => 'spreadsheet:GfVKovEtmvWtMgWDTv94kcYZLbje1O3q', 34 | :resource_type => 'spreadsheet', 35 | :title => 'Flies Spreadsheet', 36 | :document_feed_entry => dfe2, 37 | ), 38 | stub( 39 | :resource_id => 'drawing:ceKiP0rwNufPU3qguhleVIxmtXqMjaWe', 40 | :resource_type => 'drawing', 41 | :title => 'FliesBuzz Drawing', 42 | :document_feed_entry => dfe, 43 | ), 44 | stub( 45 | :resource_id => 'form:OCBeiWQf51qvmPdWVT8olFRqTM8pl5sC', 46 | :resource_type => 'form', 47 | :title => 'Spiders and Flies Form', 48 | :document_feed_entry => dfe, 49 | ), 50 | stub( 51 | :resource_id => 'document:gvZHG5YA5rkcgIXGUnDXhMOI0qE7WKnm', 52 | :resource_type => 'document', 53 | :title => 'Spiders Document', 54 | :document_feed_entry => dfe, 55 | ), 56 | stub( 57 | :resource_id => 'file:i7iqdRt3CxbJEWuo8kXIFBe9WeTnQRL3', 58 | :resource_type => 'file', 59 | :title => 'Spiders File', 60 | :document_feed_entry => dfe, 61 | ), 62 | ] 63 | 64 | @files_no_dirs = @files.select { |f| f.resource_type != 'folder' } 65 | 66 | @google_drive = stub( 67 | :files => @files 68 | ) 69 | 70 | GoogleDrive.stubs(:login).with('r.m.renfield', 'master').returns(@google_drive) 71 | end 72 | 73 | describe "#check" do 74 | before do 75 | Ryespy::Test::Redis::setup 76 | 77 | @notifier = mock() 78 | 79 | @goog_drv = Ryespy::Listener::GoogDrv.new( 80 | :username => 'r.m.renfield', 81 | :password => 'master', 82 | :notifiers => [@notifier], 83 | ) 84 | 85 | @redis = @goog_drv.instance_variable_get(:@redis) 86 | end 87 | 88 | after do 89 | @goog_drv.close 90 | 91 | Ryespy::Test::Redis::flush_namespace(@redis) 92 | end 93 | 94 | it "notifies when new files filter *" do 95 | @files_no_dirs.each do |file| 96 | @notifier.expects(:notify).with('RyespyGoogDrvJob', [file.resource_id]).once 97 | end 98 | 99 | @goog_drv.check('') 100 | end 101 | 102 | it "notifies when new files filter ^Flies\\b" do 103 | @files_no_dirs.select { |f| f.title =~ /^Flies\b/ }.each do |file| 104 | @notifier.expects(:notify).with('RyespyGoogDrvJob', [file.resource_id]).once 105 | end 106 | 107 | @goog_drv.check('^Flies\b') 108 | end 109 | 110 | it "notifies when new files filter Flies" do 111 | @files_no_dirs.select { |f| f.title =~ /Flies/ }.each do |file| 112 | @notifier.expects(:notify).with('RyespyGoogDrvJob', [file.resource_id]).once 113 | end 114 | 115 | @goog_drv.check('Flies') 116 | end 117 | 118 | it "doesn't notify when no new files" do 119 | @notifier.expects(:notify).times(3) 120 | 121 | @goog_drv.check('Spiders') 122 | @goog_drv.check('Spiders') 123 | end 124 | 125 | it "doesn't notify when no new files filter subset" do 126 | @notifier.expects(:notify).times(3) 127 | 128 | @goog_drv.check('Spiders') 129 | @goog_drv.check('Spiders\ and\ Flies') 130 | end 131 | 132 | it "notifies when new files filter distinct" do 133 | @notifier.expects(:notify).times(4) 134 | 135 | @goog_drv.check('Spiders\ and\ Flies') 136 | @goog_drv.check('Flies') 137 | end 138 | 139 | it "notifies when changed updated" do 140 | dfe = stub 141 | dfe.stubs(:css).with('updated').returns([stub(:text => '1899-03-23 01:00:00 UTC')]) 142 | dfe.stubs(:attribute).with('etag').returns('E13l2izwZbgWKfS5') 143 | 144 | @notifier.expects(:notify).times(4) 145 | 146 | @goog_drv.check('Flies') 147 | 148 | @files[1].stubs(:document_feed_entry).returns(dfe) 149 | 150 | @notifier.expects(:notify).with('RyespyGoogDrvJob', ['presentation:SuOQSdIwu3E58fq7HsbtfjG2aeuTDtOJ']).once 151 | 152 | @goog_drv.check('Flies') 153 | end 154 | 155 | it "notifies when changed etag" do 156 | dfe = stub 157 | dfe.stubs(:css).with('updated').returns([stub(:text => '1897-05-26 00:00:00 UTC')]) 158 | dfe.stubs(:attribute).with('etag').returns('qGE4UWIcITePyidC') 159 | 160 | @notifier.expects(:notify).times(4) 161 | 162 | @goog_drv.check('Flies') 163 | 164 | @files[3].stubs(:document_feed_entry).returns(dfe) 165 | 166 | @notifier.expects(:notify).with('RyespyGoogDrvJob', ['drawing:ceKiP0rwNufPU3qguhleVIxmtXqMjaWe']).once 167 | 168 | @goog_drv.check('Flies') 169 | end 170 | end 171 | 172 | end 173 | -------------------------------------------------------------------------------- /test/ryespy/listener/imap_test.rb: -------------------------------------------------------------------------------- 1 | require_relative '../../helper' 2 | 3 | require_relative '../../../lib/ryespy/listener/imap' 4 | 5 | 6 | describe Ryespy::Listener::IMAP do 7 | 8 | before do 9 | @uids = [6, 7, 42] 10 | 11 | @net_imap = stub 12 | 13 | @net_imap.stubs(:login).with('d.adams', 'solongandthanksforallthefish') 14 | @net_imap.stubs(:select).with('Dolphin') 15 | @net_imap.stubs(:uid_search).with('1:*').returns(@uids) 16 | @net_imap.stubs(:uid_search).with('43:*').returns([]) 17 | @net_imap.stubs(:disconnect) 18 | 19 | Net::IMAP.stubs(:new).with('imap.example.com', { 20 | :port => 42, 21 | :ssl => true, 22 | }).returns(@net_imap) 23 | end 24 | 25 | describe "#check" do 26 | before do 27 | Ryespy::Test::Redis::setup 28 | 29 | @notifier = mock() 30 | 31 | @imap = Ryespy::Listener::IMAP.new( 32 | :host => 'imap.example.com', 33 | :port => 42, 34 | :ssl => true, 35 | :username => 'd.adams', 36 | :password => 'solongandthanksforallthefish', 37 | :notifiers => [@notifier], 38 | ) 39 | 40 | @redis = @imap.instance_variable_get(:@redis) 41 | end 42 | 43 | after do 44 | @imap.close 45 | 46 | Ryespy::Test::Redis::flush_namespace(@redis) 47 | end 48 | 49 | it "notifies when new files" do 50 | @uids.each do |uid| 51 | @notifier.expects(:notify).with('RyespyIMAPJob', ['Dolphin', uid]).once 52 | end 53 | 54 | @imap.check('Dolphin') 55 | end 56 | 57 | it "doesn't notify when no new files" do 58 | @notifier.expects(:notify).times(3) 59 | 60 | @imap.check('Dolphin') 61 | 62 | @notifier.expects(:notify).never 63 | 64 | @imap.check('Dolphin') 65 | end 66 | end 67 | 68 | end 69 | -------------------------------------------------------------------------------- /test/ryespy/listener/rax_cf_test.rb: -------------------------------------------------------------------------------- 1 | require_relative '../../helper' 2 | 3 | require_relative '../../../lib/ryespy/listener/rax_cf' 4 | 5 | 6 | describe Ryespy::Listener::RaxCF do 7 | 8 | before do 9 | etag = 'dvYAPmJPy8nyqtR8hqqPYIagWEDuZ9FN' 10 | 11 | @files = [ 12 | stub(:content_length => 0, :content_type => 'application/directory', :key => 'abraham/', :etag => etag), 13 | stub(:content_length => 1, :content_type => 'text/plain', :key => 'abraham/a.txt', :etag => etag), 14 | stub(:content_length => 1, :content_type => 'text/plain', :key => 'abraham/b.txt', :etag => etag), 15 | stub(:content_length => 0, :content_type => 'application/directory', :key => 'a/', :etag => etag), 16 | stub(:content_length => 1, :content_type => 'text/plain', :key => 'a/abraham_README.txt', :etag => etag), 17 | stub(:content_length => 0, :content_type => 'application/directory', :key => 'van/', :etag => etag), 18 | stub(:content_length => 1, :content_type => 'text/plain', :key => 'van/van.txt', :etag => etag), 19 | ] 20 | 21 | @files_no_dirs = @files.select { |f| f.content_type != 'application/directory' } 22 | 23 | @fog_storage = stub 24 | 25 | @fog_storage_directories = stub 26 | 27 | @fog_storage_directories.stubs(:get).with('tmtiscnoa', { 28 | :prefix => 'abraham/' 29 | }).returns(stub(:files => @files.select { |f| f.key =~ /^abraham\// })) 30 | @fog_storage_directories.stubs(:get).with('tmtiscnoa', { 31 | :prefix => 'a' 32 | }).returns(stub(:files => @files.select { |f| f.key =~ /^a/ })) 33 | @fog_storage_directories.stubs(:get).with('tmtiscnoa', { 34 | :prefix => 'van/' 35 | }).returns(stub(:files => @files.select { |f| f.key =~ /^van\// })) 36 | @fog_storage_directories.stubs(:get).with('tmtiscnoa', { 37 | :prefix => '' 38 | }).returns(stub(:files => @files)) 39 | 40 | @fog_storage.stubs(:directories).returns(@fog_storage_directories) 41 | 42 | Fog::Storage.stubs(:new).with({ 43 | :provider => 'Rackspace', 44 | :rackspace_auth_url => 'https://lon.identity.api.rackspacecloud.com/v2.0', 45 | :rackspace_region => :lon, 46 | :rackspace_username => 'van.helsing', 47 | :rackspace_api_key => 'M.D., D.Ph., D.Litt., etc.', 48 | }).returns(@fog_storage) 49 | end 50 | 51 | describe "#check" do 52 | before do 53 | Ryespy::Test::Redis::setup 54 | 55 | @notifier = mock() 56 | 57 | @rax_cf = Ryespy::Listener::RaxCF.new( 58 | :endpoint => 'uk', 59 | :region => 'lon', 60 | :username => 'van.helsing', 61 | :api_key => 'M.D., D.Ph., D.Litt., etc.', 62 | :container => 'tmtiscnoa', 63 | :notifiers => [@notifier], 64 | ) 65 | 66 | @redis = @rax_cf.instance_variable_get(:@redis) 67 | end 68 | 69 | after do 70 | @rax_cf.close 71 | 72 | Ryespy::Test::Redis::flush_namespace(@redis) 73 | end 74 | 75 | it "notifies when new files prefix *" do 76 | @files_no_dirs.each do |file| 77 | @notifier.expects(:notify).with('RyespyRaxCFJob', [file.key]).once 78 | end 79 | 80 | @rax_cf.check('') 81 | end 82 | 83 | it "notifies when new files prefix van/" do 84 | @files_no_dirs.select { |f| f.key =~ /^van\// }.each do |file| 85 | @notifier.expects(:notify).with('RyespyRaxCFJob', [file.key]).once 86 | end 87 | 88 | @rax_cf.check('van/') 89 | end 90 | 91 | it "notifies when new files prefix a" do 92 | @files_no_dirs.select { |f| f.key =~ /^a/ }.each do |file| 93 | @notifier.expects(:notify).with('RyespyRaxCFJob', [file.key]).once 94 | end 95 | 96 | @rax_cf.check('a') 97 | end 98 | 99 | it "doesn't notify when no new files" do 100 | @notifier.expects(:notify).times(2) 101 | 102 | @rax_cf.check('abraham/') 103 | 104 | @notifier.expects(:notify).never 105 | 106 | @rax_cf.check('abraham/') 107 | end 108 | 109 | it "doesn't notify when no new files prefix subset" do 110 | @notifier.expects(:notify).times(3) 111 | 112 | @rax_cf.check('a') 113 | 114 | @notifier.expects(:notify).never 115 | 116 | @rax_cf.check('abraham/') 117 | end 118 | 119 | it "notifies when new files prefix distinct" do 120 | @notifier.expects(:notify).times(3) 121 | 122 | @rax_cf.check('a') 123 | 124 | @notifier.expects(:notify).times(1) 125 | 126 | @rax_cf.check('van/') 127 | end 128 | 129 | it "notifies when changed etag" do 130 | @notifier.expects(:notify).times(2) 131 | 132 | @rax_cf.check('abraham/') 133 | 134 | @files[1].stubs(:etag).returns(-2303600400) 135 | 136 | @notifier.expects(:notify).with('RyespyRaxCFJob', ['abraham/a.txt']).once 137 | 138 | @rax_cf.check('abraham/') 139 | end 140 | end 141 | 142 | end 143 | -------------------------------------------------------------------------------- /test/ryespy/notifier/sidekiq_test.rb: -------------------------------------------------------------------------------- 1 | require_relative '../../helper' 2 | 3 | require_relative '../../../lib/ryespy/notifier/sidekiq' 4 | 5 | 6 | describe Ryespy::Notifier::Sidekiq do 7 | 8 | describe "#notify" do 9 | before do 10 | @sidekiq = Ryespy::Notifier::Sidekiq.new( 11 | :namespace => Ryespy::Test::Redis::namespace 12 | ) 13 | 14 | @redis = @sidekiq.instance_variable_get(:@redis) 15 | end 16 | 17 | after do 18 | @sidekiq.close 19 | 20 | Ryespy::Test::Redis::flush_namespace(@redis) 21 | end 22 | 23 | it "writes to queues set" do 24 | @sidekiq.notify(nil, nil) 25 | 26 | @redis.smembers('queues').must_equal ['ryespy'] 27 | end 28 | 29 | it "writes to queue list" do 30 | SecureRandom.stubs(:hex).returns('9c964160d25fdf24c6549e6d') 31 | 32 | @sidekiq.notify('PlanetClass', { :brain => 'marvin' }) 33 | 34 | @redis.lrange('queue:ryespy', 0, -1).must_equal([{ 35 | 'class' => 'PlanetClass', 36 | 'args' => { 'brain' => 'marvin' }, 37 | 'queue' => 'ryespy', 38 | 'retry' => true, 39 | 'jid' => '9c964160d25fdf24c6549e6d', 40 | }.to_json]) 41 | end 42 | end 43 | 44 | end 45 | -------------------------------------------------------------------------------- /test/ryespy/version_test.rb: -------------------------------------------------------------------------------- 1 | require_relative '../helper' 2 | 3 | require_relative '../../lib/ryespy/version' 4 | 5 | 6 | describe "Ryespy::VERSION" do 7 | 8 | it "uses major.minor.patch" do 9 | Ryespy::VERSION.must_match /\A\d+\.\d+\.\d+\z/ 10 | end 11 | 12 | end 13 | --------------------------------------------------------------------------------