├── .drone.yml ├── .env.test ├── .github └── workflows │ └── ci.yml ├── .gitignore ├── .ruby-version ├── Gemfile ├── Gemfile.lock ├── LICENSE.md ├── Procfile ├── README.md ├── Rakefile ├── bin ├── console ├── pigo_darwin_arm64 ├── pigo_darwin_x86_64 ├── pigo_linux_x86_64 └── rake ├── lib ├── crawler │ ├── image │ │ ├── cache.rb │ │ ├── cascade │ │ │ └── facefinder │ │ ├── download.rb │ │ ├── download │ │ │ ├── default.rb │ │ │ ├── instagram.rb │ │ │ ├── vimeo.rb │ │ │ └── youtube.rb │ │ ├── download_cache.rb │ │ ├── helpers.rb │ │ ├── image_processor.rb │ │ ├── initializers │ │ │ ├── constants.rb │ │ │ ├── down.rb │ │ │ ├── librato.rb │ │ │ ├── sidekiq.rb │ │ │ ├── storage.rb │ │ │ └── worker_stat.rb │ │ ├── jobs │ │ │ ├── find_image.rb │ │ │ ├── process_image.rb │ │ │ └── upload_image.rb │ │ ├── meta_images.rb │ │ ├── meta_images_cache.rb │ │ └── timer.rb │ └── refresher │ │ ├── cache.rb │ │ ├── feed.rb │ │ ├── feed_status.rb │ │ ├── http_cache.rb │ │ ├── initializers │ │ ├── redis.rb │ │ └── sidekiq.rb │ │ ├── jobs │ │ ├── feed_downloader.rb │ │ └── feed_status_update.rb │ │ ├── redirect_cache.rb │ │ └── throttle.rb ├── image.rb └── refresher.rb └── test ├── cache_test.rb ├── download ├── default.rb ├── instagram.rb ├── vimeo.rb └── youtube.rb ├── download_cache_test.rb ├── download_test.rb ├── feed_downloader_test.rb ├── feed_status_test.rb ├── feed_status_update_test.rb ├── feed_test.rb ├── image_test.rb ├── jobs ├── find_image_test.rb ├── process_image_test.rb └── upload_image_test.rb ├── meta_images_cache_test.rb ├── meta_images_test.rb ├── redirect_cache_test.rb ├── redirect_test.rb ├── support └── www │ ├── atom.xml │ ├── feed.json │ ├── html.html │ └── image.jpeg ├── test_helper.rb ├── throttle_test.rb └── timer_test.rb /.drone.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/.drone.yml -------------------------------------------------------------------------------- /.env.test: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/.env.test -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/.github/workflows/ci.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/.gitignore -------------------------------------------------------------------------------- /.ruby-version: -------------------------------------------------------------------------------- 1 | 3.1.2 -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/Gemfile -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/Gemfile.lock -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/LICENSE.md -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/Procfile -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/README.md -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/Rakefile -------------------------------------------------------------------------------- /bin/console: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/bin/console -------------------------------------------------------------------------------- /bin/pigo_darwin_arm64: -------------------------------------------------------------------------------- 1 | pigo_darwin_x86_64 -------------------------------------------------------------------------------- /bin/pigo_darwin_x86_64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/bin/pigo_darwin_x86_64 -------------------------------------------------------------------------------- /bin/pigo_linux_x86_64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/bin/pigo_linux_x86_64 -------------------------------------------------------------------------------- /bin/rake: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | bundle exec rake -------------------------------------------------------------------------------- /lib/crawler/image/cache.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/lib/crawler/image/cache.rb -------------------------------------------------------------------------------- /lib/crawler/image/cascade/facefinder: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/lib/crawler/image/cascade/facefinder -------------------------------------------------------------------------------- /lib/crawler/image/download.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/lib/crawler/image/download.rb -------------------------------------------------------------------------------- /lib/crawler/image/download/default.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/lib/crawler/image/download/default.rb -------------------------------------------------------------------------------- /lib/crawler/image/download/instagram.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/lib/crawler/image/download/instagram.rb -------------------------------------------------------------------------------- /lib/crawler/image/download/vimeo.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/lib/crawler/image/download/vimeo.rb -------------------------------------------------------------------------------- /lib/crawler/image/download/youtube.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/lib/crawler/image/download/youtube.rb -------------------------------------------------------------------------------- /lib/crawler/image/download_cache.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/lib/crawler/image/download_cache.rb -------------------------------------------------------------------------------- /lib/crawler/image/helpers.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/lib/crawler/image/helpers.rb -------------------------------------------------------------------------------- /lib/crawler/image/image_processor.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/lib/crawler/image/image_processor.rb -------------------------------------------------------------------------------- /lib/crawler/image/initializers/constants.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/lib/crawler/image/initializers/constants.rb -------------------------------------------------------------------------------- /lib/crawler/image/initializers/down.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | Down.backend :http 4 | -------------------------------------------------------------------------------- /lib/crawler/image/initializers/librato.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | Librato.tracker.start! 4 | -------------------------------------------------------------------------------- /lib/crawler/image/initializers/sidekiq.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/lib/crawler/image/initializers/sidekiq.rb -------------------------------------------------------------------------------- /lib/crawler/image/initializers/storage.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/lib/crawler/image/initializers/storage.rb -------------------------------------------------------------------------------- /lib/crawler/image/initializers/worker_stat.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/lib/crawler/image/initializers/worker_stat.rb -------------------------------------------------------------------------------- /lib/crawler/image/jobs/find_image.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/lib/crawler/image/jobs/find_image.rb -------------------------------------------------------------------------------- /lib/crawler/image/jobs/process_image.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/lib/crawler/image/jobs/process_image.rb -------------------------------------------------------------------------------- /lib/crawler/image/jobs/upload_image.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/lib/crawler/image/jobs/upload_image.rb -------------------------------------------------------------------------------- /lib/crawler/image/meta_images.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/lib/crawler/image/meta_images.rb -------------------------------------------------------------------------------- /lib/crawler/image/meta_images_cache.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/lib/crawler/image/meta_images_cache.rb -------------------------------------------------------------------------------- /lib/crawler/image/timer.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/lib/crawler/image/timer.rb -------------------------------------------------------------------------------- /lib/crawler/refresher/cache.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/lib/crawler/refresher/cache.rb -------------------------------------------------------------------------------- /lib/crawler/refresher/feed.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/lib/crawler/refresher/feed.rb -------------------------------------------------------------------------------- /lib/crawler/refresher/feed_status.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/lib/crawler/refresher/feed_status.rb -------------------------------------------------------------------------------- /lib/crawler/refresher/http_cache.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/lib/crawler/refresher/http_cache.rb -------------------------------------------------------------------------------- /lib/crawler/refresher/initializers/redis.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/lib/crawler/refresher/initializers/redis.rb -------------------------------------------------------------------------------- /lib/crawler/refresher/initializers/sidekiq.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/lib/crawler/refresher/initializers/sidekiq.rb -------------------------------------------------------------------------------- /lib/crawler/refresher/jobs/feed_downloader.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/lib/crawler/refresher/jobs/feed_downloader.rb -------------------------------------------------------------------------------- /lib/crawler/refresher/jobs/feed_status_update.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/lib/crawler/refresher/jobs/feed_status_update.rb -------------------------------------------------------------------------------- /lib/crawler/refresher/redirect_cache.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/lib/crawler/refresher/redirect_cache.rb -------------------------------------------------------------------------------- /lib/crawler/refresher/throttle.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/lib/crawler/refresher/throttle.rb -------------------------------------------------------------------------------- /lib/image.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/lib/image.rb -------------------------------------------------------------------------------- /lib/refresher.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/lib/refresher.rb -------------------------------------------------------------------------------- /test/cache_test.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/test/cache_test.rb -------------------------------------------------------------------------------- /test/download/default.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/test/download/default.rb -------------------------------------------------------------------------------- /test/download/instagram.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/test/download/instagram.rb -------------------------------------------------------------------------------- /test/download/vimeo.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/test/download/vimeo.rb -------------------------------------------------------------------------------- /test/download/youtube.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/test/download/youtube.rb -------------------------------------------------------------------------------- /test/download_cache_test.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/test/download_cache_test.rb -------------------------------------------------------------------------------- /test/download_test.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/test/download_test.rb -------------------------------------------------------------------------------- /test/feed_downloader_test.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/test/feed_downloader_test.rb -------------------------------------------------------------------------------- /test/feed_status_test.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/test/feed_status_test.rb -------------------------------------------------------------------------------- /test/feed_status_update_test.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/test/feed_status_update_test.rb -------------------------------------------------------------------------------- /test/feed_test.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/test/feed_test.rb -------------------------------------------------------------------------------- /test/image_test.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/test/image_test.rb -------------------------------------------------------------------------------- /test/jobs/find_image_test.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/test/jobs/find_image_test.rb -------------------------------------------------------------------------------- /test/jobs/process_image_test.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/test/jobs/process_image_test.rb -------------------------------------------------------------------------------- /test/jobs/upload_image_test.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/test/jobs/upload_image_test.rb -------------------------------------------------------------------------------- /test/meta_images_cache_test.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/test/meta_images_cache_test.rb -------------------------------------------------------------------------------- /test/meta_images_test.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/test/meta_images_test.rb -------------------------------------------------------------------------------- /test/redirect_cache_test.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/test/redirect_cache_test.rb -------------------------------------------------------------------------------- /test/redirect_test.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/test/redirect_test.rb -------------------------------------------------------------------------------- /test/support/www/atom.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/test/support/www/atom.xml -------------------------------------------------------------------------------- /test/support/www/feed.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/test/support/www/feed.json -------------------------------------------------------------------------------- /test/support/www/html.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/test/support/www/html.html -------------------------------------------------------------------------------- /test/support/www/image.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/test/support/www/image.jpeg -------------------------------------------------------------------------------- /test/test_helper.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/test/test_helper.rb -------------------------------------------------------------------------------- /test/throttle_test.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/test/throttle_test.rb -------------------------------------------------------------------------------- /test/timer_test.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/crawler/HEAD/test/timer_test.rb --------------------------------------------------------------------------------