├── .gitignore ├── .guardrc ├── .ruby-gemset ├── .ruby-version ├── Gemfile ├── Gemfile.lock ├── Guardfile ├── README.md ├── Rakefile ├── app ├── controllers │ ├── application_controller.rb │ └── v1 │ │ ├── access_controller.rb │ │ ├── batch_controller.rb │ │ ├── match_controller.rb │ │ ├── record_controller.rb │ │ ├── search_controller.rb │ │ ├── status_controller.rb │ │ └── trends_controller.rb ├── helpers │ ├── counts_helper.rb │ ├── open_graph_helper.rb │ ├── page_helper.rb │ └── schema_org_helper.rb ├── models │ ├── api │ │ └── v1.rb │ ├── cloud.rb │ ├── crawl │ │ ├── base.rb │ │ ├── capture.rb │ │ ├── google.rb │ │ ├── sitemap.rb │ │ └── social.rb │ ├── flattener.rb │ ├── page │ │ ├── base.rb │ │ ├── parse.rb │ │ └── url.rb │ ├── persist.rb │ └── record │ │ ├── addons.rb │ │ ├── base.rb │ │ ├── export.rb │ │ ├── match.rb │ │ ├── screenshot.rb │ │ ├── search.rb │ │ ├── trends.rb │ │ └── upload.rb ├── sites │ └── .keep └── workers │ ├── crawler │ ├── base.rb │ ├── sampler.rb │ ├── scraper.rb │ ├── screener.rb │ ├── scrimper.rb │ ├── sitemapper.rb │ ├── slider.rb │ ├── socializer.rb │ ├── spider.rb │ └── stretcher.rb │ ├── mapper │ ├── base.rb │ ├── cleaner.rb │ ├── id_availability.rb │ ├── indexer.rb │ └── url_availability.rb │ ├── recorder │ ├── base.rb │ ├── collector.rb │ ├── fixer.rb │ └── uploader.rb │ ├── scheduler │ ├── base.rb │ ├── clearer.rb │ └── reindexer.rb │ ├── syncer │ ├── base.rb │ ├── mover.rb │ ├── refixer.rb │ ├── reindexer.rb │ ├── resampler.rb │ ├── rescreener.rb │ ├── rescrimper.rb │ ├── reslider.rb │ ├── resocializer.rb │ └── respider.rb │ └── worker.rb ├── bin ├── bundle ├── rails ├── rake └── spring ├── config.ru ├── config ├── application.rb ├── boot.rb ├── config.yml.example ├── database.yml ├── deploy.rb ├── environment.rb ├── environments │ ├── development.rb │ ├── production.rb │ └── test.rb ├── initializers │ ├── assets.rb │ ├── backtrace_silencers.rb │ ├── cookies_serializer.rb │ ├── elasticsearch.rb │ ├── filter_parameter_logging.rb │ ├── inflections.rb │ ├── mime_types.rb │ ├── redis.rb │ ├── session_store.rb │ ├── sidekiq.rb │ ├── vcr.rb │ └── wrap_parameters.rb ├── locales │ └── en.yml ├── routes.rb ├── secrets.yml.example ├── sidekiq-slim.yml.example ├── sidekiq.yml.example ├── sitemap.rb └── unicorn.rb ├── lib └── tasks │ ├── crawl.rake │ ├── map.rake │ ├── report.rake │ └── sync.rake ├── log └── .keep ├── production.yml ├── restart.sh ├── roles ├── build-ruby │ ├── README.md │ ├── defaults │ │ └── main.yml │ ├── handlers │ │ └── main.yml │ ├── meta │ │ └── main.yml │ ├── tasks │ │ └── main.yml │ └── vars │ │ └── main.yml ├── chruby │ ├── .gitignore │ ├── .travis.yml │ ├── LICENSE │ ├── README.md │ ├── Vagrantfile │ ├── defaults │ │ └── main.yml │ ├── meta │ │ └── main.yml │ ├── tasks │ │ └── main.yml │ ├── templates │ │ ├── chruby.fact │ │ └── chruby.sh │ └── test.yml ├── elasticsearch │ ├── .gitignore │ ├── .kitchen.yml │ ├── Gemfile │ ├── Gemfile.lock │ ├── README.md │ ├── ansible.cfg │ ├── defaults │ │ └── main.yml │ ├── files │ │ ├── scripts │ │ │ └── calculate-score.groovy │ │ └── templates │ │ │ └── basic.json │ ├── filter_plugins │ │ └── custom.py │ ├── handlers │ │ └── main.yml │ ├── meta │ │ └── main.yml │ ├── tasks │ │ ├── checkParameters.yml │ │ ├── elasticsearch-Debian-version-lock.yml │ │ ├── elasticsearch-Debian.yml │ │ ├── elasticsearch-RedHat-version-lock.yml │ │ ├── elasticsearch-RedHat.yml │ │ ├── elasticsearch-config.yml │ │ ├── elasticsearch-optional-user.yml │ │ ├── elasticsearch-plugins.yml │ │ ├── elasticsearch-scripts.yml │ │ ├── elasticsearch-service.yml │ │ ├── elasticsearch-templates.yml │ │ ├── elasticsearch-version-lock.yml │ │ ├── elasticsearch.yml │ │ ├── java.yml │ │ └── main.yml │ ├── templates │ │ ├── elasticsearch.j2 │ │ ├── elasticsearch.repo │ │ ├── elasticsearch.yml.j2 │ │ ├── init │ │ │ ├── debian │ │ │ │ └── elasticsearch.j2 │ │ │ └── redhat │ │ │ │ └── elasticsearch.j2 │ │ ├── logging.yml.j2 │ │ └── systemd │ │ │ └── elasticsearch.j2 │ ├── test │ │ └── integration │ │ │ ├── config-1x │ │ │ ├── config.yml │ │ │ └── serverspec │ │ │ │ └── default_spec.rb │ │ │ ├── config-2x │ │ │ ├── config.yml │ │ │ └── serverspec │ │ │ │ └── default_spec.rb │ │ │ ├── config.yml │ │ │ ├── helpers │ │ │ └── serverspec │ │ │ │ ├── Gemfile │ │ │ │ ├── config_spec.rb │ │ │ │ ├── multi_spec.rb │ │ │ │ ├── package_spec.rb │ │ │ │ ├── spec_helper.rb │ │ │ │ └── standard_spec.rb │ │ │ ├── multi-1x │ │ │ ├── multi.yml │ │ │ └── serverspec │ │ │ │ └── default_spec.rb │ │ │ ├── multi-2x │ │ │ ├── multi.yml │ │ │ └── serverspec │ │ │ │ └── default_spec.rb │ │ │ ├── multi.yml │ │ │ ├── package-1x │ │ │ ├── package.yaml │ │ │ └── serverspec │ │ │ │ └── default_spec.rb │ │ │ ├── package-2x │ │ │ ├── package.yaml │ │ │ └── serverspec │ │ │ │ └── default_spec.rb │ │ │ ├── package.yml │ │ │ ├── standard-1x │ │ │ ├── serverspec │ │ │ │ └── default_spec.rb │ │ │ └── standard.yml │ │ │ ├── standard-2x │ │ │ ├── serverspec │ │ │ │ └── default_spec.rb │ │ │ └── standard.yml │ │ │ └── standard.yml │ └── vars │ │ ├── Debian.yml │ │ ├── RedHat.yml │ │ └── main.yml ├── imagemagick │ ├── .gitignore │ ├── README.md │ ├── meta │ │ └── main.yml │ └── tasks │ │ └── main.yml ├── letsencrypt │ ├── LICENSE │ ├── README.md │ ├── defaults │ │ └── main.yml │ ├── meta │ │ ├── .galaxy_install_info │ │ └── main.yml │ └── tasks │ │ └── main.yml ├── logrotate │ ├── .travis.yml │ ├── LICENSE │ ├── README.md │ ├── defaults │ │ └── main.yml │ ├── meta │ │ └── main.yml │ ├── tasks │ │ └── main.yml │ ├── templates │ │ └── logrotate.d.j2 │ └── tests │ │ ├── inventory │ │ └── test.yml ├── nginx-unicorn │ ├── LICENSE │ ├── README.md │ ├── defaults │ │ └── main.yml │ ├── handlers │ │ └── main.yml │ ├── meta │ │ └── main.yml │ ├── tasks │ │ ├── debian.yml │ │ ├── main.yml │ │ └── redhat.yml │ └── templates │ │ └── nginx-site.j2 ├── ruby-install │ ├── .gitignore │ ├── .travis.yml │ ├── LICENSE │ ├── README.md │ ├── Vagrantfile │ ├── defaults │ │ └── main.yml │ ├── meta │ │ ├── .galaxy_install_info │ │ └── main.yml │ ├── tasks │ │ └── main.yml │ ├── templates │ │ └── ruby_install.fact │ └── test.yml ├── swapfile │ ├── LICENSE │ ├── README.md │ ├── defaults │ │ └── main.yml │ ├── handlers │ │ └── main.yml │ ├── meta │ │ └── main.yml │ └── tasks │ │ └── main.yml └── ubuntu-common │ ├── README.md │ ├── defaults │ └── main.yml │ ├── meta │ └── main.yml │ ├── tasks │ └── main.yml │ └── templates │ ├── locale │ └── sources.list ├── run └── test ├── controllers └── .keep ├── fixtures └── .keep ├── helpers └── .keep ├── integration └── .keep ├── mailers └── .keep ├── models ├── .keep └── url_test.rb └── test_helper.rb /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/.gitignore -------------------------------------------------------------------------------- /.guardrc: -------------------------------------------------------------------------------- 1 | require File.expand_path('../config/environment', __FILE__) 2 | -------------------------------------------------------------------------------- /.ruby-gemset: -------------------------------------------------------------------------------- 1 | skynet 2 | -------------------------------------------------------------------------------- /.ruby-version: -------------------------------------------------------------------------------- 1 | 2.7.2 2 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/Gemfile -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/Gemfile.lock -------------------------------------------------------------------------------- /Guardfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/Guardfile -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/README.md -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/Rakefile -------------------------------------------------------------------------------- /app/controllers/application_controller.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/controllers/application_controller.rb -------------------------------------------------------------------------------- /app/controllers/v1/access_controller.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/controllers/v1/access_controller.rb -------------------------------------------------------------------------------- /app/controllers/v1/batch_controller.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/controllers/v1/batch_controller.rb -------------------------------------------------------------------------------- /app/controllers/v1/match_controller.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/controllers/v1/match_controller.rb -------------------------------------------------------------------------------- /app/controllers/v1/record_controller.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/controllers/v1/record_controller.rb -------------------------------------------------------------------------------- /app/controllers/v1/search_controller.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/controllers/v1/search_controller.rb -------------------------------------------------------------------------------- /app/controllers/v1/status_controller.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/controllers/v1/status_controller.rb -------------------------------------------------------------------------------- /app/controllers/v1/trends_controller.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/controllers/v1/trends_controller.rb -------------------------------------------------------------------------------- /app/helpers/counts_helper.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/helpers/counts_helper.rb -------------------------------------------------------------------------------- /app/helpers/open_graph_helper.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/helpers/open_graph_helper.rb -------------------------------------------------------------------------------- /app/helpers/page_helper.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/helpers/page_helper.rb -------------------------------------------------------------------------------- /app/helpers/schema_org_helper.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/helpers/schema_org_helper.rb -------------------------------------------------------------------------------- /app/models/api/v1.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/models/api/v1.rb -------------------------------------------------------------------------------- /app/models/cloud.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/models/cloud.rb -------------------------------------------------------------------------------- /app/models/crawl/base.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/models/crawl/base.rb -------------------------------------------------------------------------------- /app/models/crawl/capture.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/models/crawl/capture.rb -------------------------------------------------------------------------------- /app/models/crawl/google.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/models/crawl/google.rb -------------------------------------------------------------------------------- /app/models/crawl/sitemap.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/models/crawl/sitemap.rb -------------------------------------------------------------------------------- /app/models/crawl/social.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/models/crawl/social.rb -------------------------------------------------------------------------------- /app/models/flattener.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/models/flattener.rb -------------------------------------------------------------------------------- /app/models/page/base.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/models/page/base.rb -------------------------------------------------------------------------------- /app/models/page/parse.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/models/page/parse.rb -------------------------------------------------------------------------------- /app/models/page/url.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/models/page/url.rb -------------------------------------------------------------------------------- /app/models/persist.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/models/persist.rb -------------------------------------------------------------------------------- /app/models/record/addons.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/models/record/addons.rb -------------------------------------------------------------------------------- /app/models/record/base.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/models/record/base.rb -------------------------------------------------------------------------------- /app/models/record/export.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/models/record/export.rb -------------------------------------------------------------------------------- /app/models/record/match.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/models/record/match.rb -------------------------------------------------------------------------------- /app/models/record/screenshot.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/models/record/screenshot.rb -------------------------------------------------------------------------------- /app/models/record/search.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/models/record/search.rb -------------------------------------------------------------------------------- /app/models/record/trends.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/models/record/trends.rb -------------------------------------------------------------------------------- /app/models/record/upload.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/models/record/upload.rb -------------------------------------------------------------------------------- /app/sites/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/workers/crawler/base.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/workers/crawler/base.rb -------------------------------------------------------------------------------- /app/workers/crawler/sampler.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/workers/crawler/sampler.rb -------------------------------------------------------------------------------- /app/workers/crawler/scraper.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/workers/crawler/scraper.rb -------------------------------------------------------------------------------- /app/workers/crawler/screener.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/workers/crawler/screener.rb -------------------------------------------------------------------------------- /app/workers/crawler/scrimper.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/workers/crawler/scrimper.rb -------------------------------------------------------------------------------- /app/workers/crawler/sitemapper.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/workers/crawler/sitemapper.rb -------------------------------------------------------------------------------- /app/workers/crawler/slider.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/workers/crawler/slider.rb -------------------------------------------------------------------------------- /app/workers/crawler/socializer.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/workers/crawler/socializer.rb -------------------------------------------------------------------------------- /app/workers/crawler/spider.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/workers/crawler/spider.rb -------------------------------------------------------------------------------- /app/workers/crawler/stretcher.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/workers/crawler/stretcher.rb -------------------------------------------------------------------------------- /app/workers/mapper/base.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/workers/mapper/base.rb -------------------------------------------------------------------------------- /app/workers/mapper/cleaner.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/workers/mapper/cleaner.rb -------------------------------------------------------------------------------- /app/workers/mapper/id_availability.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/workers/mapper/id_availability.rb -------------------------------------------------------------------------------- /app/workers/mapper/indexer.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/workers/mapper/indexer.rb -------------------------------------------------------------------------------- /app/workers/mapper/url_availability.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/workers/mapper/url_availability.rb -------------------------------------------------------------------------------- /app/workers/recorder/base.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/workers/recorder/base.rb -------------------------------------------------------------------------------- /app/workers/recorder/collector.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/workers/recorder/collector.rb -------------------------------------------------------------------------------- /app/workers/recorder/fixer.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/workers/recorder/fixer.rb -------------------------------------------------------------------------------- /app/workers/recorder/uploader.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/workers/recorder/uploader.rb -------------------------------------------------------------------------------- /app/workers/scheduler/base.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/workers/scheduler/base.rb -------------------------------------------------------------------------------- /app/workers/scheduler/clearer.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/workers/scheduler/clearer.rb -------------------------------------------------------------------------------- /app/workers/scheduler/reindexer.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/workers/scheduler/reindexer.rb -------------------------------------------------------------------------------- /app/workers/syncer/base.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/workers/syncer/base.rb -------------------------------------------------------------------------------- /app/workers/syncer/mover.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/workers/syncer/mover.rb -------------------------------------------------------------------------------- /app/workers/syncer/refixer.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/workers/syncer/refixer.rb -------------------------------------------------------------------------------- /app/workers/syncer/reindexer.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/workers/syncer/reindexer.rb -------------------------------------------------------------------------------- /app/workers/syncer/resampler.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/workers/syncer/resampler.rb -------------------------------------------------------------------------------- /app/workers/syncer/rescreener.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/workers/syncer/rescreener.rb -------------------------------------------------------------------------------- /app/workers/syncer/rescrimper.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/workers/syncer/rescrimper.rb -------------------------------------------------------------------------------- /app/workers/syncer/reslider.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/workers/syncer/reslider.rb -------------------------------------------------------------------------------- /app/workers/syncer/resocializer.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/workers/syncer/resocializer.rb -------------------------------------------------------------------------------- /app/workers/syncer/respider.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/workers/syncer/respider.rb -------------------------------------------------------------------------------- /app/workers/worker.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/app/workers/worker.rb -------------------------------------------------------------------------------- /bin/bundle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/bin/bundle -------------------------------------------------------------------------------- /bin/rails: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/bin/rails -------------------------------------------------------------------------------- /bin/rake: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/bin/rake -------------------------------------------------------------------------------- /bin/spring: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/bin/spring -------------------------------------------------------------------------------- /config.ru: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/config.ru -------------------------------------------------------------------------------- /config/application.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/config/application.rb -------------------------------------------------------------------------------- /config/boot.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/config/boot.rb -------------------------------------------------------------------------------- /config/config.yml.example: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/config/config.yml.example -------------------------------------------------------------------------------- /config/database.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/config/database.yml -------------------------------------------------------------------------------- /config/deploy.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/config/deploy.rb -------------------------------------------------------------------------------- /config/environment.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/config/environment.rb -------------------------------------------------------------------------------- /config/environments/development.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/config/environments/development.rb -------------------------------------------------------------------------------- /config/environments/production.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/config/environments/production.rb -------------------------------------------------------------------------------- /config/environments/test.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/config/environments/test.rb -------------------------------------------------------------------------------- /config/initializers/assets.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/config/initializers/assets.rb -------------------------------------------------------------------------------- /config/initializers/backtrace_silencers.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/config/initializers/backtrace_silencers.rb -------------------------------------------------------------------------------- /config/initializers/cookies_serializer.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/config/initializers/cookies_serializer.rb -------------------------------------------------------------------------------- /config/initializers/elasticsearch.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/config/initializers/elasticsearch.rb -------------------------------------------------------------------------------- /config/initializers/filter_parameter_logging.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/config/initializers/filter_parameter_logging.rb -------------------------------------------------------------------------------- /config/initializers/inflections.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/config/initializers/inflections.rb -------------------------------------------------------------------------------- /config/initializers/mime_types.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/config/initializers/mime_types.rb -------------------------------------------------------------------------------- /config/initializers/redis.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/config/initializers/redis.rb -------------------------------------------------------------------------------- /config/initializers/session_store.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/config/initializers/session_store.rb -------------------------------------------------------------------------------- /config/initializers/sidekiq.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/config/initializers/sidekiq.rb -------------------------------------------------------------------------------- /config/initializers/vcr.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/config/initializers/vcr.rb -------------------------------------------------------------------------------- /config/initializers/wrap_parameters.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/config/initializers/wrap_parameters.rb -------------------------------------------------------------------------------- /config/locales/en.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/config/locales/en.yml -------------------------------------------------------------------------------- /config/routes.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/config/routes.rb -------------------------------------------------------------------------------- /config/secrets.yml.example: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/config/secrets.yml.example -------------------------------------------------------------------------------- /config/sidekiq-slim.yml.example: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/config/sidekiq-slim.yml.example -------------------------------------------------------------------------------- /config/sidekiq.yml.example: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/config/sidekiq.yml.example -------------------------------------------------------------------------------- /config/sitemap.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/config/sitemap.rb -------------------------------------------------------------------------------- /config/unicorn.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/config/unicorn.rb -------------------------------------------------------------------------------- /lib/tasks/crawl.rake: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/lib/tasks/crawl.rake -------------------------------------------------------------------------------- /lib/tasks/map.rake: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/lib/tasks/map.rake -------------------------------------------------------------------------------- /lib/tasks/report.rake: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/lib/tasks/report.rake -------------------------------------------------------------------------------- /lib/tasks/sync.rake: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/lib/tasks/sync.rake -------------------------------------------------------------------------------- /log/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /production.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/production.yml -------------------------------------------------------------------------------- /restart.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/restart.sh -------------------------------------------------------------------------------- /roles/build-ruby/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/build-ruby/README.md -------------------------------------------------------------------------------- /roles/build-ruby/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: 2.2.1 3 | -------------------------------------------------------------------------------- /roles/build-ruby/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # handlers file for build-ruby 3 | -------------------------------------------------------------------------------- /roles/build-ruby/meta/main.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/build-ruby/meta/main.yml -------------------------------------------------------------------------------- /roles/build-ruby/tasks/main.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/build-ruby/tasks/main.yml -------------------------------------------------------------------------------- /roles/build-ruby/vars/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # vars file for build-ruby 3 | -------------------------------------------------------------------------------- /roles/chruby/.gitignore: -------------------------------------------------------------------------------- 1 | .vagrant 2 | -------------------------------------------------------------------------------- /roles/chruby/.travis.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/chruby/.travis.yml -------------------------------------------------------------------------------- /roles/chruby/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/chruby/LICENSE -------------------------------------------------------------------------------- /roles/chruby/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/chruby/README.md -------------------------------------------------------------------------------- /roles/chruby/Vagrantfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/chruby/Vagrantfile -------------------------------------------------------------------------------- /roles/chruby/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | chruby_version: '0.3.9' 3 | -------------------------------------------------------------------------------- /roles/chruby/meta/main.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/chruby/meta/main.yml -------------------------------------------------------------------------------- /roles/chruby/tasks/main.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/chruby/tasks/main.yml -------------------------------------------------------------------------------- /roles/chruby/templates/chruby.fact: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/chruby/templates/chruby.fact -------------------------------------------------------------------------------- /roles/chruby/templates/chruby.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/chruby/templates/chruby.sh -------------------------------------------------------------------------------- /roles/chruby/test.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/chruby/test.yml -------------------------------------------------------------------------------- /roles/elasticsearch/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/.gitignore -------------------------------------------------------------------------------- /roles/elasticsearch/.kitchen.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/.kitchen.yml -------------------------------------------------------------------------------- /roles/elasticsearch/Gemfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/Gemfile -------------------------------------------------------------------------------- /roles/elasticsearch/Gemfile.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/Gemfile.lock -------------------------------------------------------------------------------- /roles/elasticsearch/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/README.md -------------------------------------------------------------------------------- /roles/elasticsearch/ansible.cfg: -------------------------------------------------------------------------------- 1 | [defaults] -------------------------------------------------------------------------------- /roles/elasticsearch/defaults/main.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/defaults/main.yml -------------------------------------------------------------------------------- /roles/elasticsearch/files/scripts/calculate-score.groovy: -------------------------------------------------------------------------------- 1 | log(_score * 2) + my_modifier -------------------------------------------------------------------------------- /roles/elasticsearch/files/templates/basic.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/files/templates/basic.json -------------------------------------------------------------------------------- /roles/elasticsearch/filter_plugins/custom.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/filter_plugins/custom.py -------------------------------------------------------------------------------- /roles/elasticsearch/handlers/main.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/handlers/main.yml -------------------------------------------------------------------------------- /roles/elasticsearch/meta/main.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/meta/main.yml -------------------------------------------------------------------------------- /roles/elasticsearch/tasks/checkParameters.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/tasks/checkParameters.yml -------------------------------------------------------------------------------- /roles/elasticsearch/tasks/elasticsearch-Debian-version-lock.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/tasks/elasticsearch-Debian-version-lock.yml -------------------------------------------------------------------------------- /roles/elasticsearch/tasks/elasticsearch-Debian.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/tasks/elasticsearch-Debian.yml -------------------------------------------------------------------------------- /roles/elasticsearch/tasks/elasticsearch-RedHat-version-lock.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/tasks/elasticsearch-RedHat-version-lock.yml -------------------------------------------------------------------------------- /roles/elasticsearch/tasks/elasticsearch-RedHat.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/tasks/elasticsearch-RedHat.yml -------------------------------------------------------------------------------- /roles/elasticsearch/tasks/elasticsearch-config.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/tasks/elasticsearch-config.yml -------------------------------------------------------------------------------- /roles/elasticsearch/tasks/elasticsearch-optional-user.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/tasks/elasticsearch-optional-user.yml -------------------------------------------------------------------------------- /roles/elasticsearch/tasks/elasticsearch-plugins.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/tasks/elasticsearch-plugins.yml -------------------------------------------------------------------------------- /roles/elasticsearch/tasks/elasticsearch-scripts.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/tasks/elasticsearch-scripts.yml -------------------------------------------------------------------------------- /roles/elasticsearch/tasks/elasticsearch-service.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/tasks/elasticsearch-service.yml -------------------------------------------------------------------------------- /roles/elasticsearch/tasks/elasticsearch-templates.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/tasks/elasticsearch-templates.yml -------------------------------------------------------------------------------- /roles/elasticsearch/tasks/elasticsearch-version-lock.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/tasks/elasticsearch-version-lock.yml -------------------------------------------------------------------------------- /roles/elasticsearch/tasks/elasticsearch.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/tasks/elasticsearch.yml -------------------------------------------------------------------------------- /roles/elasticsearch/tasks/java.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/tasks/java.yml -------------------------------------------------------------------------------- /roles/elasticsearch/tasks/main.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/tasks/main.yml -------------------------------------------------------------------------------- /roles/elasticsearch/templates/elasticsearch.j2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/templates/elasticsearch.j2 -------------------------------------------------------------------------------- /roles/elasticsearch/templates/elasticsearch.repo: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/templates/elasticsearch.repo -------------------------------------------------------------------------------- /roles/elasticsearch/templates/elasticsearch.yml.j2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/templates/elasticsearch.yml.j2 -------------------------------------------------------------------------------- /roles/elasticsearch/templates/init/debian/elasticsearch.j2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/templates/init/debian/elasticsearch.j2 -------------------------------------------------------------------------------- /roles/elasticsearch/templates/init/redhat/elasticsearch.j2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/templates/init/redhat/elasticsearch.j2 -------------------------------------------------------------------------------- /roles/elasticsearch/templates/logging.yml.j2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/templates/logging.yml.j2 -------------------------------------------------------------------------------- /roles/elasticsearch/templates/systemd/elasticsearch.j2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/templates/systemd/elasticsearch.j2 -------------------------------------------------------------------------------- /roles/elasticsearch/test/integration/config-1x/config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - host: test-kitchen 3 | -------------------------------------------------------------------------------- /roles/elasticsearch/test/integration/config-1x/serverspec/default_spec.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/test/integration/config-1x/serverspec/default_spec.rb -------------------------------------------------------------------------------- /roles/elasticsearch/test/integration/config-2x/config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - host: test-kitchen 3 | -------------------------------------------------------------------------------- /roles/elasticsearch/test/integration/config-2x/serverspec/default_spec.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/test/integration/config-2x/serverspec/default_spec.rb -------------------------------------------------------------------------------- /roles/elasticsearch/test/integration/config.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/test/integration/config.yml -------------------------------------------------------------------------------- /roles/elasticsearch/test/integration/helpers/serverspec/Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gem 'rspec-retry' 4 | -------------------------------------------------------------------------------- /roles/elasticsearch/test/integration/helpers/serverspec/config_spec.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/test/integration/helpers/serverspec/config_spec.rb -------------------------------------------------------------------------------- /roles/elasticsearch/test/integration/helpers/serverspec/multi_spec.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/test/integration/helpers/serverspec/multi_spec.rb -------------------------------------------------------------------------------- /roles/elasticsearch/test/integration/helpers/serverspec/package_spec.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/test/integration/helpers/serverspec/package_spec.rb -------------------------------------------------------------------------------- /roles/elasticsearch/test/integration/helpers/serverspec/spec_helper.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/test/integration/helpers/serverspec/spec_helper.rb -------------------------------------------------------------------------------- /roles/elasticsearch/test/integration/helpers/serverspec/standard_spec.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/test/integration/helpers/serverspec/standard_spec.rb -------------------------------------------------------------------------------- /roles/elasticsearch/test/integration/multi-1x/multi.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - host: test-kitchen 3 | -------------------------------------------------------------------------------- /roles/elasticsearch/test/integration/multi-1x/serverspec/default_spec.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/test/integration/multi-1x/serverspec/default_spec.rb -------------------------------------------------------------------------------- /roles/elasticsearch/test/integration/multi-2x/multi.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - host: test-kitchen 3 | -------------------------------------------------------------------------------- /roles/elasticsearch/test/integration/multi-2x/serverspec/default_spec.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/test/integration/multi-2x/serverspec/default_spec.rb -------------------------------------------------------------------------------- /roles/elasticsearch/test/integration/multi.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/test/integration/multi.yml -------------------------------------------------------------------------------- /roles/elasticsearch/test/integration/package-1x/package.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - host: test-kitchen 3 | -------------------------------------------------------------------------------- /roles/elasticsearch/test/integration/package-1x/serverspec/default_spec.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/test/integration/package-1x/serverspec/default_spec.rb -------------------------------------------------------------------------------- /roles/elasticsearch/test/integration/package-2x/package.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - host: test-kitchen 3 | -------------------------------------------------------------------------------- /roles/elasticsearch/test/integration/package-2x/serverspec/default_spec.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/test/integration/package-2x/serverspec/default_spec.rb -------------------------------------------------------------------------------- /roles/elasticsearch/test/integration/package.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/test/integration/package.yml -------------------------------------------------------------------------------- /roles/elasticsearch/test/integration/standard-1x/serverspec/default_spec.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/test/integration/standard-1x/serverspec/default_spec.rb -------------------------------------------------------------------------------- /roles/elasticsearch/test/integration/standard-1x/standard.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - host: test-kitchen 3 | -------------------------------------------------------------------------------- /roles/elasticsearch/test/integration/standard-2x/serverspec/default_spec.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/test/integration/standard-2x/serverspec/default_spec.rb -------------------------------------------------------------------------------- /roles/elasticsearch/test/integration/standard-2x/standard.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - host: test-kitchen 3 | -------------------------------------------------------------------------------- /roles/elasticsearch/test/integration/standard.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/test/integration/standard.yml -------------------------------------------------------------------------------- /roles/elasticsearch/vars/Debian.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/vars/Debian.yml -------------------------------------------------------------------------------- /roles/elasticsearch/vars/RedHat.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/vars/RedHat.yml -------------------------------------------------------------------------------- /roles/elasticsearch/vars/main.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/elasticsearch/vars/main.yml -------------------------------------------------------------------------------- /roles/imagemagick/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/imagemagick/.gitignore -------------------------------------------------------------------------------- /roles/imagemagick/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/imagemagick/README.md -------------------------------------------------------------------------------- /roles/imagemagick/meta/main.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/imagemagick/meta/main.yml -------------------------------------------------------------------------------- /roles/imagemagick/tasks/main.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/imagemagick/tasks/main.yml -------------------------------------------------------------------------------- /roles/letsencrypt/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/letsencrypt/LICENSE -------------------------------------------------------------------------------- /roles/letsencrypt/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/letsencrypt/README.md -------------------------------------------------------------------------------- /roles/letsencrypt/defaults/main.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/letsencrypt/defaults/main.yml -------------------------------------------------------------------------------- /roles/letsencrypt/meta/.galaxy_install_info: -------------------------------------------------------------------------------- 1 | {install_date: 'Thu Feb 4 19:04:37 2016', version: master} 2 | -------------------------------------------------------------------------------- /roles/letsencrypt/meta/main.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/letsencrypt/meta/main.yml -------------------------------------------------------------------------------- /roles/letsencrypt/tasks/main.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/letsencrypt/tasks/main.yml -------------------------------------------------------------------------------- /roles/logrotate/.travis.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/logrotate/.travis.yml -------------------------------------------------------------------------------- /roles/logrotate/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/logrotate/LICENSE -------------------------------------------------------------------------------- /roles/logrotate/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/logrotate/README.md -------------------------------------------------------------------------------- /roles/logrotate/defaults/main.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/logrotate/defaults/main.yml -------------------------------------------------------------------------------- /roles/logrotate/meta/main.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/logrotate/meta/main.yml -------------------------------------------------------------------------------- /roles/logrotate/tasks/main.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/logrotate/tasks/main.yml -------------------------------------------------------------------------------- /roles/logrotate/templates/logrotate.d.j2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/logrotate/templates/logrotate.d.j2 -------------------------------------------------------------------------------- /roles/logrotate/tests/inventory: -------------------------------------------------------------------------------- 1 | localhost 2 | -------------------------------------------------------------------------------- /roles/logrotate/tests/test.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/logrotate/tests/test.yml -------------------------------------------------------------------------------- /roles/nginx-unicorn/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/nginx-unicorn/LICENSE -------------------------------------------------------------------------------- /roles/nginx-unicorn/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/nginx-unicorn/README.md -------------------------------------------------------------------------------- /roles/nginx-unicorn/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | nginx_sites: [] 3 | -------------------------------------------------------------------------------- /roles/nginx-unicorn/handlers/main.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/nginx-unicorn/handlers/main.yml -------------------------------------------------------------------------------- /roles/nginx-unicorn/meta/main.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/nginx-unicorn/meta/main.yml -------------------------------------------------------------------------------- /roles/nginx-unicorn/tasks/debian.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/nginx-unicorn/tasks/debian.yml -------------------------------------------------------------------------------- /roles/nginx-unicorn/tasks/main.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/nginx-unicorn/tasks/main.yml -------------------------------------------------------------------------------- /roles/nginx-unicorn/tasks/redhat.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/nginx-unicorn/tasks/redhat.yml -------------------------------------------------------------------------------- /roles/nginx-unicorn/templates/nginx-site.j2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/nginx-unicorn/templates/nginx-site.j2 -------------------------------------------------------------------------------- /roles/ruby-install/.gitignore: -------------------------------------------------------------------------------- 1 | .vagrant 2 | -------------------------------------------------------------------------------- /roles/ruby-install/.travis.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/ruby-install/.travis.yml -------------------------------------------------------------------------------- /roles/ruby-install/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/ruby-install/LICENSE -------------------------------------------------------------------------------- /roles/ruby-install/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/ruby-install/README.md -------------------------------------------------------------------------------- /roles/ruby-install/Vagrantfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/ruby-install/Vagrantfile -------------------------------------------------------------------------------- /roles/ruby-install/defaults/main.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/ruby-install/defaults/main.yml -------------------------------------------------------------------------------- /roles/ruby-install/meta/.galaxy_install_info: -------------------------------------------------------------------------------- 1 | {install_date: 'Thu Oct 2 20:55:31 2014', version: v0.1.0} 2 | -------------------------------------------------------------------------------- /roles/ruby-install/meta/main.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/ruby-install/meta/main.yml -------------------------------------------------------------------------------- /roles/ruby-install/tasks/main.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/ruby-install/tasks/main.yml -------------------------------------------------------------------------------- /roles/ruby-install/templates/ruby_install.fact: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/ruby-install/templates/ruby_install.fact -------------------------------------------------------------------------------- /roles/ruby-install/test.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/ruby-install/test.yml -------------------------------------------------------------------------------- /roles/swapfile/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/swapfile/LICENSE -------------------------------------------------------------------------------- /roles/swapfile/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/swapfile/README.md -------------------------------------------------------------------------------- /roles/swapfile/defaults/main.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/swapfile/defaults/main.yml -------------------------------------------------------------------------------- /roles/swapfile/handlers/main.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/swapfile/handlers/main.yml -------------------------------------------------------------------------------- /roles/swapfile/meta/main.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/swapfile/meta/main.yml -------------------------------------------------------------------------------- /roles/swapfile/tasks/main.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/swapfile/tasks/main.yml -------------------------------------------------------------------------------- /roles/ubuntu-common/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/ubuntu-common/README.md -------------------------------------------------------------------------------- /roles/ubuntu-common/defaults/main.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/ubuntu-common/defaults/main.yml -------------------------------------------------------------------------------- /roles/ubuntu-common/meta/main.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/ubuntu-common/meta/main.yml -------------------------------------------------------------------------------- /roles/ubuntu-common/tasks/main.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/ubuntu-common/tasks/main.yml -------------------------------------------------------------------------------- /roles/ubuntu-common/templates/locale: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/ubuntu-common/templates/locale -------------------------------------------------------------------------------- /roles/ubuntu-common/templates/sources.list: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/roles/ubuntu-common/templates/sources.list -------------------------------------------------------------------------------- /run: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/run -------------------------------------------------------------------------------- /test/controllers/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/fixtures/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/helpers/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/integration/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/mailers/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/models/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/models/url_test.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/test/models/url_test.rb -------------------------------------------------------------------------------- /test/test_helper.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bastosmichael/rails_website_crawler/HEAD/test/test_helper.rb --------------------------------------------------------------------------------