├── .github └── workflows │ ├── gem.yml │ └── ruby.yml ├── .gitignore ├── .rspec ├── .rubocop.yml ├── Gemfile ├── LICENSE ├── README.md ├── Rakefile ├── bin ├── console └── setup ├── docker-compose.yml ├── docker └── Dockerfile ├── exe └── miteru ├── lefthook.yml ├── lib ├── miteru.rb └── miteru │ ├── cache.rb │ ├── cli │ ├── application.rb │ ├── base.rb │ └── database.rb │ ├── commands │ ├── database.rb │ ├── main.rb │ ├── sidekiq.rb │ └── web.rb │ ├── concerns │ ├── database_connectable.rb │ ├── error_unwrappable.rb │ └── url_truncatable.rb │ ├── config.rb │ ├── crawler.rb │ ├── database.rb │ ├── downloader.rb │ ├── errors.rb │ ├── feeds │ ├── ayashige.rb │ ├── base.rb │ ├── phishing_database.rb │ ├── tweetfeed.rb │ ├── urlscan.rb │ └── urlscan_pro.rb │ ├── helpers.rb │ ├── http.rb │ ├── kit.rb │ ├── mixin.rb │ ├── notifiers │ ├── base.rb │ ├── slack.rb │ └── urlscan.rb │ ├── orchestrator.rb │ ├── record.rb │ ├── service.rb │ ├── sidekiq │ ├── application.rb │ └── jobs.rb │ ├── version.rb │ ├── web │ └── application.rb │ └── website.rb ├── miteru.gemspec ├── renovate.json └── spec ├── cli_spec.rb ├── crawler_spec.rb ├── downloader_spec.rb ├── feeds └── feed_spec.rb ├── fixtures ├── index.html ├── test.tar.gz └── test.zip ├── kit_spec.rb ├── miteru_spec.rb ├── orchestrator_spec.rb ├── spec_helper.rb ├── support └── shared_contexts │ ├── fake_http_server_context.rb │ └── mocked_logger_context.rb └── website_spec.rb /.github/workflows/gem.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/.github/workflows/gem.yml -------------------------------------------------------------------------------- /.github/workflows/ruby.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/.github/workflows/ruby.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/.gitignore -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/.rspec -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | Style/StringLiterals: 2 | EnforcedStyle: double_quotes 3 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/Gemfile -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/README.md -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/Rakefile -------------------------------------------------------------------------------- /bin/console: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/bin/console -------------------------------------------------------------------------------- /bin/setup: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/bin/setup -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/docker-compose.yml -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/docker/Dockerfile -------------------------------------------------------------------------------- /exe/miteru: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/exe/miteru -------------------------------------------------------------------------------- /lefthook.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/lefthook.yml -------------------------------------------------------------------------------- /lib/miteru.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/lib/miteru.rb -------------------------------------------------------------------------------- /lib/miteru/cache.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/lib/miteru/cache.rb -------------------------------------------------------------------------------- /lib/miteru/cli/application.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/lib/miteru/cli/application.rb -------------------------------------------------------------------------------- /lib/miteru/cli/base.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/lib/miteru/cli/base.rb -------------------------------------------------------------------------------- /lib/miteru/cli/database.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/lib/miteru/cli/database.rb -------------------------------------------------------------------------------- /lib/miteru/commands/database.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/lib/miteru/commands/database.rb -------------------------------------------------------------------------------- /lib/miteru/commands/main.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/lib/miteru/commands/main.rb -------------------------------------------------------------------------------- /lib/miteru/commands/sidekiq.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/lib/miteru/commands/sidekiq.rb -------------------------------------------------------------------------------- /lib/miteru/commands/web.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/lib/miteru/commands/web.rb -------------------------------------------------------------------------------- /lib/miteru/concerns/database_connectable.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/lib/miteru/concerns/database_connectable.rb -------------------------------------------------------------------------------- /lib/miteru/concerns/error_unwrappable.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/lib/miteru/concerns/error_unwrappable.rb -------------------------------------------------------------------------------- /lib/miteru/concerns/url_truncatable.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/lib/miteru/concerns/url_truncatable.rb -------------------------------------------------------------------------------- /lib/miteru/config.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/lib/miteru/config.rb -------------------------------------------------------------------------------- /lib/miteru/crawler.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/lib/miteru/crawler.rb -------------------------------------------------------------------------------- /lib/miteru/database.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/lib/miteru/database.rb -------------------------------------------------------------------------------- /lib/miteru/downloader.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/lib/miteru/downloader.rb -------------------------------------------------------------------------------- /lib/miteru/errors.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/lib/miteru/errors.rb -------------------------------------------------------------------------------- /lib/miteru/feeds/ayashige.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/lib/miteru/feeds/ayashige.rb -------------------------------------------------------------------------------- /lib/miteru/feeds/base.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/lib/miteru/feeds/base.rb -------------------------------------------------------------------------------- /lib/miteru/feeds/phishing_database.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/lib/miteru/feeds/phishing_database.rb -------------------------------------------------------------------------------- /lib/miteru/feeds/tweetfeed.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/lib/miteru/feeds/tweetfeed.rb -------------------------------------------------------------------------------- /lib/miteru/feeds/urlscan.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/lib/miteru/feeds/urlscan.rb -------------------------------------------------------------------------------- /lib/miteru/feeds/urlscan_pro.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/lib/miteru/feeds/urlscan_pro.rb -------------------------------------------------------------------------------- /lib/miteru/helpers.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/lib/miteru/helpers.rb -------------------------------------------------------------------------------- /lib/miteru/http.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/lib/miteru/http.rb -------------------------------------------------------------------------------- /lib/miteru/kit.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/lib/miteru/kit.rb -------------------------------------------------------------------------------- /lib/miteru/mixin.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/lib/miteru/mixin.rb -------------------------------------------------------------------------------- /lib/miteru/notifiers/base.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/lib/miteru/notifiers/base.rb -------------------------------------------------------------------------------- /lib/miteru/notifiers/slack.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/lib/miteru/notifiers/slack.rb -------------------------------------------------------------------------------- /lib/miteru/notifiers/urlscan.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/lib/miteru/notifiers/urlscan.rb -------------------------------------------------------------------------------- /lib/miteru/orchestrator.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/lib/miteru/orchestrator.rb -------------------------------------------------------------------------------- /lib/miteru/record.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/lib/miteru/record.rb -------------------------------------------------------------------------------- /lib/miteru/service.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/lib/miteru/service.rb -------------------------------------------------------------------------------- /lib/miteru/sidekiq/application.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/lib/miteru/sidekiq/application.rb -------------------------------------------------------------------------------- /lib/miteru/sidekiq/jobs.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/lib/miteru/sidekiq/jobs.rb -------------------------------------------------------------------------------- /lib/miteru/version.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Miteru 4 | VERSION = "3.1.0" 5 | end 6 | -------------------------------------------------------------------------------- /lib/miteru/web/application.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/lib/miteru/web/application.rb -------------------------------------------------------------------------------- /lib/miteru/website.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/lib/miteru/website.rb -------------------------------------------------------------------------------- /miteru.gemspec: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/miteru.gemspec -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/renovate.json -------------------------------------------------------------------------------- /spec/cli_spec.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/spec/cli_spec.rb -------------------------------------------------------------------------------- /spec/crawler_spec.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/spec/crawler_spec.rb -------------------------------------------------------------------------------- /spec/downloader_spec.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/spec/downloader_spec.rb -------------------------------------------------------------------------------- /spec/feeds/feed_spec.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/spec/feeds/feed_spec.rb -------------------------------------------------------------------------------- /spec/fixtures/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/spec/fixtures/index.html -------------------------------------------------------------------------------- /spec/fixtures/test.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/spec/fixtures/test.tar.gz -------------------------------------------------------------------------------- /spec/fixtures/test.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/spec/fixtures/test.zip -------------------------------------------------------------------------------- /spec/kit_spec.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/spec/kit_spec.rb -------------------------------------------------------------------------------- /spec/miteru_spec.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/spec/miteru_spec.rb -------------------------------------------------------------------------------- /spec/orchestrator_spec.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/spec/orchestrator_spec.rb -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/spec/spec_helper.rb -------------------------------------------------------------------------------- /spec/support/shared_contexts/fake_http_server_context.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/spec/support/shared_contexts/fake_http_server_context.rb -------------------------------------------------------------------------------- /spec/support/shared_contexts/mocked_logger_context.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/spec/support/shared_contexts/mocked_logger_context.rb -------------------------------------------------------------------------------- /spec/website_spec.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninoseki/miteru/HEAD/spec/website_spec.rb --------------------------------------------------------------------------------