├── .Rbuildignore ├── .Rprofile ├── .github ├── .gitignore ├── CONTRIBUTING.md └── workflows │ ├── R-CMD-check.yaml │ ├── rhub.yaml │ └── test-coverage.yaml ├── .gitignore ├── CRAN-SUBMISSION ├── DESCRIPTION ├── LICENSE ├── NAMESPACE ├── NEWS.md ├── R ├── as_list.R ├── fix_url.R ├── get_robotstxt.R ├── get_robotstxt_http_get.R ├── get_robotstxts.R ├── guess_domain.R ├── http_domain_changed.R ├── http_subdomain_changed.R ├── http_was_redirected.R ├── is_suspect_robotstxt.R ├── is_valid_robotstxt.R ├── list_merge.R ├── null_to_default.R ├── parse_robotstxt.R ├── parse_url.R ├── paths_allowed.R ├── paths_allowed_worker_spiderbar.R ├── pipe.R ├── print_robotstxt.R ├── print_robotstxt_text.R ├── remove_domain.R ├── request_handler_handler.R ├── robotstxt.R ├── rt_cache.R ├── rt_get_comments.R ├── rt_get_fields.R ├── rt_get_fields_worker.R ├── rt_get_useragent.R ├── rt_request_handler.R ├── rt_request_handler_defaults.R ├── sanitize_path.R └── tools.R ├── README.Rmd ├── README.md ├── _pkgdown.yml ├── benchmarks └── spiderbar_and_futures.r ├── cran-comments.md ├── data-raw └── logo │ ├── robotstxt-logo.jpeg │ ├── robotstxt.jpeg │ └── robotstxt.png ├── inst ├── http_requests │ ├── http_404.rds │ ├── http_client_error.rds │ ├── http_domain_change.rds │ ├── http_html_content.rds │ ├── http_ok_1.rds │ ├── http_ok_2.rds │ ├── http_ok_3.rds │ ├── http_ok_4.rds │ ├── http_redirect_www.rds │ └── http_server_error.rds ├── robotstxts │ ├── allow_single_bot.txt │ ├── crawl_delay.txt │ ├── disallow_all_for_BadBot.txt │ ├── disallow_all_for_all.txt │ ├── disallow_some_for_all.txt │ ├── disallow_two_at_once.txt │ ├── empty.txt │ ├── host.txt │ ├── rbloggers.txt │ ├── robots_amazon.txt │ ├── robots_bundestag.txt │ ├── robots_cdc.txt │ ├── robots_cdc2.txt │ ├── robots_commented_token.txt │ ├── robots_facebook.txt │ ├── robots_facebook_unsupported.txt │ ├── robots_google.txt │ ├── robots_new_york_times.txt │ ├── robots_pmeissner.txt │ ├── robots_spiegel.txt │ ├── robots_wikipedia.txt │ ├── robots_wikipedia_20170706.txt │ ├── robots_yahoo.txt │ ├── selfhtml_Example.txt │ └── testing_comments.txt └── urls.txt ├── man ├── as.list.robotstxt_text.Rd ├── figures │ └── logo.jpeg ├── fix_url.Rd ├── get_robotstxt.Rd ├── get_robotstxt_http_get.Rd ├── get_robotstxts.Rd ├── guess_domain.Rd ├── http_domain_changed.Rd ├── http_subdomain_changed.Rd ├── http_was_redirected.Rd ├── is_suspect_robotstxt.Rd ├── is_valid_robotstxt.Rd ├── list_merge.Rd ├── named_list.Rd ├── null_to_default.Rd ├── parse_robotstxt.Rd ├── parse_url.Rd ├── paths_allowed.Rd ├── paths_allowed_worker_spiderbar.Rd ├── pipe.Rd ├── print.robotstxt.Rd ├── print.robotstxt_text.Rd ├── remove_domain.Rd ├── request_handler_handler.Rd ├── robotstxt.Rd ├── rt_cache.Rd ├── rt_get_comments.Rd ├── rt_get_fields.Rd ├── rt_get_fields_worker.Rd ├── rt_get_rtxt.Rd ├── rt_get_useragent.Rd ├── rt_list_rtxt.Rd ├── rt_request_handler.Rd └── sanitize_path.Rd ├── robotstxt.Rproj ├── tests ├── testthat.R └── testthat │ ├── _snaps │ ├── http_event_handling.md │ └── paths_allowed.md │ ├── test_attribute_handling.R │ ├── test_get_robotstxt.R │ ├── test_http_event_handling.R │ ├── test_issue50.R │ ├── test_parser.R │ ├── test_path_examples_from_rfc.R │ ├── test_paths_allowed.R │ ├── test_robotstxt.R │ └── test_tools.R └── vignettes ├── style.css └── using_robotstxt.Rmd /.Rbuildignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/.Rbuildignore -------------------------------------------------------------------------------- /.Rprofile: -------------------------------------------------------------------------------- 1 | Sys.setenv("rpkg_use_internet_for_testing" = TRUE) 2 | -------------------------------------------------------------------------------- /.github/.gitignore: -------------------------------------------------------------------------------- 1 | *.html 2 | -------------------------------------------------------------------------------- /.github/CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/.github/CONTRIBUTING.md -------------------------------------------------------------------------------- /.github/workflows/R-CMD-check.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/.github/workflows/R-CMD-check.yaml -------------------------------------------------------------------------------- /.github/workflows/rhub.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/.github/workflows/rhub.yaml -------------------------------------------------------------------------------- /.github/workflows/test-coverage.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/.github/workflows/test-coverage.yaml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/.gitignore -------------------------------------------------------------------------------- /CRAN-SUBMISSION: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/CRAN-SUBMISSION -------------------------------------------------------------------------------- /DESCRIPTION: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/DESCRIPTION -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/LICENSE -------------------------------------------------------------------------------- /NAMESPACE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/NAMESPACE -------------------------------------------------------------------------------- /NEWS.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/NEWS.md -------------------------------------------------------------------------------- /R/as_list.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/R/as_list.R -------------------------------------------------------------------------------- /R/fix_url.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/R/fix_url.R -------------------------------------------------------------------------------- /R/get_robotstxt.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/R/get_robotstxt.R -------------------------------------------------------------------------------- /R/get_robotstxt_http_get.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/R/get_robotstxt_http_get.R -------------------------------------------------------------------------------- /R/get_robotstxts.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/R/get_robotstxts.R -------------------------------------------------------------------------------- /R/guess_domain.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/R/guess_domain.R -------------------------------------------------------------------------------- /R/http_domain_changed.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/R/http_domain_changed.R -------------------------------------------------------------------------------- /R/http_subdomain_changed.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/R/http_subdomain_changed.R -------------------------------------------------------------------------------- /R/http_was_redirected.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/R/http_was_redirected.R -------------------------------------------------------------------------------- /R/is_suspect_robotstxt.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/R/is_suspect_robotstxt.R -------------------------------------------------------------------------------- /R/is_valid_robotstxt.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/R/is_valid_robotstxt.R -------------------------------------------------------------------------------- /R/list_merge.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/R/list_merge.R -------------------------------------------------------------------------------- /R/null_to_default.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/R/null_to_default.R -------------------------------------------------------------------------------- /R/parse_robotstxt.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/R/parse_robotstxt.R -------------------------------------------------------------------------------- /R/parse_url.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/R/parse_url.R -------------------------------------------------------------------------------- /R/paths_allowed.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/R/paths_allowed.R -------------------------------------------------------------------------------- /R/paths_allowed_worker_spiderbar.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/R/paths_allowed_worker_spiderbar.R -------------------------------------------------------------------------------- /R/pipe.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/R/pipe.R -------------------------------------------------------------------------------- /R/print_robotstxt.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/R/print_robotstxt.R -------------------------------------------------------------------------------- /R/print_robotstxt_text.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/R/print_robotstxt_text.R -------------------------------------------------------------------------------- /R/remove_domain.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/R/remove_domain.R -------------------------------------------------------------------------------- /R/request_handler_handler.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/R/request_handler_handler.R -------------------------------------------------------------------------------- /R/robotstxt.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/R/robotstxt.R -------------------------------------------------------------------------------- /R/rt_cache.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/R/rt_cache.R -------------------------------------------------------------------------------- /R/rt_get_comments.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/R/rt_get_comments.R -------------------------------------------------------------------------------- /R/rt_get_fields.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/R/rt_get_fields.R -------------------------------------------------------------------------------- /R/rt_get_fields_worker.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/R/rt_get_fields_worker.R -------------------------------------------------------------------------------- /R/rt_get_useragent.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/R/rt_get_useragent.R -------------------------------------------------------------------------------- /R/rt_request_handler.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/R/rt_request_handler.R -------------------------------------------------------------------------------- /R/rt_request_handler_defaults.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/R/rt_request_handler_defaults.R -------------------------------------------------------------------------------- /R/sanitize_path.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/R/sanitize_path.R -------------------------------------------------------------------------------- /R/tools.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/R/tools.R -------------------------------------------------------------------------------- /README.Rmd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/README.Rmd -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/README.md -------------------------------------------------------------------------------- /_pkgdown.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/_pkgdown.yml -------------------------------------------------------------------------------- /benchmarks/spiderbar_and_futures.r: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/benchmarks/spiderbar_and_futures.r -------------------------------------------------------------------------------- /cran-comments.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/cran-comments.md -------------------------------------------------------------------------------- /data-raw/logo/robotstxt-logo.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/data-raw/logo/robotstxt-logo.jpeg -------------------------------------------------------------------------------- /data-raw/logo/robotstxt.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/data-raw/logo/robotstxt.jpeg -------------------------------------------------------------------------------- /data-raw/logo/robotstxt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/data-raw/logo/robotstxt.png -------------------------------------------------------------------------------- /inst/http_requests/http_404.rds: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/inst/http_requests/http_404.rds -------------------------------------------------------------------------------- /inst/http_requests/http_client_error.rds: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/inst/http_requests/http_client_error.rds -------------------------------------------------------------------------------- /inst/http_requests/http_domain_change.rds: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/inst/http_requests/http_domain_change.rds -------------------------------------------------------------------------------- /inst/http_requests/http_html_content.rds: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/inst/http_requests/http_html_content.rds -------------------------------------------------------------------------------- /inst/http_requests/http_ok_1.rds: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/inst/http_requests/http_ok_1.rds -------------------------------------------------------------------------------- /inst/http_requests/http_ok_2.rds: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/inst/http_requests/http_ok_2.rds -------------------------------------------------------------------------------- /inst/http_requests/http_ok_3.rds: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/inst/http_requests/http_ok_3.rds -------------------------------------------------------------------------------- /inst/http_requests/http_ok_4.rds: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/inst/http_requests/http_ok_4.rds -------------------------------------------------------------------------------- /inst/http_requests/http_redirect_www.rds: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/inst/http_requests/http_redirect_www.rds -------------------------------------------------------------------------------- /inst/http_requests/http_server_error.rds: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/inst/http_requests/http_server_error.rds -------------------------------------------------------------------------------- /inst/robotstxts/allow_single_bot.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/inst/robotstxts/allow_single_bot.txt -------------------------------------------------------------------------------- /inst/robotstxts/crawl_delay.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/inst/robotstxts/crawl_delay.txt -------------------------------------------------------------------------------- /inst/robotstxts/disallow_all_for_BadBot.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/inst/robotstxts/disallow_all_for_BadBot.txt -------------------------------------------------------------------------------- /inst/robotstxts/disallow_all_for_all.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/inst/robotstxts/disallow_all_for_all.txt -------------------------------------------------------------------------------- /inst/robotstxts/disallow_some_for_all.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/inst/robotstxts/disallow_some_for_all.txt -------------------------------------------------------------------------------- /inst/robotstxts/disallow_two_at_once.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/inst/robotstxts/disallow_two_at_once.txt -------------------------------------------------------------------------------- /inst/robotstxts/empty.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /inst/robotstxts/host.txt: -------------------------------------------------------------------------------- 1 | # comment 2 | 3 | Host: www.whatever.com 4 | -------------------------------------------------------------------------------- /inst/robotstxts/rbloggers.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/inst/robotstxts/rbloggers.txt -------------------------------------------------------------------------------- /inst/robotstxts/robots_amazon.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/inst/robotstxts/robots_amazon.txt -------------------------------------------------------------------------------- /inst/robotstxts/robots_bundestag.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/inst/robotstxts/robots_bundestag.txt -------------------------------------------------------------------------------- /inst/robotstxts/robots_cdc.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/inst/robotstxts/robots_cdc.txt -------------------------------------------------------------------------------- /inst/robotstxts/robots_cdc2.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/inst/robotstxts/robots_cdc2.txt -------------------------------------------------------------------------------- /inst/robotstxts/robots_commented_token.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/inst/robotstxts/robots_commented_token.txt -------------------------------------------------------------------------------- /inst/robotstxts/robots_facebook.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/inst/robotstxts/robots_facebook.txt -------------------------------------------------------------------------------- /inst/robotstxts/robots_facebook_unsupported.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/inst/robotstxts/robots_facebook_unsupported.txt -------------------------------------------------------------------------------- /inst/robotstxts/robots_google.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/inst/robotstxts/robots_google.txt -------------------------------------------------------------------------------- /inst/robotstxts/robots_new_york_times.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/inst/robotstxts/robots_new_york_times.txt -------------------------------------------------------------------------------- /inst/robotstxts/robots_pmeissner.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/inst/robotstxts/robots_pmeissner.txt -------------------------------------------------------------------------------- /inst/robotstxts/robots_spiegel.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/inst/robotstxts/robots_spiegel.txt -------------------------------------------------------------------------------- /inst/robotstxts/robots_wikipedia.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/inst/robotstxts/robots_wikipedia.txt -------------------------------------------------------------------------------- /inst/robotstxts/robots_wikipedia_20170706.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/inst/robotstxts/robots_wikipedia_20170706.txt -------------------------------------------------------------------------------- /inst/robotstxts/robots_yahoo.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/inst/robotstxts/robots_yahoo.txt -------------------------------------------------------------------------------- /inst/robotstxts/selfhtml_Example.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/inst/robotstxts/selfhtml_Example.txt -------------------------------------------------------------------------------- /inst/robotstxts/testing_comments.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/inst/robotstxts/testing_comments.txt -------------------------------------------------------------------------------- /inst/urls.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/inst/urls.txt -------------------------------------------------------------------------------- /man/as.list.robotstxt_text.Rd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/man/as.list.robotstxt_text.Rd -------------------------------------------------------------------------------- /man/figures/logo.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/man/figures/logo.jpeg -------------------------------------------------------------------------------- /man/fix_url.Rd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/man/fix_url.Rd -------------------------------------------------------------------------------- /man/get_robotstxt.Rd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/man/get_robotstxt.Rd -------------------------------------------------------------------------------- /man/get_robotstxt_http_get.Rd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/man/get_robotstxt_http_get.Rd -------------------------------------------------------------------------------- /man/get_robotstxts.Rd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/man/get_robotstxts.Rd -------------------------------------------------------------------------------- /man/guess_domain.Rd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/man/guess_domain.Rd -------------------------------------------------------------------------------- /man/http_domain_changed.Rd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/man/http_domain_changed.Rd -------------------------------------------------------------------------------- /man/http_subdomain_changed.Rd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/man/http_subdomain_changed.Rd -------------------------------------------------------------------------------- /man/http_was_redirected.Rd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/man/http_was_redirected.Rd -------------------------------------------------------------------------------- /man/is_suspect_robotstxt.Rd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/man/is_suspect_robotstxt.Rd -------------------------------------------------------------------------------- /man/is_valid_robotstxt.Rd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/man/is_valid_robotstxt.Rd -------------------------------------------------------------------------------- /man/list_merge.Rd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/man/list_merge.Rd -------------------------------------------------------------------------------- /man/named_list.Rd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/man/named_list.Rd -------------------------------------------------------------------------------- /man/null_to_default.Rd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/man/null_to_default.Rd -------------------------------------------------------------------------------- /man/parse_robotstxt.Rd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/man/parse_robotstxt.Rd -------------------------------------------------------------------------------- /man/parse_url.Rd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/man/parse_url.Rd -------------------------------------------------------------------------------- /man/paths_allowed.Rd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/man/paths_allowed.Rd -------------------------------------------------------------------------------- /man/paths_allowed_worker_spiderbar.Rd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/man/paths_allowed_worker_spiderbar.Rd -------------------------------------------------------------------------------- /man/pipe.Rd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/man/pipe.Rd -------------------------------------------------------------------------------- /man/print.robotstxt.Rd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/man/print.robotstxt.Rd -------------------------------------------------------------------------------- /man/print.robotstxt_text.Rd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/man/print.robotstxt_text.Rd -------------------------------------------------------------------------------- /man/remove_domain.Rd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/man/remove_domain.Rd -------------------------------------------------------------------------------- /man/request_handler_handler.Rd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/man/request_handler_handler.Rd -------------------------------------------------------------------------------- /man/robotstxt.Rd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/man/robotstxt.Rd -------------------------------------------------------------------------------- /man/rt_cache.Rd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/man/rt_cache.Rd -------------------------------------------------------------------------------- /man/rt_get_comments.Rd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/man/rt_get_comments.Rd -------------------------------------------------------------------------------- /man/rt_get_fields.Rd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/man/rt_get_fields.Rd -------------------------------------------------------------------------------- /man/rt_get_fields_worker.Rd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/man/rt_get_fields_worker.Rd -------------------------------------------------------------------------------- /man/rt_get_rtxt.Rd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/man/rt_get_rtxt.Rd -------------------------------------------------------------------------------- /man/rt_get_useragent.Rd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/man/rt_get_useragent.Rd -------------------------------------------------------------------------------- /man/rt_list_rtxt.Rd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/man/rt_list_rtxt.Rd -------------------------------------------------------------------------------- /man/rt_request_handler.Rd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/man/rt_request_handler.Rd -------------------------------------------------------------------------------- /man/sanitize_path.Rd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/man/sanitize_path.Rd -------------------------------------------------------------------------------- /robotstxt.Rproj: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/robotstxt.Rproj -------------------------------------------------------------------------------- /tests/testthat.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/tests/testthat.R -------------------------------------------------------------------------------- /tests/testthat/_snaps/http_event_handling.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/tests/testthat/_snaps/http_event_handling.md -------------------------------------------------------------------------------- /tests/testthat/_snaps/paths_allowed.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/tests/testthat/_snaps/paths_allowed.md -------------------------------------------------------------------------------- /tests/testthat/test_attribute_handling.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/tests/testthat/test_attribute_handling.R -------------------------------------------------------------------------------- /tests/testthat/test_get_robotstxt.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/tests/testthat/test_get_robotstxt.R -------------------------------------------------------------------------------- /tests/testthat/test_http_event_handling.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/tests/testthat/test_http_event_handling.R -------------------------------------------------------------------------------- /tests/testthat/test_issue50.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/tests/testthat/test_issue50.R -------------------------------------------------------------------------------- /tests/testthat/test_parser.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/tests/testthat/test_parser.R -------------------------------------------------------------------------------- /tests/testthat/test_path_examples_from_rfc.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/tests/testthat/test_path_examples_from_rfc.R -------------------------------------------------------------------------------- /tests/testthat/test_paths_allowed.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/tests/testthat/test_paths_allowed.R -------------------------------------------------------------------------------- /tests/testthat/test_robotstxt.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/tests/testthat/test_robotstxt.R -------------------------------------------------------------------------------- /tests/testthat/test_tools.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/tests/testthat/test_tools.R -------------------------------------------------------------------------------- /vignettes/style.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/vignettes/style.css -------------------------------------------------------------------------------- /vignettes/using_robotstxt.Rmd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ropensci/robotstxt/HEAD/vignettes/using_robotstxt.Rmd --------------------------------------------------------------------------------