├── .github └── workflows │ └── php.yml ├── .gitignore ├── .php-cs-fixer.dist.php ├── .travis.yml ├── LICENSE ├── README.md ├── composer.json ├── phpunit.xml ├── psalm.xml ├── src └── RobotsTxtParser │ ├── RobotsTxtParser.php │ └── RobotsTxtValidator.php └── tests ├── RobotsTxtParser ├── AbsolutePathTest.php ├── CleanParamTest.php ├── CommentsTest.php ├── CommonTest.php ├── CrawlDelayTest.php ├── DifferentCasesTest.php ├── EmptyDisallowTest.php ├── HostTest.php ├── HttpTest.php ├── Issue13Test.php ├── Issue48Test.php ├── MultipleUserAgentsRulesTest.php ├── NoIndexTest.php ├── RelativePathTest.php ├── RobotsTxtParserTest.php ├── RobotsTxtValidatorTest.php ├── RulesTest.php ├── TyposDirectiveTest.php ├── WhitespacesTest.php ├── data │ ├── robots1 │ │ ├── expectedAllow │ │ ├── expectedDisallow │ │ └── robots.txt │ ├── robots2 │ │ ├── expectedAllow │ │ ├── expectedDisallow │ │ └── robots.txt │ ├── robots3 │ │ ├── expectedAllow │ │ ├── expectedDisallow │ │ └── robots.txt │ ├── robots4 │ │ ├── expectedAllow │ │ ├── expectedDisallow │ │ └── robots.txt │ ├── robots5 │ │ ├── expectedAllow │ │ ├── expectedDisallow │ │ └── robots.txt │ ├── robots6 │ │ ├── expectedAllow │ │ ├── expectedDisallow │ │ └── robots.txt │ ├── robots7 │ │ ├── expectedAllow │ │ ├── expectedDisallow │ │ └── robots.txt │ └── robots8 │ │ ├── expectedAllow │ │ ├── expectedDisallow │ │ └── robots.txt ├── robots.txt │ └── goldmansachs.com └── t1gorTest.php └── bootstrap.php /.github/workflows/php.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/.github/workflows/php.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .* 2 | !.keepdir 3 | !.travis.yml 4 | vendor 5 | composer.lock 6 | build/ 7 | -------------------------------------------------------------------------------- /.php-cs-fixer.dist.php: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/.php-cs-fixer.dist.php -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/.travis.yml -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/README.md -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/composer.json -------------------------------------------------------------------------------- /phpunit.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/phpunit.xml -------------------------------------------------------------------------------- /psalm.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/psalm.xml -------------------------------------------------------------------------------- /src/RobotsTxtParser/RobotsTxtParser.php: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/src/RobotsTxtParser/RobotsTxtParser.php -------------------------------------------------------------------------------- /src/RobotsTxtParser/RobotsTxtValidator.php: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/src/RobotsTxtParser/RobotsTxtValidator.php -------------------------------------------------------------------------------- /tests/RobotsTxtParser/AbsolutePathTest.php: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/tests/RobotsTxtParser/AbsolutePathTest.php -------------------------------------------------------------------------------- /tests/RobotsTxtParser/CleanParamTest.php: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/tests/RobotsTxtParser/CleanParamTest.php -------------------------------------------------------------------------------- /tests/RobotsTxtParser/CommentsTest.php: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/tests/RobotsTxtParser/CommentsTest.php -------------------------------------------------------------------------------- /tests/RobotsTxtParser/CommonTest.php: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/tests/RobotsTxtParser/CommonTest.php -------------------------------------------------------------------------------- /tests/RobotsTxtParser/CrawlDelayTest.php: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/tests/RobotsTxtParser/CrawlDelayTest.php -------------------------------------------------------------------------------- /tests/RobotsTxtParser/DifferentCasesTest.php: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/tests/RobotsTxtParser/DifferentCasesTest.php -------------------------------------------------------------------------------- /tests/RobotsTxtParser/EmptyDisallowTest.php: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/tests/RobotsTxtParser/EmptyDisallowTest.php -------------------------------------------------------------------------------- /tests/RobotsTxtParser/HostTest.php: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/tests/RobotsTxtParser/HostTest.php -------------------------------------------------------------------------------- /tests/RobotsTxtParser/HttpTest.php: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/tests/RobotsTxtParser/HttpTest.php -------------------------------------------------------------------------------- /tests/RobotsTxtParser/Issue13Test.php: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/tests/RobotsTxtParser/Issue13Test.php -------------------------------------------------------------------------------- /tests/RobotsTxtParser/Issue48Test.php: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/tests/RobotsTxtParser/Issue48Test.php -------------------------------------------------------------------------------- /tests/RobotsTxtParser/MultipleUserAgentsRulesTest.php: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/tests/RobotsTxtParser/MultipleUserAgentsRulesTest.php -------------------------------------------------------------------------------- /tests/RobotsTxtParser/NoIndexTest.php: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/tests/RobotsTxtParser/NoIndexTest.php -------------------------------------------------------------------------------- /tests/RobotsTxtParser/RelativePathTest.php: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/tests/RobotsTxtParser/RelativePathTest.php -------------------------------------------------------------------------------- /tests/RobotsTxtParser/RobotsTxtParserTest.php: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/tests/RobotsTxtParser/RobotsTxtParserTest.php -------------------------------------------------------------------------------- /tests/RobotsTxtParser/RobotsTxtValidatorTest.php: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/tests/RobotsTxtParser/RobotsTxtValidatorTest.php -------------------------------------------------------------------------------- /tests/RobotsTxtParser/RulesTest.php: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/tests/RobotsTxtParser/RulesTest.php -------------------------------------------------------------------------------- /tests/RobotsTxtParser/TyposDirectiveTest.php: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/tests/RobotsTxtParser/TyposDirectiveTest.php -------------------------------------------------------------------------------- /tests/RobotsTxtParser/WhitespacesTest.php: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/tests/RobotsTxtParser/WhitespacesTest.php -------------------------------------------------------------------------------- /tests/RobotsTxtParser/data/robots1/expectedAllow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/tests/RobotsTxtParser/data/robots1/expectedAllow -------------------------------------------------------------------------------- /tests/RobotsTxtParser/data/robots1/expectedDisallow: -------------------------------------------------------------------------------- 1 | http://example.com/page.htm -------------------------------------------------------------------------------- /tests/RobotsTxtParser/data/robots1/robots.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/tests/RobotsTxtParser/data/robots1/robots.txt -------------------------------------------------------------------------------- /tests/RobotsTxtParser/data/robots2/expectedAllow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/tests/RobotsTxtParser/data/robots2/expectedAllow -------------------------------------------------------------------------------- /tests/RobotsTxtParser/data/robots2/expectedDisallow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/tests/RobotsTxtParser/data/robots2/expectedDisallow -------------------------------------------------------------------------------- /tests/RobotsTxtParser/data/robots2/robots.txt: -------------------------------------------------------------------------------- 1 | User-Agent: * 2 | Disallow: /fish -------------------------------------------------------------------------------- /tests/RobotsTxtParser/data/robots3/expectedAllow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/tests/RobotsTxtParser/data/robots3/expectedAllow -------------------------------------------------------------------------------- /tests/RobotsTxtParser/data/robots3/expectedDisallow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/tests/RobotsTxtParser/data/robots3/expectedDisallow -------------------------------------------------------------------------------- /tests/RobotsTxtParser/data/robots3/robots.txt: -------------------------------------------------------------------------------- 1 | User-Agent: * 2 | Disallow: /fish* -------------------------------------------------------------------------------- /tests/RobotsTxtParser/data/robots4/expectedAllow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/tests/RobotsTxtParser/data/robots4/expectedAllow -------------------------------------------------------------------------------- /tests/RobotsTxtParser/data/robots4/expectedDisallow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/tests/RobotsTxtParser/data/robots4/expectedDisallow -------------------------------------------------------------------------------- /tests/RobotsTxtParser/data/robots4/robots.txt: -------------------------------------------------------------------------------- 1 | User-Agent: * 2 | Disallow: /fish/ -------------------------------------------------------------------------------- /tests/RobotsTxtParser/data/robots5/expectedAllow: -------------------------------------------------------------------------------- 1 | / 2 | /windows.PHP -------------------------------------------------------------------------------- /tests/RobotsTxtParser/data/robots5/expectedDisallow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/tests/RobotsTxtParser/data/robots5/expectedDisallow -------------------------------------------------------------------------------- /tests/RobotsTxtParser/data/robots5/robots.txt: -------------------------------------------------------------------------------- 1 | User-Agent: * 2 | Disallow: /*.php -------------------------------------------------------------------------------- /tests/RobotsTxtParser/data/robots6/expectedAllow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/tests/RobotsTxtParser/data/robots6/expectedAllow -------------------------------------------------------------------------------- /tests/RobotsTxtParser/data/robots6/expectedDisallow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/tests/RobotsTxtParser/data/robots6/expectedDisallow -------------------------------------------------------------------------------- /tests/RobotsTxtParser/data/robots6/robots.txt: -------------------------------------------------------------------------------- 1 | User-Agent: * 2 | Disallow: /*.php$ -------------------------------------------------------------------------------- /tests/RobotsTxtParser/data/robots7/expectedAllow: -------------------------------------------------------------------------------- 1 | /Fish.PHP -------------------------------------------------------------------------------- /tests/RobotsTxtParser/data/robots7/expectedDisallow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/tests/RobotsTxtParser/data/robots7/expectedDisallow -------------------------------------------------------------------------------- /tests/RobotsTxtParser/data/robots7/robots.txt: -------------------------------------------------------------------------------- 1 | User-Agent: * 2 | Disallow: /fish*.php -------------------------------------------------------------------------------- /tests/RobotsTxtParser/data/robots8/expectedAllow: -------------------------------------------------------------------------------- 1 | /allowPath -------------------------------------------------------------------------------- /tests/RobotsTxtParser/data/robots8/expectedDisallow: -------------------------------------------------------------------------------- 1 | /Fiche_renseign_progest/ 2 | /intranet_dcga/ -------------------------------------------------------------------------------- /tests/RobotsTxtParser/data/robots8/robots.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/tests/RobotsTxtParser/data/robots8/robots.txt -------------------------------------------------------------------------------- /tests/RobotsTxtParser/robots.txt/goldmansachs.com: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/tests/RobotsTxtParser/robots.txt/goldmansachs.com -------------------------------------------------------------------------------- /tests/RobotsTxtParser/t1gorTest.php: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/tests/RobotsTxtParser/t1gorTest.php -------------------------------------------------------------------------------- /tests/bootstrap.php: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bopoda/robots-txt-parser/HEAD/tests/bootstrap.php --------------------------------------------------------------------------------