├── .gitignore ├── tests ├── fixtures │ ├── robots-txt-files │ │ ├── withBom.txt │ │ ├── sitemapAsLastLineInSingleRecord.txt │ │ ├── sitemapWithinSingleRecord.txt │ │ ├── contains-invalid-lines.txt │ │ ├── newscientist.com.txt │ │ ├── stackoverflow.com.txt │ │ └── google.com.txt │ └── user-agent-strings.json ├── File │ ├── AbstractFileTest.php │ ├── FileTest.php │ └── ParserTest.php ├── Directive │ ├── UserAgentDirectiveTest.php │ ├── ValueTest.php │ ├── ValidatorTest.php │ ├── DirectiveTest.php │ └── FactoryTest.php ├── DirectiveList │ ├── UserAgentDirectiveListTest.php │ └── DirectiveListTest.php ├── Record │ └── RecordTest.php └── Inspector │ ├── GetDirectivesTest.php │ └── IsAllowedTest.php ├── .travis.yml ├── phpstan.neon ├── src ├── Directive │ ├── DirectiveInterface.php │ ├── UserAgentDirective.php │ ├── Factory.php │ ├── Value.php │ ├── Validator.php │ └── Directive.php ├── Record │ └── Record.php ├── File │ ├── File.php │ └── Parser.php ├── DirectiveList │ ├── UserAgentDirectiveList.php │ └── DirectiveList.php └── Inspector │ ├── UrlMatcher.php │ └── Inspector.php ├── phpunit.xml.dist ├── LICENSE ├── composer.json ├── README.md └── composer.lock /.gitignore: -------------------------------------------------------------------------------- 1 | /build/ 2 | /vendor/ 3 | .phpunit.result.cache 4 | -------------------------------------------------------------------------------- /tests/fixtures/robots-txt-files/withBom.txt: -------------------------------------------------------------------------------- 1 | User-agent: * 2 | Disallow: / 3 | Allow: /humans.txt 4 | 5 | Sitemap: http://tsdme.nl/sitemap.txt -------------------------------------------------------------------------------- /tests/fixtures/robots-txt-files/sitemapAsLastLineInSingleRecord.txt: -------------------------------------------------------------------------------- 1 | User-Agent: * 2 | Disallow: /cms/ 3 | Sitemap: http://example.com/sitemap.xml -------------------------------------------------------------------------------- /tests/fixtures/robots-txt-files/sitemapWithinSingleRecord.txt: -------------------------------------------------------------------------------- 1 | User-Agent: * 2 | Disallow: /pathone 3 | Sitemap: http://example.com/sitemap.xml 4 | Disallow: /pathtwo -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: php 2 | php: 3 | - 7.2 4 | 5 | install: 6 | - composer install 7 | 8 | script: 9 | - composer ci 10 | 11 | cache: 12 | directories: 13 | - $HOME/.composer/cache/files 14 | -------------------------------------------------------------------------------- /tests/fixtures/robots-txt-files/contains-invalid-lines.txt: -------------------------------------------------------------------------------- 1 | User-agent: * 2 | Allow: / 3 | foo 4 | 5 | User-agent: bar 6 | Allow: /bar 7 | #bar 8 | 9 | User-agent: foo 10 | Allow: /foo 11 | 12 | # comment preceding sitemap directive 13 | Sitemap: /sitemap.xml -------------------------------------------------------------------------------- /tests/fixtures/robots-txt-files/newscientist.com.txt: -------------------------------------------------------------------------------- 1 | User-agent: * 2 | Disallow: /article/in* 3 | 4 | Sitemap: http://www.newscientist.com/sitemap_index.xml.gz 5 | 6 | User-Agent: Zibber 7 | Disallow: /auth/ 8 | Disallow: /commenting/ 9 | Disallow: /contact/ 10 | Disallow: /info/ 11 | Disallow: /search* 12 | Disallow: /subs/ 13 | 14 | Disallow /search*offset=* 15 | Allow /search*offset=10 16 | Allow /search*offset=20 -------------------------------------------------------------------------------- /tests/File/AbstractFileTest.php: -------------------------------------------------------------------------------- 1 | file = new File(); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /phpstan.neon: -------------------------------------------------------------------------------- 1 | parameters: 2 | inferPrivatePropertyTypeFromConstructor: true 3 | ignoreErrors: 4 | # Return type of 'array' of test data providers 5 | - 6 | message: '#DataProvider\(\) return type has no value type specified in iterable type array#' 7 | path: 'tests' 8 | 9 | # Test methods with intentionally no return type 10 | - 11 | message: '#::test.+\(\) has no return typehint specified#' 12 | path: 'tests' 13 | -------------------------------------------------------------------------------- /src/Directive/DirectiveInterface.php: -------------------------------------------------------------------------------- 1 | userAgentDirective = new UserAgentDirective('*'); 19 | } 20 | 21 | public function testCastUserAgentDirectiveToString() 22 | { 23 | $this->assertEquals('user-agent:*', (string)$this->userAgentDirective); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /tests/fixtures/user-agent-strings.json: -------------------------------------------------------------------------------- 1 | { 2 | "bingbot-1": "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)", 3 | "bingbot-2": "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)", 4 | "bingbot-3": "Mozilla/5.0 (Windows Phone 8.1; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 530) like Gecko (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)", 5 | "googlebot-1": "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", 6 | "googlebot-2": "Googlebot/2.1 (+http://www.google.com/bot.html)", 7 | "slurp": "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 8 | } -------------------------------------------------------------------------------- /src/Directive/UserAgentDirective.php: -------------------------------------------------------------------------------- 1 | get() === $comparator->get(); 29 | } 30 | 31 | public function get(): string 32 | { 33 | return trim(parent::get()); 34 | } 35 | 36 | public function __toString(): string 37 | { 38 | return trim(parent::__toString()); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /phpunit.xml.dist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 14 | 15 | 16 | 17 | tests/ 18 | 19 | 20 | 21 | 22 | ./src 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012 Jon Cram 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is furnished 8 | to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. -------------------------------------------------------------------------------- /src/Directive/Validator.php: -------------------------------------------------------------------------------- 1 | 0; 34 | } 35 | 36 | private static function doesDirectiveStringContainSeparatorBetweenFieldAndValue(string $directiveString): bool 37 | { 38 | return substr($directiveString, 0, 1) != Directive::FIELD_VALUE_SEPARATOR; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "webignition/robots-txt-file", 3 | "description": "Models a robots.txt file", 4 | "keywords": ["robots.txt", "parser"], 5 | "homepage": "https://github.com/webignition/robots-txt-file", 6 | "type": "library", 7 | "license": "MIT", 8 | "authors": [ 9 | { 10 | "name": "Jon Cram", 11 | "email": "webignition@gmail.com" 12 | } 13 | ], 14 | "autoload": { 15 | "psr-4": { 16 | "webignition\\RobotsTxt\\": "src/", 17 | "webignition\\RobotsTxt\\Tests\\": "tests/" 18 | } 19 | }, 20 | "scripts": { 21 | "cs": "./vendor/bin/phpcs src tests --colors --standard=PSR12", 22 | "static-analysis": "./vendor/bin/phpstan analyse src tests --level=7", 23 | "test": "./vendor/bin/phpunit --colors=always", 24 | "ci": [ 25 | "@composer cs", 26 | "@composer static-analysis", 27 | "@composer test" 28 | ] 29 | }, 30 | "require": { 31 | "php": ">=7.2.0", 32 | "ext-json": "*", 33 | "ext-mbstring": "*", 34 | "webignition/disallowed-character-terminated-string": ">=2,<3" 35 | }, 36 | "require-dev": { 37 | "phpunit/phpunit": "^8.0", 38 | "squizlabs/php_codesniffer": "^3.5", 39 | "phpstan/phpstan": "^0.12.3" 40 | }, 41 | "minimum-stability":"stable", 42 | "prefer-stable":true 43 | } 44 | -------------------------------------------------------------------------------- /src/Directive/Directive.php: -------------------------------------------------------------------------------- 1 | " 16 | * 17 | * @var Value 18 | */ 19 | private $value = null; 20 | 21 | /** 22 | * @param string $field 23 | * @param string $value 24 | */ 25 | public function __construct(string $field = '', string $value = '') 26 | { 27 | $this->field = mb_strtolower($field); 28 | $this->value = new Value($value); 29 | } 30 | 31 | public function getField(): string 32 | { 33 | return $this->field; 34 | } 35 | 36 | public function getValue(): Value 37 | { 38 | return $this->value; 39 | } 40 | 41 | public function __toString(): string 42 | { 43 | return $this->getField() . self::FIELD_VALUE_SEPARATOR . $this->getValue(); 44 | } 45 | 46 | public function equals(DirectiveInterface $comparator): bool 47 | { 48 | if ($this->getField() != $comparator->getField()) { 49 | return false; 50 | } 51 | 52 | if (!$this->getValue()->equals($comparator->getValue())) { 53 | return false; 54 | } 55 | 56 | return true; 57 | } 58 | 59 | public function isType(string $type): bool 60 | { 61 | return strtolower($this->getField()) == strtolower($type); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /tests/Directive/ValueTest.php: -------------------------------------------------------------------------------- 1 | assertEquals($expectedStringValue, (string) $directiveValue); 19 | } 20 | 21 | public function directiveStringValueDataProvider(): array 22 | { 23 | return [ 24 | 'generic value 1' => [ 25 | 'directiveStringValue' => 'value1', 26 | 'expectedStringValue' => 'value1', 27 | ], 28 | 'generic value 2' => [ 29 | 'directiveStringValue' => 'value2', 30 | 'expectedStringValue' => 'value2', 31 | ], 32 | 'with line return' => [ 33 | 'directiveStringValue' => 'foo' . "\n" . 'bar', 34 | 'expectedStringValue' => 'foo', 35 | ], 36 | 'with carriage return' => [ 37 | 'directiveStringValue' => 'foo' . "\r" . 'bar', 38 | 'expectedStringValue' => 'foo', 39 | ], 40 | 'with comment' => [ 41 | 'directiveStringValue' => 'foo #bar', 42 | 'expectedStringValue' => 'foo', 43 | ], 44 | ]; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/Record/Record.php: -------------------------------------------------------------------------------- 1 | userAgentDirectiveList)) { 33 | $this->userAgentDirectiveList = new UserAgentDirectiveList(); 34 | } 35 | 36 | return $this->userAgentDirectiveList; 37 | } 38 | 39 | public function getDirectiveList(): DirectiveList 40 | { 41 | if (is_null($this->directiveList)) { 42 | $this->directiveList = new DirectiveList(); 43 | } 44 | 45 | return $this->directiveList; 46 | } 47 | 48 | public function __toString(): string 49 | { 50 | $stringRepresentation = ''; 51 | 52 | $directives = array_merge( 53 | $this->getUserAgentDirectiveList()->getDirectives(), 54 | $this->getDirectiveList()->getDirectives() 55 | ); 56 | 57 | foreach ($directives as $directive) { 58 | $stringRepresentation .= $directive . "\n"; 59 | } 60 | 61 | return trim($stringRepresentation); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /tests/Directive/ValidatorTest.php: -------------------------------------------------------------------------------- 1 | assertTrue(Validator::isDirectiveStringValid($directiveString)); 17 | } 18 | 19 | public function validDirectiveDataProvider(): array 20 | { 21 | return [ 22 | 'generic field only' => [ 23 | 'directiveString' => 'foo:', 24 | ], 25 | 'generic field and value' => [ 26 | 'directiveString' => 'foo:bar', 27 | ], 28 | 'generic field only with comment' => [ 29 | 'directiveString' => 'foo: # comment', 30 | ], 31 | 'generic field and value with comment' => [ 32 | 'directiveString' => 'foo:bar # comment', 33 | ], 34 | ]; 35 | } 36 | 37 | /** 38 | * @dataProvider invalidDirectiveDataProvider 39 | */ 40 | public function testIsNotValid(string $directiveString) 41 | { 42 | $this->assertFalse(Validator::isDirectiveStringValid($directiveString)); 43 | } 44 | 45 | public function invalidDirectiveDataProvider(): array 46 | { 47 | return [ 48 | 'empty' => [ 49 | 'directiveString' => '', 50 | ], 51 | 'no field:value separator' => [ 52 | 'directiveString' => 'foo', 53 | ], 54 | 'starts with field:value separator' => [ 55 | 'directiveString' => ':foo', 56 | ], 57 | ]; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/File/File.php: -------------------------------------------------------------------------------- 1 | nonGroupDirectives)) { 46 | $this->nonGroupDirectives = new DirectiveList(); 47 | } 48 | 49 | return $this->nonGroupDirectives; 50 | } 51 | 52 | public function addRecord(Record $record): void 53 | { 54 | $this->records[] = $record; 55 | } 56 | 57 | /** 58 | * @return Record[] 59 | */ 60 | public function getRecords(): array 61 | { 62 | return $this->records; 63 | } 64 | 65 | public function __toString(): string 66 | { 67 | $string = ''; 68 | foreach ($this->records as $record) { 69 | $string .= $record . "\n\n"; 70 | } 71 | 72 | $string .= (string)$this->getNonGroupDirectives(); 73 | 74 | return trim($string); 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /tests/fixtures/robots-txt-files/stackoverflow.com.txt: -------------------------------------------------------------------------------- 1 | User-Agent: * 2 | Disallow: /posts/ 3 | Disallow: /posts? 4 | Disallow: /ask/ 5 | Disallow: /ask? 6 | Disallow: /questions/ask/ 7 | Disallow: /questions/ask? 8 | Disallow: /search/ 9 | Disallow: /search? 10 | Disallow: /feeds/ 11 | Disallow: /feeds? 12 | Disallow: /users/login/ 13 | Disallow: /users/login? 14 | Disallow: /users/logout/ 15 | Disallow: /users/logout? 16 | Disallow: /users/filter/ 17 | Disallow: /users/filter? 18 | Disallow: /users/authenticate/ 19 | Disallow: /users/authenticate? 20 | Disallow: /users/flag-weight/ 21 | Disallow: /users/flag-summary/ 22 | Disallow: /users/flair/ 23 | Disallow: /users/flair? 24 | Disallow: /users/activity/ 25 | Disallow: /users/stats/ 26 | Disallow: /users/*?tab=accounts 27 | Disallow: /users/rep/show 28 | Disallow: /users/rep/show? 29 | Disallow: /unanswered/ 30 | Disallow: /unanswered? 31 | Disallow: /u/ 32 | Disallow: /messages/ 33 | Disallow: /api/ 34 | Disallow: /review/ 35 | Disallow: /*/ivc/* 36 | Disallow: /*?lastactivity 37 | Disallow: /users/login/global/request/ 38 | Disallow: /users/login/global/request? 39 | Disallow: /questions/*answertab=* 40 | Disallow: /questions/tagged/*+* 41 | Disallow: /questions/tagged/*%20* 42 | Disallow: /questions/*/answer/submit 43 | Disallow: /tags/*+* 44 | Disallow: /tags/*%20* 45 | Disallow: /suggested-edits/ 46 | Disallow: /suggested-edits? 47 | Disallow: /ajax/ 48 | Disallow: /plugins/ 49 | Allow: / 50 | 51 | # 52 | # beware, the sections below WILL NOT INHERIT from the above! 53 | # http://www.google.com/support/webmasters/bin/answer.py?hl=en&answer=40360 54 | # 55 | 56 | # 57 | # disallow adsense bot, as we no longer do adsense. 58 | # 59 | User-agent: Mediapartners-Google 60 | Disallow: / 61 | 62 | # 63 | # Yahoo bot is evil. 64 | # 65 | User-agent: Slurp 66 | Disallow: / 67 | 68 | # 69 | # Yahoo Pipes is for feeds not web pages. 70 | # 71 | User-agent: Yahoo Pipes 1.0 72 | Disallow: / 73 | 74 | # 75 | # This isn't really an image 76 | # 77 | User-agent: Googlebot-Image 78 | Disallow: /*/ivc/* 79 | Disallow: /users/flair/ 80 | 81 | # 82 | # this technically isn't valid, since for some godforsaken reason 83 | # sitemap paths must be ABSOLUTE and not relative. 84 | # 85 | Sitemap: /sitemap.xml -------------------------------------------------------------------------------- /src/DirectiveList/UserAgentDirectiveList.php: -------------------------------------------------------------------------------- 1 | getDirectives(); 33 | 34 | foreach ($userAgentDirectives as $userAgent) { 35 | $userAgents[] = (string)$userAgent->getValue(); 36 | } 37 | 38 | return $userAgents; 39 | } 40 | 41 | /** 42 | * @return DirectiveInterface[]|UserAgentDirective[] 43 | */ 44 | public function getDirectives(): array 45 | { 46 | $userAgents = parent::getDirectives(); 47 | 48 | if (empty($userAgents)) { 49 | $userAgents[] = new UserAgentDirective(UserAgentDirective::DEFAULT_USER_AGENT); 50 | } 51 | 52 | return $userAgents; 53 | } 54 | 55 | public function contains(DirectiveInterface $directive): bool 56 | { 57 | if ($directive instanceof UserAgentDirective) { 58 | return parent::contains($directive); 59 | } 60 | 61 | return false; 62 | } 63 | 64 | public function match(string $userAgentString): ?string 65 | { 66 | foreach ($this->getValues() as $userAgentIdentifier) { 67 | if ($userAgentIdentifier === UserAgentDirective::DEFAULT_USER_AGENT) { 68 | continue; 69 | } 70 | 71 | if (substr_count($userAgentString, $userAgentIdentifier)) { 72 | return $userAgentIdentifier; 73 | } 74 | } 75 | 76 | return null; 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /tests/Directive/DirectiveTest.php: -------------------------------------------------------------------------------- 1 | assertEquals($expectedString, (string)$directive); 19 | } 20 | 21 | public function castToStringDataProvider(): array 22 | { 23 | return [ 24 | [ 25 | 'field' => 'allow', 26 | 'value' => '/path-1', 27 | 'expectedString' => 'allow:/path-1', 28 | ], 29 | [ 30 | 'field' => 'disallow', 31 | 'value' => '/path-2', 32 | 'expectedString' => 'disallow:/path-2', 33 | ], 34 | ]; 35 | } 36 | 37 | public function testEquals() 38 | { 39 | $directive1 = new Directive('field1', 'value1'); 40 | $directive2 = new Directive('field1', 'value1'); 41 | $directive3 = new Directive('field3', 'value3'); 42 | 43 | $this->assertTrue($directive1->equals($directive2)); 44 | $this->assertFalse($directive1->equals($directive3)); 45 | } 46 | 47 | /** 48 | * @dataProvider isTypeDataProvider 49 | */ 50 | public function testIsType(string $field, string $expectedFieldType) 51 | { 52 | $directive = new Directive($field, 'foo'); 53 | 54 | $this->assertTrue($directive->isType($expectedFieldType)); 55 | } 56 | 57 | public function isTypeDataProvider(): array 58 | { 59 | return [ 60 | 'generic field 1' => [ 61 | 'field' => 'field1', 62 | 'expectedType' => 'field1', 63 | ], 64 | 'generic field 2' => [ 65 | 'field' => 'field2', 66 | 'expectedType' => 'field2', 67 | ], 68 | 'sitemap' => [ 69 | 'field' => Directive::TYPE_SITEMAP, 70 | 'expectedType' => Directive::TYPE_SITEMAP, 71 | ], 72 | ]; 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /tests/File/FileTest.php: -------------------------------------------------------------------------------- 1 | getDirectiveList()->add(new Directive('allow', '/allowed-path')); 17 | 18 | $this->file->addRecord($record); 19 | 20 | $fileRecords = $this->file->getRecords(); 21 | 22 | $this->assertCount(1, $fileRecords); 23 | $this->assertInstanceOf(Record::class, $fileRecords[0]); 24 | } 25 | 26 | public function testCastToStringWithDefaultUserAgent() 27 | { 28 | $record = new Record(); 29 | $record->getDirectiveList()->add(new Directive('allow', '/allowed-path')); 30 | 31 | $this->file->addRecord($record); 32 | 33 | $this->assertEquals('user-agent:*' . "\n" . 'allow:/allowed-path', (string)$this->file); 34 | } 35 | 36 | public function testCastToStringWithSpecificUserAgent() 37 | { 38 | $record = new Record(); 39 | $record->getDirectiveList()->add(new Directive('allow', '/allowed-path')); 40 | $record->getUserAgentDirectiveList()->add(new UserAgentDirective('googlebot')); 41 | 42 | $this->file->addRecord($record); 43 | 44 | $this->assertEquals('user-agent:googlebot' . "\n" . 'allow:/allowed-path', (string)$this->file); 45 | } 46 | 47 | public function testCastToStringWithMultipleRecords() 48 | { 49 | $record1 = new Record(); 50 | $record1->getDirectiveList()->add(new Directive('allow', '/allowed-path')); 51 | $record1->getUserAgentDirectiveList()->add(new UserAgentDirective('googlebot')); 52 | 53 | $record2 = new Record(); 54 | $record2->getDirectiveList()->add(new Directive('disallow', '/')); 55 | $record2->getUserAgentDirectiveList()->add(new UserAgentDirective(('slurp'))); 56 | 57 | $this->file->addRecord($record1); 58 | $this->file->addRecord($record2); 59 | 60 | $this->assertEquals( 61 | 'user-agent:googlebot' . "\n" . 'allow:/allowed-path' . "\n\n" . 'user-agent:slurp' . "\n" . 'disallow:/', 62 | (string)$this->file 63 | ); 64 | } 65 | 66 | public function testCastToStringWithDirectivesOnly() 67 | { 68 | $this->file->getNonGroupDirectives()->add(new Directive('sitemap', 'http://www.example.com/sitemap.xml')); 69 | 70 | $this->assertEquals('sitemap:http://www.example.com/sitemap.xml', (string)$this->file); 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/Inspector/UrlMatcher.php: -------------------------------------------------------------------------------- 1 | decodeUrlPath((string)$directive->getValue()); 25 | 26 | if (empty($decodedDirectivePath)) { 27 | return false; 28 | } 29 | $decodedRelativeUrl = $this->decodeUrlPath($urlPath); 30 | 31 | return preg_match($this->createRegex($decodedDirectivePath), $decodedRelativeUrl) > 0; 32 | } 33 | 34 | /** 35 | * Decode a URL path without decoding encoded forward slashes 36 | * 37 | * @param string $urlPath 38 | * 39 | * @return string 40 | */ 41 | private function decodeUrlPath(string $urlPath): string 42 | { 43 | $urlPath = str_replace(strtoupper(self::URL_ENCODED_SLASH), self::URL_ENCODED_SLASH, $urlPath); 44 | $urlPathParts = explode(self::URL_ENCODED_SLASH, $urlPath); 45 | 46 | array_walk($urlPathParts, function (&$urlPathPart) { 47 | $urlPathPart = rawurldecode($urlPathPart); 48 | }); 49 | 50 | return implode(self::URL_ENCODED_SLASH, $urlPathParts); 51 | } 52 | 53 | /** 54 | * Transform a directive URL path into an equivalent regex 55 | * 56 | * @param string $directiveUrlPath 57 | * 58 | * @return string 59 | */ 60 | private function createRegex(string $directiveUrlPath): string 61 | { 62 | $hasMustEndWithWildcard = substr($directiveUrlPath, -1) === self::MUST_END_WITH_WILDCARD; 63 | if ($hasMustEndWithWildcard) { 64 | $directiveUrlPath = rtrim($directiveUrlPath, self::MUST_END_WITH_WILDCARD); 65 | } 66 | 67 | $directiveUrlPath = rtrim($directiveUrlPath, self::ANY_CHARACTER_WILDCARD); 68 | $directiveValueParts = explode(self::ANY_CHARACTER_WILDCARD, $directiveUrlPath); 69 | 70 | array_walk($directiveValueParts, function (&$directiveValue) { 71 | $directiveValue = preg_quote($directiveValue, '/'); 72 | }); 73 | 74 | return '/^' . implode(".*", $directiveValueParts) . ($hasMustEndWithWildcard ? '$' : '') . '/'; 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /tests/DirectiveList/UserAgentDirectiveListTest.php: -------------------------------------------------------------------------------- 1 | userAgentDirectiveList = new UserAgentDirectiveList(); 20 | } 21 | 22 | public function testAdd() 23 | { 24 | $this->userAgentDirectiveList->add(new UserAgentDirective('googlebot')); 25 | $this->assertEquals(array('googlebot'), $this->userAgentDirectiveList->getValues()); 26 | 27 | $this->userAgentDirectiveList->add(new UserAgentDirective('slURp')); 28 | $this->assertEquals(array('googlebot', 'slurp'), $this->userAgentDirectiveList->getValues()); 29 | } 30 | 31 | public function testContains() 32 | { 33 | $userAgentDirective1 = new UserAgentDirective('agent1'); 34 | $userAgentDirective2 = new UserAgentDirective('agent2'); 35 | $userAgentDirective3 = new UserAgentDirective('agent3'); 36 | 37 | $this->userAgentDirectiveList->add($userAgentDirective1); 38 | $this->userAgentDirectiveList->add($userAgentDirective2); 39 | $this->userAgentDirectiveList->add($userAgentDirective3); 40 | 41 | $this->assertTrue($this->userAgentDirectiveList->contains($userAgentDirective1)); 42 | $this->assertTrue($this->userAgentDirectiveList->contains($userAgentDirective2)); 43 | $this->assertTrue($this->userAgentDirectiveList->contains($userAgentDirective3)); 44 | } 45 | 46 | public function testRemove() 47 | { 48 | $userAgentDirective1 = new UserAgentDirective('agent1'); 49 | $userAgentDirective2 = new UserAgentDirective('agent2'); 50 | $userAgentDirective3 = new UserAgentDirective('agent3'); 51 | 52 | $this->userAgentDirectiveList->add($userAgentDirective1); 53 | $this->userAgentDirectiveList->add($userAgentDirective2); 54 | $this->userAgentDirectiveList->add($userAgentDirective3); 55 | 56 | $this->assertEquals(array('agent1', 'agent2', 'agent3'), $this->userAgentDirectiveList->getValues()); 57 | 58 | $this->userAgentDirectiveList->remove($userAgentDirective1); 59 | $this->assertEquals(array('agent2', 'agent3'), $this->userAgentDirectiveList->getValues()); 60 | 61 | $this->userAgentDirectiveList->remove($userAgentDirective2); 62 | $this->assertEquals(array('agent3'), $this->userAgentDirectiveList->getValues()); 63 | 64 | $this->userAgentDirectiveList->remove($userAgentDirective3); 65 | $this->assertEquals(array('*'), $this->userAgentDirectiveList->getValues()); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /tests/Directive/FactoryTest.php: -------------------------------------------------------------------------------- 1 | assertNull(Factory::create($directiveString)); 21 | } 22 | 23 | /** 24 | * @return array 25 | */ 26 | public function invalidDirectiveStringDataProvider() 27 | { 28 | return [ 29 | 'empty' => [ 30 | 'directiveString' => '', 31 | ], 32 | 'no field:value separator' => [ 33 | 'directiveString' => 'foo', 34 | ], 35 | 'starts with field:value separator' => [ 36 | 'directiveString' => ':foo', 37 | ], 38 | ]; 39 | } 40 | 41 | /** 42 | * @dataProvider userAgentDirectiveDataProvider 43 | */ 44 | public function testCreateUserAgentDirective(string $directiveString, string $expectedValue) 45 | { 46 | $directive = Factory::create($directiveString); 47 | $this->assertInstanceOf(UserAgentDirective::class, $directive); 48 | 49 | $this->assertEquals(self::FIELD_USER_AGENT_DIRECTIVE, $directive->getField()); 50 | $this->assertEquals($expectedValue, (string)$directive->getValue()); 51 | } 52 | 53 | public function userAgentDirectiveDataProvider(): array 54 | { 55 | return [ 56 | [ 57 | 'directiveString' => self::FIELD_USER_AGENT_DIRECTIVE . ':foo', 58 | 'expectedValue' => 'foo', 59 | ], 60 | [ 61 | 'directiveString' => self::FIELD_USER_AGENT_DIRECTIVE . ':bar', 62 | 'expectedValue' => 'bar', 63 | ], 64 | 65 | ]; 66 | } 67 | 68 | /** 69 | * @dataProvider disallowDirectiveDataProvider 70 | */ 71 | public function testCreateDisallowDirective(string $directiveString, string $expectedValue) 72 | { 73 | $directive = Factory::create($directiveString); 74 | 75 | $this->assertSame($expectedValue, $directive->getValue()->get()); 76 | } 77 | 78 | public function disallowDirectiveDataProvider(): array 79 | { 80 | return [ 81 | [ 82 | 'directiveString' => self::FIELD_DISALLOW_DIRECTIVE . ':/', 83 | 'expectedValue' => '/', 84 | ], 85 | [ 86 | 'directiveString' => self::FIELD_DISALLOW_DIRECTIVE . ':', 87 | 'expectedValue' => '', 88 | ], 89 | 90 | ]; 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /tests/Record/RecordTest.php: -------------------------------------------------------------------------------- 1 | record = new Record(); 21 | } 22 | 23 | public function testCastToStringWithDefaultUserAgentList() 24 | { 25 | $this->assertEquals('user-agent:*', (string)$this->record); 26 | } 27 | 28 | public function testCastToStringWithMultipleUserAgents() 29 | { 30 | $this->record->getUserAgentDirectiveList()->add(new UserAgentDirective('googlebot')); 31 | $this->record->getUserAgentDirectiveList()->add(new UserAgentDirective('slurp')); 32 | 33 | $this->assertEquals('user-agent:googlebot' . "\n" . 'user-agent:slurp', (string)$this->record); 34 | } 35 | 36 | public function testCastToStringWithAllowDisallowOnly() 37 | { 38 | $this->record->getDirectiveList()->add(new Directive('allow', '/allowed-path')); 39 | $this->record->getDirectiveList()->add(new Directive('disallow', '/disallowed-path')); 40 | 41 | $this->assertEquals( 42 | 'user-agent:*' . "\n" . 'allow:/allowed-path' . "\n" . 'disallow:/disallowed-path', 43 | (string)$this->record 44 | ); 45 | } 46 | 47 | public function testCastToStringWithMultipleUserAgentsAndAllowDisallow() 48 | { 49 | $this->record->getUserAgentDirectiveList()->add(new UserAgentDirective('googlebot')); 50 | $this->record->getUserAgentDirectiveList()->add(new UserAgentDirective('slurp')); 51 | 52 | $this->record->getDirectiveList()->add(new Directive('allow', '/allowed-path')); 53 | $this->record->getDirectiveList()->add(new Directive('disallow', '/disallowed-path')); 54 | 55 | $this->assertEquals( 56 | 'user-agent:googlebot' . "\n" . 57 | 'user-agent:slurp' . "\n" . 58 | 'allow:/allowed-path' . "\n" . 59 | 'disallow:/disallowed-path', 60 | (string)$this->record 61 | ); 62 | } 63 | 64 | public function testInclusionOfDefaultUserAgent() 65 | { 66 | $this->assertEquals(array('*'), $this->record->getUserAgentDirectiveList()->getValues()); 67 | } 68 | 69 | public function testAddUserAgent() 70 | { 71 | $this->record->getUserAgentDirectiveList()->add(new UserAgentDirective('googlebot')); 72 | 73 | $this->assertEquals(array('googlebot'), $this->record->getUserAgentDirectiveList()->getValues()); 74 | } 75 | 76 | public function testRemoveUserAgent() 77 | { 78 | $googlebotDirective = new UserAgentDirective('googlebot'); 79 | 80 | $this->record->getUserAgentDirectiveList()->add($googlebotDirective); 81 | $this->record->getUserAgentDirectiveList()->remove($googlebotDirective); 82 | 83 | $this->assertEquals(array('*'), $this->record->getUserAgentDirectiveList()->getValues()); 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/DirectiveList/DirectiveList.php: -------------------------------------------------------------------------------- 1 | contains($directive)) { 19 | $this->directives[] = $directive; 20 | } 21 | } 22 | 23 | public function remove(DirectiveInterface $directive): void 24 | { 25 | $directivePosition = null; 26 | foreach ($this->directives as $userAgentIndex => $existingUserAgent) { 27 | if ($directive->equals($existingUserAgent)) { 28 | $directivePosition = $userAgentIndex; 29 | } 30 | } 31 | 32 | if (!is_null($directivePosition)) { 33 | unset($this->directives[$directivePosition]); 34 | } 35 | } 36 | 37 | /** 38 | * @return string[] 39 | */ 40 | public function getValues(): array 41 | { 42 | $directives = array(); 43 | foreach ($this->directives as $directive) { 44 | $directives[] = (string)$directive; 45 | } 46 | 47 | return $directives; 48 | } 49 | 50 | /** 51 | * @return DirectiveInterface[] 52 | */ 53 | public function getDirectives(): array 54 | { 55 | return $this->directives; 56 | } 57 | 58 | public function first(): ?DirectiveInterface 59 | { 60 | return $this->directives[0] ?? null; 61 | } 62 | 63 | public function contains(DirectiveInterface $directive): bool 64 | { 65 | foreach ($this->directives as $existingDirective) { 66 | if ($directive->equals($existingDirective)) { 67 | return true; 68 | } 69 | } 70 | 71 | return false; 72 | } 73 | 74 | public function containsField(string $fieldName): bool 75 | { 76 | $fieldName = strtolower(trim($fieldName)); 77 | 78 | foreach ($this->directives as $directive) { 79 | if ($directive->getField() == $fieldName) { 80 | return true; 81 | } 82 | } 83 | 84 | return false; 85 | } 86 | 87 | public function __toString(): string 88 | { 89 | $string = ''; 90 | $directives = $this->getDirectives(); 91 | 92 | foreach ($directives as $directive) { 93 | $string .= $directive . "\n"; 94 | } 95 | 96 | return trim($string); 97 | } 98 | 99 | public function getByField(string $field): DirectiveList 100 | { 101 | $directives = $this->getDirectives(); 102 | 103 | foreach ($directives as $directiveIndex => $directive) { 104 | if (!$directive->isType($field)) { 105 | unset($directives[$directiveIndex]); 106 | } 107 | } 108 | 109 | $directiveList = new DirectiveList(); 110 | foreach ($directives as $directive) { 111 | $directiveList->add($directive); 112 | } 113 | 114 | return $directiveList; 115 | } 116 | 117 | public function getLength(): int 118 | { 119 | return count($this->directives); 120 | } 121 | 122 | public function isEmpty(): bool 123 | { 124 | $isEmpty = true; 125 | 126 | foreach ($this->getDirectives() as $directive) { 127 | if (!$isEmpty) { 128 | continue; 129 | } 130 | 131 | if (!empty((string)$directive->getValue())) { 132 | $isEmpty = false; 133 | } 134 | } 135 | 136 | return $isEmpty; 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /tests/File/ParserTest.php: -------------------------------------------------------------------------------- 1 | parser = new Parser(); 27 | $this->dataSourceBasePath = __DIR__ . '/../fixtures/robots-txt-files'; 28 | } 29 | 30 | public function testParsingStackoverflowDotCom() 31 | { 32 | $this->setParserSourceFromDataFile('stackoverflow.com.txt'); 33 | $file = $this->parser->getFile(); 34 | $inspector = new Inspector($file); 35 | 36 | $this->assertNotEmpty((string)$file); 37 | 38 | $records = $file->getRecords(); 39 | 40 | $this->assertCount(5, $records); 41 | 42 | $record1 = $records[0]; 43 | 44 | $this->assertEquals(array('*'), $record1->getUserAgentDirectiveList()->getValues()); 45 | $this->assertCount(48, $record1->getDirectiveList()->getDirectives()); 46 | $this->assertTrue($record1->getDirectiveList()->contains( 47 | new Directive('disallow', '/users/login/global/request/') 48 | )); 49 | 50 | $inspector->setUserAgent('googlebot-image'); 51 | $this->assertEquals( 52 | 'disallow:/*/ivc/*' . "\n" . 'disallow:/users/flair/', 53 | (string)$inspector->getDirectives() 54 | ); 55 | $this->assertEquals( 56 | 'sitemap:/sitemap.xml', 57 | (string)$file->getNonGroupDirectives()->getByField('sitemap') 58 | ); 59 | } 60 | 61 | public function testParsingWithSitemapAsLastLineInSingleRecord() 62 | { 63 | $this->setParserSourceFromDataFile('sitemapAsLastLineInSingleRecord.txt'); 64 | 65 | $file = $this->parser->getFile(); 66 | 67 | $this->assertCount(1, $file->getRecords()); 68 | $this->assertCount(1, $file->getNonGroupDirectives()->getValues()); 69 | 70 | $this->assertEquals( 71 | 'sitemap:http://example.com/sitemap.xml', 72 | (string)$file->getNonGroupDirectives()->getByField('sitemap') 73 | ); 74 | } 75 | 76 | public function testParsingWithSitemapWithinSingleRecord() 77 | { 78 | $this->setParserSourceFromDataFile('sitemapWithinSingleRecord.txt'); 79 | 80 | $file = $this->parser->getFile(); 81 | $inspector = new Inspector($file); 82 | 83 | $this->assertCount(1, $file->getRecords()); 84 | $this->assertCount(2, $inspector->getDirectives()->getValues()); 85 | $this->assertCount(1, $file->getNonGroupDirectives()->getValues()); 86 | 87 | $this->assertEquals( 88 | 'sitemap:http://example.com/sitemap.xml', 89 | (string)$file->getNonGroupDirectives()->getByField('sitemap') 90 | ); 91 | } 92 | 93 | public function testParsingInvalidSitemap() 94 | { 95 | $this->setParserSourceFromDataFile('newscientist.com.txt'); 96 | 97 | $file = $this->parser->getFile(); 98 | $inspector = new Inspector($file); 99 | 100 | $this->assertCount(2, $file->getRecords()); 101 | $this->assertCount(1, $inspector->getDirectives()->getValues()); 102 | 103 | $inspector->setUserAgent('zibber'); 104 | $this->assertCount(6, $inspector->getDirectives()->getValues()); 105 | $this->assertCount(1, $file->getNonGroupDirectives()->getValues()); 106 | $this->assertCount(1, $file->getNonGroupDirectives()->getByField('sitemap')->getValues()); 107 | } 108 | 109 | public function testParsingWithStartingBOM() 110 | { 111 | $this->setParserSourceFromDataFile('withBom.txt'); 112 | 113 | $file = $this->parser->getFile(); 114 | $inspector = new Inspector($file); 115 | 116 | $this->assertCount(1, $file->getRecords()); 117 | $this->assertCount(2, $inspector->getDirectives()->getValues()); 118 | 119 | $this->assertEquals(array( 120 | 'disallow:/', 121 | 'allow:/humans.txt' 122 | ), $inspector->getDirectives()->getValues()); 123 | } 124 | 125 | public function testParsingInvalidLines() 126 | { 127 | $this->setParserSourceFromDataFile('contains-invalid-lines.txt'); 128 | 129 | $file = $this->parser->getFile(); 130 | $inspector = new Inspector($file); 131 | 132 | $this->assertCount(3, $file->getRecords()); 133 | 134 | $inspector->setUserAgent('*'); 135 | $this->assertCount(1, $inspector->getDirectives()->getValues()); 136 | $this->assertEquals([ 137 | 'allow:/', 138 | ], $inspector->getDirectives()->getValues()); 139 | 140 | $inspector->setUserAgent('foo'); 141 | $this->assertCount(1, $inspector->getDirectives()->getValues()); 142 | $this->assertEquals([ 143 | 'allow:/foo', 144 | ], $inspector->getDirectives()->getValues()); 145 | 146 | $inspector->setUserAgent('bar'); 147 | $this->assertCount(1, $inspector->getDirectives()->getValues()); 148 | $this->assertEquals([ 149 | 'allow:/bar', 150 | ], $inspector->getDirectives()->getValues()); 151 | 152 | $sitemapDirective = $file->getNonGroupDirectives()->getDirectives()[0]; 153 | $this->assertEquals('sitemap', $sitemapDirective->getField()); 154 | $this->assertEquals('/sitemap.xml', $sitemapDirective->getValue()); 155 | } 156 | 157 | private function setParserSourceFromDataFile(string $relativePath): void 158 | { 159 | $this->parser->setSource((string) file_get_contents($this->dataSourceBasePath . '/' . $relativePath)); 160 | } 161 | } 162 | -------------------------------------------------------------------------------- /tests/DirectiveList/DirectiveListTest.php: -------------------------------------------------------------------------------- 1 | directiveList = new DirectiveList(); 20 | } 21 | 22 | public function testAddWhereDirectiveIsNotAlreadyInList() 23 | { 24 | $directive = new Directive('field1', 'value1'); 25 | $this->directiveList->add($directive); 26 | 27 | $this->assertEquals(1, $this->directiveList->getLength()); 28 | $this->assertTrue($this->directiveList->contains($directive)); 29 | } 30 | 31 | public function testAddWhereDirectiveIsAlreadyInList() 32 | { 33 | $directive = new Directive('field1', 'value1'); 34 | $this->directiveList->add($directive); 35 | $this->directiveList->add($directive); 36 | 37 | $this->assertEquals(1, $this->directiveList->getLength()); 38 | $this->assertTrue($this->directiveList->contains($directive)); 39 | } 40 | 41 | public function testContains() 42 | { 43 | $directive1 = new Directive('field1', 'value1'); 44 | $directive2 = new Directive('field2', 'value2'); 45 | $directive3 = new Directive('field3', 'value3'); 46 | 47 | $this->directiveList->add($directive1); 48 | $this->directiveList->add($directive2); 49 | 50 | $this->assertTrue($this->directiveList->contains($directive1)); 51 | $this->assertTrue($this->directiveList->contains($directive2)); 52 | $this->assertFalse($this->directiveList->contains($directive3)); 53 | } 54 | 55 | public function testCastToString() 56 | { 57 | $this->directiveList->add(new Directive('field1', 'value1')); 58 | $this->assertEquals('field1:value1', (string)$this->directiveList); 59 | 60 | $this->directiveList->add(new Directive('field2', 'value2')); 61 | $this->assertEquals('field1:value1' . "\n" . 'field2:value2', (string)$this->directiveList); 62 | } 63 | 64 | public function testContainsField() 65 | { 66 | $directive = new Directive('field1', 'value1'); 67 | 68 | $this->directiveList->add($directive); 69 | 70 | $this->assertTrue($this->directiveList->containsField('field1')); 71 | $this->assertFalse($this->directiveList->containsField('field2')); 72 | } 73 | 74 | public function testFirst() 75 | { 76 | $this->directiveList->add(new Directive('field1', 'value1')); 77 | $this->directiveList->add(new Directive('field2', 'value2')); 78 | 79 | $this->assertEquals('field1:value1', (string)$this->directiveList->first()); 80 | } 81 | 82 | public function testRemove() 83 | { 84 | $directive1 = new Directive('field1', 'value1'); 85 | $directive2 = new Directive('field2', 'value2'); 86 | $directive3 = new Directive('field3', 'value3'); 87 | 88 | $this->directiveList->add($directive1); 89 | $this->directiveList->add($directive2); 90 | $this->directiveList->add($directive3); 91 | $this->assertEquals( 92 | array('field1:value1', 'field2:value2', 'field3:value3'), 93 | $this->directiveList->getValues() 94 | ); 95 | 96 | $this->directiveList->remove($directive1); 97 | $this->assertEquals(array('field2:value2', 'field3:value3'), $this->directiveList->getValues()); 98 | 99 | $this->directiveList->remove($directive2); 100 | $this->assertEquals(array('field3:value3'), $this->directiveList->getValues()); 101 | 102 | $this->directiveList->remove($directive3); 103 | $this->assertEquals(array(), $this->directiveList->getValues()); 104 | } 105 | 106 | /** 107 | * @dataProvider getByFieldDataProvider 108 | * 109 | * @param array> $directives 110 | * @param string $field 111 | * @param string $expectedDirectiveListString 112 | */ 113 | public function testGetByField(array $directives, string $field, string $expectedDirectiveListString) 114 | { 115 | foreach ($directives as $directive) { 116 | $this->directiveList->add(new Directive($directive['field'], $directive['value'])); 117 | } 118 | 119 | $this->assertEquals( 120 | $expectedDirectiveListString, 121 | (string)$this->directiveList->getByField($field) 122 | ); 123 | } 124 | 125 | public function getByFieldDataProvider(): array 126 | { 127 | return [ 128 | 'none for empty list' => [ 129 | 'directives' => [], 130 | 'field' => 'foo', 131 | 'expectedDirectiveListString' => '', 132 | ], 133 | 'none for no matches' => [ 134 | 'directives' => [ 135 | [ 136 | 'field' => 'allow', 137 | 'value' => '/foo', 138 | ], 139 | ], 140 | 'field' => 'foo', 141 | 'expectedDirectiveListString' => '', 142 | ], 143 | 'one match' => [ 144 | 'directives' => [ 145 | [ 146 | 'field' => 'allow', 147 | 'value' => '/foo', 148 | ], 149 | ], 150 | 'field' => 'allow', 151 | 'expectedDirectiveListString' => 'allow:/foo', 152 | ], 153 | 'many matches' => [ 154 | 'directives' => [ 155 | [ 156 | 'field' => 'allow', 157 | 'value' => '/foo', 158 | ], 159 | [ 160 | 'field' => 'disallow', 161 | 'value' => '/bar', 162 | ], 163 | [ 164 | 'field' => 'allow', 165 | 'value' => '/foobar', 166 | ], 167 | [ 168 | 'field' => 'disallow', 169 | 'value' => '/fizz', 170 | ], 171 | [ 172 | 'field' => 'allow', 173 | 'value' => '/buzz', 174 | ], 175 | ], 176 | 'field' => 'allow', 177 | 'expectedDirectiveListString' => 'allow:/foo' . "\n" . 'allow:/foobar' . "\n" . 'allow:/buzz', 178 | ], 179 | ]; 180 | } 181 | } 182 | -------------------------------------------------------------------------------- /src/Inspector/Inspector.php: -------------------------------------------------------------------------------- 1 | file = $file; 30 | $this->setUserAgent($userAgent); 31 | } 32 | 33 | public function setUserAgent(string $userAgent = '*'): void 34 | { 35 | $this->userAgent = trim(mb_strtolower($userAgent)); 36 | } 37 | 38 | public function getDirectives(): DirectiveList 39 | { 40 | $isDefaultUserAgent = $this->userAgent === UserAgentDirective::DEFAULT_USER_AGENT; 41 | $defaultUserAgentDirectives = $this->getDirectivesForDefaultUserAgent(); 42 | 43 | if ($isDefaultUserAgent) { 44 | return $defaultUserAgentDirectives; 45 | } 46 | 47 | $records = $this->file->getRecords(); 48 | $matchedDirectiveLists = []; 49 | 50 | foreach ($records as $record) { 51 | $userAgentDirectiveListMatch = $record->getUserAgentDirectiveList()->match($this->userAgent); 52 | 53 | if (!is_null($userAgentDirectiveListMatch)) { 54 | $matchedDirectiveLists[$userAgentDirectiveListMatch] = $record->getDirectiveList(); 55 | } 56 | } 57 | 58 | $matchCount = count($matchedDirectiveLists); 59 | 60 | if ($matchCount === 0) { 61 | return $defaultUserAgentDirectives; 62 | } 63 | 64 | if ($matchCount === 1) { 65 | $directiveList = reset($matchedDirectiveLists); 66 | 67 | return $directiveList instanceof DirectiveList ? $directiveList : new DirectiveList(); 68 | } 69 | 70 | return $matchedDirectiveLists[ 71 | $this->findBestUserAgentStringToUserAgentIdentifierMatch(array_keys($matchedDirectiveLists)) 72 | ]; 73 | } 74 | 75 | /** 76 | * A urlPath is allowed if either: 77 | * - there are no matching disallow directives 78 | * - the longest matching allow path is greater in length than the longest matching disallow path 79 | * 80 | * Note: Google webmaster docs state that the allow/disallow precedence for paths containing wildcards 81 | * is undefined. 82 | * Many robots txt checking tools appear to use the 'longest rule wins' option regardless of 83 | * wildcards. It is this approach that is taken here. 84 | * 85 | * @param string $urlPath 86 | * 87 | * @return bool 88 | */ 89 | public function isAllowed(string $urlPath): bool 90 | { 91 | $matchingDisallowDirectives = $this->getMatchingAllowDisallowDirectivePaths( 92 | $urlPath, 93 | DirectiveInterface::TYPE_DISALLOW 94 | ); 95 | 96 | $matchingAllowDirectives = $this->getMatchingAllowDisallowDirectivePaths( 97 | $urlPath, 98 | DirectiveInterface::TYPE_ALLOW 99 | ); 100 | 101 | if (empty($matchingDisallowDirectives)) { 102 | return true; 103 | } 104 | 105 | if (empty($matchingAllowDirectives)) { 106 | return false; 107 | } 108 | 109 | $longestDisallowPath = $this->getLongestPath($matchingDisallowDirectives); 110 | $longestAllowPath = $this->getLongestPath($matchingAllowDirectives); 111 | 112 | if ($longestAllowPath === $longestDisallowPath) { 113 | return true; 114 | } 115 | 116 | $disallowPathLength = strlen($longestDisallowPath); 117 | $allowPathLength = strlen($longestAllowPath); 118 | 119 | if ($disallowPathLength === $allowPathLength) { 120 | return false; 121 | } 122 | 123 | return $allowPathLength > $disallowPathLength; 124 | } 125 | 126 | /** 127 | * @param string[] $paths 128 | * 129 | * @return string 130 | */ 131 | private function getLongestPath(array $paths): string 132 | { 133 | return array_reduce($paths, function (?string $a, string $b) { 134 | return strlen((string) $a) > strlen((string) $b) ? $a : $b; 135 | }); 136 | } 137 | 138 | /** 139 | * @param string $urlPath 140 | * @param string $type 141 | * 142 | * @return string[] 143 | */ 144 | private function getMatchingAllowDisallowDirectivePaths(string $urlPath, string $type): array 145 | { 146 | $matchingDirectives = []; 147 | $directives = $this->getDirectives(); 148 | $disallowDirectives = $directives->getByField($type); 149 | 150 | if ($disallowDirectives->isEmpty()) { 151 | return $matchingDirectives; 152 | } 153 | 154 | $matcher = new UrlMatcher(); 155 | 156 | foreach ($disallowDirectives->getDirectives() as $disallowDirective) { 157 | if ($matcher->matches($disallowDirective, $urlPath)) { 158 | $matchingDirectives[] = (string)$disallowDirective->getValue(); 159 | } 160 | } 161 | 162 | return $matchingDirectives; 163 | } 164 | 165 | private function getDirectivesForDefaultUserAgent(): DirectiveList 166 | { 167 | $defaultUserAgentDirective = new UserAgentDirective(UserAgentDirective::DEFAULT_USER_AGENT); 168 | 169 | foreach ($this->file->getRecords() as $record) { 170 | if ($record->getUserAgentDirectiveList()->contains($defaultUserAgentDirective)) { 171 | return $record->getDirectiveList(); 172 | } 173 | } 174 | 175 | return new DirectiveList(); 176 | } 177 | 178 | /** 179 | * @param string[] $userAgentIdentifiers 180 | * 181 | * @return string 182 | */ 183 | private function findBestUserAgentStringToUserAgentIdentifierMatch(array $userAgentIdentifiers): string 184 | { 185 | $scores = array(); 186 | $longestUserAgentIdentifier = ''; 187 | $highestScore = 0; 188 | $highestScoringUserAgentIdentifier = ''; 189 | 190 | foreach ($userAgentIdentifiers as $userAgentIdentifier) { 191 | $scores[$userAgentIdentifier] = 0; 192 | 193 | if ($this->userAgent === $userAgentIdentifier) { 194 | $scores[$userAgentIdentifier]++; 195 | } 196 | 197 | if (mb_strlen($userAgentIdentifier) > mb_strlen($longestUserAgentIdentifier)) { 198 | $longestUserAgentIdentifier = $userAgentIdentifier; 199 | } 200 | } 201 | 202 | $scores[$longestUserAgentIdentifier]++; 203 | 204 | foreach ($scores as $userAgentIdentifier => $score) { 205 | if ($score > $highestScore) { 206 | $highestScore = $score; 207 | $highestScoringUserAgentIdentifier = $userAgentIdentifier; 208 | } 209 | } 210 | 211 | return $highestScoringUserAgentIdentifier; 212 | } 213 | } 214 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # robots-txt-file [![Build Status](https://secure.travis-ci.org/webignition/robots-txt-file.png?branch=master)](http://travis-ci.org/webignition/robots-txt-file) 2 | 3 | - [Introduction](#introduction) 4 | - [Overview](#overview) 5 | - [Robots.txt file format refresher](#robots.txt-file-format-refresher) 6 | - [Usage](#usage) 7 | - [Parsing a robots.txt file from a string into a model](#parsing-a-robots.txt-file-from-a-string-into-a-model) 8 | - [Inspecting a model to get directives for a user agent](#inspecting-a-model-to-get-directives-for-a-user-agent) 9 | - [Check if a user agent is allowed to access a url path](#check-if-a-user-agent-is-allowed-to-access-a-url-path) 10 | - [Extract sitemap URLs](#extract-sitemap-urls) 11 | - [Filtering directives for a user agent to a specific field type](#filtering-directives-for-a-user-agent-to-a-specific-field-type) 12 | - [Building](#building) 13 | - [Using as a library in a project](#using-as-a-library-in-a-project) 14 | - [Developing](#developing) 15 | - [Testing](#testing) 16 | 17 | ## Introduction 18 | 19 | ### Overview 20 | 21 | Handles [robots.txt][1] files: 22 | 23 | - parse a robots.txt file into a model 24 | - get directives for a user agent 25 | - check if a user agent is allowed to access a url path 26 | - extract sitemap URLs 27 | - programmatically create a model and cast to a string 28 | 29 | ### Robots.txt file format refresher 30 | 31 | Let's quickly go over the format of a robots.txt file so that you can understand what you can get out of a `\webignition\RobotsTxt\File\File` object. 32 | 33 | A robots.txt file contains a collection of **records**. A record provides a set of **directives** to a specified user agent. A directive instructs a user agent to do something (or not do something). A blank line is used to separate records. 34 | 35 | Here's an example with two records: 36 | 37 | User-agent: Slurp 38 | Disallow: / 39 | 40 | User-Agent: * 41 | Disallow: /private 42 | 43 | This instructs the user agent 'Slurp' that it is not allowed access to '/' (i.e. the whole site), and this instructs all other user agents that they are not allowed access to '/private'. 44 | 45 | A robots.txt file can optionally contain directives that apply to all user agents irrespective of the specified records. These are included as a set of a directives that are not part of a record. A common use is the `sitemap` directive. 46 | 47 | Here's an example with directives that apply to everyone and everything: 48 | 49 | User-agent: Slurp 50 | Disallow: / 51 | 52 | User-Agent: * 53 | Disallow: /private 54 | 55 | Sitemap: http://example.com/sitemap.xml 56 | 57 | ## Usage 58 | 59 | ### Parsing a robots.txt file from a string into a model 60 | ```php 61 | setSource(file_get_contents('http://example.com/robots.txt')); 66 | 67 | $robotsTxtFile = $parser->getFile(); 68 | 69 | // Get an array of records 70 | $robotsTxtFile->getRecords(); 71 | 72 | // Get the list of record-independent directives (such as sitemap directives): 73 | $robotsTxtFile->getNonGroupDirectives()->get(); 74 | ``` 75 | 76 | This might not be too useful on it's own. You'd normally be retrieving information from a robots.txt file because 77 | you are a crawler and need to know what you are allowed to access (or disallowed) or because you're a tool or 78 | service that needs to locate a site's sitemap.xml file. 79 | 80 | ### Inspecting a model to get directives for a user agent 81 | 82 | Let's say we're the 'Slurp' user agent and we want to know what's been specified for us: 83 | 84 | ```php 85 | setSource(file_get_contents('http://example.com/robots.txt')); 91 | 92 | $inspector = new Inspector($parser->getFile()); 93 | $inspector->setUserAgent('slurp'); 94 | 95 | $slurpDirectiveList = $inspector->getDirectives(); 96 | ``` 97 | 98 | Ok, now we have a [DirectiveList](https://github.com/webignition/robots-txt-file/blob/master/src/webignition/RobotsTxt/DirectiveList/DirectiveList.php) 99 | containing a collection of directives. We can call `$directiveList->get()` to get the directives applicable to us. 100 | 101 | This raw set of directives is available in the model because it is 102 | there in the source robots.txt file. Often this raw data isn't 103 | immediately useful as-is. Maybe we want to inspect it further? 104 | 105 | ### Check if a user agent is allowed to access a url path 106 | 107 | That's more like it, let's inspect some of that data in the model. 108 | 109 | ```php 110 | setSource(file_get_contents('http://example.com/robots.txt')); 116 | 117 | $inspector = new Inspector($parser->getFile()); 118 | $inspector->setUserAgent('slurp'); 119 | 120 | if ($inspector->isAllowed('/foo')) { 121 | // Do whatever is needed access to /foo is allowed 122 | } 123 | ``` 124 | 125 | ### Extract sitemap URLs 126 | 127 | A robots.txt file can list the URLs of all relevant sitemaps. These directives 128 | are not specific to a user agent. 129 | 130 | Let's say we're an automated web frontend testing service and we need to find a site's sitemap.xml to find a list 131 | of URLs that need testing. We know the site's domain and we know where to look for the robots.txt file and we know 132 | that this might specify the location of the sitemap.xml file. 133 | 134 | ```php 135 | setSource(file_get_contents('http://example.com/robots.txt')); 140 | 141 | $robotsTxtFile = $parser->getFile(); 142 | 143 | $sitemapDirectives = $robotsTxtFile->getNonGroupDirectives()->getByField('sitemap'); 144 | $sitemapUrl = (string)$sitemapDirectives->first()->getValue(); 145 | ``` 146 | 147 | Cool, we've found the URL for the first sitemap listed in the robots.txt file. 148 | There may be many, although just the one is most common. 149 | 150 | ### Filtering directives for a user agent to a specific field type 151 | 152 | Let's get all the `disallow` directives for Slurp: 153 | 154 | ```php 155 | setSource(file_get_contents('http://example.com/robots.txt')); 161 | 162 | $robotsTxtFile = $parser->getFile(); 163 | 164 | $inspector = new Inspector($robotsTxtFile); 165 | $inspector->setUserAgent('slurp'); 166 | 167 | $slurpDisallowDirectiveList = $inspector->getDirectives()->getByField('disallow'); 168 | ``` 169 | 170 | ## Building 171 | 172 | ### Using as a library in a project 173 | If used as a dependency by another project, update that project's composer.json 174 | and update your dependencies. 175 | 176 | "require": { 177 | "webignition/robots-txt-file": "*" 178 | } 179 | 180 | This will get you the latest version. Check the [list of releases](https://github.com/webignition/robots-txt-file/releases) for specific versions. 181 | 182 | ### Developing 183 | This project has external dependencies managed with [composer][3]. Get and install this first. 184 | 185 | # Make a suitable project directory 186 | mkdir ~/robots-txt-file && cd ~/robots-txt-file 187 | 188 | # Clone repository 189 | git clone git@github.com:webignition/robots-txt-file.git. 190 | 191 | # Retrieve/update dependencies 192 | composer update 193 | 194 | # Run code sniffer and unit tests 195 | composer cs 196 | composer test 197 | 198 | ## Testing 199 | Have look at the [project on travis][4] for the latest build status, or give the tests 200 | a go yourself. 201 | 202 | cd ~/robots-txt-file 203 | composer test 204 | 205 | 206 | [1]: http://www.robotstxt.org/ 207 | [2]: https://github.com/webignition/robots-txt-parser 208 | [3]: http://getcomposer.org 209 | [4]: http://travis-ci.org/webignition/robots-txt-file/builds 210 | -------------------------------------------------------------------------------- /src/File/Parser.php: -------------------------------------------------------------------------------- 1 | 77 | */ 78 | private $nonGroupFieldNames = array( 79 | DirectiveInterface::TYPE_SITEMAP => true 80 | ); 81 | 82 | /** 83 | * @param string $source 84 | */ 85 | public function setSource(string $source): void 86 | { 87 | $this->source = $this->prepareSource($source); 88 | } 89 | 90 | private function prepareSource(string $source): string 91 | { 92 | $source = trim($source); 93 | 94 | if (substr($source, 0, strlen(self::UTF8_BOM)) == self::UTF8_BOM) { 95 | $source = substr($source, strlen(self::UTF8_BOM)); 96 | } 97 | 98 | return $source; 99 | } 100 | 101 | public function getFile(): File 102 | { 103 | if (is_null($this->file)) { 104 | $this->file = new File(); 105 | $this->parse(); 106 | } 107 | 108 | return $this->file; 109 | } 110 | 111 | private function parse(): void 112 | { 113 | $this->currentState = self::STARTING_STATE; 114 | $this->sourceLines = explode("\n", trim($this->source)); 115 | $this->sourceLineCount = count($this->sourceLines); 116 | 117 | while ($this->sourceLineIndex < $this->sourceLineCount) { 118 | $this->parseCurrentLine(); 119 | } 120 | 121 | if ($this->hasCurrentRecord()) { 122 | $this->file->addRecord($this->currentRecord); 123 | } 124 | } 125 | 126 | private function parseCurrentLine(): void 127 | { 128 | switch ($this->currentState) { 129 | case self::STATE_UNKNOWN: 130 | $this->deriveStateFromCurrentLine(); 131 | $this->previousState = self::STATE_UNKNOWN; 132 | break; 133 | 134 | case self::STATE_STARTING_RECORD: 135 | $this->currentRecord = new Record(); 136 | $this->currentState = self::STATE_ADDING_TO_RECORD; 137 | $this->previousState = self::STATE_STARTING_RECORD; 138 | break; 139 | 140 | case self::STATE_ADDING_TO_RECORD: 141 | if ($this->isCurrentLineANonGroupDirective()) { 142 | $this->currentState = self::STATE_ADDING_TO_FILE; 143 | $this->previousState = self::STATE_ADDING_TO_RECORD; 144 | 145 | return; 146 | } 147 | 148 | if ($this->isCurrentLineADirective()) { 149 | $directive = DirectiveFactory::create($this->getCurrentLine()); 150 | 151 | if ($directive->isType(DirectiveInterface::TYPE_USER_AGENT)) { 152 | $this->currentRecord->getUserAgentDirectiveList()->add($directive); 153 | } else { 154 | $this->currentRecord->getDirectiveList()->add($directive); 155 | } 156 | } else { 157 | if (!empty($this->currentRecord)) { 158 | $this->file->addRecord($this->currentRecord); 159 | $this->currentRecord = null; 160 | } 161 | 162 | $this->currentState = self::STATE_UNKNOWN; 163 | } 164 | 165 | $this->sourceLineIndex++; 166 | 167 | break; 168 | 169 | case self::STATE_ADDING_TO_FILE: 170 | $directive = DirectiveFactory::create($this->getCurrentLine()); 171 | 172 | if (!is_null($directive)) { 173 | $this->file->getNonGroupDirectives()->add($directive); 174 | } 175 | 176 | $this->currentState = ($this->previousState == self::STATE_ADDING_TO_RECORD) 177 | ? self::STATE_ADDING_TO_RECORD 178 | : self::STATE_UNKNOWN; 179 | $this->previousState = self::STATE_ADDING_TO_FILE; 180 | $this->sourceLineIndex++; 181 | 182 | break; 183 | 184 | default: 185 | } 186 | } 187 | 188 | private function getCurrentLine(): string 189 | { 190 | return isset($this->sourceLines[$this->sourceLineIndex]) 191 | ? trim($this->sourceLines[$this->sourceLineIndex]) 192 | : ''; 193 | } 194 | 195 | private function hasCurrentRecord(): bool 196 | { 197 | return !is_null($this->currentRecord); 198 | } 199 | 200 | private function isCurrentLineBlank(): bool 201 | { 202 | return $this->getCurrentLine() == ''; 203 | } 204 | 205 | private function isCurrentLineAComment(): bool 206 | { 207 | return substr($this->getCurrentLine(), 0, 1) == self::COMMENT_START_CHARACTER; 208 | } 209 | 210 | private function isCurrentLineADirective(): bool 211 | { 212 | if ($this->isCurrentLineBlank()) { 213 | return false; 214 | } 215 | 216 | if ($this->isCurrentLineAComment()) { 217 | return false; 218 | } 219 | 220 | $directive = DirectiveFactory::create($this->getCurrentLine()); 221 | 222 | if (empty($directive)) { 223 | return false; 224 | } 225 | 226 | return true; 227 | } 228 | 229 | private function isCurrentLineANonGroupDirective(): bool 230 | { 231 | if (!$this->isCurrentLineADirective()) { 232 | return false; 233 | } 234 | 235 | $directive = DirectiveFactory::create($this->getCurrentLine()); 236 | 237 | if (empty($directive)) { 238 | return false; 239 | } 240 | 241 | return array_key_exists($directive->getField(), $this->nonGroupFieldNames); 242 | } 243 | 244 | private function deriveStateFromCurrentLine(): void 245 | { 246 | if (!$this->isCurrentLineADirective()) { 247 | $this->sourceLineIndex++; 248 | $this->currentState = self::STATE_UNKNOWN; 249 | 250 | return; 251 | } 252 | 253 | $directive = DirectiveFactory::create($this->getCurrentLine()); 254 | 255 | if (!is_null($directive) && $directive->isType(DirectiveInterface::TYPE_USER_AGENT)) { 256 | $this->currentState = self::STATE_STARTING_RECORD; 257 | 258 | return; 259 | } 260 | 261 | $this->currentState = self::STATE_ADDING_TO_FILE; 262 | 263 | return; 264 | } 265 | } 266 | -------------------------------------------------------------------------------- /tests/fixtures/robots-txt-files/google.com.txt: -------------------------------------------------------------------------------- 1 | User-agent: * 2 | Disallow: /search 3 | Disallow: /sdch 4 | Disallow: /groups 5 | Disallow: /images 6 | Disallow: /catalogs 7 | Allow: /catalogs/about 8 | Allow: /catalogs/p? 9 | Disallow: /catalogues 10 | Disallow: /news 11 | Allow: /news/directory 12 | Disallow: /nwshp 13 | Disallow: /setnewsprefs? 14 | Disallow: /index.html? 15 | Disallow: /? 16 | Allow: /?hl= 17 | Disallow: /?hl=*& 18 | Disallow: /addurl/image? 19 | Disallow: /pagead/ 20 | Disallow: /relpage/ 21 | Disallow: /relcontent 22 | Disallow: /imgres 23 | Disallow: /imglanding 24 | Disallow: /sbd 25 | Disallow: /keyword/ 26 | Disallow: /u/ 27 | Disallow: /univ/ 28 | Disallow: /cobrand 29 | Disallow: /custom 30 | Disallow: /advanced_group_search 31 | Disallow: /googlesite 32 | Disallow: /preferences 33 | Disallow: /setprefs 34 | Disallow: /swr 35 | Disallow: /url 36 | Disallow: /default 37 | Disallow: /m? 38 | Disallow: /m/? 39 | Disallow: /m/blogs? 40 | Disallow: /m/directions? 41 | Disallow: /m/ig 42 | Disallow: /m/images? 43 | Disallow: /m/imgres? 44 | Disallow: /m/local? 45 | Disallow: /m/movies? 46 | Disallow: /m/news? 47 | Disallow: /m/news/i? 48 | Disallow: /m/place? 49 | Disallow: /m/products? 50 | Disallow: /m/products/ 51 | Disallow: /m/setnewsprefs? 52 | Disallow: /m/search? 53 | Disallow: /m/swmloptin? 54 | Disallow: /m/trends 55 | Disallow: /m/video? 56 | Disallow: /wml? 57 | Disallow: /wml/? 58 | Disallow: /wml/search? 59 | Disallow: /xhtml? 60 | Disallow: /xhtml/? 61 | Disallow: /xhtml/search? 62 | Disallow: /xml? 63 | Disallow: /imode? 64 | Disallow: /imode/? 65 | Disallow: /imode/search? 66 | Disallow: /jsky? 67 | Disallow: /jsky/? 68 | Disallow: /jsky/search? 69 | Disallow: /pda? 70 | Disallow: /pda/? 71 | Disallow: /pda/search? 72 | Disallow: /sprint_xhtml 73 | Disallow: /sprint_wml 74 | Disallow: /pqa 75 | Disallow: /palm 76 | Disallow: /gwt/ 77 | Disallow: /purchases 78 | Disallow: /hws 79 | Disallow: /bsd? 80 | Disallow: /linux? 81 | Disallow: /mac? 82 | Disallow: /microsoft? 83 | Disallow: /unclesam? 84 | Disallow: /answers/search?q= 85 | Disallow: /local? 86 | Disallow: /local_url 87 | Disallow: /shihui? 88 | Disallow: /shihui/ 89 | Disallow: /froogle? 90 | Disallow: /products? 91 | Disallow: /products/ 92 | Disallow: /froogle_ 93 | Disallow: /product_ 94 | Disallow: /products_ 95 | Disallow: /products; 96 | Disallow: /print 97 | Disallow: /books/ 98 | Disallow: /bkshp?*q=* 99 | Disallow: /books?*q=* 100 | Disallow: /books?*output=* 101 | Disallow: /books?*pg=* 102 | Disallow: /books?*jtp=* 103 | Disallow: /books?*jscmd=* 104 | Disallow: /books?*buy=* 105 | Disallow: /books?*zoom=* 106 | Allow: /books?*q=related:* 107 | Allow: /books?*q=editions:* 108 | Allow: /books?*q=subject:* 109 | Allow: /books/about 110 | Allow: /booksrightsholders 111 | Allow: /books?*zoom=1* 112 | Allow: /books?*zoom=5* 113 | Disallow: /ebooks/ 114 | Disallow: /ebooks?*q=* 115 | Disallow: /ebooks?*output=* 116 | Disallow: /ebooks?*pg=* 117 | Disallow: /ebooks?*jscmd=* 118 | Disallow: /ebooks?*buy=* 119 | Disallow: /ebooks?*zoom=* 120 | Allow: /ebooks?*q=related:* 121 | Allow: /ebooks?*q=editions:* 122 | Allow: /ebooks?*q=subject:* 123 | Allow: /ebooks?*zoom=1* 124 | Allow: /ebooks?*zoom=5* 125 | Disallow: /patents? 126 | Allow: /patents?id= 127 | Allow: /patents?vid= 128 | Disallow: /scholar 129 | Disallow: /citations? 130 | Allow: /citations?user= 131 | Allow: /citations?view_op=new_profile 132 | Allow: /citations?view_op=top_venues 133 | Disallow: /complete 134 | Disallow: /s? 135 | Disallow: /sponsoredlinks 136 | Disallow: /videosearch? 137 | Disallow: /videopreview? 138 | Disallow: /videoprograminfo? 139 | Disallow: /maps? 140 | Disallow: /mapstt? 141 | Disallow: /mapslt? 142 | Disallow: /maps/stk/ 143 | Disallow: /maps/br? 144 | Disallow: /mapabcpoi? 145 | Disallow: /maphp? 146 | Disallow: /mapprint? 147 | Disallow: /maps/api/js/StaticMapService.GetMapImage? 148 | Disallow: /maps/api/staticmap? 149 | Disallow: /mld? 150 | Disallow: /staticmap? 151 | Disallow: /places/ 152 | Allow: /places/$ 153 | Disallow: /maps/place 154 | Disallow: /help/maps/streetview/partners/welcome/ 155 | Disallow: /lochp? 156 | Disallow: /center 157 | Disallow: /ie? 158 | Disallow: /sms/demo? 159 | Disallow: /katrina? 160 | Disallow: /blogsearch? 161 | Disallow: /blogsearch/ 162 | Disallow: /blogsearch_feeds 163 | Disallow: /advanced_blog_search 164 | Disallow: /reader/ 165 | Allow: /reader/play 166 | Disallow: /uds/ 167 | Disallow: /chart? 168 | Disallow: /transit? 169 | Disallow: /mbd? 170 | Disallow: /extern_js/ 171 | Disallow: /calendar/feeds/ 172 | Disallow: /calendar/ical/ 173 | Disallow: /cl2/feeds/ 174 | Disallow: /cl2/ical/ 175 | Disallow: /coop/directory 176 | Disallow: /coop/manage 177 | Disallow: /trends? 178 | Disallow: /trends/music? 179 | Disallow: /trends/hottrends? 180 | Disallow: /trends/viz? 181 | Disallow: /notebook/search? 182 | Disallow: /musica 183 | Disallow: /musicad 184 | Disallow: /musicas 185 | Disallow: /musicl 186 | Disallow: /musics 187 | Disallow: /musicsearch 188 | Disallow: /musicsp 189 | Disallow: /musiclp 190 | Disallow: /browsersync 191 | Disallow: /call 192 | Disallow: /archivesearch? 193 | Disallow: /archivesearch/url 194 | Disallow: /archivesearch/advanced_search 195 | Disallow: /base/reportbadoffer 196 | Disallow: /urchin_test/ 197 | Disallow: /movies? 198 | Disallow: /codesearch? 199 | Disallow: /codesearch/feeds/search? 200 | Disallow: /wapsearch? 201 | Disallow: /safebrowsing 202 | Allow: /safebrowsing/diagnostic 203 | Allow: /safebrowsing/report_badware/ 204 | Allow: /safebrowsing/report_error/ 205 | Allow: /safebrowsing/report_phish/ 206 | Disallow: /reviews/search? 207 | Disallow: /orkut/albums 208 | Allow: /jsapi 209 | Disallow: /views? 210 | Disallow: /c/ 211 | Disallow: /cbk 212 | Allow: /cbk?output=tile&cb_client=maps_sv 213 | Disallow: /recharge/dashboard/car 214 | Disallow: /recharge/dashboard/static/ 215 | Disallow: /translate_a/ 216 | Disallow: /translate_c 217 | Disallow: /translate_f 218 | Disallow: /translate_static/ 219 | Disallow: /translate_suggestion 220 | Disallow: /profiles/me 221 | Allow: /profiles 222 | Disallow: /s2/profiles/me 223 | Allow: /s2/profiles 224 | Allow: /s2/photos 225 | Allow: /s2/static 226 | Disallow: /s2 227 | Disallow: /transconsole/portal/ 228 | Disallow: /gcc/ 229 | Disallow: /aclk 230 | Disallow: /cse? 231 | Disallow: /cse/home 232 | Disallow: /cse/panel 233 | Disallow: /cse/manage 234 | Disallow: /tbproxy/ 235 | Disallow: /imesync/ 236 | Disallow: /shenghuo/search? 237 | Disallow: /support/forum/search? 238 | Disallow: /reviews/polls/ 239 | Disallow: /hosted/images/ 240 | Disallow: /ppob/? 241 | Disallow: /ppob? 242 | Disallow: /ig/add? 243 | Disallow: /adwordsresellers 244 | Disallow: /accounts/o8 245 | Allow: /accounts/o8/id 246 | Disallow: /topicsearch?q= 247 | Disallow: /xfx7/ 248 | Disallow: /squared/api 249 | Disallow: /squared/search 250 | Disallow: /squared/table 251 | Disallow: /toolkit/ 252 | Allow: /toolkit/*.html 253 | Disallow: /globalmarketfinder/ 254 | Allow: /globalmarketfinder/*.html 255 | Disallow: /qnasearch? 256 | Disallow: /app/updates 257 | Disallow: /sidewiki/entry/ 258 | Disallow: /quality_form? 259 | Disallow: /labs/popgadget/search 260 | Disallow: /buzz/post 261 | Disallow: /compressiontest/ 262 | Disallow: /analytics/reporting/ 263 | Disallow: /analytics/admin/ 264 | Disallow: /analytics/web/ 265 | Disallow: /analytics/feeds/ 266 | Disallow: /analytics/settings/ 267 | Disallow: /alerts/ 268 | Disallow: /ads/preferences/ 269 | Allow: /ads/preferences/html/ 270 | Allow: /ads/preferences/plugin 271 | Disallow: /settings/ads/onweb/ 272 | Disallow: /phone/compare/? 273 | Allow: /alerts/manage 274 | Disallow: /travel/clk 275 | Disallow: /hotelfinder/rpc 276 | Disallow: /flights/rpc 277 | Disallow: /commercesearch/services/ 278 | Disallow: /evaluation/ 279 | Disallow: /webstore/search 280 | Disallow: /chrome/browser/mobile/tour 281 | Sitemap: http://www.gstatic.com/s2/sitemaps/profiles-sitemap.xml 282 | Sitemap: http://www.google.com/hostednews/sitemap_index.xml 283 | Sitemap: http://www.google.com/ventures/sitemap_ventures.xml 284 | Sitemap: http://www.google.com/sitemaps_webmasters.xml 285 | Sitemap: http://www.gstatic.com/trends/websites/sitemaps/sitemapindex.xml 286 | Sitemap: http://www.gstatic.com/dictionary/static/sitemaps/sitemap_index.xml -------------------------------------------------------------------------------- /tests/Inspector/GetDirectivesTest.php: -------------------------------------------------------------------------------- 1 | 28 | */ 29 | private $userAgentStringFixtures = []; 30 | 31 | /** 32 | * @var File 33 | */ 34 | protected $file; 35 | 36 | protected function setUp(): void 37 | { 38 | $this->file = new File(); 39 | } 40 | 41 | /** 42 | * @dataProvider getDirectivesForDefaultFileDataProvider 43 | */ 44 | public function testGetDirectivesForDefaultFile(string $userAgentString, string $expectedDirectives) 45 | { 46 | $this->createDefaultFile(); 47 | $inspector = new Inspector($this->file); 48 | $inspector->setUserAgent($userAgentString); 49 | 50 | $this->assertEquals( 51 | $expectedDirectives, 52 | (string) $inspector->getDirectives() 53 | ); 54 | } 55 | 56 | public function getDirectivesForDefaultFileDataProvider(): array 57 | { 58 | return [ 59 | 'googlebot-lowercase' => [ 60 | 'userAgentString' => 'googlebot', 61 | 'expectedDirectives' => implode("\n", $this->getExpectedGooglebotDirectives()) 62 | ], 63 | 'googlebot-uppercase' => [ 64 | 'userAgentString' => 'GOOGLEBOT', 65 | 'expectedDirectives' => implode("\n", $this->getExpectedGooglebotDirectives()) 66 | ], 67 | 'googlebot-mixedcase' => [ 68 | 'userAgentString' => 'GOOGLEbot', 69 | 'expectedDirectives' => implode("\n", $this->getExpectedGooglebotDirectives()) 70 | ], 71 | 'googlebot-news' => [ 72 | 'userAgentString' => 'googlebot-news', 73 | 'expectedDirectives' => implode("\n", $this->getExpectedGooglebotNewsDirectives()) 74 | ], 75 | 'no specific agent' => [ 76 | 'userAgentString' => '*', 77 | 'expectedDirectives' => implode("\n", $this->getExpectedAllAgentsDirectives()) 78 | ], 79 | 'specific agent not present' => [ 80 | 'userAgentString' => 'foo', 81 | 'expectedDirectives' => implode("\n", $this->getExpectedAllAgentsDirectives()) 82 | ], 83 | 'full googlebot string variant 1' => [ 84 | 'userAgentString' => $this->getUserAgentStringFixture('googlebot-1'), 85 | 'expectedDirectives' => implode("\n", $this->getExpectedGooglebotDirectives()) 86 | ], 87 | 'full googlebot string variant 2' => [ 88 | 'userAgentString' => $this->getUserAgentStringFixture('googlebot-2'), 89 | 'expectedDirectives' => implode("\n", $this->getExpectedGooglebotDirectives()) 90 | ], 91 | 'bingbot variant 1' => [ 92 | 'userAgentString' => $this->getUserAgentStringFixture('bingbot-1'), 93 | 'expectedDirectives' => implode("\n", $this->getExpectedBingbotSlurpDirectives()) 94 | ], 95 | 'bingbot variant 2' => [ 96 | 'userAgentString' => $this->getUserAgentStringFixture('bingbot-2'), 97 | 'expectedDirectives' => implode("\n", $this->getExpectedBingbotSlurpDirectives()) 98 | ], 99 | 'bingbot variant 3' => [ 100 | 'userAgentString' => $this->getUserAgentStringFixture('bingbot-3'), 101 | 'expectedDirectives' => implode("\n", $this->getExpectedBingbotSlurpDirectives()) 102 | ], 103 | 'slurp' => [ 104 | 'userAgentString' => $this->getUserAgentStringFixture('slurp'), 105 | 'expectedDirectives' => implode("\n", $this->getExpectedBingbotSlurpDirectives()) 106 | ], 107 | ]; 108 | } 109 | 110 | protected function createDefaultFile(): void 111 | { 112 | $defaultAgentRecord = new Record(); 113 | $defaultAgentRecord->getUserAgentDirectiveList()->add(new UserAgentDirective('*')); 114 | $defaultAgentRecord->getDirectiveList()->add(new Directive( 115 | self::FIELD_ALLOW, 116 | self::VALUE_ALL_AGENTS_0 117 | )); 118 | $defaultAgentRecord->getDirectiveList()->add(new Directive( 119 | self::FIELD_DISALLOW, 120 | self::VALUE_ALL_AGENTS_1 121 | )); 122 | 123 | $googlebotRecord = new Record(); 124 | $googlebotRecord->getUserAgentDirectiveList()->add(new UserAgentDirective('googlebot')); 125 | $googlebotRecord->getDirectiveList()->add(new Directive( 126 | self::FIELD_ALLOW, 127 | self::VALUE_GOOGLEBOT_0 128 | )); 129 | $googlebotRecord->getDirectiveList()->add(new Directive( 130 | self::FIELD_DISALLOW, 131 | self::VALUE_GOOGLEBOT_1 132 | )); 133 | 134 | $googlebotNewsRecord = new Record(); 135 | $googlebotNewsRecord->getUserAgentDirectiveList()->add(new UserAgentDirective('googlebot-news')); 136 | $googlebotNewsRecord->getDirectiveList()->add(new Directive( 137 | self::FIELD_ALLOW, 138 | self::VALUE_GOOGLEBOT_NEWS_0 139 | )); 140 | $googlebotNewsRecord->getDirectiveList()->add(new Directive( 141 | self::FIELD_DISALLOW, 142 | self::VALUE_GOOGLEBOT_NEWS_1 143 | )); 144 | 145 | $bingbotAndSlurpRecord = new Record(); 146 | $bingbotAndSlurpRecord->getUserAgentDirectiveList()->add(new UserAgentDirective('bingbot')); 147 | $bingbotAndSlurpRecord->getUserAgentDirectiveList()->add(new UserAgentDirective('slurp')); 148 | $bingbotAndSlurpRecord->getDirectiveList()->add(new Directive( 149 | self::FIELD_ALLOW, 150 | self::VALUE_BINGBOT_SLURP_0 151 | )); 152 | $bingbotAndSlurpRecord->getDirectiveList()->add(new Directive( 153 | self::FIELD_DISALLOW, 154 | self::VALUE_BINGBOT_SLURP_1 155 | )); 156 | 157 | $this->file->addRecord($defaultAgentRecord); 158 | $this->file->addRecord($googlebotRecord); 159 | $this->file->addRecord($googlebotNewsRecord); 160 | $this->file->addRecord($bingbotAndSlurpRecord); 161 | } 162 | 163 | /** 164 | * @return string[] 165 | */ 166 | private function getExpectedAllAgentsDirectives(): array 167 | { 168 | $expectedDirectives = []; 169 | 170 | $expectedDirectives[] = self::FIELD_ALLOW . ':' . self::VALUE_ALL_AGENTS_0; 171 | $expectedDirectives[] = self::FIELD_DISALLOW . ':' . self::VALUE_ALL_AGENTS_1; 172 | 173 | return $expectedDirectives; 174 | } 175 | 176 | /** 177 | * @return string[] 178 | */ 179 | private function getExpectedGooglebotDirectives(): array 180 | { 181 | $expectedDirectives = []; 182 | 183 | $expectedDirectives[] = self::FIELD_ALLOW . ':' . self::VALUE_GOOGLEBOT_0; 184 | $expectedDirectives[] = self::FIELD_DISALLOW . ':' . self::VALUE_GOOGLEBOT_1; 185 | 186 | return $expectedDirectives; 187 | } 188 | 189 | /** 190 | * @return string[] 191 | */ 192 | private function getExpectedGooglebotNewsDirectives(): array 193 | { 194 | $expectedDirectives = []; 195 | 196 | $expectedDirectives[] = self::FIELD_ALLOW . ':' . self::VALUE_GOOGLEBOT_NEWS_0; 197 | $expectedDirectives[] = self::FIELD_DISALLOW . ':' . self::VALUE_GOOGLEBOT_NEWS_1; 198 | 199 | return $expectedDirectives; 200 | } 201 | 202 | /** 203 | * @return string[] 204 | */ 205 | private function getExpectedBingbotSlurpDirectives(): array 206 | { 207 | $expectedDirectives = []; 208 | 209 | $expectedDirectives[] = self::FIELD_ALLOW . ':' . self::VALUE_BINGBOT_SLURP_0; 210 | $expectedDirectives[] = self::FIELD_DISALLOW . ':' . self::VALUE_BINGBOT_SLURP_1; 211 | 212 | return $expectedDirectives; 213 | } 214 | 215 | private function getUserAgentStringFixture(string $fixtureIdentifier): string 216 | { 217 | if (empty($this->userAgentStringFixtures)) { 218 | $path = __DIR__ . '/../fixtures/user-agent-strings.json'; 219 | $this->userAgentStringFixtures = json_decode((string) file_get_contents($path), true); 220 | } 221 | 222 | return $this->userAgentStringFixtures[$fixtureIdentifier]; 223 | } 224 | } 225 | -------------------------------------------------------------------------------- /tests/Inspector/IsAllowedTest.php: -------------------------------------------------------------------------------- 1 | setSource(''); 16 | 17 | $file = $parser->getFile(); 18 | $inspector = new Inspector($file); 19 | 20 | $this->assertTrue($inspector->isAllowed('/foo')); 21 | } 22 | 23 | /** 24 | * @dataProvider emptyDisallowDirectiveDataProvider 25 | * 26 | * @param string[] $emptyDisallowDirectiveStrings 27 | */ 28 | public function testIsAllowedWhenOnlyEmptyDisallowIsPresent(array $emptyDisallowDirectiveStrings) 29 | { 30 | $parser = new Parser(); 31 | $parser->setSource('user-agent: *' . "\n" . implode("\n", $emptyDisallowDirectiveStrings)); 32 | 33 | $file = $parser->getFile(); 34 | $inspector = new Inspector($file); 35 | 36 | $this->assertTrue($inspector->isAllowed('/foo')); 37 | } 38 | 39 | public function emptyDisallowDirectiveDataProvider(): array 40 | { 41 | return [ 42 | [ 43 | 'emptyDisallowDirectiveStrings' => ['disallow:'], 44 | ], 45 | [ 46 | 'emptyDisallowDirectiveStrings' => ['disallow:', 'disallow:'], 47 | ], 48 | [ 49 | 'emptyDisallowDirectiveStrings' => ['disallow:', 'disallow:', 'disallow:'], 50 | ], 51 | ]; 52 | } 53 | 54 | /** 55 | * @dataProvider noMatchesDataProvider 56 | */ 57 | public function testIsAllowedWithNoMatchingDisallowDirectives(string $directivePath, string $urlPath) 58 | { 59 | $parser = new Parser(); 60 | $parser->setSource('user-agent: *' . "\n" . 'disallow: ' . $directivePath); 61 | 62 | $file = $parser->getFile(); 63 | $inspector = new Inspector($file); 64 | 65 | $this->assertTrue($inspector->isAllowed($urlPath)); 66 | } 67 | 68 | /** 69 | * robots-txt-rfc-draft-* data sets taken from: 70 | * http://www.robotstxt.org/norobots-rfc.txt 71 | * 72 | * Directive path URL path Matches 73 | * /tmp/ /tmp no 74 | * /a%2fb.html /a/b.html no 75 | * /a/b.html /a%2fb.html no 76 | * 77 | * google-webmasters-* data sets taken from: 78 | * https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt#url-matching-based-on-path-values 79 | * 80 | * Directive path URL path Matches 81 | * /fish /Fish.asp no 82 | * /catfish no 83 | * /?id=fish no 84 | * 85 | * /fish* /Fish.asp no 86 | * /catfish no 87 | * /?id=fish no 88 | * 89 | * /fish/ /fish no 90 | * /fish.html no 91 | * /Fish/Salmon.asp no 92 | * 93 | * /*.php / no 94 | * /windows.PHP no 95 | * 96 | * /*.php$ /filename.php?parameters 97 | * /filename.php/ 98 | * /filename.php5 99 | * /windows.PHP 100 | * 101 | * /fish*.php /Fish.PHP 102 | * 103 | * @return array 104 | */ 105 | public function noMatchesDataProvider(): array 106 | { 107 | return [ 108 | 'robots-txt-rfc-draft-1' => [ 109 | 'directivePath' => '/tmp/', 110 | 'urlPath' => '/tmp', 111 | ], 112 | 'robots-txt-rfc-draft-2' => [ 113 | 'directivePath' => '/a%2fb.html', 114 | 'urlPath' => '/a/b.html', 115 | ], 116 | 'robots-txt-rfc-draft-3' => [ 117 | 'directivePath' => '/a/b.html', 118 | 'urlPath' => '/a%2fb.html', 119 | ], 120 | '/google-webmasters-1' => [ 121 | 'directivePath' => '/fish', 122 | 'urlPath' => '/Fish.asp', 123 | ], 124 | '/google-webmasters-2' => [ 125 | 'directivePath' => '/fish', 126 | 'urlPath' => '/catfish', 127 | ], 128 | '/google-webmasters-3' => [ 129 | 'directivePath' => '/fish', 130 | 'urlPath' => '/?id=fish', 131 | ], 132 | '/google-webmasters-4' => [ 133 | 'directivePath' => '/fish*', 134 | 'urlPath' => '/Fish.asp', 135 | ], 136 | '/google-webmasters-5' => [ 137 | 'directivePath' => '/fish*', 138 | 'urlPath' => '/catfish', 139 | ], 140 | '/google-webmasters-6' => [ 141 | 'directivePath' => '/fish*', 142 | 'urlPath' => '/?id=fish', 143 | ], 144 | '/google-webmasters-8' => [ 145 | 'directivePath' => '/fish/', 146 | 'urlPath' => '/fish.html', 147 | ], 148 | '/google-webmasters-9' => [ 149 | 'directivePath' => '/fish/', 150 | 'urlPath' => '/Fish/Salmon.asp', 151 | ], 152 | '/google-webmasters-10' => [ 153 | 'directivePath' => '/*.php', 154 | 'urlPath' => '/', 155 | ], 156 | '/google-webmasters-11' => [ 157 | 'directivePath' => '/*.php', 158 | 'urlPath' => 'windows.PHP', 159 | ], 160 | '/google-webmasters-12' => [ 161 | 'directivePath' => '/*.php$', 162 | 'urlPath' => '/filename.php?parameters', 163 | ], 164 | '/google-webmasters-13' => [ 165 | 'directivePath' => '/*.php$', 166 | 'urlPath' => '/filename.php/', 167 | ], 168 | '/google-webmasters-14' => [ 169 | 'directivePath' => '/*.php$', 170 | 'urlPath' => '/filename.php5', 171 | ], 172 | '/google-webmasters-15' => [ 173 | 'directivePath' => '/*.php$', 174 | 'urlPath' => '/windows.PHP', 175 | ], 176 | '/google-webmasters-16' => [ 177 | 'directivePath' => '/fish*.php', 178 | 'urlPath' => '/Fish.PHP', 179 | ], 180 | ]; 181 | } 182 | 183 | /** 184 | * @dataProvider matchesDataProvider 185 | */ 186 | public function testIsNotAllowedWithMatchingDisallowDirectives(string $directivePath, string $urlPath) 187 | { 188 | $parser = new Parser(); 189 | $parser->setSource('user-agent: *' . "\n" . 'disallow: ' . $directivePath); 190 | 191 | $file = $parser->getFile(); 192 | $inspector = new Inspector($file); 193 | 194 | $this->assertFalse($inspector->isAllowed($urlPath)); 195 | } 196 | 197 | /** 198 | * robots-txt-rfc-draft-* data sets taken from: 199 | * http://www.robotstxt.org/norobots-rfc.txt 200 | * 201 | * Directive path URL path Matches 202 | * /tmp /tmp yes 203 | * /tmp /tmp.html yes 204 | * /tmp /tmp/a.html yes 205 | * /tmp/ /tmp/ yes 206 | * /tmp/ /tmp/a.html yes 207 | 208 | * /a%3cd.html /a%3cd.html yes 209 | * /a%3Cd.html /a%3cd.html yes 210 | * /a%3cd.html /a%3Cd.html yes 211 | * /a%3Cd.html /a%3Cd.html yes 212 | 213 | * /a%2fb.html /a%2fb.html yes 214 | * /a/b.html /a/b.html yes 215 | 216 | * /%7ejoe/index.html /~joe/index.html yes 217 | * /~joe/index.html /%7Ejoe/index.html yes 218 | * 219 | * google-webmasters-* data sets taken from: 220 | * https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt#url-matching-based-on-path-values 221 | * 222 | * Directive path URL path Matches 223 | * / yes 224 | * /* yes 225 | * /fish /fish 226 | * /fish.html 227 | * /fish/salmon.html 228 | * /fishheads 229 | * /fishheads/yummy.html 230 | * /fish.php?id=anything 231 | * /fish* /fish 232 | * /fish.html 233 | * /fish/salmon.html 234 | * /fishheads 235 | * /fishheads/yummy.html 236 | * /fish.php?id=anything 237 | * 238 | * /fish/ /fish/ 239 | * /fish/?id=anything 240 | * /fish/salmon.htm 241 | * 242 | * /*.php /filename.php 243 | * /folder/filename.php 244 | * /folder/filename.php?parameters 245 | * /folder/any.php.file.html 246 | * /filename.php/ 247 | * 248 | * /*.php$ /filename.php 249 | * /folder/filename.php 250 | * 251 | * /fish*.php /fish.php 252 | * /fishheads/catfish.php?parameters 253 | * 254 | * @return array 255 | */ 256 | public function matchesDataProvider(): array 257 | { 258 | return [ 259 | 'robots-txt-rfc-draft-1' => [ 260 | 'directivePath' => '/tmp', 261 | 'urlPath' => '/tmp', 262 | ], 263 | 'robots-txt-rfc-draft-2' => [ 264 | 'directivePath' => '/tmp', 265 | 'urlPath' => '/tmp.html', 266 | ], 267 | 'robots-txt-rfc-draft-3' => [ 268 | 'directivePath' => '/tmp', 269 | 'urlPath' => '/tmp/a.html', 270 | ], 271 | 'robots-txt-rfc-draft-4' => [ 272 | 'directivePath' => '/tmp/', 273 | 'urlPath' => '/tmp/', 274 | ], 275 | 'robots-txt-rfc-draft-5' => [ 276 | 'directivePath' => '/tmp/', 277 | 'urlPath' => '/tmp/a.html', 278 | ], 279 | 'robots-txt-rfc-draft-6' => [ 280 | 'directivePath' => '/a%3cd.html', 281 | 'urlPath' => '/a%3cd.html', 282 | ], 283 | 'robots-txt-rfc-draft-7' => [ 284 | 'directivePath' => '/a%3Cd.html', 285 | 'urlPath' => '/a%3cd.html', 286 | ], 287 | 'robots-txt-rfc-draft-8' => [ 288 | 'directivePath' => '/a%3cd.html', 289 | 'urlPath' => '/a%3Cd.html', 290 | ], 291 | 'robots-txt-rfc-draft-9' => [ 292 | 'directivePath' => '/a%3Cd.html', 293 | 'urlPath' => '/a%3Cd.html', 294 | ], 295 | 'robots-txt-rfc-draft-10' => [ 296 | 'directivePath' => '/a%2fb.html', 297 | 'urlPath' => '/a%2fb.html', 298 | ], 299 | 'robots-txt-rfc-draft-11' => [ 300 | 'directivePath' => '/a/b.html', 301 | 'urlPath' => '/a/b.html ', 302 | ], 303 | 'robots-txt-rfc-draft-12' => [ 304 | 'directivePath' => '/%7ejoe/index.html', 305 | 'urlPath' => '/~joe/index.html', 306 | ], 307 | 'robots-txt-rfc-draft-13' => [ 308 | 'directivePath' => '/~joe/index.html', 309 | 'urlPath' => '/%7Ejoe/index.html', 310 | ], 311 | 'google-webmasters-1' => [ 312 | 'directivePath' => '/', 313 | 'urlPath' => '/foo', 314 | ], 315 | 'google-webmasters-2' => [ 316 | 'directivePath' => '/*', 317 | 'urlPath' => '/foo', 318 | ], 319 | 'google-webmasters-4' => [ 320 | 'directivePath' => '/fish', 321 | 'urlPath' => '/fish.html', 322 | ], 323 | 'google-webmasters-5' => [ 324 | 'directivePath' => '/fish', 325 | 'urlPath' => '/fish/salmon.html', 326 | ], 327 | 'google-webmasters-6' => [ 328 | 'directivePath' => '/fish', 329 | 'urlPath' => '/fishheads', 330 | ], 331 | 'google-webmasters-7' => [ 332 | 'directivePath' => '/fish', 333 | 'urlPath' => '/fishheads/yummy.html', 334 | ], 335 | 'google-webmasters-8' => [ 336 | 'directivePath' => '/fish', 337 | 'urlPath' => '/fish.php?id=anything', 338 | ], 339 | 'google-webmasters-9' => [ 340 | 'directivePath' => '/fish*', 341 | 'urlPath' => '/fish', 342 | ], 343 | 'google-webmasters-10' => [ 344 | 'directivePath' => '/fish*', 345 | 'urlPath' => '/fish.html', 346 | ], 347 | 'google-webmasters-11' => [ 348 | 'directivePath' => '/fish*', 349 | 'urlPath' => '/fish/salmon.html', 350 | ], 351 | 'google-webmasters-12' => [ 352 | 'directivePath' => '/fish*', 353 | 'urlPath' => '/fishheads', 354 | ], 355 | 'google-webmasters-13' => [ 356 | 'directivePath' => '/fish*', 357 | 'urlPath' => '/fishheads/yummy.html', 358 | ], 359 | 'google-webmasters-14' => [ 360 | 'directivePath' => '/fish*', 361 | 'urlPath' => '/fish.php?id=anything', 362 | ], 363 | 'google-webmasters-16' => [ 364 | 'directivePath' => '/fish/', 365 | 'urlPath' => '/fish/?id=anything', 366 | ], 367 | 'google-webmasters-17' => [ 368 | 'directivePath' => '/fish/', 369 | 'urlPath' => '/fish/salmon.htm', 370 | ], 371 | 'google-webmasters-18' => [ 372 | 'directivePath' => '/*.php', 373 | 'urlPath' => '/filename.php', 374 | ], 375 | 'google-webmasters-19' => [ 376 | 'directivePath' => '/*.php', 377 | 'urlPath' => '/folder/filename.php', 378 | ], 379 | 'google-webmasters-20' => [ 380 | 'directivePath' => '/*.php', 381 | 'urlPath' => '/folder/filename.php?parameters', 382 | ], 383 | 'google-webmasters-21' => [ 384 | 'directivePath' => '/*.php', 385 | 'urlPath' => '/folder/any.php.file.html', 386 | ], 387 | 'google-webmasters-22' => [ 388 | 'directivePath' => '/*.php', 389 | 'urlPath' => '/filename.php/', 390 | ], 391 | 'google-webmasters-23' => [ 392 | 'directivePath' => '/*.php$', 393 | 'urlPath' => '/filename.php', 394 | ], 395 | 'google-webmasters-24' => [ 396 | 'directivePath' => '/*.php$', 397 | 'urlPath' => '/folder/filename.php', 398 | ], 399 | 'google-webmasters-25' => [ 400 | 'directivePath' => '/fish*.php', 401 | 'urlPath' => '/fish.php', 402 | ], 403 | 'google-webmasters-26' => [ 404 | 'directivePath' => '/fish*.php', 405 | 'urlPath' => '/fishheads/catfish.php?parameters', 406 | ], 407 | ]; 408 | } 409 | 410 | /** 411 | * @dataProvider allowDisallowDirectiveResolutionDataProvider 412 | * 413 | * @param string[] $directiveStrings 414 | * @param string $urlPath 415 | * @param bool $expectedAllowed 416 | */ 417 | public function testMatchingAllowAndDisallowDirectiveResolution( 418 | array $directiveStrings, 419 | string $urlPath, 420 | bool $expectedAllowed 421 | ) { 422 | $parser = new Parser(); 423 | $parser->setSource('user-agent: *' . "\n" . implode("\n", $directiveStrings)); 424 | 425 | $file = $parser->getFile(); 426 | $inspector = new Inspector($file); 427 | 428 | $this->assertEquals($expectedAllowed, $inspector->isAllowed($urlPath)); 429 | } 430 | 431 | /** 432 | * Data sets derived from: 433 | * - https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt#order-of-precedence-for-group-member-records 434 | * - studying the behaviour of google webmasters robots txt checker 435 | * 436 | * @return array 437 | */ 438 | public function allowDisallowDirectiveResolutionDataProvider(): array 439 | { 440 | return [ 441 | 'longer patternless allow supercedes patternless disallow' => [ 442 | 'directiveStrings' => [ 443 | 'allow: /folder/', 444 | 'disallow: /folder', 445 | 446 | ], 447 | 'urlPath' => '/folder/page', 448 | 'expectedAllowed' => true, 449 | ], 450 | 'longer patternless disallow supercedes patternless allow' => [ 451 | 'directiveStrings' => [ 452 | 'allow: /folder', 453 | 'disallow: /folder/', 454 | 455 | ], 456 | 'urlPath' => '/folder/page', 457 | 'expectedAllowed' => false, 458 | ], 459 | 'allow supercedes disallow if both are identical' => [ 460 | 'directiveStrings' => [ 461 | 'allow: /folder', 462 | 'disallow: /folder', 463 | 464 | ], 465 | 'urlPath' => '/folder/page', 466 | 'expectedAllowed' => true, 467 | ], 468 | 'disallow supercedes allow if both are of the same length' => [ 469 | 'directiveStrings' => [ 470 | 'allow: /folder', 471 | 'disallow: /*/page', 472 | 473 | ], 474 | 'urlPath' => '/folders/page', 475 | 'expectedAllowed' => false, 476 | ], 477 | 'longer patterned allow supercedes shorter disallow' => [ 478 | 'directiveStrings' => [ 479 | 'allow: /$', 480 | 'disallow: /', 481 | 482 | ], 483 | 'urlPath' => '/', 484 | 'expectedAllowed' => true, 485 | ], 486 | 'only disallow matches' => [ 487 | 'directiveStrings' => [ 488 | 'allow: /$', 489 | 'disallow: /', 490 | 491 | ], 492 | 'urlPath' => '/page.htm', 493 | 'expectedAllowed' => false, 494 | ], 495 | ]; 496 | } 497 | } 498 | -------------------------------------------------------------------------------- /composer.lock: -------------------------------------------------------------------------------- 1 | { 2 | "_readme": [ 3 | "This file locks the dependencies of your project to a known state", 4 | "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", 5 | "This file is @generated automatically" 6 | ], 7 | "content-hash": "451cee603be8491c9a245349e31569fc", 8 | "packages": [ 9 | { 10 | "name": "webignition/disallowed-character-terminated-string", 11 | "version": "2.0", 12 | "source": { 13 | "type": "git", 14 | "url": "https://github.com/webignition/disallowed-character-terminated-string.git", 15 | "reference": "1c35b8bacbb2e76837c0aa8538dc2468a1f10e6e" 16 | }, 17 | "dist": { 18 | "type": "zip", 19 | "url": "https://api.github.com/repos/webignition/disallowed-character-terminated-string/zipball/1c35b8bacbb2e76837c0aa8538dc2468a1f10e6e", 20 | "reference": "1c35b8bacbb2e76837c0aa8538dc2468a1f10e6e", 21 | "shasum": "" 22 | }, 23 | "require": { 24 | "php": ">=7.2" 25 | }, 26 | "require-dev": { 27 | "phpstan/phpstan": "^0.12.3", 28 | "phpunit/phpunit": "~8.0", 29 | "squizlabs/php_codesniffer": "^3.5" 30 | }, 31 | "type": "library", 32 | "autoload": { 33 | "psr-4": { 34 | "webignition\\DisallowedCharacterTerminatedString\\": "src" 35 | } 36 | }, 37 | "notification-url": "https://packagist.org/downloads/", 38 | "license": [ 39 | "MIT" 40 | ], 41 | "authors": [ 42 | { 43 | "name": "Jon Cram", 44 | "email": "webignition@gmail.com" 45 | } 46 | ], 47 | "description": "A string terminated by one or more disallowed characters", 48 | "homepage": "https://github.com/webignition/disallowed-character-terminated-string", 49 | "keywords": [ 50 | "string", 51 | "terminated" 52 | ], 53 | "time": "2019-12-20T15:52:44+00:00" 54 | } 55 | ], 56 | "packages-dev": [ 57 | { 58 | "name": "doctrine/instantiator", 59 | "version": "1.3.0", 60 | "source": { 61 | "type": "git", 62 | "url": "https://github.com/doctrine/instantiator.git", 63 | "reference": "ae466f726242e637cebdd526a7d991b9433bacf1" 64 | }, 65 | "dist": { 66 | "type": "zip", 67 | "url": "https://api.github.com/repos/doctrine/instantiator/zipball/ae466f726242e637cebdd526a7d991b9433bacf1", 68 | "reference": "ae466f726242e637cebdd526a7d991b9433bacf1", 69 | "shasum": "" 70 | }, 71 | "require": { 72 | "php": "^7.1" 73 | }, 74 | "require-dev": { 75 | "doctrine/coding-standard": "^6.0", 76 | "ext-pdo": "*", 77 | "ext-phar": "*", 78 | "phpbench/phpbench": "^0.13", 79 | "phpstan/phpstan-phpunit": "^0.11", 80 | "phpstan/phpstan-shim": "^0.11", 81 | "phpunit/phpunit": "^7.0" 82 | }, 83 | "type": "library", 84 | "extra": { 85 | "branch-alias": { 86 | "dev-master": "1.2.x-dev" 87 | } 88 | }, 89 | "autoload": { 90 | "psr-4": { 91 | "Doctrine\\Instantiator\\": "src/Doctrine/Instantiator/" 92 | } 93 | }, 94 | "notification-url": "https://packagist.org/downloads/", 95 | "license": [ 96 | "MIT" 97 | ], 98 | "authors": [ 99 | { 100 | "name": "Marco Pivetta", 101 | "email": "ocramius@gmail.com", 102 | "homepage": "http://ocramius.github.com/" 103 | } 104 | ], 105 | "description": "A small, lightweight utility to instantiate objects in PHP without invoking their constructors", 106 | "homepage": "https://www.doctrine-project.org/projects/instantiator.html", 107 | "keywords": [ 108 | "constructor", 109 | "instantiate" 110 | ], 111 | "time": "2019-10-21T16:45:58+00:00" 112 | }, 113 | { 114 | "name": "myclabs/deep-copy", 115 | "version": "1.9.4", 116 | "source": { 117 | "type": "git", 118 | "url": "https://github.com/myclabs/DeepCopy.git", 119 | "reference": "579bb7356d91f9456ccd505f24ca8b667966a0a7" 120 | }, 121 | "dist": { 122 | "type": "zip", 123 | "url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/579bb7356d91f9456ccd505f24ca8b667966a0a7", 124 | "reference": "579bb7356d91f9456ccd505f24ca8b667966a0a7", 125 | "shasum": "" 126 | }, 127 | "require": { 128 | "php": "^7.1" 129 | }, 130 | "replace": { 131 | "myclabs/deep-copy": "self.version" 132 | }, 133 | "require-dev": { 134 | "doctrine/collections": "^1.0", 135 | "doctrine/common": "^2.6", 136 | "phpunit/phpunit": "^7.1" 137 | }, 138 | "type": "library", 139 | "autoload": { 140 | "psr-4": { 141 | "DeepCopy\\": "src/DeepCopy/" 142 | }, 143 | "files": [ 144 | "src/DeepCopy/deep_copy.php" 145 | ] 146 | }, 147 | "notification-url": "https://packagist.org/downloads/", 148 | "license": [ 149 | "MIT" 150 | ], 151 | "description": "Create deep copies (clones) of your objects", 152 | "keywords": [ 153 | "clone", 154 | "copy", 155 | "duplicate", 156 | "object", 157 | "object graph" 158 | ], 159 | "time": "2019-12-15T19:12:40+00:00" 160 | }, 161 | { 162 | "name": "nikic/php-parser", 163 | "version": "v4.3.0", 164 | "source": { 165 | "type": "git", 166 | "url": "https://github.com/nikic/PHP-Parser.git", 167 | "reference": "9a9981c347c5c49d6dfe5cf826bb882b824080dc" 168 | }, 169 | "dist": { 170 | "type": "zip", 171 | "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/9a9981c347c5c49d6dfe5cf826bb882b824080dc", 172 | "reference": "9a9981c347c5c49d6dfe5cf826bb882b824080dc", 173 | "shasum": "" 174 | }, 175 | "require": { 176 | "ext-tokenizer": "*", 177 | "php": ">=7.0" 178 | }, 179 | "require-dev": { 180 | "ircmaxell/php-yacc": "0.0.5", 181 | "phpunit/phpunit": "^6.5 || ^7.0 || ^8.0" 182 | }, 183 | "bin": [ 184 | "bin/php-parse" 185 | ], 186 | "type": "library", 187 | "extra": { 188 | "branch-alias": { 189 | "dev-master": "4.3-dev" 190 | } 191 | }, 192 | "autoload": { 193 | "psr-4": { 194 | "PhpParser\\": "lib/PhpParser" 195 | } 196 | }, 197 | "notification-url": "https://packagist.org/downloads/", 198 | "license": [ 199 | "BSD-3-Clause" 200 | ], 201 | "authors": [ 202 | { 203 | "name": "Nikita Popov" 204 | } 205 | ], 206 | "description": "A PHP parser written in PHP", 207 | "keywords": [ 208 | "parser", 209 | "php" 210 | ], 211 | "time": "2019-11-08T13:50:10+00:00" 212 | }, 213 | { 214 | "name": "phar-io/manifest", 215 | "version": "1.0.3", 216 | "source": { 217 | "type": "git", 218 | "url": "https://github.com/phar-io/manifest.git", 219 | "reference": "7761fcacf03b4d4f16e7ccb606d4879ca431fcf4" 220 | }, 221 | "dist": { 222 | "type": "zip", 223 | "url": "https://api.github.com/repos/phar-io/manifest/zipball/7761fcacf03b4d4f16e7ccb606d4879ca431fcf4", 224 | "reference": "7761fcacf03b4d4f16e7ccb606d4879ca431fcf4", 225 | "shasum": "" 226 | }, 227 | "require": { 228 | "ext-dom": "*", 229 | "ext-phar": "*", 230 | "phar-io/version": "^2.0", 231 | "php": "^5.6 || ^7.0" 232 | }, 233 | "type": "library", 234 | "extra": { 235 | "branch-alias": { 236 | "dev-master": "1.0.x-dev" 237 | } 238 | }, 239 | "autoload": { 240 | "classmap": [ 241 | "src/" 242 | ] 243 | }, 244 | "notification-url": "https://packagist.org/downloads/", 245 | "license": [ 246 | "BSD-3-Clause" 247 | ], 248 | "authors": [ 249 | { 250 | "name": "Arne Blankerts", 251 | "email": "arne@blankerts.de", 252 | "role": "Developer" 253 | }, 254 | { 255 | "name": "Sebastian Heuer", 256 | "email": "sebastian@phpeople.de", 257 | "role": "Developer" 258 | }, 259 | { 260 | "name": "Sebastian Bergmann", 261 | "email": "sebastian@phpunit.de", 262 | "role": "Developer" 263 | } 264 | ], 265 | "description": "Component for reading phar.io manifest information from a PHP Archive (PHAR)", 266 | "time": "2018-07-08T19:23:20+00:00" 267 | }, 268 | { 269 | "name": "phar-io/version", 270 | "version": "2.0.1", 271 | "source": { 272 | "type": "git", 273 | "url": "https://github.com/phar-io/version.git", 274 | "reference": "45a2ec53a73c70ce41d55cedef9063630abaf1b6" 275 | }, 276 | "dist": { 277 | "type": "zip", 278 | "url": "https://api.github.com/repos/phar-io/version/zipball/45a2ec53a73c70ce41d55cedef9063630abaf1b6", 279 | "reference": "45a2ec53a73c70ce41d55cedef9063630abaf1b6", 280 | "shasum": "" 281 | }, 282 | "require": { 283 | "php": "^5.6 || ^7.0" 284 | }, 285 | "type": "library", 286 | "autoload": { 287 | "classmap": [ 288 | "src/" 289 | ] 290 | }, 291 | "notification-url": "https://packagist.org/downloads/", 292 | "license": [ 293 | "BSD-3-Clause" 294 | ], 295 | "authors": [ 296 | { 297 | "name": "Arne Blankerts", 298 | "email": "arne@blankerts.de", 299 | "role": "Developer" 300 | }, 301 | { 302 | "name": "Sebastian Heuer", 303 | "email": "sebastian@phpeople.de", 304 | "role": "Developer" 305 | }, 306 | { 307 | "name": "Sebastian Bergmann", 308 | "email": "sebastian@phpunit.de", 309 | "role": "Developer" 310 | } 311 | ], 312 | "description": "Library for handling version information and constraints", 313 | "time": "2018-07-08T19:19:57+00:00" 314 | }, 315 | { 316 | "name": "phpdocumentor/reflection-common", 317 | "version": "2.0.0", 318 | "source": { 319 | "type": "git", 320 | "url": "https://github.com/phpDocumentor/ReflectionCommon.git", 321 | "reference": "63a995caa1ca9e5590304cd845c15ad6d482a62a" 322 | }, 323 | "dist": { 324 | "type": "zip", 325 | "url": "https://api.github.com/repos/phpDocumentor/ReflectionCommon/zipball/63a995caa1ca9e5590304cd845c15ad6d482a62a", 326 | "reference": "63a995caa1ca9e5590304cd845c15ad6d482a62a", 327 | "shasum": "" 328 | }, 329 | "require": { 330 | "php": ">=7.1" 331 | }, 332 | "require-dev": { 333 | "phpunit/phpunit": "~6" 334 | }, 335 | "type": "library", 336 | "extra": { 337 | "branch-alias": { 338 | "dev-master": "2.x-dev" 339 | } 340 | }, 341 | "autoload": { 342 | "psr-4": { 343 | "phpDocumentor\\Reflection\\": "src/" 344 | } 345 | }, 346 | "notification-url": "https://packagist.org/downloads/", 347 | "license": [ 348 | "MIT" 349 | ], 350 | "authors": [ 351 | { 352 | "name": "Jaap van Otterdijk", 353 | "email": "opensource@ijaap.nl" 354 | } 355 | ], 356 | "description": "Common reflection classes used by phpdocumentor to reflect the code structure", 357 | "homepage": "http://www.phpdoc.org", 358 | "keywords": [ 359 | "FQSEN", 360 | "phpDocumentor", 361 | "phpdoc", 362 | "reflection", 363 | "static analysis" 364 | ], 365 | "time": "2018-08-07T13:53:10+00:00" 366 | }, 367 | { 368 | "name": "phpdocumentor/reflection-docblock", 369 | "version": "4.3.3", 370 | "source": { 371 | "type": "git", 372 | "url": "https://github.com/phpDocumentor/ReflectionDocBlock.git", 373 | "reference": "2ecaa9fef01634c83bfa8dc1fe35fb5cef223a62" 374 | }, 375 | "dist": { 376 | "type": "zip", 377 | "url": "https://api.github.com/repos/phpDocumentor/ReflectionDocBlock/zipball/2ecaa9fef01634c83bfa8dc1fe35fb5cef223a62", 378 | "reference": "2ecaa9fef01634c83bfa8dc1fe35fb5cef223a62", 379 | "shasum": "" 380 | }, 381 | "require": { 382 | "php": "^7.0", 383 | "phpdocumentor/reflection-common": "^1.0.0 || ^2.0.0", 384 | "phpdocumentor/type-resolver": "~0.4 || ^1.0.0", 385 | "webmozart/assert": "^1.0" 386 | }, 387 | "require-dev": { 388 | "doctrine/instantiator": "^1.0.5", 389 | "mockery/mockery": "^1.0", 390 | "phpunit/phpunit": "^6.4" 391 | }, 392 | "type": "library", 393 | "extra": { 394 | "branch-alias": { 395 | "dev-master": "4.x-dev" 396 | } 397 | }, 398 | "autoload": { 399 | "psr-4": { 400 | "phpDocumentor\\Reflection\\": [ 401 | "src/" 402 | ] 403 | } 404 | }, 405 | "notification-url": "https://packagist.org/downloads/", 406 | "license": [ 407 | "MIT" 408 | ], 409 | "authors": [ 410 | { 411 | "name": "Mike van Riel", 412 | "email": "me@mikevanriel.com" 413 | } 414 | ], 415 | "description": "With this component, a library can provide support for annotations via DocBlocks or otherwise retrieve information that is embedded in a DocBlock.", 416 | "time": "2019-12-20T13:40:23+00:00" 417 | }, 418 | { 419 | "name": "phpdocumentor/type-resolver", 420 | "version": "1.0.1", 421 | "source": { 422 | "type": "git", 423 | "url": "https://github.com/phpDocumentor/TypeResolver.git", 424 | "reference": "2e32a6d48972b2c1976ed5d8967145b6cec4a4a9" 425 | }, 426 | "dist": { 427 | "type": "zip", 428 | "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/2e32a6d48972b2c1976ed5d8967145b6cec4a4a9", 429 | "reference": "2e32a6d48972b2c1976ed5d8967145b6cec4a4a9", 430 | "shasum": "" 431 | }, 432 | "require": { 433 | "php": "^7.1", 434 | "phpdocumentor/reflection-common": "^2.0" 435 | }, 436 | "require-dev": { 437 | "ext-tokenizer": "^7.1", 438 | "mockery/mockery": "~1", 439 | "phpunit/phpunit": "^7.0" 440 | }, 441 | "type": "library", 442 | "extra": { 443 | "branch-alias": { 444 | "dev-master": "1.x-dev" 445 | } 446 | }, 447 | "autoload": { 448 | "psr-4": { 449 | "phpDocumentor\\Reflection\\": "src" 450 | } 451 | }, 452 | "notification-url": "https://packagist.org/downloads/", 453 | "license": [ 454 | "MIT" 455 | ], 456 | "authors": [ 457 | { 458 | "name": "Mike van Riel", 459 | "email": "me@mikevanriel.com" 460 | } 461 | ], 462 | "description": "A PSR-5 based resolver of Class names, Types and Structural Element Names", 463 | "time": "2019-08-22T18:11:29+00:00" 464 | }, 465 | { 466 | "name": "phpspec/prophecy", 467 | "version": "1.10.0", 468 | "source": { 469 | "type": "git", 470 | "url": "https://github.com/phpspec/prophecy.git", 471 | "reference": "d638ebbb58daba25a6a0dc7969e1358a0e3c6682" 472 | }, 473 | "dist": { 474 | "type": "zip", 475 | "url": "https://api.github.com/repos/phpspec/prophecy/zipball/d638ebbb58daba25a6a0dc7969e1358a0e3c6682", 476 | "reference": "d638ebbb58daba25a6a0dc7969e1358a0e3c6682", 477 | "shasum": "" 478 | }, 479 | "require": { 480 | "doctrine/instantiator": "^1.0.2", 481 | "php": "^5.3|^7.0", 482 | "phpdocumentor/reflection-docblock": "^2.0|^3.0.2|^4.0|^5.0", 483 | "sebastian/comparator": "^1.2.3|^2.0|^3.0", 484 | "sebastian/recursion-context": "^1.0|^2.0|^3.0" 485 | }, 486 | "require-dev": { 487 | "phpspec/phpspec": "^2.5 || ^3.2", 488 | "phpunit/phpunit": "^4.8.35 || ^5.7 || ^6.5 || ^7.1" 489 | }, 490 | "type": "library", 491 | "extra": { 492 | "branch-alias": { 493 | "dev-master": "1.10.x-dev" 494 | } 495 | }, 496 | "autoload": { 497 | "psr-4": { 498 | "Prophecy\\": "src/Prophecy" 499 | } 500 | }, 501 | "notification-url": "https://packagist.org/downloads/", 502 | "license": [ 503 | "MIT" 504 | ], 505 | "authors": [ 506 | { 507 | "name": "Konstantin Kudryashov", 508 | "email": "ever.zet@gmail.com", 509 | "homepage": "http://everzet.com" 510 | }, 511 | { 512 | "name": "Marcello Duarte", 513 | "email": "marcello.duarte@gmail.com" 514 | } 515 | ], 516 | "description": "Highly opinionated mocking framework for PHP 5.3+", 517 | "homepage": "https://github.com/phpspec/prophecy", 518 | "keywords": [ 519 | "Double", 520 | "Dummy", 521 | "fake", 522 | "mock", 523 | "spy", 524 | "stub" 525 | ], 526 | "time": "2019-12-17T16:54:23+00:00" 527 | }, 528 | { 529 | "name": "phpstan/phpstan", 530 | "version": "0.12.3", 531 | "source": { 532 | "type": "git", 533 | "url": "https://github.com/phpstan/phpstan.git", 534 | "reference": "c15a6ea55da71d8133399306f560cfe4d30301b7" 535 | }, 536 | "dist": { 537 | "type": "zip", 538 | "url": "https://api.github.com/repos/phpstan/phpstan/zipball/c15a6ea55da71d8133399306f560cfe4d30301b7", 539 | "reference": "c15a6ea55da71d8133399306f560cfe4d30301b7", 540 | "shasum": "" 541 | }, 542 | "require": { 543 | "nikic/php-parser": "^4.3.0", 544 | "php": "^7.1" 545 | }, 546 | "bin": [ 547 | "phpstan", 548 | "phpstan.phar" 549 | ], 550 | "type": "library", 551 | "extra": { 552 | "branch-alias": { 553 | "dev-master": "0.12-dev" 554 | } 555 | }, 556 | "autoload": { 557 | "files": [ 558 | "bootstrap.php" 559 | ] 560 | }, 561 | "notification-url": "https://packagist.org/downloads/", 562 | "license": [ 563 | "MIT" 564 | ], 565 | "description": "PHPStan - PHP Static Analysis Tool", 566 | "time": "2019-12-14T13:41:17+00:00" 567 | }, 568 | { 569 | "name": "phpunit/php-code-coverage", 570 | "version": "7.0.10", 571 | "source": { 572 | "type": "git", 573 | "url": "https://github.com/sebastianbergmann/php-code-coverage.git", 574 | "reference": "f1884187926fbb755a9aaf0b3836ad3165b478bf" 575 | }, 576 | "dist": { 577 | "type": "zip", 578 | "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/f1884187926fbb755a9aaf0b3836ad3165b478bf", 579 | "reference": "f1884187926fbb755a9aaf0b3836ad3165b478bf", 580 | "shasum": "" 581 | }, 582 | "require": { 583 | "ext-dom": "*", 584 | "ext-xmlwriter": "*", 585 | "php": "^7.2", 586 | "phpunit/php-file-iterator": "^2.0.2", 587 | "phpunit/php-text-template": "^1.2.1", 588 | "phpunit/php-token-stream": "^3.1.1", 589 | "sebastian/code-unit-reverse-lookup": "^1.0.1", 590 | "sebastian/environment": "^4.2.2", 591 | "sebastian/version": "^2.0.1", 592 | "theseer/tokenizer": "^1.1.3" 593 | }, 594 | "require-dev": { 595 | "phpunit/phpunit": "^8.2.2" 596 | }, 597 | "suggest": { 598 | "ext-xdebug": "^2.7.2" 599 | }, 600 | "type": "library", 601 | "extra": { 602 | "branch-alias": { 603 | "dev-master": "7.0-dev" 604 | } 605 | }, 606 | "autoload": { 607 | "classmap": [ 608 | "src/" 609 | ] 610 | }, 611 | "notification-url": "https://packagist.org/downloads/", 612 | "license": [ 613 | "BSD-3-Clause" 614 | ], 615 | "authors": [ 616 | { 617 | "name": "Sebastian Bergmann", 618 | "email": "sebastian@phpunit.de", 619 | "role": "lead" 620 | } 621 | ], 622 | "description": "Library that provides collection, processing, and rendering functionality for PHP code coverage information.", 623 | "homepage": "https://github.com/sebastianbergmann/php-code-coverage", 624 | "keywords": [ 625 | "coverage", 626 | "testing", 627 | "xunit" 628 | ], 629 | "time": "2019-11-20T13:55:58+00:00" 630 | }, 631 | { 632 | "name": "phpunit/php-file-iterator", 633 | "version": "2.0.2", 634 | "source": { 635 | "type": "git", 636 | "url": "https://github.com/sebastianbergmann/php-file-iterator.git", 637 | "reference": "050bedf145a257b1ff02746c31894800e5122946" 638 | }, 639 | "dist": { 640 | "type": "zip", 641 | "url": "https://api.github.com/repos/sebastianbergmann/php-file-iterator/zipball/050bedf145a257b1ff02746c31894800e5122946", 642 | "reference": "050bedf145a257b1ff02746c31894800e5122946", 643 | "shasum": "" 644 | }, 645 | "require": { 646 | "php": "^7.1" 647 | }, 648 | "require-dev": { 649 | "phpunit/phpunit": "^7.1" 650 | }, 651 | "type": "library", 652 | "extra": { 653 | "branch-alias": { 654 | "dev-master": "2.0.x-dev" 655 | } 656 | }, 657 | "autoload": { 658 | "classmap": [ 659 | "src/" 660 | ] 661 | }, 662 | "notification-url": "https://packagist.org/downloads/", 663 | "license": [ 664 | "BSD-3-Clause" 665 | ], 666 | "authors": [ 667 | { 668 | "name": "Sebastian Bergmann", 669 | "email": "sebastian@phpunit.de", 670 | "role": "lead" 671 | } 672 | ], 673 | "description": "FilterIterator implementation that filters files based on a list of suffixes.", 674 | "homepage": "https://github.com/sebastianbergmann/php-file-iterator/", 675 | "keywords": [ 676 | "filesystem", 677 | "iterator" 678 | ], 679 | "time": "2018-09-13T20:33:42+00:00" 680 | }, 681 | { 682 | "name": "phpunit/php-text-template", 683 | "version": "1.2.1", 684 | "source": { 685 | "type": "git", 686 | "url": "https://github.com/sebastianbergmann/php-text-template.git", 687 | "reference": "31f8b717e51d9a2afca6c9f046f5d69fc27c8686" 688 | }, 689 | "dist": { 690 | "type": "zip", 691 | "url": "https://api.github.com/repos/sebastianbergmann/php-text-template/zipball/31f8b717e51d9a2afca6c9f046f5d69fc27c8686", 692 | "reference": "31f8b717e51d9a2afca6c9f046f5d69fc27c8686", 693 | "shasum": "" 694 | }, 695 | "require": { 696 | "php": ">=5.3.3" 697 | }, 698 | "type": "library", 699 | "autoload": { 700 | "classmap": [ 701 | "src/" 702 | ] 703 | }, 704 | "notification-url": "https://packagist.org/downloads/", 705 | "license": [ 706 | "BSD-3-Clause" 707 | ], 708 | "authors": [ 709 | { 710 | "name": "Sebastian Bergmann", 711 | "email": "sebastian@phpunit.de", 712 | "role": "lead" 713 | } 714 | ], 715 | "description": "Simple template engine.", 716 | "homepage": "https://github.com/sebastianbergmann/php-text-template/", 717 | "keywords": [ 718 | "template" 719 | ], 720 | "time": "2015-06-21T13:50:34+00:00" 721 | }, 722 | { 723 | "name": "phpunit/php-timer", 724 | "version": "2.1.2", 725 | "source": { 726 | "type": "git", 727 | "url": "https://github.com/sebastianbergmann/php-timer.git", 728 | "reference": "1038454804406b0b5f5f520358e78c1c2f71501e" 729 | }, 730 | "dist": { 731 | "type": "zip", 732 | "url": "https://api.github.com/repos/sebastianbergmann/php-timer/zipball/1038454804406b0b5f5f520358e78c1c2f71501e", 733 | "reference": "1038454804406b0b5f5f520358e78c1c2f71501e", 734 | "shasum": "" 735 | }, 736 | "require": { 737 | "php": "^7.1" 738 | }, 739 | "require-dev": { 740 | "phpunit/phpunit": "^7.0" 741 | }, 742 | "type": "library", 743 | "extra": { 744 | "branch-alias": { 745 | "dev-master": "2.1-dev" 746 | } 747 | }, 748 | "autoload": { 749 | "classmap": [ 750 | "src/" 751 | ] 752 | }, 753 | "notification-url": "https://packagist.org/downloads/", 754 | "license": [ 755 | "BSD-3-Clause" 756 | ], 757 | "authors": [ 758 | { 759 | "name": "Sebastian Bergmann", 760 | "email": "sebastian@phpunit.de", 761 | "role": "lead" 762 | } 763 | ], 764 | "description": "Utility class for timing", 765 | "homepage": "https://github.com/sebastianbergmann/php-timer/", 766 | "keywords": [ 767 | "timer" 768 | ], 769 | "time": "2019-06-07T04:22:29+00:00" 770 | }, 771 | { 772 | "name": "phpunit/php-token-stream", 773 | "version": "3.1.1", 774 | "source": { 775 | "type": "git", 776 | "url": "https://github.com/sebastianbergmann/php-token-stream.git", 777 | "reference": "995192df77f63a59e47f025390d2d1fdf8f425ff" 778 | }, 779 | "dist": { 780 | "type": "zip", 781 | "url": "https://api.github.com/repos/sebastianbergmann/php-token-stream/zipball/995192df77f63a59e47f025390d2d1fdf8f425ff", 782 | "reference": "995192df77f63a59e47f025390d2d1fdf8f425ff", 783 | "shasum": "" 784 | }, 785 | "require": { 786 | "ext-tokenizer": "*", 787 | "php": "^7.1" 788 | }, 789 | "require-dev": { 790 | "phpunit/phpunit": "^7.0" 791 | }, 792 | "type": "library", 793 | "extra": { 794 | "branch-alias": { 795 | "dev-master": "3.1-dev" 796 | } 797 | }, 798 | "autoload": { 799 | "classmap": [ 800 | "src/" 801 | ] 802 | }, 803 | "notification-url": "https://packagist.org/downloads/", 804 | "license": [ 805 | "BSD-3-Clause" 806 | ], 807 | "authors": [ 808 | { 809 | "name": "Sebastian Bergmann", 810 | "email": "sebastian@phpunit.de" 811 | } 812 | ], 813 | "description": "Wrapper around PHP's tokenizer extension.", 814 | "homepage": "https://github.com/sebastianbergmann/php-token-stream/", 815 | "keywords": [ 816 | "tokenizer" 817 | ], 818 | "time": "2019-09-17T06:23:10+00:00" 819 | }, 820 | { 821 | "name": "phpunit/phpunit", 822 | "version": "8.5.0", 823 | "source": { 824 | "type": "git", 825 | "url": "https://github.com/sebastianbergmann/phpunit.git", 826 | "reference": "3ee1c1fd6fc264480c25b6fb8285edefe1702dab" 827 | }, 828 | "dist": { 829 | "type": "zip", 830 | "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/3ee1c1fd6fc264480c25b6fb8285edefe1702dab", 831 | "reference": "3ee1c1fd6fc264480c25b6fb8285edefe1702dab", 832 | "shasum": "" 833 | }, 834 | "require": { 835 | "doctrine/instantiator": "^1.2.0", 836 | "ext-dom": "*", 837 | "ext-json": "*", 838 | "ext-libxml": "*", 839 | "ext-mbstring": "*", 840 | "ext-xml": "*", 841 | "ext-xmlwriter": "*", 842 | "myclabs/deep-copy": "^1.9.1", 843 | "phar-io/manifest": "^1.0.3", 844 | "phar-io/version": "^2.0.1", 845 | "php": "^7.2", 846 | "phpspec/prophecy": "^1.8.1", 847 | "phpunit/php-code-coverage": "^7.0.7", 848 | "phpunit/php-file-iterator": "^2.0.2", 849 | "phpunit/php-text-template": "^1.2.1", 850 | "phpunit/php-timer": "^2.1.2", 851 | "sebastian/comparator": "^3.0.2", 852 | "sebastian/diff": "^3.0.2", 853 | "sebastian/environment": "^4.2.2", 854 | "sebastian/exporter": "^3.1.1", 855 | "sebastian/global-state": "^3.0.0", 856 | "sebastian/object-enumerator": "^3.0.3", 857 | "sebastian/resource-operations": "^2.0.1", 858 | "sebastian/type": "^1.1.3", 859 | "sebastian/version": "^2.0.1" 860 | }, 861 | "require-dev": { 862 | "ext-pdo": "*" 863 | }, 864 | "suggest": { 865 | "ext-soap": "*", 866 | "ext-xdebug": "*", 867 | "phpunit/php-invoker": "^2.0.0" 868 | }, 869 | "bin": [ 870 | "phpunit" 871 | ], 872 | "type": "library", 873 | "extra": { 874 | "branch-alias": { 875 | "dev-master": "8.5-dev" 876 | } 877 | }, 878 | "autoload": { 879 | "classmap": [ 880 | "src/" 881 | ] 882 | }, 883 | "notification-url": "https://packagist.org/downloads/", 884 | "license": [ 885 | "BSD-3-Clause" 886 | ], 887 | "authors": [ 888 | { 889 | "name": "Sebastian Bergmann", 890 | "email": "sebastian@phpunit.de", 891 | "role": "lead" 892 | } 893 | ], 894 | "description": "The PHP Unit Testing framework.", 895 | "homepage": "https://phpunit.de/", 896 | "keywords": [ 897 | "phpunit", 898 | "testing", 899 | "xunit" 900 | ], 901 | "time": "2019-12-06T05:41:38+00:00" 902 | }, 903 | { 904 | "name": "sebastian/code-unit-reverse-lookup", 905 | "version": "1.0.1", 906 | "source": { 907 | "type": "git", 908 | "url": "https://github.com/sebastianbergmann/code-unit-reverse-lookup.git", 909 | "reference": "4419fcdb5eabb9caa61a27c7a1db532a6b55dd18" 910 | }, 911 | "dist": { 912 | "type": "zip", 913 | "url": "https://api.github.com/repos/sebastianbergmann/code-unit-reverse-lookup/zipball/4419fcdb5eabb9caa61a27c7a1db532a6b55dd18", 914 | "reference": "4419fcdb5eabb9caa61a27c7a1db532a6b55dd18", 915 | "shasum": "" 916 | }, 917 | "require": { 918 | "php": "^5.6 || ^7.0" 919 | }, 920 | "require-dev": { 921 | "phpunit/phpunit": "^5.7 || ^6.0" 922 | }, 923 | "type": "library", 924 | "extra": { 925 | "branch-alias": { 926 | "dev-master": "1.0.x-dev" 927 | } 928 | }, 929 | "autoload": { 930 | "classmap": [ 931 | "src/" 932 | ] 933 | }, 934 | "notification-url": "https://packagist.org/downloads/", 935 | "license": [ 936 | "BSD-3-Clause" 937 | ], 938 | "authors": [ 939 | { 940 | "name": "Sebastian Bergmann", 941 | "email": "sebastian@phpunit.de" 942 | } 943 | ], 944 | "description": "Looks up which function or method a line of code belongs to", 945 | "homepage": "https://github.com/sebastianbergmann/code-unit-reverse-lookup/", 946 | "time": "2017-03-04T06:30:41+00:00" 947 | }, 948 | { 949 | "name": "sebastian/comparator", 950 | "version": "3.0.2", 951 | "source": { 952 | "type": "git", 953 | "url": "https://github.com/sebastianbergmann/comparator.git", 954 | "reference": "5de4fc177adf9bce8df98d8d141a7559d7ccf6da" 955 | }, 956 | "dist": { 957 | "type": "zip", 958 | "url": "https://api.github.com/repos/sebastianbergmann/comparator/zipball/5de4fc177adf9bce8df98d8d141a7559d7ccf6da", 959 | "reference": "5de4fc177adf9bce8df98d8d141a7559d7ccf6da", 960 | "shasum": "" 961 | }, 962 | "require": { 963 | "php": "^7.1", 964 | "sebastian/diff": "^3.0", 965 | "sebastian/exporter": "^3.1" 966 | }, 967 | "require-dev": { 968 | "phpunit/phpunit": "^7.1" 969 | }, 970 | "type": "library", 971 | "extra": { 972 | "branch-alias": { 973 | "dev-master": "3.0-dev" 974 | } 975 | }, 976 | "autoload": { 977 | "classmap": [ 978 | "src/" 979 | ] 980 | }, 981 | "notification-url": "https://packagist.org/downloads/", 982 | "license": [ 983 | "BSD-3-Clause" 984 | ], 985 | "authors": [ 986 | { 987 | "name": "Jeff Welch", 988 | "email": "whatthejeff@gmail.com" 989 | }, 990 | { 991 | "name": "Volker Dusch", 992 | "email": "github@wallbash.com" 993 | }, 994 | { 995 | "name": "Bernhard Schussek", 996 | "email": "bschussek@2bepublished.at" 997 | }, 998 | { 999 | "name": "Sebastian Bergmann", 1000 | "email": "sebastian@phpunit.de" 1001 | } 1002 | ], 1003 | "description": "Provides the functionality to compare PHP values for equality", 1004 | "homepage": "https://github.com/sebastianbergmann/comparator", 1005 | "keywords": [ 1006 | "comparator", 1007 | "compare", 1008 | "equality" 1009 | ], 1010 | "time": "2018-07-12T15:12:46+00:00" 1011 | }, 1012 | { 1013 | "name": "sebastian/diff", 1014 | "version": "3.0.2", 1015 | "source": { 1016 | "type": "git", 1017 | "url": "https://github.com/sebastianbergmann/diff.git", 1018 | "reference": "720fcc7e9b5cf384ea68d9d930d480907a0c1a29" 1019 | }, 1020 | "dist": { 1021 | "type": "zip", 1022 | "url": "https://api.github.com/repos/sebastianbergmann/diff/zipball/720fcc7e9b5cf384ea68d9d930d480907a0c1a29", 1023 | "reference": "720fcc7e9b5cf384ea68d9d930d480907a0c1a29", 1024 | "shasum": "" 1025 | }, 1026 | "require": { 1027 | "php": "^7.1" 1028 | }, 1029 | "require-dev": { 1030 | "phpunit/phpunit": "^7.5 || ^8.0", 1031 | "symfony/process": "^2 || ^3.3 || ^4" 1032 | }, 1033 | "type": "library", 1034 | "extra": { 1035 | "branch-alias": { 1036 | "dev-master": "3.0-dev" 1037 | } 1038 | }, 1039 | "autoload": { 1040 | "classmap": [ 1041 | "src/" 1042 | ] 1043 | }, 1044 | "notification-url": "https://packagist.org/downloads/", 1045 | "license": [ 1046 | "BSD-3-Clause" 1047 | ], 1048 | "authors": [ 1049 | { 1050 | "name": "Kore Nordmann", 1051 | "email": "mail@kore-nordmann.de" 1052 | }, 1053 | { 1054 | "name": "Sebastian Bergmann", 1055 | "email": "sebastian@phpunit.de" 1056 | } 1057 | ], 1058 | "description": "Diff implementation", 1059 | "homepage": "https://github.com/sebastianbergmann/diff", 1060 | "keywords": [ 1061 | "diff", 1062 | "udiff", 1063 | "unidiff", 1064 | "unified diff" 1065 | ], 1066 | "time": "2019-02-04T06:01:07+00:00" 1067 | }, 1068 | { 1069 | "name": "sebastian/environment", 1070 | "version": "4.2.3", 1071 | "source": { 1072 | "type": "git", 1073 | "url": "https://github.com/sebastianbergmann/environment.git", 1074 | "reference": "464c90d7bdf5ad4e8a6aea15c091fec0603d4368" 1075 | }, 1076 | "dist": { 1077 | "type": "zip", 1078 | "url": "https://api.github.com/repos/sebastianbergmann/environment/zipball/464c90d7bdf5ad4e8a6aea15c091fec0603d4368", 1079 | "reference": "464c90d7bdf5ad4e8a6aea15c091fec0603d4368", 1080 | "shasum": "" 1081 | }, 1082 | "require": { 1083 | "php": "^7.1" 1084 | }, 1085 | "require-dev": { 1086 | "phpunit/phpunit": "^7.5" 1087 | }, 1088 | "suggest": { 1089 | "ext-posix": "*" 1090 | }, 1091 | "type": "library", 1092 | "extra": { 1093 | "branch-alias": { 1094 | "dev-master": "4.2-dev" 1095 | } 1096 | }, 1097 | "autoload": { 1098 | "classmap": [ 1099 | "src/" 1100 | ] 1101 | }, 1102 | "notification-url": "https://packagist.org/downloads/", 1103 | "license": [ 1104 | "BSD-3-Clause" 1105 | ], 1106 | "authors": [ 1107 | { 1108 | "name": "Sebastian Bergmann", 1109 | "email": "sebastian@phpunit.de" 1110 | } 1111 | ], 1112 | "description": "Provides functionality to handle HHVM/PHP environments", 1113 | "homepage": "http://www.github.com/sebastianbergmann/environment", 1114 | "keywords": [ 1115 | "Xdebug", 1116 | "environment", 1117 | "hhvm" 1118 | ], 1119 | "time": "2019-11-20T08:46:58+00:00" 1120 | }, 1121 | { 1122 | "name": "sebastian/exporter", 1123 | "version": "3.1.2", 1124 | "source": { 1125 | "type": "git", 1126 | "url": "https://github.com/sebastianbergmann/exporter.git", 1127 | "reference": "68609e1261d215ea5b21b7987539cbfbe156ec3e" 1128 | }, 1129 | "dist": { 1130 | "type": "zip", 1131 | "url": "https://api.github.com/repos/sebastianbergmann/exporter/zipball/68609e1261d215ea5b21b7987539cbfbe156ec3e", 1132 | "reference": "68609e1261d215ea5b21b7987539cbfbe156ec3e", 1133 | "shasum": "" 1134 | }, 1135 | "require": { 1136 | "php": "^7.0", 1137 | "sebastian/recursion-context": "^3.0" 1138 | }, 1139 | "require-dev": { 1140 | "ext-mbstring": "*", 1141 | "phpunit/phpunit": "^6.0" 1142 | }, 1143 | "type": "library", 1144 | "extra": { 1145 | "branch-alias": { 1146 | "dev-master": "3.1.x-dev" 1147 | } 1148 | }, 1149 | "autoload": { 1150 | "classmap": [ 1151 | "src/" 1152 | ] 1153 | }, 1154 | "notification-url": "https://packagist.org/downloads/", 1155 | "license": [ 1156 | "BSD-3-Clause" 1157 | ], 1158 | "authors": [ 1159 | { 1160 | "name": "Sebastian Bergmann", 1161 | "email": "sebastian@phpunit.de" 1162 | }, 1163 | { 1164 | "name": "Jeff Welch", 1165 | "email": "whatthejeff@gmail.com" 1166 | }, 1167 | { 1168 | "name": "Volker Dusch", 1169 | "email": "github@wallbash.com" 1170 | }, 1171 | { 1172 | "name": "Adam Harvey", 1173 | "email": "aharvey@php.net" 1174 | }, 1175 | { 1176 | "name": "Bernhard Schussek", 1177 | "email": "bschussek@gmail.com" 1178 | } 1179 | ], 1180 | "description": "Provides the functionality to export PHP variables for visualization", 1181 | "homepage": "http://www.github.com/sebastianbergmann/exporter", 1182 | "keywords": [ 1183 | "export", 1184 | "exporter" 1185 | ], 1186 | "time": "2019-09-14T09:02:43+00:00" 1187 | }, 1188 | { 1189 | "name": "sebastian/global-state", 1190 | "version": "3.0.0", 1191 | "source": { 1192 | "type": "git", 1193 | "url": "https://github.com/sebastianbergmann/global-state.git", 1194 | "reference": "edf8a461cf1d4005f19fb0b6b8b95a9f7fa0adc4" 1195 | }, 1196 | "dist": { 1197 | "type": "zip", 1198 | "url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/edf8a461cf1d4005f19fb0b6b8b95a9f7fa0adc4", 1199 | "reference": "edf8a461cf1d4005f19fb0b6b8b95a9f7fa0adc4", 1200 | "shasum": "" 1201 | }, 1202 | "require": { 1203 | "php": "^7.2", 1204 | "sebastian/object-reflector": "^1.1.1", 1205 | "sebastian/recursion-context": "^3.0" 1206 | }, 1207 | "require-dev": { 1208 | "ext-dom": "*", 1209 | "phpunit/phpunit": "^8.0" 1210 | }, 1211 | "suggest": { 1212 | "ext-uopz": "*" 1213 | }, 1214 | "type": "library", 1215 | "extra": { 1216 | "branch-alias": { 1217 | "dev-master": "3.0-dev" 1218 | } 1219 | }, 1220 | "autoload": { 1221 | "classmap": [ 1222 | "src/" 1223 | ] 1224 | }, 1225 | "notification-url": "https://packagist.org/downloads/", 1226 | "license": [ 1227 | "BSD-3-Clause" 1228 | ], 1229 | "authors": [ 1230 | { 1231 | "name": "Sebastian Bergmann", 1232 | "email": "sebastian@phpunit.de" 1233 | } 1234 | ], 1235 | "description": "Snapshotting of global state", 1236 | "homepage": "http://www.github.com/sebastianbergmann/global-state", 1237 | "keywords": [ 1238 | "global state" 1239 | ], 1240 | "time": "2019-02-01T05:30:01+00:00" 1241 | }, 1242 | { 1243 | "name": "sebastian/object-enumerator", 1244 | "version": "3.0.3", 1245 | "source": { 1246 | "type": "git", 1247 | "url": "https://github.com/sebastianbergmann/object-enumerator.git", 1248 | "reference": "7cfd9e65d11ffb5af41198476395774d4c8a84c5" 1249 | }, 1250 | "dist": { 1251 | "type": "zip", 1252 | "url": "https://api.github.com/repos/sebastianbergmann/object-enumerator/zipball/7cfd9e65d11ffb5af41198476395774d4c8a84c5", 1253 | "reference": "7cfd9e65d11ffb5af41198476395774d4c8a84c5", 1254 | "shasum": "" 1255 | }, 1256 | "require": { 1257 | "php": "^7.0", 1258 | "sebastian/object-reflector": "^1.1.1", 1259 | "sebastian/recursion-context": "^3.0" 1260 | }, 1261 | "require-dev": { 1262 | "phpunit/phpunit": "^6.0" 1263 | }, 1264 | "type": "library", 1265 | "extra": { 1266 | "branch-alias": { 1267 | "dev-master": "3.0.x-dev" 1268 | } 1269 | }, 1270 | "autoload": { 1271 | "classmap": [ 1272 | "src/" 1273 | ] 1274 | }, 1275 | "notification-url": "https://packagist.org/downloads/", 1276 | "license": [ 1277 | "BSD-3-Clause" 1278 | ], 1279 | "authors": [ 1280 | { 1281 | "name": "Sebastian Bergmann", 1282 | "email": "sebastian@phpunit.de" 1283 | } 1284 | ], 1285 | "description": "Traverses array structures and object graphs to enumerate all referenced objects", 1286 | "homepage": "https://github.com/sebastianbergmann/object-enumerator/", 1287 | "time": "2017-08-03T12:35:26+00:00" 1288 | }, 1289 | { 1290 | "name": "sebastian/object-reflector", 1291 | "version": "1.1.1", 1292 | "source": { 1293 | "type": "git", 1294 | "url": "https://github.com/sebastianbergmann/object-reflector.git", 1295 | "reference": "773f97c67f28de00d397be301821b06708fca0be" 1296 | }, 1297 | "dist": { 1298 | "type": "zip", 1299 | "url": "https://api.github.com/repos/sebastianbergmann/object-reflector/zipball/773f97c67f28de00d397be301821b06708fca0be", 1300 | "reference": "773f97c67f28de00d397be301821b06708fca0be", 1301 | "shasum": "" 1302 | }, 1303 | "require": { 1304 | "php": "^7.0" 1305 | }, 1306 | "require-dev": { 1307 | "phpunit/phpunit": "^6.0" 1308 | }, 1309 | "type": "library", 1310 | "extra": { 1311 | "branch-alias": { 1312 | "dev-master": "1.1-dev" 1313 | } 1314 | }, 1315 | "autoload": { 1316 | "classmap": [ 1317 | "src/" 1318 | ] 1319 | }, 1320 | "notification-url": "https://packagist.org/downloads/", 1321 | "license": [ 1322 | "BSD-3-Clause" 1323 | ], 1324 | "authors": [ 1325 | { 1326 | "name": "Sebastian Bergmann", 1327 | "email": "sebastian@phpunit.de" 1328 | } 1329 | ], 1330 | "description": "Allows reflection of object attributes, including inherited and non-public ones", 1331 | "homepage": "https://github.com/sebastianbergmann/object-reflector/", 1332 | "time": "2017-03-29T09:07:27+00:00" 1333 | }, 1334 | { 1335 | "name": "sebastian/recursion-context", 1336 | "version": "3.0.0", 1337 | "source": { 1338 | "type": "git", 1339 | "url": "https://github.com/sebastianbergmann/recursion-context.git", 1340 | "reference": "5b0cd723502bac3b006cbf3dbf7a1e3fcefe4fa8" 1341 | }, 1342 | "dist": { 1343 | "type": "zip", 1344 | "url": "https://api.github.com/repos/sebastianbergmann/recursion-context/zipball/5b0cd723502bac3b006cbf3dbf7a1e3fcefe4fa8", 1345 | "reference": "5b0cd723502bac3b006cbf3dbf7a1e3fcefe4fa8", 1346 | "shasum": "" 1347 | }, 1348 | "require": { 1349 | "php": "^7.0" 1350 | }, 1351 | "require-dev": { 1352 | "phpunit/phpunit": "^6.0" 1353 | }, 1354 | "type": "library", 1355 | "extra": { 1356 | "branch-alias": { 1357 | "dev-master": "3.0.x-dev" 1358 | } 1359 | }, 1360 | "autoload": { 1361 | "classmap": [ 1362 | "src/" 1363 | ] 1364 | }, 1365 | "notification-url": "https://packagist.org/downloads/", 1366 | "license": [ 1367 | "BSD-3-Clause" 1368 | ], 1369 | "authors": [ 1370 | { 1371 | "name": "Jeff Welch", 1372 | "email": "whatthejeff@gmail.com" 1373 | }, 1374 | { 1375 | "name": "Sebastian Bergmann", 1376 | "email": "sebastian@phpunit.de" 1377 | }, 1378 | { 1379 | "name": "Adam Harvey", 1380 | "email": "aharvey@php.net" 1381 | } 1382 | ], 1383 | "description": "Provides functionality to recursively process PHP variables", 1384 | "homepage": "http://www.github.com/sebastianbergmann/recursion-context", 1385 | "time": "2017-03-03T06:23:57+00:00" 1386 | }, 1387 | { 1388 | "name": "sebastian/resource-operations", 1389 | "version": "2.0.1", 1390 | "source": { 1391 | "type": "git", 1392 | "url": "https://github.com/sebastianbergmann/resource-operations.git", 1393 | "reference": "4d7a795d35b889bf80a0cc04e08d77cedfa917a9" 1394 | }, 1395 | "dist": { 1396 | "type": "zip", 1397 | "url": "https://api.github.com/repos/sebastianbergmann/resource-operations/zipball/4d7a795d35b889bf80a0cc04e08d77cedfa917a9", 1398 | "reference": "4d7a795d35b889bf80a0cc04e08d77cedfa917a9", 1399 | "shasum": "" 1400 | }, 1401 | "require": { 1402 | "php": "^7.1" 1403 | }, 1404 | "type": "library", 1405 | "extra": { 1406 | "branch-alias": { 1407 | "dev-master": "2.0-dev" 1408 | } 1409 | }, 1410 | "autoload": { 1411 | "classmap": [ 1412 | "src/" 1413 | ] 1414 | }, 1415 | "notification-url": "https://packagist.org/downloads/", 1416 | "license": [ 1417 | "BSD-3-Clause" 1418 | ], 1419 | "authors": [ 1420 | { 1421 | "name": "Sebastian Bergmann", 1422 | "email": "sebastian@phpunit.de" 1423 | } 1424 | ], 1425 | "description": "Provides a list of PHP built-in functions that operate on resources", 1426 | "homepage": "https://www.github.com/sebastianbergmann/resource-operations", 1427 | "time": "2018-10-04T04:07:39+00:00" 1428 | }, 1429 | { 1430 | "name": "sebastian/type", 1431 | "version": "1.1.3", 1432 | "source": { 1433 | "type": "git", 1434 | "url": "https://github.com/sebastianbergmann/type.git", 1435 | "reference": "3aaaa15fa71d27650d62a948be022fe3b48541a3" 1436 | }, 1437 | "dist": { 1438 | "type": "zip", 1439 | "url": "https://api.github.com/repos/sebastianbergmann/type/zipball/3aaaa15fa71d27650d62a948be022fe3b48541a3", 1440 | "reference": "3aaaa15fa71d27650d62a948be022fe3b48541a3", 1441 | "shasum": "" 1442 | }, 1443 | "require": { 1444 | "php": "^7.2" 1445 | }, 1446 | "require-dev": { 1447 | "phpunit/phpunit": "^8.2" 1448 | }, 1449 | "type": "library", 1450 | "extra": { 1451 | "branch-alias": { 1452 | "dev-master": "1.1-dev" 1453 | } 1454 | }, 1455 | "autoload": { 1456 | "classmap": [ 1457 | "src/" 1458 | ] 1459 | }, 1460 | "notification-url": "https://packagist.org/downloads/", 1461 | "license": [ 1462 | "BSD-3-Clause" 1463 | ], 1464 | "authors": [ 1465 | { 1466 | "name": "Sebastian Bergmann", 1467 | "email": "sebastian@phpunit.de", 1468 | "role": "lead" 1469 | } 1470 | ], 1471 | "description": "Collection of value objects that represent the types of the PHP type system", 1472 | "homepage": "https://github.com/sebastianbergmann/type", 1473 | "time": "2019-07-02T08:10:15+00:00" 1474 | }, 1475 | { 1476 | "name": "sebastian/version", 1477 | "version": "2.0.1", 1478 | "source": { 1479 | "type": "git", 1480 | "url": "https://github.com/sebastianbergmann/version.git", 1481 | "reference": "99732be0ddb3361e16ad77b68ba41efc8e979019" 1482 | }, 1483 | "dist": { 1484 | "type": "zip", 1485 | "url": "https://api.github.com/repos/sebastianbergmann/version/zipball/99732be0ddb3361e16ad77b68ba41efc8e979019", 1486 | "reference": "99732be0ddb3361e16ad77b68ba41efc8e979019", 1487 | "shasum": "" 1488 | }, 1489 | "require": { 1490 | "php": ">=5.6" 1491 | }, 1492 | "type": "library", 1493 | "extra": { 1494 | "branch-alias": { 1495 | "dev-master": "2.0.x-dev" 1496 | } 1497 | }, 1498 | "autoload": { 1499 | "classmap": [ 1500 | "src/" 1501 | ] 1502 | }, 1503 | "notification-url": "https://packagist.org/downloads/", 1504 | "license": [ 1505 | "BSD-3-Clause" 1506 | ], 1507 | "authors": [ 1508 | { 1509 | "name": "Sebastian Bergmann", 1510 | "email": "sebastian@phpunit.de", 1511 | "role": "lead" 1512 | } 1513 | ], 1514 | "description": "Library that helps with managing the version number of Git-hosted PHP projects", 1515 | "homepage": "https://github.com/sebastianbergmann/version", 1516 | "time": "2016-10-03T07:35:21+00:00" 1517 | }, 1518 | { 1519 | "name": "squizlabs/php_codesniffer", 1520 | "version": "3.5.3", 1521 | "source": { 1522 | "type": "git", 1523 | "url": "https://github.com/squizlabs/PHP_CodeSniffer.git", 1524 | "reference": "557a1fc7ac702c66b0bbfe16ab3d55839ef724cb" 1525 | }, 1526 | "dist": { 1527 | "type": "zip", 1528 | "url": "https://api.github.com/repos/squizlabs/PHP_CodeSniffer/zipball/557a1fc7ac702c66b0bbfe16ab3d55839ef724cb", 1529 | "reference": "557a1fc7ac702c66b0bbfe16ab3d55839ef724cb", 1530 | "shasum": "" 1531 | }, 1532 | "require": { 1533 | "ext-simplexml": "*", 1534 | "ext-tokenizer": "*", 1535 | "ext-xmlwriter": "*", 1536 | "php": ">=5.4.0" 1537 | }, 1538 | "require-dev": { 1539 | "phpunit/phpunit": "^4.0 || ^5.0 || ^6.0 || ^7.0" 1540 | }, 1541 | "bin": [ 1542 | "bin/phpcs", 1543 | "bin/phpcbf" 1544 | ], 1545 | "type": "library", 1546 | "extra": { 1547 | "branch-alias": { 1548 | "dev-master": "3.x-dev" 1549 | } 1550 | }, 1551 | "notification-url": "https://packagist.org/downloads/", 1552 | "license": [ 1553 | "BSD-3-Clause" 1554 | ], 1555 | "authors": [ 1556 | { 1557 | "name": "Greg Sherwood", 1558 | "role": "lead" 1559 | } 1560 | ], 1561 | "description": "PHP_CodeSniffer tokenizes PHP, JavaScript and CSS files and detects violations of a defined set of coding standards.", 1562 | "homepage": "https://github.com/squizlabs/PHP_CodeSniffer", 1563 | "keywords": [ 1564 | "phpcs", 1565 | "standards" 1566 | ], 1567 | "time": "2019-12-04T04:46:47+00:00" 1568 | }, 1569 | { 1570 | "name": "symfony/polyfill-ctype", 1571 | "version": "v1.13.1", 1572 | "source": { 1573 | "type": "git", 1574 | "url": "https://github.com/symfony/polyfill-ctype.git", 1575 | "reference": "f8f0b461be3385e56d6de3dbb5a0df24c0c275e3" 1576 | }, 1577 | "dist": { 1578 | "type": "zip", 1579 | "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/f8f0b461be3385e56d6de3dbb5a0df24c0c275e3", 1580 | "reference": "f8f0b461be3385e56d6de3dbb5a0df24c0c275e3", 1581 | "shasum": "" 1582 | }, 1583 | "require": { 1584 | "php": ">=5.3.3" 1585 | }, 1586 | "suggest": { 1587 | "ext-ctype": "For best performance" 1588 | }, 1589 | "type": "library", 1590 | "extra": { 1591 | "branch-alias": { 1592 | "dev-master": "1.13-dev" 1593 | } 1594 | }, 1595 | "autoload": { 1596 | "psr-4": { 1597 | "Symfony\\Polyfill\\Ctype\\": "" 1598 | }, 1599 | "files": [ 1600 | "bootstrap.php" 1601 | ] 1602 | }, 1603 | "notification-url": "https://packagist.org/downloads/", 1604 | "license": [ 1605 | "MIT" 1606 | ], 1607 | "authors": [ 1608 | { 1609 | "name": "Gert de Pagter", 1610 | "email": "BackEndTea@gmail.com" 1611 | }, 1612 | { 1613 | "name": "Symfony Community", 1614 | "homepage": "https://symfony.com/contributors" 1615 | } 1616 | ], 1617 | "description": "Symfony polyfill for ctype functions", 1618 | "homepage": "https://symfony.com", 1619 | "keywords": [ 1620 | "compatibility", 1621 | "ctype", 1622 | "polyfill", 1623 | "portable" 1624 | ], 1625 | "time": "2019-11-27T13:56:44+00:00" 1626 | }, 1627 | { 1628 | "name": "theseer/tokenizer", 1629 | "version": "1.1.3", 1630 | "source": { 1631 | "type": "git", 1632 | "url": "https://github.com/theseer/tokenizer.git", 1633 | "reference": "11336f6f84e16a720dae9d8e6ed5019efa85a0f9" 1634 | }, 1635 | "dist": { 1636 | "type": "zip", 1637 | "url": "https://api.github.com/repos/theseer/tokenizer/zipball/11336f6f84e16a720dae9d8e6ed5019efa85a0f9", 1638 | "reference": "11336f6f84e16a720dae9d8e6ed5019efa85a0f9", 1639 | "shasum": "" 1640 | }, 1641 | "require": { 1642 | "ext-dom": "*", 1643 | "ext-tokenizer": "*", 1644 | "ext-xmlwriter": "*", 1645 | "php": "^7.0" 1646 | }, 1647 | "type": "library", 1648 | "autoload": { 1649 | "classmap": [ 1650 | "src/" 1651 | ] 1652 | }, 1653 | "notification-url": "https://packagist.org/downloads/", 1654 | "license": [ 1655 | "BSD-3-Clause" 1656 | ], 1657 | "authors": [ 1658 | { 1659 | "name": "Arne Blankerts", 1660 | "email": "arne@blankerts.de", 1661 | "role": "Developer" 1662 | } 1663 | ], 1664 | "description": "A small library for converting tokenized PHP source code into XML and potentially other formats", 1665 | "time": "2019-06-13T22:48:21+00:00" 1666 | }, 1667 | { 1668 | "name": "webmozart/assert", 1669 | "version": "1.6.0", 1670 | "source": { 1671 | "type": "git", 1672 | "url": "https://github.com/webmozart/assert.git", 1673 | "reference": "573381c0a64f155a0d9a23f4b0c797194805b925" 1674 | }, 1675 | "dist": { 1676 | "type": "zip", 1677 | "url": "https://api.github.com/repos/webmozart/assert/zipball/573381c0a64f155a0d9a23f4b0c797194805b925", 1678 | "reference": "573381c0a64f155a0d9a23f4b0c797194805b925", 1679 | "shasum": "" 1680 | }, 1681 | "require": { 1682 | "php": "^5.3.3 || ^7.0", 1683 | "symfony/polyfill-ctype": "^1.8" 1684 | }, 1685 | "conflict": { 1686 | "vimeo/psalm": "<3.6.0" 1687 | }, 1688 | "require-dev": { 1689 | "phpunit/phpunit": "^4.8.36 || ^7.5.13" 1690 | }, 1691 | "type": "library", 1692 | "autoload": { 1693 | "psr-4": { 1694 | "Webmozart\\Assert\\": "src/" 1695 | } 1696 | }, 1697 | "notification-url": "https://packagist.org/downloads/", 1698 | "license": [ 1699 | "MIT" 1700 | ], 1701 | "authors": [ 1702 | { 1703 | "name": "Bernhard Schussek", 1704 | "email": "bschussek@gmail.com" 1705 | } 1706 | ], 1707 | "description": "Assertions to validate method input/output with nice error messages.", 1708 | "keywords": [ 1709 | "assert", 1710 | "check", 1711 | "validate" 1712 | ], 1713 | "time": "2019-11-24T13:36:37+00:00" 1714 | } 1715 | ], 1716 | "aliases": [], 1717 | "minimum-stability": "stable", 1718 | "stability-flags": [], 1719 | "prefer-stable": true, 1720 | "prefer-lowest": false, 1721 | "platform": { 1722 | "php": ">=7.2.0", 1723 | "ext-json": "*", 1724 | "ext-mbstring": "*" 1725 | }, 1726 | "platform-dev": [] 1727 | } 1728 | --------------------------------------------------------------------------------