├── .gitignore ├── .git-blame-ignore-refs ├── .github ├── FUNDING.yml └── workflows │ └── run_tests.yml ├── README.md ├── tests ├── Fixtures │ └── TestSpider.php ├── TestCase.php ├── RegisterDefaultNamespaceTest.php ├── SpiderMakeCommandTest.php └── __snapshots__ │ └── files │ └── SpiderMakeCommandTest__testCreateNewSpiderClass__1.php ├── phpunit.xml ├── .php-cs-fixer.php ├── LICENSE.md ├── src ├── Commands │ ├── stubs │ │ └── spider.stub │ └── SpiderMakeCommand.php └── RoachServiceProvider.php ├── config └── roach.php ├── composer.json └── CHANGELOG.md /.gitignore: -------------------------------------------------------------------------------- 1 | .build/ 2 | .phpunit.cache 3 | vendor/ -------------------------------------------------------------------------------- /.git-blame-ignore-refs: -------------------------------------------------------------------------------- 1 | # Update copyright year 2 | 0d689bc5f73ad96763ba09c228db9c4228a8f0bf -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: ksassnowski 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Web Scraping for Laravel 2 | 3 | This is the Laravel adapter for [Roach](https://roach-php.dev), the complete web scraping toolkit for PHP. 4 | 5 | ## Installation 6 | 7 | Install the package via composer 8 | 9 | ```bash 10 | composer require roach-php/laravel 11 | ``` 12 | 13 | ## Documentation 14 | 15 | Check out the [full documentation](https://roach-php.dev/docs/laravel) to get up and running. 16 | 17 | ## Credits 18 | 19 | - [Kai Sassnowski](https://github.com/ksassnowski) 20 | - [All contributors](https://github.com/roach-php/laravel/contributors) 21 | 22 | 23 | ## License 24 | 25 | MIT 26 | -------------------------------------------------------------------------------- /tests/Fixtures/TestSpider.php: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | tests 6 | 7 | 8 | 9 | 10 | src 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /tests/TestCase.php: -------------------------------------------------------------------------------- 1 | withHeader($header) 16 | ->withRules(Config\Rules::fromArray([ 17 | 'php_unit_test_class_requires_covers' => false, 18 | 'phpdoc_to_property_type' => false, 19 | ])); 20 | 21 | $config = Config\Factory::fromRuleSet($ruleSet); 22 | 23 | $config->getFinder() 24 | ->in(__DIR__) 25 | ->exclude(['config', 'src/Commands/stubs', 'tests/__snapshots__']); 26 | $config->setCacheFile(__DIR__ . '/.build/php-cs-fixer/.php-cs-fixer.cache'); 27 | 28 | return $config; -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) Kai Sassnowski me@kai-sassnowski.com 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /tests/RegisterDefaultNamespaceTest.php: -------------------------------------------------------------------------------- 1 | 'RoachPHP\Laravel\Tests\Fixtures']); 35 | $runner = Roach::fake(); 36 | 37 | Artisan::call('roach:run', ['spider' => 'TestSpider']); 38 | 39 | $runner->assertRunWasStarted(TestSpider::class); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/Commands/stubs/spider.stub: -------------------------------------------------------------------------------- 1 | 42 | */ 43 | public function parse(Response $response): Generator 44 | { 45 | // todo... 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /tests/SpiderMakeCommandTest.php: -------------------------------------------------------------------------------- 1 | 'App\Spiders\Test']); 31 | } 32 | 33 | protected function tearDown(): void 34 | { 35 | File::deleteDirectory(__DIR__ . '/Fixtures/Spiders'); 36 | 37 | parent::tearDown(); 38 | } 39 | 40 | public function testCreateNewSpiderClass(): void 41 | { 42 | $this->artisan('roach:spider', ['name' => 'ExampleSpider']); 43 | 44 | $this->assertMatchesFileSnapshot(app_path('Spiders/Test/ExampleSpider.php')); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /tests/__snapshots__/files/SpiderMakeCommandTest__testCreateNewSpiderClass__1.php: -------------------------------------------------------------------------------- 1 | 42 | */ 43 | public function parse(Response $response): Generator 44 | { 45 | // todo... 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/Commands/SpiderMakeCommand.php: -------------------------------------------------------------------------------- 1 | ArrayRequestScheduler::class, 20 | 21 | /* 22 | |-------------------------------------------------------------------------- 23 | | HTTP Client 24 | |-------------------------------------------------------------------------- 25 | | 26 | | The HTTP client implementation Roach uses to dispatch new request 27 | | during a run. 28 | | 29 | | Needs to implement the RoachPHP\Http\ClientInterface interface. 30 | | 31 | */ 32 | 'client' => Client::class, 33 | 34 | /* 35 | |-------------------------------------------------------------------------- 36 | | Default Spider Namespace 37 | |-------------------------------------------------------------------------- 38 | | 39 | | The default namespace the `roach:run` and `roach:spider` commands use 40 | | to determine the namespace of spider classes. This should not contain 41 | | leading or trailing backslashes. 42 | | 43 | */ 44 | 'default_spider_namespace' => 'App\Spiders', 45 | ]; 46 | -------------------------------------------------------------------------------- /.github/workflows/run_tests.yml: -------------------------------------------------------------------------------- 1 | name: run-tests 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | test: 7 | runs-on: ${{ matrix.os }} 8 | strategy: 9 | matrix: 10 | php: [8.1, 8.2, 8.3, 8.4] 11 | laravel: [10.*, 11.*, 12.*] 12 | dependency-version: [prefer-stable] 13 | os: [ubuntu-latest] 14 | include: 15 | - laravel: 10.* 16 | testbench: ^8.0 17 | - laravel: 11.* 18 | testbench: ^9.0 19 | - laravel: 12.* 20 | testbench: ^10.0 21 | exclude: 22 | - php: 8.1 23 | laravel: 12.* 24 | - php: 8.1 25 | laravel: 11.* 26 | - php: 8.4 27 | laravel: 10.* 28 | 29 | name: P${{ matrix.php }} - L${{ matrix.laravel}} - ${{ matrix.dependency-version }} - ${{ matrix.os }} 30 | 31 | steps: 32 | - name: Setup PHP 33 | uses: shivammathur/setup-php@v2 34 | with: 35 | php-version: ${{ matrix.php }} 36 | coverage: none 37 | 38 | - name: Checkout code 39 | uses: actions/checkout@v3 40 | 41 | - name: Cache dependencies 42 | uses: actions/cache@v4 43 | with: 44 | path: ~/.composer/cache/files 45 | key: dependencies-laravel-${{ matrix.laravel }}-php-${{ matrix.php }}-composer-${{ hashFiles('composer.json') }} 46 | 47 | - name: Install dependencies 48 | run: | 49 | composer require "laravel/framework:${{ matrix.laravel }}" "orchestra/testbench:${{ matrix.testbench }}" --no-interaction --no-update 50 | composer update --${{ matrix.dependency-version }} --prefer-dist --no-interaction 51 | 52 | - name: Execute tests 53 | run: vendor/bin/phpunit 54 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "roach-php/laravel", 3 | "description": "Laravel adapter for roach-php/core", 4 | "license": "MIT", 5 | "type": "library", 6 | "authors": [ 7 | { 8 | "name": "Kai Sassnowski", 9 | "email": "me@kai-sassnowski.com" 10 | } 11 | ], 12 | "require": { 13 | "php": "~8.1.0 || ~8.2.0 || ~8.3.0 || ~8.4.0", 14 | "laravel/framework": "^10.0 || ^11.0 || ^12.0", 15 | "roach-php/core": "^2.0 || ^3.0 || dev-main", 16 | "spatie/laravel-package-tools": "^1.11" 17 | }, 18 | "require-dev": { 19 | "ergebnis/composer-normalize": "^2.15", 20 | "ergebnis/php-cs-fixer-config": "^6.19", 21 | "orchestra/testbench": "^7.0 || ^8.0 || ^9.0 || ^10.0", 22 | "phpunit/phpunit": "^10.0 || ^11.0", 23 | "roave/security-advisories": "dev-latest", 24 | "spatie/phpunit-snapshot-assertions": "^5.0" 25 | }, 26 | "minimum-stability": "dev", 27 | "prefer-stable": true, 28 | "autoload": { 29 | "psr-4": { 30 | "RoachPHP\\Laravel\\": "src/" 31 | } 32 | }, 33 | "autoload-dev": { 34 | "psr-4": { 35 | "RoachPHP\\Laravel\\Tests\\": "tests/" 36 | } 37 | }, 38 | "config": { 39 | "allow-plugins": { 40 | "ergebnis/composer-normalize": true 41 | } 42 | }, 43 | "extra": { 44 | "laravel": { 45 | "providers": [ 46 | "RoachPHP\\Laravel\\RoachServiceProvider" 47 | ] 48 | } 49 | }, 50 | "scripts": { 51 | "post-install-cmd": [ 52 | "composer normalize" 53 | ], 54 | "post-update-cmd": [ 55 | "composer normalize" 56 | ], 57 | "coding-standards": [ 58 | "mkdir -p .build/php-cs-fixer", 59 | "php-cs-fixer fix --diff --verbose" 60 | ] 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | All notable changes to this project will be documented in this file. 3 | 4 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 5 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 6 | 7 | ## [3.2.0] - 2025-03-21 8 | 9 | ### Added 10 | 11 | - Added support for Laravel 12 12 | - Added support for PHP 8.4 13 | 14 | ## [3.1.0] – 2024-03-22 15 | 16 | ### Added 17 | 18 | - Added support for Laravel 11 19 | 20 | ## [3.0.0] – 2024-01-05 21 | 22 | ### Added 23 | 24 | - Added support for PHP 8.3 25 | 26 | ### Removed 27 | 28 | - Dropped Laravel 9 support 29 | 30 | ## [2.0.0] – 2022-02-06 31 | 32 | ### Added 33 | 34 | - Added support for Laravel 10 35 | 36 | ### Removed 37 | 38 | - Dropped support for PHP 8.0 39 | 40 | ## [1.0.0] - 2022-04-19 41 | 42 | ### Added 43 | 44 | - Added support for `roach-php/core:^1.0` 45 | - Added `force (-f)` optioni to `roach:spider` command (@josezenem, #9) 46 | - Added `default_spider_namespace` configuration option. This option is used by both the 47 | `roach:spider` and `roach:run` commands 48 | 49 | ### Fixed 50 | 51 | - Fixed bug where `artisan roach:run` command would sometimes not correctly parse namespaces (fixes [#11](https://github.com/roach-php/laravel/issues/11)) 52 | 53 | ### Removed 54 | 55 | - Drop support for Laravel 8 56 | 57 | ## [0.3.1] - 2022-02-01 58 | 59 | ### Fixed 60 | 61 | - Fixed default container bindings not being registered correctly if package config has not been published (fixes #5) 62 | 63 | ## [0.3.0] - 2022-01-30 64 | 65 | ### Added 66 | 67 | - Added support for Laravel 9 68 | 69 | ## [0.2.1] - 2022-01-04 70 | 71 | ### Fixed 72 | 73 | - Update incorrect namespace in spider stub (#4). Credits @RyanPriceDotCa 74 | 75 | ## [0.2.0] - 2021-12-28 76 | 77 | ### Changed 78 | 79 | - Bump `roach-php/core` dependency to `^0.2.0` which includes a middleware to execute 80 | Javascript on a page. 81 | 82 | ## [0.1.0] - 2021-12-27 83 | 84 | ### Added 85 | 86 | - Initial release 87 | 88 | [3.2.0]: https://github.com/roach-php/laravel/compare/3.1.0...3.2.0 89 | [3.1.0]: https://github.com/roach-php/laravel/compare/3.0.0...3.1.0 90 | [3.0.0]: https://github.com/roach-php/laravel/compare/2.0.0...3.0.0 91 | [2.0.0]: https://github.com/roach-php/laravel/compare/1.0.0...2.0.0 92 | [1.0.0]: https://github.com/roach-php/laravel/compare/0.3.1...1.0.0 93 | [0.3.1]: https://github.com/roach-php/laravel/compare/0.3.0...0.3.1 94 | [0.3.0]: https://github.com/roach-php/laravel/compare/0.2.1...0.3.0 95 | [0.2.1]: https://github.com/roach-php/laravel/compare/0.2.0...0.2.1 96 | [0.2.0]: https://github.com/roach-php/laravel/compare/0.1.0...0.2.0 97 | -------------------------------------------------------------------------------- /src/RoachServiceProvider.php: -------------------------------------------------------------------------------- 1 | name('roach') 45 | ->hasConfigFile() 46 | ->hasCommands([ 47 | SpiderMakeCommand::class, 48 | Repl::class, 49 | RunSpiderCommand::class, 50 | ]); 51 | } 52 | 53 | public function registeringPackage(): void 54 | { 55 | $this->app->bind(EngineInterface::class, Engine::class); 56 | $this->app->bind(RunnerInterface::class, Runner::class); 57 | $this->app->singleton(EventDispatcher::class, EventDispatcher::class); 58 | $this->app->singleton(EventDispatcherInterface::class, EventDispatcher::class); 59 | $this->app->bind( 60 | ClientInterface::class, 61 | static fn (Application $app) => $app->make(config('roach.client')), 62 | ); 63 | $this->app->bind( 64 | RequestSchedulerInterface::class, 65 | static fn (Application $app) => $app->make(config('roach.request_queue')), 66 | ); 67 | $this->app->bind(ClockInterface::class, SystemClock::class); 68 | $this->app->bind(ItemPipelineInterface::class, ItemPipeline::class); 69 | $this->app->bind( 70 | NamespaceResolverInterface::class, 71 | static fn (Application $app) => new DefaultNamespaceResolverDecorator( 72 | new StaticNamespaceResolver(), 73 | config('roach.default_spider_namespace') ?: 'App\Spiders', 74 | ), 75 | ); 76 | } 77 | 78 | public function bootingPackage(): void 79 | { 80 | Roach::useContainer($this->app); 81 | } 82 | } 83 | --------------------------------------------------------------------------------