├── LICENSE.md ├── README.md ├── composer.json ├── config └── .gitkeep └── src ├── Commands └── Crawl.php ├── CrawlObserver.php ├── CrawlRenderServiceProvider.php └── Middleware └── CrawlerPrerenderMiddleware.php /LICENSE.md: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | 3 | Copyright (c) Appstract 4 | 5 | > Permission is hereby granted, free of charge, to any person obtaining a copy 6 | > of this software and associated documentation files (the "Software"), to deal 7 | > in the Software without restriction, including without limitation the rights 8 | > to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | > copies of the Software, and to permit persons to whom the Software is 10 | > furnished to do so, subject to the following conditions: 11 | > 12 | > The above copyright notice and this permission notice shall be included in 13 | > all copies or substantial portions of the Software. 14 | > 15 | > THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | > IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | > FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | > AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | > LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | > OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | > THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # [Wip] Laravel Crawler Pre-render 2 | 3 | [![Latest Version on Packagist](https://img.shields.io/packagist/v/appstract/laravel-crawl-render.svg?style=flat-square)](https://packagist.org/packages/appstract/laravel-crawl-render) 4 | [![Total Downloads](https://img.shields.io/packagist/dt/appstract/laravel-crawl-render.svg?style=flat-square)](https://packagist.org/packages/appstract/laravel-crawl-render) 5 | [![Software License](https://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat-square)](LICENSE.md) 6 | 7 | A simple package to pre-render your Javascript site for web crawlers like Google bot to improve your SEO. 8 | 9 | It uses Spatie's [Crawler](https://github.com/spatie/crawler) and [Browsershot](https://github.com/spatie/browsershot) to crawl your website and store the HTML. 10 | Crawlers (detected by [Jaybizzle's Crawler Detect](https://github.com/JayBizzle/Crawler-Detect)) are getting the pre-rendered version served. 11 | 12 | ## Installation 13 | 14 | You can install the package via composer: 15 | 16 | ``` bash 17 | composer require appstract/laravel-crawl-render 18 | ``` 19 | 20 | You need Puppeteer to be installed for Browsershot to work, see: [https://github.com/spatie/browsershot#requirements](https://github.com/spatie/browsershot#requirements) 21 | 22 | ## Usage 23 | First add the middleware to any routes you want to Pre-render. 24 | 25 | ``` php 26 | \Appstract\CrawlRender\Middleware\CrawlerPrerenderMiddleware::class 27 | ``` 28 | 29 | Then run the crawler to pre-render your site: 30 | ``` php 31 | php artisan prerender:crawl 32 | ``` 33 | 34 | You can run this command regularly, for example after a deploy or with a schedule: 35 | 36 | ``` php 37 | // app/console/Kernel.php 38 | protected function schedule(Schedule $schedule) 39 | { 40 | $schedule->command('prerender:crawl')->daily()->at('02:00'); 41 | } 42 | ``` 43 | 44 | ## Testing 45 | 46 | ``` bash 47 | composer test 48 | ``` 49 | 50 | ## Contributing 51 | 52 | Contributions are welcome, [thanks to y'all](https://github.com/appstract/laravel-crawl-render/graphs/contributors) :) 53 | 54 | ## About Appstract 55 | 56 | Appstract is a small team from The Netherlands. We create (open source) tools for webdevelopment and write about related subjects on [Medium](https://medium.com/appstract). You can [follow us on Twitter](https://twitter.com/teamappstract), [buy us a beer](https://www.paypal.me/teamappstract/10) or [support us on Patreon](https://www.patreon.com/appstract). 57 | 58 | ## License 59 | 60 | The MIT License (MIT). Please see [License File](LICENSE.md) for more information. 61 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "appstract/laravel-crawl-render", 3 | "description": "Pre-render your site for web crawlers", 4 | "keywords": [ 5 | "appstract", 6 | "crawler", 7 | "render", 8 | "pre-render", 9 | "vue" 10 | ], 11 | "homepage": "https://github.com/appstract/laravel-crawl-render", 12 | "license": "MIT", 13 | "authors": [ 14 | { 15 | "name": "Olav van Schie", 16 | "email": "hello@appstract.team", 17 | "homepage": "https://appstract.team", 18 | "role": "Developer" 19 | } 20 | ], 21 | "require": { 22 | "php": "^7.1", 23 | "spatie/crawler": "^4.0", 24 | "jaybizzle/laravel-crawler-detect": "1.*" 25 | }, 26 | "require-dev": { 27 | "phpunit/phpunit": "5.*" 28 | }, 29 | "autoload": { 30 | "psr-4": { 31 | "Appstract\\CrawlRender\\": "src" 32 | } 33 | }, 34 | "autoload-dev": { 35 | "psr-4": { 36 | "Appstract\\CrawlRender\\Test\\": "tests" 37 | } 38 | }, 39 | "scripts": { 40 | "test": "vendor/bin/phpunit" 41 | }, 42 | "config": { 43 | "sort-packages": true 44 | }, 45 | "extra": { 46 | "laravel": { 47 | "providers": [ 48 | "Appstract\\CrawlRender\\CrawlRenderServiceProvider" 49 | ] 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /config/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/appstract/laravel-crawl-render/934ba0bb0989a46bc6cedcf071c69ab8413e507f/config/.gitkeep -------------------------------------------------------------------------------- /src/Commands/Crawl.php: -------------------------------------------------------------------------------- 1 | writeln('Removing old stuff'); 38 | Storage::disk('local')->deleteDirectory('prerendered'); 39 | 40 | $output->writeln('Starting crawler'); 41 | 42 | Crawler::create() 43 | ->setCrawlObserver(new CrawlObserver($output)) 44 | ->setCrawlProfile(new CrawlInternalUrls(config('app.url'))) 45 | ->executeJavaScript() 46 | //->setBrowsershot((new Browsershot())->waitUntilNetworkIdle()) 47 | ->setBrowsershot((new Browsershot())->setDelay(1500)) 48 | ->startCrawling(config('app.url')); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/CrawlObserver.php: -------------------------------------------------------------------------------- 1 | consoleOutput = $consoleOutput; 19 | } 20 | 21 | /** 22 | * Called when the crawler will crawl the url. 23 | * 24 | * @param \Psr\Http\Message\UriInterface $url 25 | */ 26 | public function willCrawl(UriInterface $url) 27 | { 28 | $this->consoleOutput->writeln('Crawling: '.$url->getPath()); 29 | } 30 | 31 | /** 32 | * Called when the crawler has crawled the given url successfully. 33 | * 34 | * @param \Psr\Http\Message\UriInterface $url 35 | * @param \Psr\Http\Message\ResponseInterface $response 36 | * @param \Psr\Http\Message\UriInterface|null $foundOnUrl 37 | */ 38 | public function crawled(UriInterface $url, ResponseInterface $response, ?UriInterface $foundOnUrl = null) 39 | { 40 | Storage::disk('local')->put('prerendered/'.$url->getPath().'/dom.html', 41 | //Browsershot::url($url)->waitUntilNetworkIdle()->bodyHtml() 42 | Browsershot::url($url)->setDelay(1500)->bodyHtml() 43 | ); 44 | } 45 | 46 | /** 47 | * Called when the crawler had a problem crawling the given url. 48 | * 49 | * @param \Psr\Http\Message\UriInterface $url 50 | * @param \GuzzleHttp\Exception\RequestException $requestException 51 | * @param \Psr\Http\Message\UriInterface|null $foundOnUrl 52 | */ 53 | public function crawlFailed(UriInterface $url, RequestException $requestException, ?UriInterface $foundOnUrl = null) 54 | { 55 | } 56 | 57 | /** 58 | * Called when the crawl has ended. 59 | */ 60 | public function finishedCrawling() 61 | { 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/CrawlRenderServiceProvider.php: -------------------------------------------------------------------------------- 1 | app->runningInConsole()) { 15 | $this->commands([ 16 | Commands\Crawl::class, 17 | ]); 18 | } 19 | } 20 | 21 | /** 22 | * Register the application services. 23 | */ 24 | public function register() 25 | { 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/Middleware/CrawlerPrerenderMiddleware.php: -------------------------------------------------------------------------------- 1 | hasPrerenderedView($request->getPathInfo())) { 23 | $response->setContent( 24 | $this->getPrerenderedView($request->getPathInfo()) 25 | ); 26 | } 27 | 28 | return $response; 29 | } 30 | 31 | /** 32 | * @param $path 33 | * 34 | * @return mixed 35 | */ 36 | protected function hasPrerenderedView($path) 37 | { 38 | return Storage::disk('local')->exists('prerendered'.$path.'/dom.html'); 39 | } 40 | 41 | /** 42 | * @param $path 43 | * 44 | * @return mixed 45 | */ 46 | protected function getPrerenderedView($path) 47 | { 48 | return Storage::disk('local')->get('prerendered'.$path.'/dom.html'); 49 | } 50 | } 51 | --------------------------------------------------------------------------------