├── LICENSE
├── README.md
├── composer.json
├── logs
└── v2.0.0.txt
└── src
├── BlazarEventServiceProvider.php
├── BlazarServiceProvider.php
├── Commands
└── BlazarFlush.php
├── Events
├── PreRendEvent.php
└── PreRendEventQ.php
├── Listeners
├── PreRendListener.php
└── PreRendListenerQ.php
├── Middleware
├── Blazar.php
└── DontPreRender.php
├── Traits
├── Helpers.php
├── Listeners.php
└── Middleware.php
├── config
└── blazar.php
└── exec-chrome.js
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2017 Muah
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | > ### This project is abandoned due to impracticality, i would highly recommend to try using ServiceWorkers and here is [a little article](https://ctf0.wordpress.com/2018/07/14/laravel-and-pwa/) to get you started.
2 |
3 |
4 |
5 | # Blazar
6 |
7 | [](https://packagist.org/packages/ctf0/blazar) [](https://packagist.org/packages/ctf0/blazar)
8 |
9 | Automate pre-rendering pages on the fly through utilizing [puppeteer](https://github.com/GoogleChrome/puppeteer) which runs in the background when needed without adding any overhead to the server nor to the user experience.
10 |
11 |
12 |
13 | ## Installation
14 |
15 | - install [puppeteer](https://github.com/GoogleChrome/puppeteer#installation)
16 |
17 | - `composer require ctf0/blazar`
18 |
19 | - (Laravel < 5.5) add the service provider to `config/app.php`
20 |
21 | ```php
22 | 'providers' => [
23 | ctf0\Blazar\BlazarServiceProvider::class,
24 | ]
25 | ```
26 |
27 | - publish the package assets
28 |
29 | `php artisan vendor:publish --provider="ctf0\Blazar\BlazarServiceProvider"`
30 |
31 | - add the middlewares to `app/Http/Kernel.php`
32 |
33 | ```php
34 | protected $middlewareGroups = [
35 | // ...
36 | \ctf0\Blazar\Middleware\Blazar::class,
37 | ];
38 |
39 | protected $routeMiddleware = [
40 | // ...
41 | 'dont-pre-render' => \ctf0\Blazar\Middleware\DontPreRender::class,
42 | ];
43 | ```
44 |
45 | - the package use caching through **Redis** to store the rendered results, so make sure to check the [docs](https://laravel.com/docs/5.4/redis) for installation & configuration.
46 |
47 |
48 |
49 | ## Config
50 | **config/blazar.php**
51 |
52 | ```php
53 | return [
54 | /*
55 | * puppeteer bin path
56 | */
57 | 'puppeteer_path' => '',
58 |
59 | /*
60 | * puppeteer script path
61 | *
62 | * leave it empty to the use the one from the package
63 | */
64 | 'script_path' => '',
65 |
66 | /*
67 | * prerender the page only if the url is being visited from a bot/crawler
68 | */
69 | 'bots_only' => false,
70 |
71 | /*
72 | * log the url when its processed by puppeteer
73 | */
74 | 'debug' => true,
75 |
76 | /**
77 | * clear user cache on logout
78 | */
79 | 'clear_user_cache' => true
80 | ];
81 | ```
82 |
83 |
84 |
85 | ## Usage
86 |
87 | - we use [Queues](https://laravel.com/docs/5.4/events#queued-event-listeners) to **pre-render the visited pages** in the background for more than one reason
88 |
89 | >- avoid latency when the page is being accessed for the first time.
90 | >- don't keep the user waiting in case `puppeteer` took long to render the page or when something goes wrong.
91 | >- after `puppeteer` has finished rendering, the page is cached to optimize server load even further.
92 | >- make your website **SEO friendly**, because instead of serving the normal pages that usually produce issues for crawlers, we are now serving the **pre-renderd version**. [ReadMore](#-render-pages-automatically)
93 | >- even for websites with some traffic, we are still going to process each visited page without any problems.
94 |
95 | #### # Render Pages Automatically
96 |
97 | Atm in order to ***pre-render*** any page, it have to be visited first but if you want to make sure that all is working from day one, you can use the excellent package [laravel-link-checker](https://packagist.org/packages/spatie/laravel-link-checker) by **Spatie**
98 |
99 | - which by simply running `php artisan link-checker:run` you will
100 |
101 | >- check which "url/link" on the website is not working.
102 | >- **pre-render** all pages at once.
103 |
104 | #### # Flushing The Cache
105 |
106 | - to clear the whole package cache, you can use
107 |
108 | ```bash
109 | php artisan blazar:flush
110 | ```
111 |
112 | or from within your app
113 |
114 | ```php
115 | Artisan::call('blazar:flush');
116 | ```
117 |
118 | #### # Bots Only
119 |
120 | > we now use [CrawlerDetect](https://github.com/JayBizzle/Laravel-Crawler-Detect) instead of relying on '\?\_escaped_fragment_'
121 |
122 | if you decided to pre-render the pages for bots only, no need to the run the queue as the page will remain busy **"stalled response"** until rendered by `puppeteer`, which happens on the fly.
123 |
124 | however because we are caching the result, so this will only happen once per page.
125 |
126 | also note that we are saving the page cache equal to the url so even if you switched off the `bots_only` option, if the page is cached, we will always serve the cached result.
127 |
128 |
129 |
130 | ## Notes
131 |
132 | #### # Queues
133 |
134 | the worker should only fires when a url is visited & if this url is not cached,
135 | however if you have an unfinished old process, the queue will start processing pages on its own, so to fix that, simply restart the queue server `beanstalkd, redis, etc...`
136 |
137 | ```bash
138 | # ex. using Homebrew
139 |
140 | brew services restart beanstalkd
141 | ```
142 |
143 | #### # Auth
144 |
145 | as i dont know how to make laravel think that a page visited through puppeteer is the same as the current logged in user.
146 |
147 | so trying to pre-render pages with **`auth` middleware** will be cached as if the user was redirected to the home page or whatever you've set to **redirectTo** under
148 | `Constollers/Auth/LoginController.php` & `Middleware/RedirectIfAuthenticated.php`
149 |
150 | so to solve that, simply add **`dont-pre-render` middleware** to those routes and everything will work as expected.
151 | also make sure to add the same middleware to any route that needs fresh csrf-token for each user **"login, register, etc.."** to avoid getting `CSRF Token Mismatch` for other users trying to use those pages.
152 |
153 | #### # More Reading
154 | - https://vuejsdevelopers.com/2017/04/09/vue-laravel-fake-server-side-rendering/
155 | - https://vuejsdevelopers.com/2017/11/06/vue-js-laravel-server-side-rendering/
156 |
--------------------------------------------------------------------------------
/composer.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "ctf0/blazar",
3 | "description": "pre-render pages on the fly",
4 | "homepage": "https://github.com/ctf0/Blazar",
5 | "license": "MIT",
6 | "keywords": [
7 | "ctf0",
8 | "Blazar",
9 | "laravel",
10 | "phantomjs",
11 | "pre-render",
12 | "prerender"
13 | ],
14 | "authors": [
15 | {
16 | "name": "Muah",
17 | "email": "muah003@gmail.com"
18 | }
19 | ],
20 | "require": {
21 | "php" : "~7.0",
22 | "illuminate/support": "~5.4.0|~5.5.0|~5.6.0",
23 | "jaybizzle/laravel-crawler-detect": "^1.0",
24 | "ctf0/package-changelog": "^1.0"
25 | },
26 | "autoload": {
27 | "psr-4": {
28 | "ctf0\\Blazar\\": "src"
29 | }
30 | },
31 | "extra": {
32 | "laravel": {
33 | "providers": [
34 | "ctf0\\Blazar\\BlazarServiceProvider"
35 | ]
36 | },
37 | "changeLog": "logs"
38 | },
39 | "config": {
40 | "sort-packages": true
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/logs/v2.0.0.txt:
--------------------------------------------------------------------------------
1 | - use puppeteer instead of phantomjs
2 | - update readme
--------------------------------------------------------------------------------
/src/BlazarEventServiceProvider.php:
--------------------------------------------------------------------------------
1 | ['ctf0\Blazar\Listeners\PreRendListener'],
15 | 'ctf0\Blazar\Events\PreRendEventQ' => ['ctf0\Blazar\Listeners\PreRendListenerQ'],
16 | ];
17 |
18 | public function boot()
19 | {
20 | parent::boot();
21 |
22 | if (config('blazar.clear_user_cache')) {
23 | Event::listen('Illuminate\Auth\Events\Logout', function ($event) {
24 | $id = $event->user->id;
25 |
26 | return $this->preCacheStore()->tags($this->cacheName($id, true))->flush();
27 | });
28 | }
29 | }
30 | }
31 |
--------------------------------------------------------------------------------
/src/BlazarServiceProvider.php:
--------------------------------------------------------------------------------
1 | publishes([
17 | __DIR__ . '/config' => config_path(),
18 | ], 'config');
19 | }
20 |
21 | /**
22 | * Register any package services.
23 | */
24 | public function register()
25 | {
26 | // commands
27 | $this->commands([
28 | Commands\BlazarFlush::class,
29 | ]);
30 |
31 | // events & listeners
32 | $this->app->register(BlazarEventServiceProvider::class);
33 |
34 | // packages
35 | $this->app->register(\Jaybizzle\LaravelCrawlerDetect\LaravelCrawlerDetectServiceProvider::class);
36 | AliasLoader::getInstance()->alias('Crawler', 'Jaybizzle\LaravelCrawlerDetect\Facades\LaravelCrawlerDetect');
37 | $this->app->register(\ctf0\PackageChangeLog\PackageChangeLogServiceProvider::class);
38 | }
39 | }
40 |
--------------------------------------------------------------------------------
/src/Commands/BlazarFlush.php:
--------------------------------------------------------------------------------
1 | keys('*blazar*');
33 | foreach ($keys as $key) {
34 | $redis->del($key);
35 | }
36 |
37 | $this->info('Blazar cache was cleared');
38 | }
39 | }
40 |
--------------------------------------------------------------------------------
/src/Events/PreRendEvent.php:
--------------------------------------------------------------------------------
1 | url = $url;
16 | }
17 | }
18 |
--------------------------------------------------------------------------------
/src/Events/PreRendEventQ.php:
--------------------------------------------------------------------------------
1 | url = $url;
18 | $this->token = $token;
19 | $this->userId = $userId;
20 | }
21 | }
22 |
--------------------------------------------------------------------------------
/src/Listeners/PreRendListener.php:
--------------------------------------------------------------------------------
1 | url;
16 | $cache_name = $this->cacheName($url);
17 |
18 | $this->cacheResult(
19 | $url,
20 | $cache_name,
21 | $this->runChrome($this->prepareUrlForShell($url))
22 | );
23 | }
24 |
25 | /**
26 | * cache result.
27 | *
28 | * @param [type] $url [description]
29 | * @param [type] $cache_name [description]
30 | * @param [type] $output [description]
31 | *
32 | * @return [type] [description]
33 | */
34 | protected function cacheResult($url, $cache_name, $output)
35 | {
36 | // couldnt open url
37 | if (str_contains($output, 'Something Went Wrong')) {
38 | $this->debugLog("Bot-$url : $output");
39 |
40 | return;
41 | }
42 |
43 | // log result
44 | if ($this->debug) {
45 | $this->debugLog("Bot-$url : Processed By Puppeteer");
46 | }
47 |
48 | // save to cache
49 | return $this->preCacheStore()->rememberForever($cache_name, function () use ($output) {
50 | return $output;
51 | });
52 | }
53 | }
54 |
--------------------------------------------------------------------------------
/src/Listeners/PreRendListenerQ.php:
--------------------------------------------------------------------------------
1 | url;
17 | $token = $event->token;
18 | $userId = $event->userId;
19 | $cache_name = $this->cacheName($url);
20 |
21 | $this->cacheResult(
22 | $url,
23 | $cache_name,
24 | $userId,
25 | $this->replaceToken(
26 | $token,
27 | $this->runChrome($this->prepareUrlForShell($url), $token, $userId)
28 | )
29 | );
30 | }
31 |
32 | /**
33 | * cache result.
34 | *
35 | * @param [type] $url [description]
36 | * @param [type] $cache_name [description]
37 | * @param [type] $userId [description]
38 | * @param [type] $output [description]
39 | *
40 | * @return [type] [description]
41 | */
42 | protected function cacheResult($url, $cache_name, $userId, $output)
43 | {
44 | // couldnt open url
45 | if (str_contains($output, 'Something Went Wrong')) {
46 | $this->debugLog("$url : $output");
47 |
48 | return;
49 | }
50 |
51 | // log result
52 | if ($this->debug) {
53 | $this->debugLog("$url : Processed By Puppeteer");
54 | }
55 |
56 | // save to cache
57 | if ($userId) {
58 | return $this->preCacheStore()
59 | ->tags($this->cacheName($userId, true))
60 | ->rememberForever($cache_name, function () use ($output) {
61 | return $output;
62 | });
63 | }
64 |
65 | return $this->preCacheStore()
66 | ->rememberForever($cache_name, function () use ($output) {
67 | return $output;
68 | });
69 | }
70 |
71 | /**
72 | * replace "Puppeteer" csrf_token with "current user".
73 | *
74 | * @param [type] $token [description]
75 | * @param [type] $output [description]
76 | *
77 | * @return [type] [description]
78 | */
79 | protected function replaceToken($token, $output)
80 | {
81 | $pattern = [
82 | '//' => "",
83 | '//' => "",
84 | ];
85 |
86 | return preg_replace(array_keys($pattern), array_values($pattern), $output);
87 | }
88 | }
89 |
--------------------------------------------------------------------------------
/src/Middleware/Blazar.php:
--------------------------------------------------------------------------------
1 | header('user-id')) {
21 | Auth::loginUsingId($id);
22 | }
23 |
24 | $response = $next($request);
25 |
26 | if ($this->dontPre($response)) {
27 | return $response;
28 | }
29 |
30 | if ($this->isPreRendable($request, $response)) {
31 | $url = $this->formatUrlQuery($request);
32 | $cache_store = $this->preCacheStore();
33 | $cache_name = $this->cacheName($url);
34 |
35 | config('blazar.bots_only') && Crawler::isCrawler()
36 | ? $this->preBots($response, $url, $cache_store, $cache_name)
37 | : $this->preAll($response, $url, $cache_store, $cache_name);
38 | }
39 |
40 | return $response;
41 | }
42 |
43 | /**
44 | * Bots.
45 | *
46 | * @param [type] $response [description]
47 | * @param [type] $url [description]
48 | * @param [type] $cache_store [description]
49 | * @param [type] $cache_name [description]
50 | *
51 | * @return [type] [description]
52 | */
53 | protected function preBots($response, $url, $cache_store, $cache_name)
54 | {
55 | if ($cache_store->has($cache_name)) {
56 | $response->setContent($cache_store->get($cache_name));
57 | } else {
58 | $this->preRenderedResponse($url, null, true);
59 | $response->setContent($cache_store->get($cache_name));
60 | }
61 | }
62 |
63 | /**
64 | * All.
65 | *
66 | * @param [type] $response [description]
67 | * @param [type] $url [description]
68 | * @param [type] $cache_store [description]
69 | * @param [type] $cache_name [description]
70 | *
71 | * @return [type] [description]
72 | */
73 | protected function preAll($response, $url, $cache_store, $cache_name)
74 | {
75 | $userId = auth()->check() ? auth()->user()->id : null;
76 |
77 | if (is_null($userId)) {
78 | if ($cache_store->has($cache_name)) {
79 | $response->setContent($cache_store->get($cache_name));
80 | } else {
81 | $this->preRenderedResponse($url);
82 | }
83 | } else {
84 | $tags = $cache_store->tags($this->cacheName($userId, true));
85 |
86 | if ($tags->has($cache_name)) {
87 | $response->setContent($tags->get($cache_name));
88 | } else {
89 | $this->preRenderedResponse($url, $userId);
90 | }
91 | }
92 | }
93 |
94 | /**
95 | * main op.
96 | *
97 | * @param [type] $url [description]
98 | * @param [type] $userId [description]
99 | * @param [type] $bots [description]
100 | *
101 | * @return [type] [description]
102 | */
103 | protected function preRenderedResponse($url, $userId = null, $bots = null)
104 | {
105 | $bots
106 | ? event(new PreRendEvent($url))
107 | : event(new PreRendEventQ($url, csrf_token(), $userId));
108 | }
109 | }
110 |
--------------------------------------------------------------------------------
/src/Middleware/DontPreRender.php:
--------------------------------------------------------------------------------
1 | headers->set('dont-pre-render', true);
22 |
23 | return $response;
24 | }
25 | }
26 |
--------------------------------------------------------------------------------
/src/Traits/Helpers.php:
--------------------------------------------------------------------------------
1 | cachePrefix() . $item
21 | : $this->cachePrefix() . md5($item);
22 | }
23 | }
24 |
--------------------------------------------------------------------------------
/src/Traits/Listeners.php:
--------------------------------------------------------------------------------
1 | puppet = config('blazar.puppeteer_path');
14 | $this->script = config('blazar.script_path');
15 | $this->debug = config('blazar.debug');
16 | }
17 |
18 | /**
19 | * helpers.
20 | *
21 | * @param [type] $url [description]
22 | *
23 | * @return [type] [description]
24 | */
25 | protected function debugLog($url)
26 | {
27 | logger($url);
28 | }
29 |
30 | /**
31 | * escape special chars for shell.
32 | *
33 | * @param [type] $url [description]
34 | *
35 | * @return [type] [description]
36 | */
37 | protected function prepareUrlForShell($url)
38 | {
39 | $pattern = [
40 | '/\?/' => "\?",
41 | '/\=/' => "\=",
42 | '/\&/' => "\&",
43 | ];
44 |
45 | return preg_replace(array_keys($pattern), array_values($pattern), $url);
46 | }
47 |
48 | /**
49 | * process with puppeteer.
50 | *
51 | * @param [type] $url [description]
52 | * @param [type] $token [description]
53 | * @param [type] $user_id [description]
54 | *
55 | * @return [type] [description]
56 | */
57 | protected function runChrome($url, $token = null, $user_id = null)
58 | {
59 | $puppet = $this->puppet;
60 | $script = $this->script ?: dirname(__DIR__) . '/exec-chrome.js';
61 | $cmnd = "node '$script' '$puppet' '$url'";
62 |
63 | // $this->debugLog("node '$script' '$puppet' '$url' '$token' '$user_id'");
64 |
65 | return $token
66 | ? shell_exec($cmnd . " '$token' '$user_id'")
67 | : shell_exec($cmnd);
68 | }
69 | }
70 |
--------------------------------------------------------------------------------
/src/Traits/Middleware.php:
--------------------------------------------------------------------------------
1 | headers->get('dont-pre-render');
17 | }
18 |
19 | /**
20 | * https://laravel-news.com/cache-query-params.
21 | *
22 | * @param [type] $request [description]
23 | *
24 | * @return [type] [description]
25 | */
26 | protected function formatUrlQuery($request)
27 | {
28 | $url = $request->url();
29 |
30 | if ($query = $request->query()) {
31 | $str = str_contains($query, '_escaped_fragment')
32 | ? preg_replace('/(\?|\&)_escaped_fragment_?/', '', $query)
33 | : $query;
34 |
35 | ksort($str);
36 | $rebuild_query = http_build_query($str);
37 |
38 | return "{$url}?{$rebuild_query}";
39 | }
40 |
41 | return $url;
42 | }
43 |
44 | /**
45 | * check if we can pre-render.
46 | *
47 | * @param [type] $request [description]
48 | * @param [type] $response [description]
49 | *
50 | * @return bool [description]
51 | */
52 | protected function isPreRendable($request, $response)
53 | {
54 | return !str_contains($request->header('User-Agent'), 'HeadlessChrome') &&
55 | !$request->ajax() &&
56 | !$request->pjax() &&
57 | $request->isMethodCacheable() &&
58 | $response->isSuccessful();
59 | }
60 | }
61 |
--------------------------------------------------------------------------------
/src/config/blazar.php:
--------------------------------------------------------------------------------
1 | '/usr/local/lib/node_modules/puppeteer',
11 |
12 | /*
13 | * puppeteer script path
14 | * leave it empty to the use the one from the package
15 | */
16 | 'script_path' => '',
17 |
18 | /*
19 | * prerender the page only if the url is being visited from a bot/crawler
20 | * https://github.com/JayBizzle/Laravel-Crawler-Detect
21 | */
22 | 'bots_only' => false,
23 |
24 | /*
25 | * log the url when its processed by puppeteer
26 | */
27 | 'debug' => false,
28 |
29 | /*
30 | * clear user cache on logout
31 | */
32 | 'clear_user_cache' => false,
33 | ];
34 |
--------------------------------------------------------------------------------
/src/exec-chrome.js:
--------------------------------------------------------------------------------
1 | // Arguments
2 | const args = process.argv
3 | let path = args[2]
4 | let url = args[3]
5 | let token = args[4]
6 | let userId = args[5]
7 |
8 | const puppeteer = require(path)
9 |
10 | ;(async () => {
11 | let browser
12 | let page
13 | let html
14 |
15 | try {
16 | browser = await puppeteer.launch({
17 | ignoreHTTPSErrors: true
18 | })
19 |
20 | page = await browser.newPage()
21 |
22 | if (token) {
23 | page.setExtraHTTPHeaders({
24 | 'X-CSRF-TOKEN': token,
25 | 'user-id': userId
26 | })
27 | }
28 |
29 | await page.goto(url, {waitUntil: 'networkidle0'})
30 |
31 | html = await page.content()
32 | console.log(html)
33 |
34 | await browser.close()
35 |
36 | } catch (exception) {
37 | if (browser) {
38 | await browser.close()
39 | }
40 |
41 | console.error(`Something Went Wrong: ${exception}`)
42 |
43 | process.exit(1)
44 | }
45 | })()
46 |
--------------------------------------------------------------------------------