├── .doclintrc
├── .gitignore
├── phpstan.neon.dist
├── .gitattributes
├── code-of-conduct.md
├── .github
├── ISSUE_TEMPLATE
│ ├── 3_blank_issue.md
│ ├── config.yml
│ ├── 2_feature_request.yml
│ └── 1_bug_report.yml
├── workflows
│ ├── ci.yml
│ ├── keepalive.yml
│ ├── merge-up.yml
│ ├── add-prs-to-project.yml
│ ├── tag-patch-release.yml
│ └── dispatch-ci.yml
└── PULL_REQUEST_TEMPLATE.md
├── templates
└── SilverStripe
│ └── StaticPublishQueue
│ └── HTMLRedirection.ss
├── .editorconfig
├── _config
└── staticpublishqueue.yml
├── phpunit.xml.dist
├── src
├── Job
│ ├── DeleteWholeCache.php
│ ├── GenerateStaticCacheJob.php
│ ├── DeleteStaticCacheJob.php
│ └── StaticCacheFullBuildJob.php
├── Service
│ ├── UrlBundleInterface.php
│ └── UrlBundleService.php
├── Contract
│ ├── StaticPublisher.php
│ ├── StaticPublishingTrigger.php
│ └── StaticallyPublishable.php
├── Dev
│ └── StaticPublisherState.php
├── Task
│ └── StaticCacheFullBuildTask.php
├── Job.php
├── Publisher.php
├── Extension
│ ├── Publishable
│ │ └── PublishableSiteTree.php
│ └── Engine
│ │ └── SiteTreePublishingEngine.php
└── Publisher
│ └── FilesystemPublisher.php
├── phpcs.xml.dist
├── LICENSE
├── README.md
├── composer.json
└── includes
├── staticrequesthandler.php
└── functions.php
/.doclintrc:
--------------------------------------------------------------------------------
1 | docs/en/
2 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | /vendor/
2 | /composer.lock
3 | /resources/
4 |
--------------------------------------------------------------------------------
/phpstan.neon.dist:
--------------------------------------------------------------------------------
1 | parameters:
2 | paths:
3 | - src
4 |
--------------------------------------------------------------------------------
/.gitattributes:
--------------------------------------------------------------------------------
1 | /tests export-ignore
2 | /docs export-ignore
3 | /.travis.yml export-ignore
4 |
--------------------------------------------------------------------------------
/code-of-conduct.md:
--------------------------------------------------------------------------------
1 | When having discussions about this module in issues or pull request please adhere to the [SilverStripe Community Code of Conduct](https://docs.silverstripe.org/en/contributing/code_of_conduct).
2 |
--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/3_blank_issue.md:
--------------------------------------------------------------------------------
1 | ---
2 | name: Blank issue
3 | about: Only for use by maintainers
4 | ---
5 |
6 |
--------------------------------------------------------------------------------
/.github/workflows/ci.yml:
--------------------------------------------------------------------------------
1 | name: CI
2 |
3 | on:
4 | push:
5 | pull_request:
6 | workflow_dispatch:
7 |
8 | jobs:
9 | ci:
10 | name: CI
11 | # Do not run if this is a pull-request from same repo i.e. not a fork repo
12 | if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository
13 | uses: silverstripe/gha-ci/.github/workflows/ci.yml@v2
14 |
--------------------------------------------------------------------------------
/templates/SilverStripe/StaticPublishQueue/HTMLRedirection.ss:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 | This page has been moved
8 | Redirecting you to $URL
9 |
12 |
13 |
14 |
--------------------------------------------------------------------------------
/.editorconfig:
--------------------------------------------------------------------------------
1 | # For more information about the properties used in this file,
2 | # please see the EditorConfig documentation:
3 | # http://editorconfig.org
4 |
5 | [*]
6 | charset = utf-8
7 | end_of_line = lf
8 | indent_size = 4
9 | indent_style = space
10 | insert_final_newline = true
11 | trim_trailing_whitespace = true
12 |
13 | [{*.yml,package.json}]
14 | indent_size = 2
15 |
16 | # The indent size used in the package.json file cannot be changed:
17 | # https://github.com/npm/npm/pull/3180#issuecomment-16336516
18 |
--------------------------------------------------------------------------------
/_config/staticpublishqueue.yml:
--------------------------------------------------------------------------------
1 | ---
2 | Name: staticpublishqueue
3 | ---
4 | SilverStripe\Core\Injector\Injector:
5 | SilverStripe\StaticPublishQueue\Publisher:
6 | class: SilverStripe\StaticPublishQueue\Publisher\FilesystemPublisher
7 | SilverStripe\StaticPublishQueue\Service\UrlBundleInterface:
8 | class: SilverStripe\StaticPublishQueue\Service\UrlBundleService
9 |
10 | SilverStripe\CMS\Model\SiteTree:
11 | extensions:
12 | - SilverStripe\StaticPublishQueue\Extension\Engine\SiteTreePublishingEngine
13 | - SilverStripe\StaticPublishQueue\Extension\Publishable\PublishableSiteTree
14 |
--------------------------------------------------------------------------------
/.github/workflows/keepalive.yml:
--------------------------------------------------------------------------------
1 | name: Keepalive
2 |
3 | on:
4 | # At 3:15 AM UTC, on day 20 of the month
5 | schedule:
6 | - cron: '15 3 20 * *'
7 | workflow_dispatch:
8 |
9 | permissions: {}
10 |
11 | jobs:
12 | keepalive:
13 | name: Keepalive
14 | # Only run cron on the silverstripe account
15 | if: (github.event_name == 'schedule' && github.repository_owner == 'silverstripe') || (github.event_name != 'schedule')
16 | runs-on: ubuntu-latest
17 | permissions:
18 | actions: write
19 | steps:
20 | - name: Keepalive
21 | uses: silverstripe/gha-keepalive@v1
22 |
--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/config.yml:
--------------------------------------------------------------------------------
1 | blank_issues_enabled: false
2 | contact_links:
3 | - name: Security Vulnerability
4 | url: https://docs.silverstripe.org/en/contributing/issues_and_bugs/#reporting-security-issues
5 | about: ⚠️ We do not use GitHub issues to track security vulnerability reports. Click "open" on the right to see how to report security vulnerabilities.
6 | - name: Support Question
7 | url: https://www.silverstripe.org/community/
8 | about: We use GitHub issues only to discuss bugs and new features. For support questions, please use one of the support options available in our community channels.
9 |
--------------------------------------------------------------------------------
/.github/workflows/merge-up.yml:
--------------------------------------------------------------------------------
1 | name: Merge-up
2 |
3 | on:
4 | # At 6:30 AM UTC, only on Wednesday
5 | schedule:
6 | - cron: '30 6 * * 3'
7 | workflow_dispatch:
8 |
9 | permissions: {}
10 |
11 | jobs:
12 | merge-up:
13 | name: Merge-up
14 | # Only run cron on the silverstripe account
15 | if: (github.event_name == 'schedule' && github.repository_owner == 'silverstripe') || (github.event_name != 'schedule')
16 | runs-on: ubuntu-latest
17 | permissions:
18 | contents: write
19 | actions: write
20 | steps:
21 | - name: Merge-up
22 | uses: silverstripe/gha-merge-up@v2
23 |
--------------------------------------------------------------------------------
/phpunit.xml.dist:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | tests/
6 |
7 |
8 |
9 |
10 | src/
11 |
12 |
13 | tests/
14 |
15 |
16 |
17 |
--------------------------------------------------------------------------------
/src/Job/DeleteWholeCache.php:
--------------------------------------------------------------------------------
1 | isComplete = Publisher::singleton()->purgeAll();
21 | }
22 |
23 | public function processUrl(string $url, int $priority): void
24 | {
25 | // noop
26 | }
27 | }
28 |
--------------------------------------------------------------------------------
/.github/workflows/add-prs-to-project.yml:
--------------------------------------------------------------------------------
1 | name: Add new PRs to github project
2 |
3 | on:
4 | pull_request_target:
5 | types:
6 | - opened
7 | - ready_for_review
8 |
9 | permissions: {}
10 |
11 | jobs:
12 | addprtoproject:
13 | name: Add PR to GitHub Project
14 | # Only run on the silverstripe account
15 | if: github.repository_owner == 'silverstripe'
16 | runs-on: ubuntu-latest
17 | steps:
18 | - name: Add PR to github project
19 | uses: silverstripe/gha-add-pr-to-project@v1
20 | with:
21 | app_id: ${{ vars.PROJECT_PERMISSIONS_APP_ID }}
22 | private_key: ${{ secrets.PROJECT_PERMISSIONS_APP_PRIVATE_KEY }}
23 |
--------------------------------------------------------------------------------
/src/Service/UrlBundleInterface.php:
--------------------------------------------------------------------------------
1 | publishURL($url, true);
21 | $meta = is_array($meta) ? $meta : [];
22 |
23 | if (array_key_exists('success', $meta) && $meta['success']) {
24 | $this->markUrlAsProcessed($url);
25 |
26 | return;
27 | }
28 |
29 | $this->handleFailedUrl($url, $meta);
30 | }
31 | }
32 |
--------------------------------------------------------------------------------
/src/Contract/StaticPublishingTrigger.php:
--------------------------------------------------------------------------------
1 | purgeURL($url);
23 | $meta = is_array($meta) ? $meta : [];
24 |
25 | if (array_key_exists('success', $meta) && $meta['success']) {
26 | $this->markUrlAsProcessed($url);
27 |
28 | return;
29 | }
30 |
31 | $this->handleFailedUrl($url, $meta);
32 | }
33 | }
34 |
--------------------------------------------------------------------------------
/src/Contract/StaticallyPublishable.php:
--------------------------------------------------------------------------------
1 | Priority (int)
24 | */
25 | public function urlsToCache();
26 | }
27 |
--------------------------------------------------------------------------------
/.github/workflows/dispatch-ci.yml:
--------------------------------------------------------------------------------
1 | name: Dispatch CI
2 |
3 | on:
4 | # At 6:30 AM UTC, only on Friday, Saturday, and Sunday
5 | schedule:
6 | - cron: '30 6 * * 5,6,0'
7 | workflow_dispatch:
8 | inputs:
9 | major_type:
10 | description: 'Major branch type'
11 | required: true
12 | type: choice
13 | options:
14 | - 'dynamic'
15 | - 'current'
16 | - 'next'
17 | - 'previous'
18 | default: 'dynamic'
19 | minor_type:
20 | description: 'Minor branch type'
21 | required: true
22 | type: choice
23 | options:
24 | - 'dynamic'
25 | - 'next-minor'
26 | - 'next-patch'
27 | default: 'dynamic'
28 |
29 | permissions: {}
30 |
31 | jobs:
32 | dispatch-ci:
33 | name: Dispatch CI
34 | # Only run cron on the silverstripe account
35 | if: (github.event_name == 'schedule' && github.repository_owner == 'silverstripe') || (github.event_name != 'schedule')
36 | runs-on: ubuntu-latest
37 | permissions:
38 | contents: read
39 | actions: write
40 | steps:
41 | - name: Dispatch CI
42 | uses: silverstripe/gha-dispatch-ci@v1
43 | with:
44 | major_type: ${{ inputs.major_type }}
45 | minor_type: ${{ inputs.minor_type }}
46 |
--------------------------------------------------------------------------------
/phpcs.xml.dist:
--------------------------------------------------------------------------------
1 |
2 |
3 | CodeSniffer ruleset for SilverStripe coding conventions.
4 |
5 | src
6 | tests
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Copyright (c) 2016, Silverstripe Ltd.
2 | All rights reserved.
3 |
4 | Redistribution and use in source and binary forms, with or without
5 | modification, are permitted provided that the following conditions are met:
6 |
7 | * Redistributions of source code must retain the above copyright notice, this
8 | list of conditions and the following disclaimer.
9 |
10 | * Redistributions in binary form must reproduce the above copyright notice,
11 | this list of conditions and the following disclaimer in the documentation
12 | and/or other materials provided with the distribution.
13 |
14 | * Neither the name of silverstripe-cms nor the names of its
15 | contributors may be used to endorse or promote products derived from
16 | this software without specific prior written permission.
17 |
18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
22 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
25 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 |
--------------------------------------------------------------------------------
/src/Dev/StaticPublisherState.php:
--------------------------------------------------------------------------------
1 | registerService(new QueuedJobsTest_Handler(), QueuedJobHandler::class);
22 |
23 | // Disable special actions of the queue service
24 | Config::modify()->set(QueuedJobService::class, 'use_shutdown_function', false);
25 |
26 | // It seems Injector doesn't cover all cases so we force-inject a test service which is suitable for unit tests
27 | Injector::inst()->registerService(new QueuedJobsTestService(), QueuedJobService::class);
28 | SiteTreePublishingEngine::setQueueService(QueuedJobService::singleton());
29 | }
30 |
31 | public function tearDown(SapphireTest $test)
32 | {
33 | }
34 |
35 | public function setUpOnce($class)
36 | {
37 | }
38 |
39 | public function tearDownOnce($class)
40 | {
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Static Publisher with Queue
2 |
3 | [](https://github.com/silverstripe/silverstripe-staticpublishqueue/actions/workflows/ci.yml)
4 | [](https://www.silverstripe.org/software/addons/silverstripe-commercially-supported-module-list/)
5 |
6 | ## Installation
7 |
8 | ```sh
9 | composer require silverstripe/staticpublishqueue
10 | ```
11 |
12 | ## Brief
13 |
14 | This module provides an API for your project to be able to generate a static cache of your pages to enhance
15 | performance by not booting Silverstripe in order to serve requests.
16 |
17 | It generates the cache files using the [QueuedJobs module](https://docs.silverstripe.org/en/optional_features/queuedjobs/).
18 |
19 | [Documentation](https://docs.silverstripe.org/en/optional_features/staticpublishqueue/)
20 |
21 | ## Unit-testing with StaticPublisherState to disable queuedjobs for unit-tests
22 |
23 | You can use `StaticPublisherState` to disable queuejobs job queueing and logging in unit-testing to improve performance.
24 |
25 | Add the following yml to your project:
26 |
27 | ```yml
28 | ----
29 | Name: staticpublishqueue-tests
30 | Only:
31 | classexists:
32 | - 'Symbiote\QueuedJobs\Tests\QueuedJobsTest\QueuedJobsTest_Handler'
33 | - 'SilverStripe\StaticPublishQueue\Test\QueuedJobsTestService'
34 | ----
35 | SilverStripe\Core\Injector\Injector:
36 | SilverStripe\Dev\State\SapphireTestState:
37 | properties:
38 | States:
39 | staticPublisherState: '%$SilverStripe\StaticPublishQueue\Dev\StaticPublisherState'
40 | ```
41 |
--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/2_feature_request.yml:
--------------------------------------------------------------------------------
1 | name: 🚀 Feature Request
2 | description: Submit a feature request (but only if you're planning on implementing it)
3 | body:
4 | - type: markdown
5 | attributes:
6 | value: |
7 | Please only submit feature requests if you plan on implementing the feature yourself.
8 | See the [contributing code documentation](https://docs.silverstripe.org/en/contributing/code/#make-or-find-a-github-issue) for more guidelines about submitting feature requests.
9 | - type: textarea
10 | id: description
11 | attributes:
12 | label: Description
13 | description: A clear and concise description of the new feature, and why it belongs in core
14 | validations:
15 | required: true
16 | - type: textarea
17 | id: more-info
18 | attributes:
19 | label: Additional context or points of discussion
20 | description: |
21 | *Optional: Any additional context, points of discussion, etc that might help validate and refine your idea*
22 | - type: checkboxes
23 | id: validations
24 | attributes:
25 | label: Validations
26 | description: "Before submitting the issue, please confirm the following:"
27 | options:
28 | - label: You intend to implement the feature yourself
29 | required: true
30 | - label: You have read the [contributing guide](https://docs.silverstripe.org/en/contributing/code/)
31 | required: true
32 | - label: You strongly believe this feature should be in core, rather than being its own community module
33 | required: true
34 | - label: You have checked for existing issues or pull requests related to this feature (and didn't find any)
35 | required: true
36 |
--------------------------------------------------------------------------------
/composer.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "silverstripe/staticpublishqueue",
3 | "description": "Static publishing queue to create static versions of pages for enhanced performance and security",
4 | "license": "BSD-3-Clause",
5 | "type": "silverstripe-vendormodule",
6 | "keywords": [
7 | "silverstripe",
8 | "static",
9 | "html",
10 | "security",
11 | "performance",
12 | "static-publishing",
13 | "caching",
14 | "cache",
15 | "static-caching",
16 | "static-cache",
17 | "queue",
18 | "publishing"
19 | ],
20 | "require": {
21 | "php": "^8.3",
22 | "silverstripe/framework": "^6",
23 | "silverstripe/cms": "^6",
24 | "silverstripe/config": "^3",
25 | "symbiote/silverstripe-queuedjobs": "^6",
26 | "silverstripe/versioned": "^3"
27 | },
28 | "require-dev": {
29 | "silverstripe/recipe-testing": "^4",
30 | "silverstripe/standards": "^1",
31 | "silverstripe/documentation-lint": "^1",
32 | "phpstan/extension-installer": "^1.3"
33 | },
34 | "autoload": {
35 | "psr-4": {
36 | "SilverStripe\\StaticPublishQueue\\": "src/",
37 | "SilverStripe\\StaticPublishQueue\\Test\\": "tests/php/"
38 | },
39 | "files": [
40 | "includes/functions.php"
41 | ]
42 | },
43 | "include-path": [
44 | "includes/"
45 | ],
46 | "extra": [],
47 | "config": {
48 | "allow-plugins": {
49 | "composer/installers": true,
50 | "silverstripe/recipe-plugin": true,
51 | "silverstripe/vendor-plugin": true,
52 | "dealerdirect/phpcodesniffer-composer-installer": true
53 | }
54 | },
55 | "scripts": {
56 | "lint": "phpcs -s src/ tests/php/",
57 | "lint-clean": "phpcbf src/ tests/php/"
58 | },
59 | "prefer-stable": true,
60 | "minimum-stability": "dev"
61 | }
62 |
--------------------------------------------------------------------------------
/.github/PULL_REQUEST_TEMPLATE.md:
--------------------------------------------------------------------------------
1 |
6 | ## Description
7 |
11 |
12 | ## Manual testing steps
13 |
17 |
18 | ## Issues
19 |
23 | - #
24 |
25 | ## Pull request checklist
26 |
30 | - [ ] The target branch is correct
31 | - See [picking the right version](https://docs.silverstripe.org/en/contributing/code/#picking-the-right-version)
32 | - [ ] All commits are relevant to the purpose of the PR (e.g. no debug statements, unrelated refactoring, or arbitrary linting)
33 | - Small amounts of additional linting are usually okay, but if it makes it hard to concentrate on the relevant changes, ask for the unrelated changes to be reverted, and submitted as a separate PR.
34 | - [ ] The commit messages follow our [commit message guidelines](https://docs.silverstripe.org/en/contributing/code/#commit-messages)
35 | - [ ] The PR follows our [contribution guidelines](https://docs.silverstripe.org/en/contributing/code/)
36 | - [ ] Code changes follow our [coding conventions](https://docs.silverstripe.org/en/contributing/coding_conventions/)
37 | - [ ] This change is covered with tests (or tests aren't necessary for this change)
38 | - [ ] Any relevant User Help/Developer documentation is updated; for impactful changes, information is added to the changelog for the intended release
39 | - [ ] CI is green
40 |
--------------------------------------------------------------------------------
/includes/staticrequesthandler.php:
--------------------------------------------------------------------------------
1 | URLSegment),
10 | // or through URL collection (for controller method names etc.).
11 | $urlParts = @parse_url($url);
12 |
13 | // query strings are not yet supported so we need to bail is there is one present
14 | // except for some params, which we ignore
15 | if (!empty($urlParts['query'])) {
16 | parse_str($urlParts['query'], $queryParts);
17 | if (!empty($queryParts['stage']) && $queryParts['stage'] === 'Live') {
18 | unset($queryParts['stage']);
19 | }
20 | if (!empty($queryParts)) {
21 | return;
22 | }
23 | }
24 |
25 | // Remove base folders from the URL if webroot is hosted in a subfolder)
26 | $path = isset($urlParts['path']) ? $urlParts['path'] : '';
27 | if (mb_substr(mb_strtolower($path), 0, mb_strlen($baseURL)) === mb_strtolower($baseURL)) {
28 | $urlSegment = mb_substr($path, mb_strlen($baseURL));
29 | } else {
30 | $urlSegment = $path;
31 | }
32 |
33 | // Normalize URLs
34 | $urlSegment = trim($urlSegment, '/');
35 |
36 | $filename = $urlSegment ?: 'index';
37 |
38 | if ($domainBasedCaching) {
39 | if (!$urlParts) {
40 | throw new \LogicException('Unable to parse URL');
41 | }
42 | if (isset($urlParts['host'])) {
43 | $filename = $urlParts['host'] . '/' . $filename;
44 | }
45 | }
46 | $dirName = dirname($filename);
47 | $prefix = '';
48 | if ($dirName !== '/' && $dirName !== '.') {
49 | $prefix = $dirName . '/';
50 | }
51 | return $prefix . basename($filename);
52 | }
53 | }
54 |
55 | if (!function_exists('SilverStripe\\StaticPublishQueue\\PathToURL')) {
56 | function PathToURL($path, $destPath, $domainBasedCaching = false)
57 | {
58 | if (strpos($path, $destPath) === 0) {
59 | //Strip off the full path of the cache dir from the front
60 | $path = substr($path, strlen($destPath));
61 | }
62 |
63 | // Strip off the file extension and leading /
64 | $relativeURL = substr($path, 0, strrpos($path, '.'));
65 | $relativeURL = ltrim($relativeURL, '/');
66 |
67 | if ($domainBasedCaching) {
68 | // factor in the domain as the top dir
69 | if (substr($relativeURL, -6) === '/index') {
70 | $relativeURL = substr($relativeURL, 0, strlen($relativeURL) - 5);
71 | }
72 | return \SilverStripe\Control\Director::protocol() . $relativeURL;
73 | }
74 |
75 | return $relativeURL === 'index'
76 | ? \SilverStripe\Control\Director::absoluteBaseURL()
77 | : \SilverStripe\Control\Director::absoluteURL($relativeURL);
78 | }
79 | }
80 |
--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/1_bug_report.yml:
--------------------------------------------------------------------------------
1 | name: 🪳 Bug Report
2 | description: Tell us if something isn't working the way it's supposed to
3 |
4 | body:
5 | - type: markdown
6 | attributes:
7 | value: |
8 | We strongly encourage you to [submit a pull request](https://docs.silverstripe.org/en/contributing/code/) which fixes the issue.
9 | Bug reports which are accompanied with a pull request are a lot more likely to be resolved quickly.
10 | - type: input
11 | id: affected-versions
12 | attributes:
13 | label: Module version(s) affected
14 | description: |
15 | What version of _this module_ have you reproduced this bug on?
16 | Run `composer info` to see the specific version of each module installed in your project.
17 | If you don't have access to that, check inside the help menu in the bottom left of the CMS.
18 | placeholder: x.y.z
19 | validations:
20 | required: true
21 | - type: textarea
22 | id: description
23 | attributes:
24 | label: Description
25 | description: A clear and concise description of the problem
26 | validations:
27 | required: true
28 | - type: textarea
29 | id: how-to-reproduce
30 | attributes:
31 | label: How to reproduce
32 | description: |
33 | ⚠️ This is the most important part of the report ⚠️
34 | Without a way to easily reproduce your issue, there is little chance we will be able to help you and work on a fix.
35 | - Please, take the time to show us some code and/or configuration that is needed for others to reproduce the problem easily.
36 | - If the bug is too complex to reproduce with some short code samples, please reproduce it in a public repository and provide a link to the repository along with steps for setting up and reproducing the bug using that repository.
37 | - If part of the bug includes an error or exception, please provide a full stack trace.
38 | - If any user interaction is required to reproduce the bug, please add an ordered list of steps that are required to reproduce it.
39 | - Be as clear as you can, but don't miss any steps out. Simply saying "create a page" is less useful than guiding us through the steps you're taking to create a page, for example.
40 | placeholder: |
41 |
42 | #### Code sample
43 | ```php
44 |
45 | ```
46 |
47 | #### Reproduction steps
48 | 1.
49 | validations:
50 | required: true
51 | - type: textarea
52 | id: possible-solution
53 | attributes:
54 | label: Possible Solution
55 | description: |
56 | *Optional: only if you have suggestions on a fix/reason for the bug*
57 | Please consider [submitting a pull request](https://docs.silverstripe.org/en/contributing/code/) with your solution! It helps get faster feedback and greatly increases the chance of the bug being fixed.
58 | - type: textarea
59 | id: additional-context
60 | attributes:
61 | label: Additional Context
62 | description: "*Optional: any other context about the problem: log messages, screenshots, etc.*"
63 | - type: checkboxes
64 | id: validations
65 | attributes:
66 | label: Validations
67 | description: "Before submitting the issue, please make sure you do the following:"
68 | options:
69 | - label: Check that there isn't already an issue that reports the same bug
70 | required: true
71 | - label: Double check that your reproduction steps work in a fresh installation of [`silverstripe/installer`](https://github.com/silverstripe/silverstripe-installer) (with any code examples you've provided)
72 | required: true
73 |
--------------------------------------------------------------------------------
/src/Service/UrlBundleService.php:
--------------------------------------------------------------------------------
1 | urls[$url] = $url;
32 | }
33 | }
34 |
35 | public function getJobsForUrls(string $jobClass, ?string $message = null, ?DataObject $contextModel = null): array
36 | {
37 | $singleton = singleton($jobClass);
38 |
39 | if (!$singleton instanceof Job) {
40 | return [];
41 | }
42 |
43 | $urls = $this->getUrls();
44 | $urlsPerJob = $singleton->getUrlsPerJob();
45 | $batches = $urlsPerJob > 0 ? array_chunk($urls, $urlsPerJob) : [$urls];
46 | $jobs = [];
47 |
48 | foreach ($batches as $urlBatch) {
49 | $priorityUrls = $this->assignPriorityToUrls($urlBatch);
50 |
51 | /** @var Job $job */
52 | $job = Injector::inst()->create($jobClass);
53 | $job->hydrate($priorityUrls, $message);
54 |
55 | // Use this extension point to inject some additional data into the job
56 | $this->extend('updateHydratedJob', $job, $contextModel);
57 |
58 | $jobs[] = $job;
59 | }
60 |
61 | return $jobs;
62 | }
63 |
64 | /**
65 | * Get URLs for further processing
66 | */
67 | protected function getUrls(): array
68 | {
69 | $urls = [];
70 |
71 | foreach ($this->urls as $url) {
72 | $url = $this->formatUrl($url);
73 |
74 | if (!$url) {
75 | continue;
76 | }
77 |
78 | $urls[] = $url;
79 | }
80 |
81 | $urls = array_unique($urls);
82 |
83 | // Use this extension point to change the order of the URLs if needed
84 | $this->extend('updateGetUrls', $urls);
85 |
86 | return $urls;
87 | }
88 |
89 | /**
90 | * Extensibility function which allows to handle custom formatting / encoding needs for URLs
91 | * Returning "falsy" value will make the URL to be skipped
92 | */
93 | protected function formatUrl(string $url): ?string
94 | {
95 | if (UrlBundleService::config()->get('strip_stage_param')) {
96 | $url = $this->stripStageParam($url);
97 | }
98 |
99 | // Use this extension point to reformat URLs, for example encode special characters
100 | $this->extend('updateFormatUrl', $url);
101 |
102 | return $url;
103 | }
104 |
105 | /**
106 | * Add priority data to URLs
107 | */
108 | protected function assignPriorityToUrls(array $urls): array
109 | {
110 | $priority = 0;
111 | $priorityUrls = [];
112 |
113 | foreach ($urls as $url) {
114 | $priorityUrls[$url] = $priority;
115 | ++$priority;
116 | }
117 |
118 | return $priorityUrls;
119 | }
120 |
121 | /**
122 | * Any URL that we attempt to process through static publisher should always have any stage=* param removed
123 | */
124 | private function stripStageParam(string $url): string
125 | {
126 | // This will safely remove "stage" params, but keep any others. It doesn't matter where in the string "stage="
127 | // exists
128 | $url = preg_replace('/([?&])stage=[^&]+(&|$)/', '$1', $url);
129 | // Trim any trailing "?" or "&".
130 | $url = rtrim($url, '&');
131 | $url = rtrim($url, '?');
132 |
133 | return $url;
134 | }
135 | }
136 |
--------------------------------------------------------------------------------
/src/Task/StaticCacheFullBuildTask.php:
--------------------------------------------------------------------------------
1 | create(StaticCacheFullBuildJob::class);
34 | $signature = $job->getSignature();
35 |
36 | // see if we already have this job in a queue
37 | $filter = [
38 | 'Signature' => $signature,
39 | 'JobStatus' => [
40 | QueuedJob::STATUS_NEW,
41 | QueuedJob::STATUS_INIT,
42 | ],
43 | ];
44 |
45 | $existing = DataList::create(QueuedJobDescriptor::class)->filter($filter)->first();
46 |
47 | if ($existing && $existing->exists()) {
48 | $output->writeln(sprintf(
49 | 'There is already a %s in the queue, added %s %s',
50 | StaticCacheFullBuildJob::class,
51 | $existing->Created,
52 | $existing->StartAfter ? 'and set to start after ' . $existing->StartAfter : ''
53 | ));
54 |
55 | return Command::FAILURE;
56 | }
57 |
58 | if ($input->getOption('startAfter')) {
59 | $now = DBDatetime::now();
60 | $today = $now->Date();
61 | $startTime = $input->getOption('startAfter');
62 |
63 | // move to tomorrow if the starttime has passed today
64 | if ($now->Time24() > $startTime) {
65 | $timestamp = strtotime($today . ' ' . $startTime . ' +1 day');
66 | $dayWord = 'tomorrow';
67 | } else {
68 | $timestamp = strtotime($today . ' ' . $startTime);
69 | $dayWord = 'today';
70 | }
71 |
72 | $startAfter = (new DateTime())->setTimestamp($timestamp);
73 | $thisTimeTomorrow = (new DateTime())->setTimestamp(strtotime($now . ' +1 day'))->getTimestamp();
74 |
75 | // sanity check that we are in the next 24 hours - prevents some weird stuff sneaking through
76 | if ($startAfter->getTimestamp() > $thisTimeTomorrow || $startAfter->getTimestamp() < $now->getTimestamp()) {
77 | $output->writeln('Invalid startAfter parameter passed. Please ensure the time format is HHmm e.g. 1300');
78 |
79 | return Command::INVALID;
80 | }
81 |
82 | $output->writeln(sprintf(
83 | '%s queued for %s %s.',
84 | StaticCacheFullBuildJob::class,
85 | $startAfter->format('H:m'),
86 | $dayWord
87 | ));
88 | } else {
89 | $startAfter = null;
90 | $output->writeln(StaticCacheFullBuildJob::class . ' added to the queue for immediate processing');
91 | }
92 |
93 | $job->setJobData(0, 0, false, new \stdClass(), [
94 | 'Building static cache for full site',
95 | ]);
96 | QueuedJobService::singleton()->queueJob($job, $startAfter ? $startAfter->format('Y-m-d H:i:s') : null);
97 |
98 | return Command::SUCCESS;
99 | }
100 |
101 | public function getOptions(): array
102 | {
103 | return [
104 | new InputOption('startAfter', null, InputOption::VALUE_REQUIRED, 'Delay execution until this time. Must be in 24hr format e.g. 1300'),
105 | ];
106 | }
107 | }
108 |
--------------------------------------------------------------------------------
/src/Job/StaticCacheFullBuildJob.php:
--------------------------------------------------------------------------------
1 | URLsToProcess = $this->getAllLivePageURLs();
40 | $this->URLsToCleanUp = [];
41 |
42 | parent::setup();
43 |
44 | Publisher::singleton()->purgeAll();
45 | $this->addMessage(sprintf('Building %s URLS', count($this->URLsToProcess)));
46 | $this->addMessage(var_export(array_keys($this->URLsToProcess), true));
47 | }
48 |
49 | /**
50 | * Do some processing yourself!
51 | */
52 | public function process(): void
53 | {
54 | // Remove any URLs which have already been processed
55 | if ($this->ProcessedURLs) {
56 | $this->URLsToProcess = array_diff_key(
57 | $this->URLsToProcess,
58 | $this->ProcessedURLs
59 | );
60 | }
61 |
62 | $chunkSize = $this->getChunkSize();
63 | $count = 0;
64 |
65 | // Generate static cache for all live pages
66 | foreach ($this->URLsToProcess as $url => $priority) {
67 | $count += 1;
68 |
69 | if ($chunkSize > 0 && $count > $chunkSize) {
70 | return;
71 | }
72 |
73 | $this->processUrl($url, $priority);
74 | }
75 |
76 | if (count($this->URLsToProcess) === 0) {
77 | $trimSlashes = function ($value) {
78 | $value = trim($value, '/');
79 |
80 | // We want to trim the schema from the beginning as they map to the same place
81 | // anyway.
82 | $value = ltrim($value, 'http://');
83 | $value = ltrim($value, 'https://');
84 |
85 | return $value;
86 | };
87 |
88 | // List of all URLs which have a static cache file
89 | $this->publishedURLs = array_map($trimSlashes, Publisher::singleton()->getPublishedURLs());
90 |
91 | // List of all URLs which were published as a part of this job
92 | $this->ProcessedURLs = array_map($trimSlashes, $this->ProcessedURLs);
93 |
94 | // Determine stale URLs - those which were not published as a part of this job
95 | // but still have a static cache file
96 | $this->URLsToCleanUp = array_diff($this->publishedURLs, $this->ProcessedURLs);
97 |
98 | foreach ($this->URLsToCleanUp as $staleURL) {
99 | $purgeMeta = Publisher::singleton()->purgeURL($staleURL);
100 | $purgeMeta = is_array($purgeMeta) ? $purgeMeta : [];
101 |
102 | if (array_key_exists('success', $purgeMeta) && $purgeMeta['success']) {
103 | unset($this->jobData->URLsToCleanUp[$staleURL]);
104 |
105 | continue;
106 | }
107 |
108 | $this->handleFailedUrl($staleURL, $purgeMeta);
109 | }
110 | }
111 |
112 | $this->updateCompletedState();
113 | }
114 |
115 | protected function getAllLivePageURLs(): array
116 | {
117 | $urls = [];
118 | $this->extend('beforeGetAllLivePageURLs', $urls);
119 | $livePages = Versioned::get_by_stage(SiteTree::class, Versioned::LIVE);
120 |
121 | foreach ($livePages as $page) {
122 | if ($page->hasExtension(PublishableSiteTree::class) || $page instanceof StaticallyPublishable) {
123 | $urls = array_merge($urls, $page->urlsToCache());
124 | }
125 | }
126 |
127 | $this->extend('afterGetAllLivePageURLs', $urls);
128 |
129 | return $urls;
130 | }
131 |
132 | protected function processUrl(string $url, int $priority): void
133 | {
134 | $meta = Publisher::singleton()->publishURL($url, true);
135 | $meta = is_array($meta) ? $meta : [];
136 |
137 | if (array_key_exists('success', $meta) && $meta['success']) {
138 | $this->markUrlAsProcessed($url);
139 |
140 | return;
141 | }
142 |
143 | $this->handleFailedUrl($url, $meta);
144 | }
145 |
146 | protected function updateCompletedState(): void
147 | {
148 | if (count($this->URLsToProcess) > 0) {
149 | return;
150 | }
151 |
152 | if (count($this->URLsToCleanUp) > 0) {
153 | return;
154 | }
155 |
156 | $this->isComplete = true;
157 | }
158 | }
159 |
--------------------------------------------------------------------------------
/src/Job.php:
--------------------------------------------------------------------------------
1 | URLsToProcess = $urls;
67 |
68 | if (!$message) {
69 | return;
70 | }
71 |
72 | $this->messages = [
73 | sprintf('%s: %s', $message, var_export(array_keys($urls), true)),
74 | ];
75 | }
76 |
77 | /**
78 | * Static cache manipulation jobs need to run without a user
79 | * this is because we don't want any session related data to become part of URLs
80 | * For example stage GET param is injected into URLs when user is logged in
81 | * This is problematic as stage param must not be present in statically published URLs
82 | * as they always refer to live content
83 | * Including stage param in visiting URL is meant to bypass static cache and redirect to admin login
84 | * this is something we definitely don't want for statically cached pages
85 | */
86 | public function getRunAsMemberID(): ?int
87 | {
88 | return 0;
89 | }
90 |
91 | public function setup(): void
92 | {
93 | parent::setup();
94 | $this->totalSteps = count($this->URLsToProcess);
95 | }
96 |
97 | public function getSignature(): string
98 | {
99 | if (!$this->URLsToProcess) {
100 | return md5(static::class);
101 | }
102 | return md5(implode('-', [static::class, implode('-', array_keys($this->URLsToProcess))]));
103 | }
104 |
105 | public function process(): void
106 | {
107 | $chunkSize = $this->getChunkSize();
108 | $count = 0;
109 |
110 | if ($this->URLsToProcess) {
111 | foreach ($this->URLsToProcess as $url => $priority) {
112 | $count += 1;
113 |
114 | if ($chunkSize > 0 && $count > $chunkSize) {
115 | return;
116 | }
117 |
118 | $this->processUrl($url, $priority);
119 | }
120 | }
121 |
122 | $this->updateCompletedState();
123 | }
124 |
125 | public function getUrlsPerJob(): int
126 | {
127 | $urlsPerJob = (int) $this->config()->get('urls_per_job');
128 |
129 | return ($urlsPerJob > 0) ? $urlsPerJob : 0;
130 | }
131 |
132 | /**
133 | * Implement this method to process URL
134 | */
135 | abstract protected function processUrl(string $url, int $priority): void;
136 |
137 | /**
138 | * Move URL to list of processed URLs and update job step to indicate progress
139 | * indication of progress is important for jobs which take long time to process
140 | * jobs that do not indicate progress may be identified as stalled by the queue
141 | * and may end up paused
142 | */
143 | protected function markUrlAsProcessed(string $url): void
144 | {
145 | // These operation has to be done directly on the job data properties
146 | // as the magic methods won't cover array access write
147 | $this->jobData->ProcessedURLs[$url] = $url;
148 | unset($this->jobData->URLsToProcess[$url]);
149 | $this->currentStep += 1;
150 | }
151 |
152 | /**
153 | * Check if job is complete and update the job state if needed
154 | */
155 | protected function updateCompletedState(): void
156 | {
157 | if ($this->URLsToProcess && count($this->URLsToProcess) > 0) {
158 | return;
159 | }
160 |
161 | $this->isComplete = true;
162 | }
163 |
164 | protected function getChunkSize(): int
165 | {
166 | $chunkSize = (int) $this->config()->get('chunk_size');
167 |
168 | return $chunkSize > 0 ? $chunkSize : 0;
169 | }
170 |
171 | /**
172 | * This function can be overridden to handle the case of failure of specific URL processing
173 | * such case is not handled by default which results in all such errors being effectively silenced
174 | */
175 | protected function handleFailedUrl(string $url, array $meta)
176 | {
177 | // no op - override this on your job classes if needed
178 | }
179 | }
180 |
--------------------------------------------------------------------------------
/src/Publisher.php:
--------------------------------------------------------------------------------
1 | AbsoluteLink()}.
51 | */
52 | private static $domain_based_caching = false;
53 |
54 | /**
55 | * @config
56 | *
57 | * @var bool Add a timestamp to the statically published output for HTML files
58 | */
59 | private static $add_timestamp = false;
60 |
61 | /**
62 | * @param string $url
63 | *
64 | * @return HTTPResponse
65 | */
66 | public function generatePageResponse($url)
67 | {
68 | if (Director::is_relative_url($url)) {
69 | $url = Director::absoluteURL($url);
70 | }
71 | $urlParts = parse_url($url);
72 | if (!empty($urlParts['query'])) {
73 | parse_str($urlParts['query'], $getVars);
74 | } else {
75 | $getVars = [];
76 | }
77 | // back up requirements backend
78 | $origRequirements = Requirements::backend();
79 | Requirements::set_backend(Requirements_Backend::create());
80 |
81 | $origThemes = SSViewer::get_themes();
82 | $staticThemes = static::config()->get('static_publisher_themes');
83 | if ($staticThemes) {
84 | SSViewer::set_themes($staticThemes);
85 | } else {
86 | // get the themes raw from config to prevent the "running from the CMS" problem where no themes are live
87 | $rawThemes = SSViewer::config()->uninherited('themes');
88 | SSViewer::set_themes($rawThemes);
89 | }
90 | try {
91 | $ssl = Environment::getEnv('SS_STATIC_FORCE_SSL');
92 | if (!$ssl) {
93 | $ssl = $urlParts['scheme'] == 'https' ? true : false;
94 | }
95 |
96 | // try to add all the server vars that would be needed to create a static cache
97 | $request = HTTPRequestBuilder::createFromVariables(
98 | [
99 | '_SERVER' => [
100 | 'REQUEST_URI' => isset($urlParts['path']) ? $urlParts['path'] : '',
101 | 'REQUEST_METHOD' => 'GET',
102 | 'REMOTE_ADDR' => '127.0.0.1',
103 | 'HTTPS' => $ssl ? 'on' : 'off',
104 | 'QUERY_STRING' => isset($urlParts['query']) ? $urlParts['query'] : '',
105 | 'REQUEST_TIME' => DBDatetime::now()->getTimestamp(),
106 | 'REQUEST_TIME_FLOAT' => (float) DBDatetime::now()->getTimestamp(),
107 | 'HTTP_HOST' => $urlParts['host'] . (isset($urlParts['port']) ? ':' . $urlParts['port'] : ''),
108 | 'HTTP_USER_AGENT' => 'silverstripe/staticpublishqueue',
109 | ],
110 | '_GET' => $getVars,
111 | '_POST' => [],
112 | ],
113 | ''
114 | );
115 | $app = $this->getHTTPApplication();
116 | $response = $app->handle($request);
117 |
118 | if ($this->config()->get('add_timestamp')) {
119 | $response->setBody(
120 | str_replace(
121 | '