├── _config
    └── .gitkeep
├── .gitattributes
├── .gitignore
├── phpunit.xml.dist
├── .editorconfig
├── .github
    └── workflows
    │   ├── keepalive.yml
    │   └── main.yml
├── code
    ├── PopulateTask.php
    ├── extensions
    │   └── PopulateMySQLExport.php
    ├── Populate.php
    └── PopulateFactory.php
├── composer.json
├── LICENSE
├── phpcs.xml.dist
├── .scrutinizer.yml
└── README.md


/_config/.gitkeep:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/.gitattributes:
--------------------------------------------------------------------------------
1 | /tests export-ignore
2 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | /.env
2 | /.idea
3 | /assets
4 | /composer.lock
5 | /resources
6 | /vendor


--------------------------------------------------------------------------------
/phpunit.xml.dist:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <!--
 3 | Standard module phpunit configuration.
 4 | Requires PHPUnit ^5.7
 5 | -->
 6 | <phpunit bootstrap="vendor/silverstripe/framework/tests/bootstrap.php" colors="true">
 7 |     <testsuite name="Default">
 8 |         <directory>tests/php</directory>
 9 |     </testsuite>
10 | </phpunit>


--------------------------------------------------------------------------------
/.editorconfig:
--------------------------------------------------------------------------------
 1 | # For more information about the properties used in this file,
 2 | # please see the EditorConfig documentation:
 3 | # http://editorconfig.org
 4 | 
 5 | [*]
 6 | charset = utf-8
 7 | end_of_line = lf
 8 | indent_size = 4
 9 | indent_style = space
10 | insert_final_newline = true
11 | trim_trailing_whitespace = true
12 | 
13 | [{*.yml,package.json}]
14 | indent_size = 2
15 | 
16 | # The indent size used in the package.json file cannot be changed:
17 | # https://github.com/npm/npm/pull/3180#issuecomment-16336516
18 | 


--------------------------------------------------------------------------------
/.github/workflows/keepalive.yml:
--------------------------------------------------------------------------------
 1 | name: Keepalive
 2 | 
 3 | on:
 4 |   # The 4th of every month at 10:50am UTC
 5 |   schedule:
 6 |     - cron: '50 10 4 * *'
 7 |   workflow_dispatch:
 8 | 
 9 | jobs:
10 |   keepalive:
11 |     name: Keepalive
12 |     # Only run cron on the silverstripe account
13 |     if: (github.event_name == 'schedule' && github.repository_owner == 'silverstripe') || (github.event_name != 'schedule')
14 |     runs-on: ubuntu-latest
15 |     steps:
16 |       - name: Keepalive
17 |         uses: silverstripe/gha-keepalive@v1
18 | 


--------------------------------------------------------------------------------
/.github/workflows/main.yml:
--------------------------------------------------------------------------------
 1 | name: CI
 2 | 
 3 | on:
 4 |   push:
 5 |   pull_request:
 6 |   workflow_dispatch:
 7 |   # Every Tuesday at 2:20pm UTC
 8 |   schedule:
 9 |     - cron: '20 14 * * 2'
10 | 
11 | jobs:
12 |   ci:
13 |     name: CI
14 |     # Only run cron on the silverstripe account
15 |     if: (github.event_name == 'schedule' && github.repository_owner == 'silverstripe') || (github.event_name != 'schedule')
16 |     uses: silverstripe/gha-ci/.github/workflows/ci.yml@v1
17 |     with:
18 |       # Turn phpcoverage off because it causes a segfault
19 |       phpcoverage_force_off: true
20 | 


--------------------------------------------------------------------------------
/code/PopulateTask.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | namespace DNADesign\Populate;
 4 | 
 5 | use Exception;
 6 | use SilverStripe\Dev\BuildTask;
 7 | use SilverStripe\PolyExecution\PolyOutput;
 8 | use Symfony\Component\Console\Command\Command;
 9 | use Symfony\Component\Console\Input\InputInterface;
10 | 
11 | class PopulateTask extends BuildTask
12 | {
13 |     protected static string $commandName = 'PopulateTask';
14 | 
15 |     /**
16 |      * @throws Exception
17 |      */
18 |     protected function execute(InputInterface $input, PolyOutput $output): int
19 |     {
20 |         Populate::requireRecords();
21 | 
22 |         return Command::SUCCESS;
23 |     }
24 | }
25 | 


--------------------------------------------------------------------------------
/composer.json:
--------------------------------------------------------------------------------
 1 | {
 2 |     "name": "dnadesign/silverstripe-populate",
 3 |     "description": "Populate your database through YAML files",
 4 |     "type": "silverstripe-vendormodule",
 5 |     "keywords": ["silverstripe", "yaml", "fixture", "populate", "mock", "faker"],
 6 |     "license": "BSD-3-Clause",
 7 |     "authors": [{
 8 |         "name": "Will Rossiter",
 9 |         "email": "will@fullscreen.io"
10 |     }],
11 |     "require": {
12 |         "php": "^8.3",
13 |         "silverstripe/framework": "^6.0",
14 |         "silverstripe/versioned": "^3.0"
15 |     },
16 |     "require-dev": {
17 |         "silverstripe/recipe-testing": "^4.0",
18 |         "squizlabs/php_codesniffer": "^3.0"
19 |     },
20 |     "autoload": {
21 |         "psr-4": {
22 |             "DNADesign\\Populate\\": "code/",
23 |             "DNADesign\\Populate\\Tests\\": "tests/php/"
24 |         }
25 |     },
26 |     "minimum-stability": "dev",
27 |     "extra": {
28 |         "branch-alias": {
29 |             "dev-master": "4.0.x-dev"
30 |         }
31 |     },
32 |     "config": {
33 |         "allow-plugins": {
34 |             "composer/installers": true,
35 |             "silverstripe/vendor-plugin": true
36 |         }
37 |     }
38 | }
39 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | Copyright (c) 2010-2014, DNA Designed Communications Limited
 2 | All rights reserved.
 3 | 
 4 | Redistribution and use in source and binary forms, with or without modification, 
 5 | are permitted provided that the following conditions are met:
 6 | 
 7 |     * Redistributions of source code must retain the above copyright notice, 
 8 |     this list of conditions and the following disclaimer.
 9 |     * Redistributions in binary form must reproduce the above copyright notice, 
10 |     this list of conditions and the following disclaimer in the 
11 |       documentation and/or other materials provided with the distribution.
12 |     * Neither the name of DNA Designed Communications Limited nor the names of 
13 |     its contributors may be used to endorse or promote products derived from 
14 |     this software  without specific prior written permission.
15 | 
16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 
17 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 
18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 
19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 
20 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 
21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 
22 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 
23 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 
25 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 | 


--------------------------------------------------------------------------------
/phpcs.xml.dist:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <ruleset name="SilverStripe">
 3 | 	<description>CodeSniffer ruleset for SilverStripe coding conventions.</description>
 4 | 
 5 | 	<file>code</file>
 6 | 	<file>tests</file>
 7 | 
 8 | 	<!-- base rules are PSR-2 -->
 9 | 	<rule ref="PSR2" >
10 | 		<!-- Current exclusions -->
11 | 		<exclude name="PSR1.Methods.CamelCapsMethodName" />
12 | 		<exclude name="PSR1.Files.SideEffects.FoundWithSymbols" />
13 | 		<exclude name="PSR2.Classes.PropertyDeclaration" />
14 | 		<exclude name="PSR2.ControlStructures.SwitchDeclaration" /> <!-- causes php notice while linting -->
15 | 		<exclude name="PSR2.ControlStructures.SwitchDeclaration.WrongOpenercase" />
16 | 		<exclude name="PSR2.ControlStructures.SwitchDeclaration.WrongOpenerdefault" />
17 | 		<exclude name="PSR2.ControlStructures.SwitchDeclaration.TerminatingComment" />
18 | 		<exclude name="PSR2.Methods.MethodDeclaration.Underscore" />
19 | 		<exclude name="Squiz.Scope.MethodScope" />
20 | 		<exclude name="Squiz.Classes.ValidClassName.NotCamelCaps" />
21 | 		<exclude name="Generic.Files.LineLength.TooLong" />
22 | 		<exclude name="PEAR.Functions.ValidDefaultValue.NotAtEnd" />
23 | 	</rule>
24 | 
25 | 	<rule ref="Squiz.Strings.ConcatenationSpacing">
26 | 		<properties>
27 | 			<property name="spacing" value="1" />
28 | 			<property name="ignoreNewlines" value="true"/>
29 | 		</properties>
30 | 	</rule>
31 | 
32 | 	<!-- use short array syntax (less thirdparty) -->
33 | 	<rule ref="Generic.Arrays.DisallowLongArraySyntax">
34 | 		<exclude-pattern>/thirdparty/*</exclude-pattern>
35 | 	</rule>
36 | 
37 | 	<!-- include php files only -->
38 | 	<arg name="extensions" value="php,lib,inc,php5"/>
39 | </ruleset>
40 | 


--------------------------------------------------------------------------------
/.scrutinizer.yml:
--------------------------------------------------------------------------------
 1 | inherit: true
 2 | 
 3 | checks:
 4 |   php:
 5 |     verify_property_names: true
 6 |     verify_argument_usable_as_reference: true
 7 |     verify_access_scope_valid: true
 8 |     useless_calls: true
 9 |     use_statement_alias_conflict: true
10 |     variable_existence: true
11 |     unused_variables: true
12 |     unused_properties: true
13 |     unused_parameters: true
14 |     unused_methods: true
15 |     unreachable_code: true
16 |     too_many_arguments: true
17 |     sql_injection_vulnerabilities: true
18 |     simplify_boolean_return: true
19 |     side_effects_or_types: true
20 |     security_vulnerabilities: true
21 |     return_doc_comments: true
22 |     return_doc_comment_if_not_inferrable: true
23 |     require_scope_for_properties: true
24 |     require_scope_for_methods: true
25 |     require_php_tag_first: true
26 |     psr2_switch_declaration: true
27 |     psr2_class_declaration: true
28 |     property_assignments: true
29 |     prefer_while_loop_over_for_loop: true
30 |     precedence_mistakes: true
31 |     precedence_in_conditions: true
32 |     phpunit_assertions: true
33 |     php5_style_constructor: true
34 |     parse_doc_comments: true
35 |     parameter_non_unique: true
36 |     parameter_doc_comments: true
37 |     param_doc_comment_if_not_inferrable: true
38 |     optional_parameters_at_the_end: true
39 |     one_class_per_file: true
40 |     no_unnecessary_if: true
41 |     no_trailing_whitespace: true
42 |     no_property_on_interface: true
43 |     no_non_implemented_abstract_methods: true
44 |     no_error_suppression: true
45 |     no_duplicate_arguments: true
46 |     no_commented_out_code: true
47 |     newline_at_end_of_file: true
48 |     missing_arguments: true
49 |     method_calls_on_non_object: true
50 |     instanceof_class_exists: true
51 |     foreach_traversable: true
52 |     fix_line_ending: true
53 |     fix_doc_comments: true
54 |     duplication: true
55 |     deprecated_code_usage: true
56 |     deadlock_detection_in_loops: true
57 |     code_rating: true
58 |     closure_use_not_conflicting: true
59 |     catch_class_exists: true
60 |     blank_line_after_namespace_declaration: false
61 |     avoid_multiple_statements_on_same_line: true
62 |     avoid_duplicate_types: true
63 |     avoid_conflicting_incrementers: true
64 |     avoid_closing_tag: true
65 |     assignment_of_null_return: true
66 |     argument_type_checks: true
67 | 
68 | filter:
69 |   paths: [code/*, tests/*]
70 | 


--------------------------------------------------------------------------------
/code/extensions/PopulateMySQLExport.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | 
  3 | namespace DNADesign\Populate;
  4 | 
  5 | use SilverStripe\Control\Controller;
  6 | use SilverStripe\Core\Config\Config;
  7 | use SilverStripe\Core\Config\Configurable;
  8 | use SilverStripe\Core\Extension;
  9 | use SilverStripe\ORM\DB;
 10 | 
 11 | /**
 12 |  * An extension for {@link Populate} which, when applied, exports the result of
 13 |  * the database to a file path on the server through mysqldump.
 14 |  *
 15 |  * This is useful for situations like unit tests where you want to import state
 16 |  * but want the speed of using direct mysql queries.
 17 |  *
 18 |  * Opt into this extension by adding this to your populate.yml configuration.
 19 |  *
 20 |  * <code>
 21 |  *    PopulateMySQLExportExtension:
 22 |  *      export_db_path: ~/path.sql
 23 |  *
 24 |  *    Populate:
 25 |  *      extensions
 26 |  *        - PopulateMySQLExportExtension
 27 |  * </code>
 28 |  */
 29 | class PopulateMySQLExportExtension extends Extension
 30 | {
 31 |     use Configurable;
 32 | 
 33 |     /**
 34 |      * @config
 35 |      */
 36 |     private static $export_db_path;
 37 | 
 38 |     public function getPath()
 39 |     {
 40 |         $path = Config::inst()->get(__CLASS__, 'export_db_path');
 41 | 
 42 |         if (!$path) {
 43 |             $path = Controller::join_links(TEMP_FOLDER . '/populate.sql');
 44 |         } else {
 45 |             $path = (substr($path, 0, 1) !== "/") ? Controller::join_links(BASE_PATH, $path) : $path;
 46 |         }
 47 | 
 48 |         return $path;
 49 |     }
 50 | 
 51 |     /**
 52 |      *
 53 |      */
 54 |     public function onAfterPopulateRecords()
 55 |     {
 56 |         $path = $this->getPath();
 57 | 
 58 |         DB::alteration_message("Saving populate state to $path", "success");
 59 |         $result = DB::query('SHOW TABLES');
 60 |         $tables = $result->column();
 61 |         $return = '';
 62 | 
 63 |         foreach ($tables as $table) {
 64 |             $return .= 'DROP TABLE IF EXISTS `' . $table . '`;';
 65 |             $row2 = DB::query("SHOW CREATE TABLE `$table`");
 66 |             $create = $row2->nextRecord();
 67 |             $create = str_replace("\"", "`", $create ?? '');
 68 |             $return .= "\n\n" . $create['Create Table'] . ";\n\n";
 69 | 
 70 |             $result = DB::query("SELECT * FROM `$table`");
 71 | 
 72 |             while ($row = $result->nextRecord()) {
 73 |                 $return .= 'INSERT INTO ' . $table . ' VALUES(';
 74 | 
 75 |                 foreach ($row as $k => $v) {
 76 |                     $v = addslashes($v);
 77 |                     $v = str_replace("\n", "\\n", $v ?? '');
 78 | 
 79 |                     if ($v) {
 80 |                         $return .= '"' . $v . '"';
 81 |                     } else {
 82 |                         $return .= '""';
 83 |                     }
 84 | 
 85 |                     $return .= ',';
 86 |                 }
 87 | 
 88 |                 $return = rtrim($return, ',');
 89 |                 $return .= ");\n";
 90 |             }
 91 |         }
 92 | 
 93 |         $return .= "\n\n\n";
 94 | 
 95 |         $handle = fopen($path, 'w+');
 96 | 
 97 |         fwrite($handle, $return);
 98 |         fclose($handle);
 99 |     }
100 | }
101 | 


--------------------------------------------------------------------------------
/code/Populate.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | 
  3 | namespace DNADesign\Populate;
  4 | 
  5 | use Exception;
  6 | use SilverStripe\Assets\File;
  7 | use SilverStripe\Control\Director;
  8 | use SilverStripe\Core\ClassInfo;
  9 | use SilverStripe\Core\Config\Configurable;
 10 | use SilverStripe\Core\Extensible;
 11 | use SilverStripe\Core\Injector\Injector;
 12 | use SilverStripe\Dev\YamlFixture;
 13 | use SilverStripe\ORM\Connect\DatabaseException;
 14 | use SilverStripe\ORM\DataList;
 15 | use SilverStripe\ORM\DataObject;
 16 | use SilverStripe\ORM\DB;
 17 | use SilverStripe\Versioned\Versioned;
 18 | 
 19 | class Populate
 20 | {
 21 |     use Configurable;
 22 |     use Extensible;
 23 | 
 24 |     private static array $include_yaml_fixtures = [];
 25 | 
 26 |     /**
 27 |      * An array of classes to clear from the database before importing. While
 28 |      * populating SiteTree it may be worth clearing the 'SiteTree' table.
 29 |      */
 30 |     private static array $truncate_classes = [];
 31 | 
 32 |     private static array $truncate_tables = [];
 33 | 
 34 |     /**
 35 |      * Flag to determine if we're already run for this session (i.e to prevent
 36 |      * parent calls invoking {@link requireRecords} twice).
 37 |      */
 38 |     private static bool $ran = false;
 39 | 
 40 |     /**
 41 |      * Used internally to not truncate multiple tables multiple times
 42 |      */
 43 |     private static array $clearedTables = [];
 44 | 
 45 |     /**
 46 |      * @param bool $force - allows you to bypass the ran check to run this multiple times
 47 |      * @throws Exception
 48 |      */
 49 |     public static function requireRecords(bool $force = false): bool
 50 |     {
 51 |         if (self::$ran && !$force) {
 52 |             return true;
 53 |         }
 54 | 
 55 |         self::$ran = true;
 56 | 
 57 |         if (!self::canBuildOnEnvironment()) {
 58 |             throw new Exception('requireRecords can only be run in development or test environments');
 59 |         }
 60 | 
 61 |         /** @var PopulateFactory $factory */
 62 |         $factory = Injector::inst()->create(PopulateFactory::class);
 63 | 
 64 |         foreach (self::config()->get('truncate_objects') as $className) {
 65 |             self::truncateObject($className);
 66 |         }
 67 | 
 68 |         foreach (self::config()->get('truncate_tables') as $table) {
 69 |             self::truncateTable($table);
 70 |         }
 71 | 
 72 |         foreach (self::config()->get('include_yaml_fixtures') as $fixtureFile) {
 73 |             DB::alteration_message(sprintf('Processing %s', $fixtureFile), 'created');
 74 |             $fixture = new YamlFixture($fixtureFile);
 75 |             $fixture->writeInto($factory);
 76 | 
 77 |             $fixture = null;
 78 |         }
 79 | 
 80 |         $factory->processFailedFixtures();
 81 | 
 82 |         $populate = Injector::inst()->create(Populate::class);
 83 |         $populate->extend('onAfterPopulateRecords');
 84 | 
 85 |         return true;
 86 |     }
 87 | 
 88 |     /**
 89 |      * Delete all the associated tables for a class
 90 |      */
 91 |     private static function truncateObject(string $className): void
 92 |     {
 93 |         if (in_array($className, ClassInfo::subclassesFor(File::class))) {
 94 |             foreach (DataList::create($className) as $obj) {
 95 |                 /** @var File $obj */
 96 |                 $obj->deleteFile();
 97 |             }
 98 |         }
 99 | 
100 |         $tables = [];
101 | 
102 |         // All ancestors or children with tables
103 |         $withTables = array_filter(
104 |             array_merge(
105 |                 ClassInfo::ancestry($className),
106 |                 ClassInfo::subclassesFor($className)
107 |             ),
108 |             function ($next) {
109 |                 return DataObject::getSchema()->classHasTable($next);
110 |             }
111 |         );
112 | 
113 |         $classTables = [];
114 | 
115 |         foreach ($withTables as $className) {
116 |             $classTables[$className] = DataObject::getSchema()->tableName($className);
117 |         }
118 | 
119 |         // Establish tables which store object data that needs to be truncated
120 |         foreach ($classTables as $className => $baseTable) {
121 |             /** @var DataObject|Versioned $obj */
122 |             $obj = Injector::inst()->get($className);
123 | 
124 |             // Include base tables
125 |             $tables[$baseTable] = $baseTable;
126 | 
127 |             if (!$obj->hasExtension(Versioned::class)) {
128 |                 // No versioned tables to clear
129 |                 continue;
130 |             }
131 | 
132 |             $stages = $obj->getVersionedStages();
133 | 
134 |             foreach ($stages as $stage) {
135 |                 $table = $obj->stageTable($baseTable, $stage);
136 | 
137 |                 // Include staged table(s)
138 |                 $tables[$table] = $table;
139 |             }
140 | 
141 |             $versionedTable = "{$baseTable}_Versions";
142 | 
143 |             // Include versions table
144 |             $tables[$versionedTable] = $versionedTable;
145 |         }
146 | 
147 |         $populate = Injector::inst()->create(Populate::class);
148 |         $populate->extend('updateTruncateObjectTables', $tables, $className, $classTables);
149 | 
150 |         foreach ($tables as $table) {
151 |             if (!DB::get_schema()->hasTable($table)) {
152 |                 // No table to clear
153 |                 continue;
154 |             }
155 | 
156 |             self::truncateTable($table);
157 |         }
158 |     }
159 | 
160 |     /**
161 |      * Attempts to truncate a table. Outputs messages to indicate if table has
162 |      * already been truncated or cannot be truncated
163 |      */
164 |     private static function truncateTable(string $table): void
165 |     {
166 |         if (array_key_exists($table, self::$clearedTables)) {
167 |             DB::alteration_message("$table already truncated", "deleted");
168 | 
169 |             return;
170 |         }
171 | 
172 |         DB::alteration_message("Truncating table $table", "deleted");
173 | 
174 |         try {
175 |             DB::get_conn()->clearTable($table);
176 |         } catch (DatabaseException $databaseException) {
177 |             DB::alteration_message("Couldn't truncate table $table as it doesn't exist", "deleted");
178 |         }
179 | 
180 |         self::$clearedTables[$table] = true;
181 |     }
182 | 
183 |     private static function canBuildOnEnvironment(): bool
184 |     {
185 |         // Populate (by default) is allowed to run on dev and test environments
186 |         if (Director::isDev() || Director::isTest()) {
187 |             return true;
188 |         }
189 | 
190 |         // Check if developer/s have specified that Populate can run on live
191 |         return (bool) self::config()->get('allow_build_on_live');
192 |     }
193 | }
194 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # Populate Module
  2 | 
  3 | ![Build Status](https://github.com/silverstripe/silverstripe-populate/actions/workflows/main.yml/badge.svg)
  4 | 
  5 | This module provides a way to populate a database from YAML fixtures and custom
  6 | classes. For instance, when a building a web application the pages and default
  7 | objects can be defined in YAML and shared around developers. This extends the
  8 | `requireDefaultRecords` concept in SilverStripe's DataModel.
  9 | 
 10 | * [Requirements](#requirements)
 11 | * [Installation Instructions](#installation-instructions)
 12 | * [Setup](#setup)
 13 | * [Configuration options](#configuration-options)
 14 | * [YAML Format](#yaml-format)
 15 |     * [Updating Records](#updating-records)
 16 |     * [`PopulateMergeWhen`](#populatemergewhen)
 17 |     * [`PopulateMergeMatch`](#populatemergematch)
 18 |     * [`PopulateMergeAny`](#populatemergeany)
 19 |     * [Default Assets](#default-assets)
 20 | * [Extensions](#extensions)
 21 |     * [PopulateMySQLExport](#populatemysqlexport)
 22 | * [Publish configuration](#publish-configuration)
 23 | * [Allow Populate to run on "live" environments](#allow-populate-to-run-on-live-environments)
 24 | 
 25 | ## Requirements
 26 | 
 27 | * PHP 8.3
 28 | * SilverStripe [Framework ^6](https://github.com/silverstripe/silverstripe-framework)
 29 | * SilverStripe [Versioned ^3](https://github.com/silverstripe/silverstripe-versioned)
 30 | 
 31 | ## Installation Instructions
 32 | 
 33 | This module must only ever be used in your development environment, and should never be used on production. While there is code to prevent it from being run in production, it is not fool-proof and therefore you must **never run this module in production**. Install it as a dev dependency in composer like so:
 34 | ```
 35 | composer require --dev dnadesign/silverstripe-populate
 36 | ```
 37 | 
 38 | ## Setup
 39 | 
 40 | First create a new `yml` config file in your config directory `app/_config/populate.yml` (or add it to an existing `config.yml` file if you prefer).
 41 | 
 42 | ```yaml
 43 | DNADesign\Populate\Populate:
 44 |   include_yaml_fixtures:
 45 |     - 'app/fixtures/populate.yml'
 46 | ```
 47 | 
 48 | *If you're sharing test setup with populate, you can specify any number of paths to load fixtures from.*
 49 | 
 50 | An example `app/fixtures/populate.yml` might look like the following:
 51 | 
 52 | ```yaml
 53 | Page:
 54 |   home:
 55 |     Title: "Home"
 56 |     Content: "My Home Page"
 57 |     ParentID: 0
 58 | SilverStripe\Security\Member:
 59 |   admin:
 60 |     ID: 1
 61 |     Email: "admin@example.com"
 62 |     PopulateMergeMatch:
 63 |       - 'ID'
 64 |       - 'Email'
 65 | ```
 66 | 
 67 | Out of the box, the records will be created on when you run the `PopulateTask`
 68 | through `/dev/tasks/PopulateTask/`. To make it completely transparent to
 69 | developers during the application build, you can also include this to hook in on
 70 | `requireDefaultRecords` as part of `dev/build` by including the following in
 71 | one of your application models `requireDefaultRecords` methods:
 72 | 
 73 | ```php
 74 | use DNADesign\Populate\Populate;
 75 | 
 76 | class Page extends SiteTree
 77 | {
 78 |     public function requireDefaultRecords()
 79 |     {
 80 |         parent::requireDefaultRecords();
 81 |         Populate::requireRecords();
 82 |     }
 83 | }
 84 | ```
 85 | 
 86 | ## Configuration options
 87 | 
 88 | *include_yaml_fixtures*
 89 | 
 90 | An array of YAML files to parse.
 91 | 
 92 | **mysite/_config/app.yml**
 93 | 
 94 | ```yaml
 95 | DNADesign\Populate\Populate:
 96 |   include_yaml_fixtures:
 97 |     - 'app/fixtures/populate.yml'
 98 | ```
 99 | 
100 | *truncate_objects*
101 | 
102 | An array of ClassName's whose instances are to be removed from the database prior to importing. Useful to prevent multiple copies of populated content from being imported. It's recommended to truncate any objects you create, to ensure you can re-run `PopulateTask` as often as you want during development and get a consistent database state. This supports Versioned objects (like `SiteTree`) and [Fluent](https://addons.silverstripe.org/add-ons/tractorcow/silverstripe-fluent) (if the module is installed).
103 | 
104 | ```yaml
105 | DNADesign\Populate\Populate:
106 |   truncate_objects:
107 |     - Page
108 |     - SilverStripe\Assets\Image
109 | ```
110 | 
111 | *truncate_tables*
112 | 
113 | An array of tables to be truncated. Useful when there's no relation between your populated classes and the table you want truncated
114 | 
115 | ```yaml
116 | DNADesign\Populate\Populate:
117 |   truncate_tables:
118 |     - Image_Special_Table
119 | ```
120 | 
121 | See *Updating Records* if you wish to merge new and old records rather than
122 | clearing all of them.
123 | 
124 | ## YAML Format
125 | 
126 | Populate uses the same `FixtureFactory` setup as SilverStripe's unit testing
127 | framework. The basic structure of which is:
128 | 
129 | ```yaml
130 | ClassName:
131 |   somereference:
132 |     FieldName: "Value"
133 | ```
134 | 
135 | Relations are handled by referring to them by their reference value:
136 | ```yaml
137 | SilverStripe\Security\Member:
138 |     admin:
139 |       Email: "admin@example.com"
140 | 
141 | Page:
142 |   homepage:
143 |     AuthorID: =>SilverStripe\Security\Member.admin
144 | ```
145 | 
146 | See [SilverStripe's fixture documentation](https://docs.silverstripe.org/en/4/developer_guides/testing/fixtures/) for more advanced examples, including `$many_many` and `$many_many_extraFields`.
147 | 
148 | Any object which implements the `Versioned` extension will be automatically
149 | published.
150 | 
151 | Basic PHP operations can also be included in the YAML file. Any line that is
152 | wrapped in a ` character and ends with a semi colon will be evaled in the
153 | current scope of the importer.
154 | 
155 | ```yaml
156 | Page:
157 |   mythankyoupage:
158 |     ThankYouText: "`Page::config()->thank_you_text`;"
159 |     LinkedPage: "`sprintf(\"[Page](%s)\", App\\Page\\HelpPage::get()->first()->Link())`;"
160 | ```
161 | 
162 | ### Updating Records
163 | 
164 | If you do not truncate the entire table, the module will attempt to first look
165 | up an existing record and update that existing record. For this to happen the
166 | YAML must declare the fields to match in the look up. You can use several
167 | options for this.
168 | 
169 | #### `PopulateMergeWhen`
170 | 
171 | Contains a WHERE clause to match e.g `"URLSegment = 'home' AND ParentID = 0"`.
172 | 
173 | ```yaml
174 | Mysite\PageTypes\HomePage:
175 |   home:
176 |     Title: "My awesome homepage"
177 |     PopulateMergeWhen: "URLSegment = 'home' AND ParentID = 0"
178 | ```
179 | 
180 | ### `PopulateMergeMatch`
181 | 
182 | Takes a list of fields defined in the YAML and matches them based on the
183 | database to avoid repeating content
184 | 
185 | ```yaml
186 | Mysite\PageTypes\HomePage:
187 |   home:
188 |     Title: "My awesome homepage"
189 |     URLSegment: 'home'
190 |     ParentID: 0
191 |     PopulateMergeMatch:
192 |       - URLSegment
193 |       - ParentID
194 | ```
195 | 
196 | ### `PopulateMergeAny`
197 | 
198 | Takes the first record in the database and merges with that. This option is
199 | suitable for things like `SiteConfig` where you normally only contain a single
200 | record.
201 | 
202 | ```yaml
203 | SilverStripe\SiteConfig\SiteConfig:
204 |   mysiteconfig:
205 |     Tagline: "SilverStripe is awesome"
206 |     PopulateMergeAny: true
207 | ```
208 | 
209 | If the criteria meets more than 1 instance, all instances bar the first are
210 | removed from the database so ensure you criteria is specific enough to get the
211 | unique field value.
212 | 
213 | ### Default Assets
214 | 
215 | The script also handles creating default File and image records through the
216 | `PopulateFileFrom` flag. This copies the file from another path (say mysite) and
217 | puts the file inside your assets folder.
218 | 
219 | ```yaml
220 | SilverStripe\Assets\Image:
221 |   lgoptimusl3ii:
222 |     Filename: assets/shop/lgoptimusl3ii.png
223 |     PopulateFileFrom: app/images/demo/large.png
224 | 
225 | Mysite\PageTypes\Product:
226 |   lgoptimus:
227 |     ProductImage: =>SilverStripe\Assets\Image.lgoptimusl3ii
228 | ```
229 | 
230 | ## Extensions
231 | 
232 | The module also provides extensions that can be opted into depending on your
233 | project needs
234 | 
235 | ### PopulateMySQLExport
236 | 
237 | This extension outputs the result of the Populate::requireDefaultRecords() as a
238 | SQL Dump on your local machine. This speeds up the process if using Populate as
239 | part of a test suite or some other CI service as instead of manually calling
240 | the task (which will use the ORM) your test case can be fed raw MySQL to import
241 | and hopefully speed up execution times.
242 | 
243 | To apply the extension add it to Populate, configure the path, flush, then run
244 | `dev/tasks/PopulateTask`
245 | 
246 | ```yaml
247 | DNADesign\Populate\PopulateMySQLExportExtension:
248 |   export_db_path: ~/path.sql
249 | 
250 | DNADesign\Populate\Populate:
251 |   extensions
252 |     - DNADesign\Populate\PopulateMySQLExportExtension
253 | ```
254 | 
255 | ## Publish configuration
256 | 
257 | By default the module uses `publishSingle()` to publish records. If, for whatever reason, you would prefer to that the
258 | module uses `publishRecursive()`, you can enable this by settings the following configuration:
259 | 
260 | ```yaml
261 | DNADesign\Populate\Populate:
262 |   enable_publish_recursive: true
263 | ```
264 | 
265 | ## Allow Populate to run on "live" environments
266 | 
267 | **DANGER ZONE:** Please understand that you are about to provide admins with the ability to run Populate on your
268 | production environment. Before setting this configuration you should understand and accept the risks related to the
269 | loss of production data.
270 | 
271 | ```yaml
272 | DNADesign\Populate\Populate:
273 |   allow_build_on_live: true
274 | ```
275 | 
276 | ## Credits
277 | 
278 | silverstripe-populate was originally created by [wilr](https://github.com/wilr) and [DNA Design](https://www.dna.co.nz/).
279 | 


--------------------------------------------------------------------------------
/code/PopulateFactory.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | 
  3 | namespace DNADesign\Populate;
  4 | 
  5 | use Exception;
  6 | use InvalidArgumentException;
  7 | use SilverStripe\Assets\File;
  8 | use SilverStripe\Assets\Folder;
  9 | use SilverStripe\Assets\Storage\AssetStore;
 10 | use SilverStripe\Core\Injector\Injector;
 11 | use SilverStripe\Dev\FixtureBlueprint;
 12 | use SilverStripe\Dev\FixtureFactory;
 13 | use SilverStripe\ORM\DataList;
 14 | use SilverStripe\ORM\DataObject;
 15 | use SilverStripe\ORM\DB;
 16 | use SilverStripe\ORM\ValidationException;
 17 | use SilverStripe\Versioned\Versioned;
 18 | use function basename;
 19 | use function dirname;
 20 | use function file_get_contents;
 21 | use function hash_equals;
 22 | use function sha1;
 23 | use function sizeof;
 24 | use function str_replace;
 25 | 
 26 | class PopulateFactory extends FixtureFactory
 27 | {
 28 |     /**
 29 |      * List of fixtures that failed to be created due to YAML fixture lookup failures (e.g. because of a dependency that
 30 |      * isn't met at the time of creation). We re-try creation of these after all other fixtures have been created.
 31 |      */
 32 |     private array $failedFixtures = [];
 33 | 
 34 |     /**
 35 |      * Creates the object in the database as the original object will be wiped.
 36 |      *
 37 |      * @param string $name Name of the {@link FixtureBlueprint} to use, usually a DataObject subclass.
 38 |      * @param string $identifier Unique identifier for this fixture type
 39 |      * @param array $data Map of properties. Overrides default data
 40 |      * @return bool|DataObject|null
 41 |      * @throws ValidationException
 42 |      */
 43 |     public function createObject($name, $identifier, $data = null)
 44 |     {
 45 |         DB::alteration_message("Creating $identifier ($name)", "created");
 46 | 
 47 |         if ($data) {
 48 |             foreach ($data as $k => $v) {
 49 |                 if (!(is_array($v)) && preg_match('/^`(.)*`;$/', $v ?? '')) {
 50 |                     $str = substr($v, 1, -2);
 51 |                     $pv = null;
 52 | 
 53 |                     eval("\$pv = $str;");
 54 | 
 55 |                     $data[$k] = $pv;
 56 |                 }
 57 |             }
 58 |         }
 59 | 
 60 |         // for files copy the source dir if the image has a 'PopulateFileFrom'
 61 |         // Follows silverstripe/asset-admin logic, see AssetAdmin::apiCreateFile()
 62 |         if (isset($data['PopulateFileFrom'])) {
 63 |             $file = $this->populateFile($data);
 64 | 
 65 |             if ($file) {
 66 |                 // Skip the rest of this method (populateFile sets all other values on the object), just return the created file
 67 |                 if (!isset($this->fixtures[$name])) {
 68 |                     $this->fixtures[$name] = [];
 69 |                 }
 70 | 
 71 |                 $this->fixtures[$name][$identifier] = $file->ID;
 72 | 
 73 |                 return $file;
 74 |             }
 75 |         }
 76 | 
 77 |         // if any merge labels are defined then we should create the object
 78 |         // from that
 79 |         $lookup = null;
 80 | 
 81 |         if (isset($data['PopulateMergeWhen'])) {
 82 |             $lookup = DataList::create($name)->where(
 83 |                 $data['PopulateMergeWhen']
 84 |             );
 85 | 
 86 |             unset($data['PopulateMergeWhen']);
 87 |         } elseif (isset($data['PopulateMergeMatch'])) {
 88 |             $filter = [];
 89 | 
 90 |             foreach ($data['PopulateMergeMatch'] as $field) {
 91 |                 $filter[$field] = $data[$field];
 92 |             }
 93 | 
 94 |             if (!$filter) {
 95 |                 throw new Exception('Not a valid PopulateMergeMatch filter');
 96 |             }
 97 | 
 98 |             $lookup = DataList::create($name)->filter($filter);
 99 | 
100 |             unset($data['PopulateMergeMatch']);
101 |         } elseif (isset($data['PopulateMergeAny'])) {
102 |             $lookup = DataList::create($name);
103 | 
104 |             unset($data['PopulateMergeAny']);
105 |         }
106 | 
107 |         if ($lookup && $lookup->count() > 0) {
108 |             $existing = $lookup->first();
109 | 
110 |             foreach ($lookup as $old) {
111 |                 if ($old->ID == $existing->ID) {
112 |                     continue;
113 |                 }
114 | 
115 |                 if ($old->hasExtension(Versioned::class)) {
116 |                     foreach ($old->getVersionedStages() as $stage) {
117 |                         $old->deleteFromStage($stage);
118 |                     }
119 |                 }
120 | 
121 |                 $old->delete();
122 |             }
123 | 
124 |             $blueprint = new FixtureBlueprint($name);
125 |             $obj = $blueprint->createObject($identifier, $data, $this->fixtures);
126 |             $latest = $obj->toMap();
127 | 
128 |             unset($latest['ID']);
129 | 
130 |             $existing->update($latest);
131 |             $existing->write();
132 | 
133 |             $obj->delete();
134 | 
135 |             $this->fixtures[$name][$identifier] = $existing->ID;
136 | 
137 |             $obj = $existing;
138 |             $obj->flushCache();
139 |         } else {
140 |             try {
141 |                 $obj = parent::createObject($name, $identifier, $data);
142 |             } catch (InvalidArgumentException $e) {
143 |                 $this->failedFixtures[] = [
144 |                     'class' => $name,
145 |                     'id' => $identifier,
146 |                     'data' => $data,
147 |                 ];
148 | 
149 |                 DB::alteration_message(sprintf('Exception: %s', $e->getMessage()), 'error');
150 | 
151 |                 DB::alteration_message(
152 |                     sprintf('Failed to create %s (%s), queueing for later', $identifier, $name),
153 |                     'error'
154 |                 );
155 | 
156 |                 return null;
157 |             }
158 |         }
159 | 
160 |         if ($obj->hasExtension(Versioned::class)) {
161 |             if (Populate::config()->get('enable_publish_recursive')) {
162 |                 $obj->publishRecursive();
163 |             } else {
164 |                 $obj->publishSingle();
165 |             }
166 | 
167 |             $obj->flushCache();
168 |         }
169 | 
170 |         return $obj;
171 |     }
172 | 
173 |     /**
174 |      * @param bool $recurse Marker for whether we are recursing - should be false when calling from outside this method
175 |      * @throws Exception
176 |      */
177 |     public function processFailedFixtures(bool $recurse = false): void
178 |     {
179 |         if (!$this->failedFixtures) {
180 |             DB::alteration_message('No failed fixtures to process', 'created');
181 | 
182 |             return;
183 |         }
184 | 
185 |         DB::alteration_message('');
186 |         DB::alteration_message('');
187 |         DB::alteration_message(sprintf('Processing %s failed fixtures', count($this->failedFixtures)), 'created');
188 | 
189 |         $failed = $this->failedFixtures;
190 | 
191 |         // Reset $this->failedFixtures so that continual failures can be re-attempted
192 |         $this->failedFixtures = [];
193 | 
194 |         foreach ($failed as $fixture) {
195 |             // createObject returns null if the object failed to create
196 |             // This also re-populates $this->failedFixtures so we can re-compare
197 |             $obj = $this->createObject($fixture['class'], $fixture['id'], $fixture['data']);
198 | 
199 |             if (is_null($obj)) {
200 |                 DB::alteration_message(
201 |                     sprintf('Further attempt to create %s (%s) still failed', $fixture['id'], $fixture['class']),
202 |                     'error'
203 |                 );
204 |             }
205 |         }
206 | 
207 |         if (sizeof($this->failedFixtures) > 0 && sizeof($failed) > sizeof($this->failedFixtures)) {
208 |             // We made some progress because there are less failed fixtures now than there were before, so run again
209 |             $this->processFailedFixtures(true);
210 |         }
211 | 
212 |         // Our final run gets here - either we made no progress on object creation, or there were some fixtures with
213 |         // broken or circular relations that can't be resolved - list these at the end.
214 |         if (!$recurse && sizeof($this->failedFixtures) > 0) {
215 |             $message = sprintf("Some fixtures (%d) couldn't be created:", sizeof($this->failedFixtures));
216 |             DB::alteration_message("");
217 |             DB::alteration_message("");
218 |             DB::alteration_message($message, "error");
219 | 
220 |             foreach ($this->failedFixtures as $fixture) {
221 |                 DB::alteration_message(sprintf('%s (%s)', $fixture['id'], $fixture['class']));
222 |             }
223 |         }
224 |     }
225 | 
226 |     /**
227 |      * @param array $data
228 |      * @return File|bool The created (or updated) File object, or true if the file already existed
229 |      * @throws Exception If anything is missing and the file can't be processed
230 |      */
231 |     private function populateFile(array $data): File|bool
232 |     {
233 |         if (!isset($data['Filename']) || !isset($data['PopulateFileFrom'])) {
234 |             throw new Exception('When passing "PopulateFileFrom", you must also pass "Filename" with the path that you want to file to be stored at (e.g. assets/test.jpg)');
235 |         }
236 | 
237 |         $fixtureFilePath = BASE_PATH . '/' . $data['PopulateFileFrom'];
238 |         $filenameWithoutAssets = str_replace('assets/', '', $data['Filename'] ?? '');
239 | 
240 |         // Find the existing object (if one exists)
241 |         /** @var File $existingObj */
242 |         $existingObj = File::find($filenameWithoutAssets);
243 | 
244 |         if ($existingObj && $existingObj->exists()) {
245 |             $file = $existingObj;
246 | 
247 |             // If the file hashes match, and the file already exists, we don't need to update anything.
248 |             $hash = $existingObj->File->getHash();
249 | 
250 |             if (hash_equals($hash, sha1(file_get_contents($fixtureFilePath) ?? ''))) {
251 |                 return true;
252 |             }
253 |         } else {
254 |             // Create instance of file data object based on the extension of the fixture file
255 |             $fileClass = File::get_class_for_file_extension(File::get_file_extension($fixtureFilePath));
256 |             $file = Injector::inst()->create($fileClass);
257 |         }
258 | 
259 |         $folder = Folder::find_or_make(dirname($filenameWithoutAssets));
260 |         $filename = basename($filenameWithoutAssets);
261 | 
262 |         // We could just use $data['Filename'], but we need to allow for filsystem abstraction
263 |         $filePath = File::join_paths($folder->getFilename(), $filename);
264 | 
265 |         $fileCfg = [
266 |             // if there's a filename conflict we've got new content so overwrite it.
267 |             'conflict' => AssetStore::CONFLICT_OVERWRITE,
268 |             'visibility' => AssetStore::VISIBILITY_PUBLIC,
269 |         ];
270 | 
271 |         // Set any other attributes that the file may need (e.g. Title)
272 |         foreach ($data as $k => $v) {
273 |             if (in_array($k, ['PopulateFileFrom', 'Filename'])) {
274 |                 continue;
275 |             }
276 | 
277 |             $file->$k = $v;
278 |         }
279 | 
280 |         try {
281 |             $file->setFromString(file_get_contents($fixtureFilePath), $filePath, null, null, $fileCfg);
282 |             // Setting ParentID needs to come after setFromString() as (at least sometimes) setFromString() resets the
283 |             // file Parent back to the "Uploads" folder
284 |             $file->ParentID = $folder->ID;
285 |             $file->write();
286 |             $file->publishRecursive();
287 |         } catch (Exception $e) {
288 |             throw $e;
289 |             DB::alteration_message($e->getMessage(), "error");
290 |         }
291 | 
292 |         return $file;
293 |     }
294 | }
295 | 


--------------------------------------------------------------------------------