├── doc ├── img │ ├── logging.png │ ├── mapping.png │ ├── datasource.png │ ├── execution.png │ ├── datasource_asset.png │ ├── datasource_sql.png │ ├── execution_cron.png │ ├── execution_manual.png │ ├── import_preview.png │ ├── data_target_direct.png │ ├── resolver_settings.png │ ├── faq_execution_status.png │ ├── transformation_pipeline.png │ ├── data_target_classification_store.png │ ├── data_target_classification_store_batch.png │ ├── data_target_classification_store_batch_date.png │ ├── data_target_classification_store_batch_simple.png │ ├── data_target_classification_store_batch_decoding.png │ └── data_target_classification_store_batch_quantityValue.png ├── 03_Upgrade.md ├── 06_Extending │ ├── README.md │ ├── 03_Events.md │ └── 01_Extend_Custom_Strategies.md ├── 03_Configuration │ ├── README.md │ ├── 03_Import_Preview.md │ ├── 06_Mapping_Configuration │ │ ├── README.md │ │ └── 03_Data_Target │ │ │ └── README.md │ └── 07_Execution_Configuration.md ├── 06_Troubleshooting_FAQ.md └── 05_Import_Progress_and_Logging.md ├── src ├── Resources │ ├── config │ │ ├── doctrine_migrations.yml │ │ └── pimcore │ │ │ ├── routing.yml │ │ │ └── config.yml │ └── public │ │ ├── js │ │ └── pimcore │ │ │ ├── configuration │ │ │ ├── components │ │ │ │ ├── resolver │ │ │ │ │ ├── load │ │ │ │ │ │ ├── path.js │ │ │ │ │ │ ├── notLoad.js │ │ │ │ │ │ └── id.js │ │ │ │ │ ├── location │ │ │ │ │ │ ├── noChange.js │ │ │ │ │ │ ├── doNotCreate.js │ │ │ │ │ │ └── findOrCreateFolder.js │ │ │ │ │ └── publish │ │ │ │ │ │ ├── alwaysPublish.js │ │ │ │ │ │ ├── noChangePublishNew.js │ │ │ │ │ │ ├── noChangeUnpublishNew.js │ │ │ │ │ │ └── attributeBased.js │ │ │ │ ├── cleanup │ │ │ │ │ ├── delete.js │ │ │ │ │ └── unpublish.js │ │ │ │ ├── mapping │ │ │ │ │ └── operator │ │ │ │ │ │ ├── asArray.js │ │ │ │ │ │ ├── flattenArray.js │ │ │ │ │ │ ├── reduceArrayKeyValuePairs.js │ │ │ │ │ │ ├── boolean.js │ │ │ │ │ │ ├── asColor.js │ │ │ │ │ │ ├── gallery.js │ │ │ │ │ │ ├── htmlDecode.js │ │ │ │ │ │ ├── asGeobounds.js │ │ │ │ │ │ ├── asGeopoint.js │ │ │ │ │ │ ├── asGeopolygon.js │ │ │ │ │ │ ├── asCountries.js │ │ │ │ │ │ ├── asGeopolyline.js │ │ │ │ │ │ ├── imageAdvanced.js │ │ │ │ │ │ ├── inputQuantityValue.js │ │ │ │ │ │ ├── quantityValueArray.js │ │ │ │ │ │ ├── inputQuantityValueArray.js │ │ │ │ │ │ ├── combine.js │ │ │ │ │ │ ├── date.js │ │ │ │ │ │ ├── numeric.js │ │ │ │ │ │ ├── trim.js │ │ │ │ │ │ ├── loadAsset.js │ │ │ │ │ │ ├── objectField.js │ │ │ │ │ │ ├── conditionalConversion.js │ │ │ │ │ │ ├── stringReplace.js │ │ │ │ │ │ ├── explode.js │ │ │ │ │ │ └── staticText.js │ │ │ │ ├── interpreter │ │ │ │ │ ├── sql.js │ │ │ │ │ ├── json.js │ │ │ │ │ ├── xlsx.js │ │ │ │ │ └── xml.js │ │ │ │ └── loader │ │ │ │ │ └── http.js │ │ │ └── configEvents.js │ │ │ ├── helper │ │ │ └── abstractOptionType.js │ │ │ └── adapter │ │ │ └── dataImporterDataObject.js │ │ └── css │ │ └── icons.css ├── Exception │ ├── InvalidInputException.php │ ├── ElementNotFoundException.php │ ├── QueueNotEmptyException.php │ └── InvalidConfigurationException.php ├── Settings │ └── SettingsAwareInterface.php ├── DataSource │ ├── Interpreter │ │ └── SqlFileInterpreter.php │ └── Loader │ │ ├── DataLoaderInterface.php │ │ ├── AssetLoader.php │ │ ├── DataLoaderFactory.php │ │ ├── UploadLoader.php │ │ ├── PushLoader.php │ │ └── HttpLoader.php ├── Processing │ └── Scheduler │ │ ├── Exception │ │ └── InvalidScheduleException.php │ │ ├── SchedulerInterface.php │ │ ├── CronScheduler.php │ │ ├── JobScheduler.php │ │ └── SchedulerFactory.php ├── Event │ ├── DataObject │ │ ├── PostSaveEvent.php │ │ ├── PreSaveEvent.php │ │ ├── ProcessElementExceptionEvent.php │ │ └── AbstractDataObjectImportEvent.php │ └── PostPreparationEvent.php ├── Cleanup │ ├── CleanupStrategyInterface.php │ ├── DeleteStrategy.php │ ├── UnpublishStrategy.php │ └── CleanupStrategyFactory.php ├── Resolver │ ├── Location │ │ ├── NoChangeStrategy.php │ │ ├── DoNotCreateStrategy.php │ │ ├── LocationStrategyInterface.php │ │ └── StaticPathStrategy.php │ ├── Factory │ │ ├── FactoryInterface.php │ │ └── DataObjectFactory.php │ ├── Publish │ │ ├── AlwaysPublishStrategy.php │ │ ├── NoChangePublishNewStrategy.php │ │ ├── NoChangeUnpublishNewStrategy.php │ │ ├── PublishStrategyInterface.php │ │ └── AttributeBasedStrategy.php │ └── Load │ │ ├── IdStrategy.php │ │ ├── PathStrategy.php │ │ ├── NotLoadStrategy.php │ │ ├── LoadStrategyInterface.php │ │ └── AttributeStrategy.php ├── Mapping │ ├── DataTarget │ │ └── DataTargetInterface.php │ ├── Operator │ │ ├── Factory │ │ │ ├── AsArray.php │ │ │ ├── AsGeopolygon.php │ │ │ ├── AsGeopolyline.php │ │ │ ├── Boolean.php │ │ │ ├── AsColor.php │ │ │ ├── AsGeopoint.php │ │ │ ├── AsCountries.php │ │ │ ├── AsGeobounds.php │ │ │ ├── ImageAdvanced.php │ │ │ └── InputQuantityValue.php │ │ ├── AbstractOperator.php │ │ ├── OperatorInterface.php │ │ ├── Simple │ │ │ ├── Combine.php │ │ │ ├── FlattenArray.php │ │ │ ├── ReduceArrayKeyValuePairs.php │ │ │ ├── HtmlDecode.php │ │ │ └── StringReplace.php │ │ └── GeopolyAbstractOperator.php │ └── MappingConfiguration.php ├── Messenger │ └── DataImporterMessage.php ├── EventListener │ └── DataImporterListener.php ├── Maintenance │ └── RestartQueueWorkersTask.php ├── Command │ ├── ParallelizationAbstractCommand.php │ ├── PrepareImportCommand.php │ └── ParallelProcessQueueCommand.php ├── Controller │ └── ConnectionController.php ├── Migrations │ ├── Version20210305134111.php │ ├── Version20220304130000.php │ ├── Version20211201173215.php │ ├── Version20251003105903.php │ ├── Version20240715160305.php │ └── Version20211110174732.php ├── DependencyInjection │ └── CompilerPass │ │ ├── LoaderConfigurationFactoryPass.php │ │ ├── CleanupStrategyConfigurationFactoryPass.php │ │ ├── InterpreterConfigurationFactoryPass.php │ │ └── MappingConfigurationFactoryPass.php ├── Installer.php └── Preview │ └── PreviewService.php ├── SECURITY.md ├── composer.json └── README.md /doc/img/logging.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pimcore/data-importer/HEAD/doc/img/logging.png -------------------------------------------------------------------------------- /doc/img/mapping.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pimcore/data-importer/HEAD/doc/img/mapping.png -------------------------------------------------------------------------------- /doc/img/datasource.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pimcore/data-importer/HEAD/doc/img/datasource.png -------------------------------------------------------------------------------- /doc/img/execution.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pimcore/data-importer/HEAD/doc/img/execution.png -------------------------------------------------------------------------------- /doc/img/datasource_asset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pimcore/data-importer/HEAD/doc/img/datasource_asset.png -------------------------------------------------------------------------------- /doc/img/datasource_sql.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pimcore/data-importer/HEAD/doc/img/datasource_sql.png -------------------------------------------------------------------------------- /doc/img/execution_cron.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pimcore/data-importer/HEAD/doc/img/execution_cron.png -------------------------------------------------------------------------------- /doc/img/execution_manual.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pimcore/data-importer/HEAD/doc/img/execution_manual.png -------------------------------------------------------------------------------- /doc/img/import_preview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pimcore/data-importer/HEAD/doc/img/import_preview.png -------------------------------------------------------------------------------- /doc/img/data_target_direct.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pimcore/data-importer/HEAD/doc/img/data_target_direct.png -------------------------------------------------------------------------------- /doc/img/resolver_settings.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pimcore/data-importer/HEAD/doc/img/resolver_settings.png -------------------------------------------------------------------------------- /doc/img/faq_execution_status.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pimcore/data-importer/HEAD/doc/img/faq_execution_status.png -------------------------------------------------------------------------------- /doc/img/transformation_pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pimcore/data-importer/HEAD/doc/img/transformation_pipeline.png -------------------------------------------------------------------------------- /doc/img/data_target_classification_store.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pimcore/data-importer/HEAD/doc/img/data_target_classification_store.png -------------------------------------------------------------------------------- /doc/img/data_target_classification_store_batch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pimcore/data-importer/HEAD/doc/img/data_target_classification_store_batch.png -------------------------------------------------------------------------------- /doc/img/data_target_classification_store_batch_date.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pimcore/data-importer/HEAD/doc/img/data_target_classification_store_batch_date.png -------------------------------------------------------------------------------- /doc/img/data_target_classification_store_batch_simple.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pimcore/data-importer/HEAD/doc/img/data_target_classification_store_batch_simple.png -------------------------------------------------------------------------------- /doc/img/data_target_classification_store_batch_decoding.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pimcore/data-importer/HEAD/doc/img/data_target_classification_store_batch_decoding.png -------------------------------------------------------------------------------- /doc/img/data_target_classification_store_batch_quantityValue.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pimcore/data-importer/HEAD/doc/img/data_target_classification_store_batch_quantityValue.png -------------------------------------------------------------------------------- /src/Resources/config/doctrine_migrations.yml: -------------------------------------------------------------------------------- 1 | doctrine_migrations: 2 | migrations_paths: 3 | 'Pimcore\Bundle\DataImporterBundle\Migrations': '@PimcoreDataImporterBundle/Migrations' 4 | -------------------------------------------------------------------------------- /doc/03_Upgrade.md: -------------------------------------------------------------------------------- 1 | # Update Notes 2 | 3 | ## Update to Version 1.11 4 | ### General 5 | - Added support of `doctrine/dbal` `v4`, dropped support of `doctrine/dbal` `v2` 6 | 7 | ## Update to Version 1.10 8 | ### General 9 | - Dropped support of Pimcore 10, bumped minimum requirement of `pimcore/pimcore` to `^11.2`. 10 | -------------------------------------------------------------------------------- /doc/06_Extending/README.md: -------------------------------------------------------------------------------- 1 | # Extending 2 | 3 | The bundle architecture provides several extension possibilities for developers to customize the behaviour. 4 | Beside standard symfony extension and customization possibilities, there are two more specific options for extending: 5 | 6 | - [Extend via Custom Strategies](./01_Extend_Custom_Strategies.md) 7 | - [Events](./03_Events.md) 8 | -------------------------------------------------------------------------------- /src/Exception/InvalidInputException.php: -------------------------------------------------------------------------------- 1 | delete(); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/Resources/public/js/pimcore/configuration/components/resolver/load/path.js: -------------------------------------------------------------------------------- 1 | /** 2 | * This source file is available under the terms of the 3 | * Pimcore Open Core License (POCL) 4 | * Full copyright and license information is available in 5 | * LICENSE.md which is distributed with this source code. 6 | * 7 | * @copyright Copyright (c) Pimcore GmbH (http://www.pimcore.com) 8 | * @license Pimcore Open Core License (POCL) 9 | */ 10 | 11 | pimcore.registerNS('pimcore.plugin.pimcoreDataImporterBundle.configuration.components.resolver.load.path'); 12 | pimcore.plugin.pimcoreDataImporterBundle.configuration.components.resolver.load.path = Class.create(pimcore.plugin.pimcoreDataImporterBundle.configuration.components.resolver.load.id, { 13 | 14 | type: 'path' 15 | 16 | }); -------------------------------------------------------------------------------- /src/Resources/public/js/pimcore/configuration/components/cleanup/delete.js: -------------------------------------------------------------------------------- 1 | /** 2 | * This source file is available under the terms of the 3 | * Pimcore Open Core License (POCL) 4 | * Full copyright and license information is available in 5 | * LICENSE.md which is distributed with this source code. 6 | * 7 | * @copyright Copyright (c) Pimcore GmbH (http://www.pimcore.com) 8 | * @license Pimcore Open Core License (POCL) 9 | */ 10 | 11 | pimcore.registerNS('pimcore.plugin.pimcoreDataImporterBundle.configuration.components.cleanup.delete'); 12 | pimcore.plugin.pimcoreDataImporterBundle.configuration.components.cleanup.delete = Class.create(pimcore.plugin.pimcoreDataImporterBundle.configuration.components.abstractOptionType, { 13 | 14 | type: 'delete', 15 | 16 | buildSettingsForm: function() { 17 | return null; 18 | } 19 | 20 | }); -------------------------------------------------------------------------------- /src/Cleanup/UnpublishStrategy.php: -------------------------------------------------------------------------------- 1 | setPublished(false); 23 | $element->save(); 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/Resources/public/js/pimcore/configuration/components/cleanup/unpublish.js: -------------------------------------------------------------------------------- 1 | /** 2 | * This source file is available under the terms of the 3 | * Pimcore Open Core License (POCL) 4 | * Full copyright and license information is available in 5 | * LICENSE.md which is distributed with this source code. 6 | * 7 | * @copyright Copyright (c) Pimcore GmbH (http://www.pimcore.com) 8 | * @license Pimcore Open Core License (POCL) 9 | */ 10 | 11 | pimcore.registerNS("pimcore.plugin.pimcoreDataImporterBundle.configuration.components.cleanup.unpublish"); 12 | pimcore.plugin.pimcoreDataImporterBundle.configuration.components.cleanup.unpublish = Class.create(pimcore.plugin.pimcoreDataImporterBundle.configuration.components.abstractOptionType, { 13 | 14 | type: 'unpublish', 15 | 16 | buildSettingsForm: function() { 17 | return null; 18 | } 19 | 20 | }); -------------------------------------------------------------------------------- /src/Resources/public/js/pimcore/configuration/components/resolver/load/notLoad.js: -------------------------------------------------------------------------------- 1 | /** 2 | * This source file is available under the terms of the 3 | * Pimcore Open Core License (POCL) 4 | * Full copyright and license information is available in 5 | * LICENSE.md which is distributed with this source code. 6 | * 7 | * @copyright Copyright (c) Pimcore GmbH (http://www.pimcore.com) 8 | * @license Pimcore Open Core License (POCL) 9 | */ 10 | 11 | pimcore.registerNS('pimcore.plugin.pimcoreDataImporterBundle.configuration.components.resolver.load.notLoad'); 12 | pimcore.plugin.pimcoreDataImporterBundle.configuration.components.resolver.load.notLoad = Class.create(pimcore.plugin.pimcoreDataImporterBundle.configuration.components.abstractOptionType, { 13 | 14 | type: 'notLoad', 15 | 16 | buildSettingsForm: function() { 17 | 18 | return null; 19 | } 20 | 21 | }); -------------------------------------------------------------------------------- /src/Resources/public/js/pimcore/configuration/components/resolver/location/noChange.js: -------------------------------------------------------------------------------- 1 | /** 2 | * This source file is available under the terms of the 3 | * Pimcore Open Core License (POCL) 4 | * Full copyright and license information is available in 5 | * LICENSE.md which is distributed with this source code. 6 | * 7 | * @copyright Copyright (c) Pimcore GmbH (http://www.pimcore.com) 8 | * @license Pimcore Open Core License (POCL) 9 | */ 10 | 11 | pimcore.registerNS('pimcore.plugin.pimcoreDataImporterBundle.configuration.components.resolver.location.noChange'); 12 | pimcore.plugin.pimcoreDataImporterBundle.configuration.components.resolver.location.noChange = Class.create(pimcore.plugin.pimcoreDataImporterBundle.configuration.components.abstractOptionType, { 13 | 14 | type: 'noChange', 15 | 16 | buildSettingsForm: function() { 17 | return null; 18 | } 19 | 20 | }); -------------------------------------------------------------------------------- /src/Resources/public/js/pimcore/configuration/components/mapping/operator/asArray.js: -------------------------------------------------------------------------------- 1 | /** 2 | * This source file is available under the terms of the 3 | * Pimcore Open Core License (POCL) 4 | * Full copyright and license information is available in 5 | * LICENSE.md which is distributed with this source code. 6 | * 7 | * @copyright Copyright (c) Pimcore GmbH (http://www.pimcore.com) 8 | * @license Pimcore Open Core License (POCL) 9 | */ 10 | 11 | pimcore.registerNS('pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.operator.asArray'); 12 | pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.operator.asArray = Class.create(pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.abstractOperator, { 13 | 14 | type: 'asArray', 15 | 16 | getMenuGroup: function() { 17 | return this.menuGroups.dataTypes; 18 | } 19 | }); -------------------------------------------------------------------------------- /src/Resolver/Location/NoChangeStrategy.php: -------------------------------------------------------------------------------- 1 | executionType; 24 | } 25 | 26 | public function getIds(): array 27 | { 28 | return $this->ids; 29 | } 30 | 31 | public function getMessageId(): string 32 | { 33 | return $this->messageId; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/Resources/public/js/pimcore/configuration/components/mapping/operator/asGeobounds.js: -------------------------------------------------------------------------------- 1 | /** 2 | * This source file is available under the terms of the 3 | * Pimcore Open Core License (POCL) 4 | * Full copyright and license information is available in 5 | * LICENSE.md which is distributed with this source code. 6 | * 7 | * @copyright Copyright (c) Pimcore GmbH (http://www.pimcore.com) 8 | * @license Pimcore Open Core License (POCL) 9 | */ 10 | 11 | pimcore.registerNS('pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.operator.asGeobounds'); 12 | pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.operator.asGeobounds = Class.create(pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.abstractOperator, { 13 | 14 | type: 'asGeobounds', 15 | getMenuGroup: function() { 16 | return this.menuGroups.dataTypes; 17 | }, 18 | 19 | getIconClass: function() { 20 | return "pimcore_icon_geobounds"; 21 | }, 22 | }); -------------------------------------------------------------------------------- /src/Resources/public/js/pimcore/configuration/components/mapping/operator/asGeopoint.js: -------------------------------------------------------------------------------- 1 | /** 2 | * This source file is available under the terms of the 3 | * Pimcore Open Core License (POCL) 4 | * Full copyright and license information is available in 5 | * LICENSE.md which is distributed with this source code. 6 | * 7 | * @copyright Copyright (c) Pimcore GmbH (http://www.pimcore.com) 8 | * @license Pimcore Open Core License (POCL) 9 | */ 10 | 11 | pimcore.registerNS('pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.operator.asGeopoint'); 12 | pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.operator.asGeopoint = Class.create(pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.abstractOperator, { 13 | 14 | type: 'asGeopoint', 15 | getMenuGroup: function() { 16 | return this.menuGroups.dataTypes; 17 | }, 18 | 19 | getIconClass: function() { 20 | return "pimcore_icon_geopoint"; 21 | }, 22 | 23 | }); -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Reporting a Vulnerability 4 | 5 | If you think that you have found a security issue, 6 | don’t use the bug tracker and don’t publish it publicly. 7 | Instead, all security issues must be reported via a private vulnerability report. 8 | 9 | Please follow the [instructions](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/privately-reporting-a-security-vulnerability#privately-reporting-a-security-vulnerability) to submit a private report. 10 | 11 | 12 | ## Resolving Process 13 | Every submitted security issue is handled with top priority by following these steps: 14 | 15 | 1. Confirm the vulnerability 16 | 2. Determine the severity 17 | 3. Contact reporter 18 | 4. Work on a patch 19 | 5. Get a CVE identification number (may be done by the reporter or a security service provider) 20 | 6. Patch reviewing 21 | 7. Tagging a new release for supported versions 22 | 8. Publish security announcement 23 | -------------------------------------------------------------------------------- /src/Resources/public/js/pimcore/configuration/components/mapping/operator/asGeopolygon.js: -------------------------------------------------------------------------------- 1 | /** 2 | * This source file is available under the terms of the 3 | * Pimcore Open Core License (POCL) 4 | * Full copyright and license information is available in 5 | * LICENSE.md which is distributed with this source code. 6 | * 7 | * @copyright Copyright (c) Pimcore GmbH (http://www.pimcore.com) 8 | * @license Pimcore Open Core License (POCL) 9 | */ 10 | 11 | pimcore.registerNS('pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.operator.asGeopolygon'); 12 | pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.operator.asGeopolygon = Class.create(pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.abstractOperator, { 13 | 14 | type: 'asGeopolygon', 15 | getMenuGroup: function() { 16 | return this.menuGroups.dataTypes; 17 | }, 18 | 19 | getIconClass: function() { 20 | return "pimcore_icon_geopolygon"; 21 | }, 22 | }); -------------------------------------------------------------------------------- /src/Resources/public/js/pimcore/configuration/components/mapping/operator/asCountries.js: -------------------------------------------------------------------------------- 1 | /** 2 | * This source file is available under the terms of the 3 | * Pimcore Open Core License (POCL) 4 | * Full copyright and license information is available in 5 | * LICENSE.md which is distributed with this source code. 6 | * 7 | * @copyright Copyright (c) Pimcore GmbH (http://www.pimcore.com) 8 | * @license Pimcore Open Core License (POCL) 9 | */ 10 | 11 | pimcore.registerNS('pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.operator.asCountries'); 12 | pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.operator.asCountries = Class.create(pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.abstractOperator, { 13 | 14 | type: 'asCountries', 15 | 16 | getMenuGroup: function() { 17 | return this.menuGroups.dataTypes; 18 | }, 19 | 20 | getIconClass: function() { 21 | return "pimcore_icon_countrymultiselect"; 22 | } 23 | }); -------------------------------------------------------------------------------- /src/Resources/public/js/pimcore/configuration/components/mapping/operator/asGeopolyline.js: -------------------------------------------------------------------------------- 1 | /** 2 | * This source file is available under the terms of the 3 | * Pimcore Open Core License (POCL) 4 | * Full copyright and license information is available in 5 | * LICENSE.md which is distributed with this source code. 6 | * 7 | * @copyright Copyright (c) Pimcore GmbH (http://www.pimcore.com) 8 | * @license Pimcore Open Core License (POCL) 9 | */ 10 | 11 | pimcore.registerNS('pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.operator.asGeopolyline'); 12 | pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.operator.asGeopolyline = Class.create(pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.abstractOperator, { 13 | 14 | type: 'asGeopolyline', 15 | getMenuGroup: function() { 16 | return this.menuGroups.dataTypes; 17 | }, 18 | 19 | getIconClass: function() { 20 | return "pimcore_icon_geopolyline"; 21 | }, 22 | }); -------------------------------------------------------------------------------- /src/Resources/public/js/pimcore/configuration/components/mapping/operator/imageAdvanced.js: -------------------------------------------------------------------------------- 1 | /** 2 | * This source file is available under the terms of the 3 | * Pimcore Open Core License (POCL) 4 | * Full copyright and license information is available in 5 | * LICENSE.md which is distributed with this source code. 6 | * 7 | * @copyright Copyright (c) Pimcore GmbH (http://www.pimcore.com) 8 | * @license Pimcore Open Core License (POCL) 9 | */ 10 | 11 | pimcore.registerNS("pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.operator.imageAdvanced"); 12 | pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.operator.imageAdvanced = Class.create(pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.abstractOperator, { 13 | 14 | type: 'imageAdvanced', 15 | getMenuGroup: function() { 16 | return this.menuGroups.dataTypes; 17 | }, 18 | 19 | getIconClass: function() { 20 | return "pimcore_icon_hotspotimage"; 21 | }, 22 | }); -------------------------------------------------------------------------------- /src/DataSource/Loader/DataLoaderInterface.php: -------------------------------------------------------------------------------- 1 | setPublished(true); 28 | } 29 | 30 | return $element; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/Resources/public/js/pimcore/configuration/components/mapping/operator/inputQuantityValueArray.js: -------------------------------------------------------------------------------- 1 | /** 2 | * This source file is available under the terms of the 3 | * Pimcore Open Core License (POCL) 4 | * Full copyright and license information is available in 5 | * LICENSE.md which is distributed with this source code. 6 | * 7 | * @copyright Copyright (c) Pimcore GmbH (http://www.pimcore.com) 8 | * @license Pimcore Open Core License (POCL) 9 | */ 10 | 11 | pimcore.registerNS("pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.operator.inputQuantityValueArray"); 12 | pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.operator.inputQuantityValueArray = Class.create(pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.abstractOperator, { 13 | 14 | type: 'inputQuantityValueArray', 15 | 16 | getMenuGroup: function() { 17 | return this.menuGroups.dataTypes; 18 | }, 19 | 20 | getIconClass: function() { 21 | return "pimcore_icon_inputQuantityValue"; 22 | }, 23 | }); -------------------------------------------------------------------------------- /src/Resolver/Publish/NoChangePublishNewStrategy.php: -------------------------------------------------------------------------------- 1 | setPublished(true); 28 | } 29 | 30 | return $element; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/Resolver/Publish/NoChangeUnpublishNewStrategy.php: -------------------------------------------------------------------------------- 1 | setPublished(false); 28 | } 29 | 30 | return $element; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/Resources/public/js/pimcore/helper/abstractOptionType.js: -------------------------------------------------------------------------------- 1 | /** 2 | * This source file is available under the terms of the 3 | * Pimcore Open Core License (POCL) 4 | * Full copyright and license information is available in 5 | * LICENSE.md which is distributed with this source code. 6 | * 7 | * @copyright Copyright (c) Pimcore GmbH (http://www.pimcore.com) 8 | * @license Pimcore Open Core License (POCL) 9 | */ 10 | 11 | pimcore.registerNS("pimcore.plugin.pimcoreDataImporterBundle.configuration.components.abstractOptionType"); 12 | pimcore.plugin.pimcoreDataImporterBundle.configuration.components.abstractOptionType = Class.create({ 13 | 14 | type: 'abstract', 15 | data: {}, 16 | dataNamePrefix: '', 17 | configItemRootContainer: null, 18 | initContext: null, 19 | 20 | initialize: function (data, dataNamePrefix, configItemRootContainer, initContext) { 21 | 22 | this.data = data; 23 | this.dataNamePrefix = dataNamePrefix + '.'; 24 | this.configItemRootContainer = configItemRootContainer; 25 | this.initContext = initContext; 26 | } 27 | 28 | }); -------------------------------------------------------------------------------- /src/EventListener/DataImporterListener.php: -------------------------------------------------------------------------------- 1 | messengerQueueActivated) { 29 | return; 30 | } 31 | 32 | $this->dataImporterHandler->dispatchMessages($event->getExecutionType()); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/Resolver/Publish/PublishStrategyInterface.php: -------------------------------------------------------------------------------- 1 | 10 | 11 | ![Import Progress](./img/execution.png) 12 | 13 | ## Logging 14 | 15 | The importer logs overview information into Pimcore application logger and detailed debugging information into standard 16 | Pimcore/symfony loggers. 17 | 18 |
19 | 20 | ![Import Progress](./img/logging.png) 21 | 22 | The Import Logs tab of the importer configuration shows a prefiltered list of the application logger containing only 23 | entries belonging to the current import configuration. 24 | 25 | The application logger contains ... 26 | - information about started imports. 27 | - information about each imported element including the original imported data as linked file object. 28 | - archived original import files if activated. 29 | - information about any problems and errors happened. 30 | -------------------------------------------------------------------------------- /doc/03_Configuration/03_Import_Preview.md: -------------------------------------------------------------------------------- 1 | # Import Preview 2 | 3 | The import configuration allows to show a preview of the data which helps 4 | to setup the [mapping configuration](06_Mapping_Configuration/README.md) and 5 | validate the expected results. 6 | 7 |
8 | 9 | ![Import Preview](../img/import_preview.png) 10 | 11 | The preview interprets the file the same way as actual import data sources 12 | will be interpreted when executing the import and shows one record of the preview 13 | file at once. The preview does not import any data though. 14 | 15 | Any changes in file format settings result in a reloading of the uploaded preview 16 | file. Reloading the preview file or paging the records will also update the 17 | processing result preview of the [mapping configuration](06_Mapping_Configuration/README.md). 18 | 19 | There are two options for loading the preview file: 20 | - Upload an extra preview file. 21 | - Copy the preview file from the configured data source (not possible with push data source). 22 | 23 | For both options, preview files are saved per user and import configuration in Pimcores `tmp` directory and 24 | should only be a small extract of the actual import files - due to loading performance and used storage space. 25 | 26 | -------------------------------------------------------------------------------- /src/Event/PostPreparationEvent.php: -------------------------------------------------------------------------------- 1 | configName = $configName; 26 | $this->executionType = $executionType; 27 | $this->fileInterpreted = $fileInterpreted; 28 | } 29 | 30 | public function getConfigName(): string 31 | { 32 | return $this->configName; 33 | } 34 | 35 | public function getExecutionType(): string 36 | { 37 | return $this->executionType; 38 | } 39 | 40 | public function isFileInterpreted(): bool 41 | { 42 | return $this->fileInterpreted; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/Processing/Scheduler/CronScheduler.php: -------------------------------------------------------------------------------- 1 | cronDefinition = $cronDefinition; 29 | $this->modifiedAt = $modifiedAt; 30 | } 31 | 32 | public function isExecutable(?DateTime $executedAt): bool 33 | { 34 | $cron = new CronExpression($this->cronDefinition); 35 | $startAt = $executedAt ?: $this->modifiedAt; 36 | 37 | $nextRun = $cron->getNextRunDate($startAt); 38 | $now = new DateTime(); 39 | 40 | return $nextRun < $now; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/Maintenance/RestartQueueWorkersTask.php: -------------------------------------------------------------------------------- 1 | messengerQueueActivated === true) { 30 | $this->dataImporterHandler->dispatchMessages(ImportProcessingService::EXECUTION_TYPE_SEQUENTIAL); 31 | $this->dataImporterHandler->dispatchMessages(ImportProcessingService::EXECUTION_TYPE_PARALLEL); 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /doc/03_Configuration/06_Mapping_Configuration/README.md: -------------------------------------------------------------------------------- 1 | # Mapping Configuration 2 | 3 | The mapping configuration defines what data fields from the import data should be imported where and how to 4 | the Pimcore data object. 5 | 6 |
7 | 8 | ![Import Preview](../../img/mapping.png) 9 | 10 | The mapping configuration consists of multiple mapping entries. Each mapping entry provides settings for: 11 | - **Label**: Defines a name for the mapping entry. The name is used as title of the mapping entry only and has no further 12 | functionality. 13 | - **Source Attributes**: Define one or multiple fields from the import data as source for the mapping entry. Multiple fields 14 | are passed as an array to transformation pipeline and data target. The field list is based on the preview file uploaded, 15 | but also other fields can be defined by just writing the data index or field name. 16 | - **Transformation Pipeline**: The transformation pipeline transforms the data read from the source attribute(s) with operators. 17 | For details see [Transformation Pipeline](./02_Transformation_Pipeline.md). 18 | - **Data Target**: Data target definition assigns the result of the transformation pipeline to a data object field. 19 | For details see [Data Target](./03_Data_Target/README.md). 20 | -------------------------------------------------------------------------------- /src/Resolver/Load/IdStrategy.php: -------------------------------------------------------------------------------- 1 | dataObjectLoader->loadById($identifier, 30 | $this->getClassName()); 31 | } 32 | 33 | public function loadFullIdentifierList(): array 34 | { 35 | $sql = sprintf( 36 | 'SELECT `id` FROM object_%s', 37 | $this->dataObjectClassId 38 | ); 39 | 40 | return $this->db->fetchFirstColumn($sql); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/Resolver/Location/StaticPathStrategy.php: -------------------------------------------------------------------------------- 1 | path = $settings['path']; 33 | } 34 | 35 | public function updateParent(ElementInterface $element, array $inputData): ElementInterface 36 | { 37 | $element->setParent(Service::createFolderByPath($this->path)); 38 | 39 | return $element; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/Resolver/Load/PathStrategy.php: -------------------------------------------------------------------------------- 1 | dataObjectLoader->loadByPath($identifier, 30 | $this->getClassName()); 31 | } 32 | 33 | public function loadFullIdentifierList(): array 34 | { 35 | $sql = sprintf( 36 | 'SELECT CONCAT(`path`, `key`) FROM object_%s', 37 | $this->dataObjectClassId 38 | ); 39 | 40 | return $this->db->fetchFirstColumn($sql); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/Command/ParallelizationAbstractCommand.php: -------------------------------------------------------------------------------- 1 | doFetchItems($input, $output); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/Mapping/Operator/Factory/AsArray.php: -------------------------------------------------------------------------------- 1 | scheduledAt = $scheduledAt; 28 | $this->modifiedAt = $modifiedAt; 29 | } 30 | 31 | public function isExecutable(?DateTime $executedAt): bool 32 | { 33 | $now = new DateTime(); 34 | 35 | $hasExecutedInPast = $executedAt && $this->scheduledAt <= $executedAt; 36 | $isTimeToExecute = $now >= $this->scheduledAt; 37 | $isModifiedBeforeSchedule = $this->modifiedAt <= $this->scheduledAt; 38 | 39 | if ($isTimeToExecute && $isModifiedBeforeSchedule && !$hasExecutedInPast) { 40 | return true; 41 | } 42 | 43 | return false; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/Resources/public/js/pimcore/configuration/components/mapping/operator/combine.js: -------------------------------------------------------------------------------- 1 | /** 2 | * This source file is available under the terms of the 3 | * Pimcore Open Core License (POCL) 4 | * Full copyright and license information is available in 5 | * LICENSE.md which is distributed with this source code. 6 | * 7 | * @copyright Copyright (c) Pimcore GmbH (http://www.pimcore.com) 8 | * @license Pimcore Open Core License (POCL) 9 | */ 10 | 11 | pimcore.registerNS("pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.operator.combine"); 12 | pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.operator.combine = Class.create(pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.abstractOperator, { 13 | 14 | type: 'combine', 15 | 16 | getMenuGroup: function() { 17 | return this.menuGroups.dataManipulation; 18 | }, 19 | 20 | getFormItems: function() { 21 | return [ 22 | { 23 | xtype: 'textfield', 24 | fieldLabel: t('plugin_pimcore_datahub_data_importer_configpanel_transformation_pipeline_glue'), 25 | value: this.data.settings ? this.data.settings.glue : ' ', 26 | listeners: { 27 | change: this.inputChangePreviewUpdate.bind(this) 28 | }, 29 | name: 'settings.glue' 30 | } 31 | ]; 32 | } 33 | 34 | }); -------------------------------------------------------------------------------- /src/Resolver/Publish/AttributeBasedStrategy.php: -------------------------------------------------------------------------------- 1 | dataSourceIndex = $dsi; 32 | } 33 | 34 | public function updatePublishState(ElementInterface $element, bool $justCreated, array $inputData): ElementInterface 35 | { 36 | if (method_exists($element, 'setPublished')) { 37 | $element->setPublished($inputData[$this->dataSourceIndex] ?? false); 38 | } 39 | 40 | return $element; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/Mapping/Operator/Factory/AsGeopolygon.php: -------------------------------------------------------------------------------- 1 | getParameter('doctrine.connections'); 32 | 33 | if (!is_array($connections)) { 34 | throw new Exception('Doctrine connection not returned as array'); 35 | } 36 | 37 | $mappedConnections = array_map(fn ($key, $value): array => [ 38 | 'name' => $key, 39 | 'value' => $value 40 | ], array_keys($connections), $connections); 41 | 42 | return $this->json($mappedConnections); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/Resources/public/js/pimcore/configuration/components/mapping/operator/date.js: -------------------------------------------------------------------------------- 1 | /** 2 | * This source file is available under the terms of the 3 | * Pimcore Open Core License (POCL) 4 | * Full copyright and license information is available in 5 | * LICENSE.md which is distributed with this source code. 6 | * 7 | * @copyright Copyright (c) Pimcore GmbH (http://www.pimcore.com) 8 | * @license Pimcore Open Core License (POCL) 9 | */ 10 | 11 | pimcore.registerNS("pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.operator.date"); 12 | pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.operator.date = Class.create(pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.abstractOperator, { 13 | 14 | type: 'date', 15 | 16 | getMenuGroup: function() { 17 | return this.menuGroups.dataTypes; 18 | }, 19 | 20 | getIconClass: function() { 21 | return "pimcore_icon_date"; 22 | }, 23 | 24 | getFormItems: function() { 25 | return [ 26 | { 27 | xtype: 'textfield', 28 | fieldLabel: t('plugin_pimcore_datahub_data_importer_configpanel_transformation_pipeline_format'), 29 | value: this.data.settings ? this.data.settings.format : 'Y-m-d', 30 | listeners: { 31 | change: this.inputChangePreviewUpdate.bind(this) 32 | }, 33 | name: 'settings.format' 34 | } 35 | ]; 36 | } 37 | 38 | }); -------------------------------------------------------------------------------- /src/Cleanup/CleanupStrategyFactory.php: -------------------------------------------------------------------------------- 1 | cleanupStrategies = $cleanupStrategies; 32 | } 33 | 34 | /** 35 | * @param string $type 36 | * 37 | * @return CleanupStrategyInterface 38 | * 39 | * @throws InvalidConfigurationException 40 | */ 41 | public function loadCleanupStrategy(string $type) 42 | { 43 | if (empty($type) || !array_key_exists($type, $this->cleanupStrategies)) { 44 | throw new InvalidConfigurationException('Unknown loader type `' . $type . '`'); 45 | } 46 | 47 | return $this->cleanupStrategies[$type]; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /doc/03_Configuration/07_Execution_Configuration.md: -------------------------------------------------------------------------------- 1 | # Execution Configuration 2 | 3 | Depending on the configuration there are different options to start an actual import. 4 | 5 | When data source is `push`, the import process starts every time data is pushed to the corresponding 6 | endpoint. There is no further configuration possible or needed. 7 | 8 | For all the other data sources, Pimcore needs a trigger to start the import process. Following triggers 9 | are available. 10 | 11 | ### Manual Execution 12 | Clicking on the `Manual Execution Start` button in the configuration editor manually starts the import. 13 | 14 | ![Manual Execution Start](../img/execution_manual.png) 15 | 16 | ### Cron-based Execution 17 | Besides the `Manual Execution Start` button there is also the possibility to configure a cron definition 18 | based on which the import starts on a regular base, e.g. every 10 Minutes, once a day, etc. 19 | 20 | ![Cron Definition](../img/execution_cron.png) 21 | 22 | See for example [Crontab Guru](https://crontab.guru/) for possibilities and a generator for generating 23 | cron definitions. 24 | 25 | > Make sure `datahub:data-importer:execute-cron` command is setup properly during installation. 26 | 27 | ### Command-based Execution 28 | It is also possible to imports with executing the `datahub:data-importer:prepare-import` command on the commandline. 29 | 30 | 31 | 32 | For further details on import execution also see [Import Execution Details](../04_Import_Execution_Details.md). 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /src/Resources/public/js/pimcore/adapter/dataImporterDataObject.js: -------------------------------------------------------------------------------- 1 | /** 2 | * This source file is available under the terms of the 3 | * Pimcore Open Core License (POCL) 4 | * Full copyright and license information is available in 5 | * LICENSE.md which is distributed with this source code. 6 | * 7 | * @copyright Copyright (c) Pimcore GmbH (http://www.pimcore.com) 8 | * @license Pimcore Open Core License (POCL) 9 | */ 10 | 11 | pimcore.registerNS("pimcore.plugin.datahub.adapter.dataImporterDataObject"); 12 | pimcore.plugin.datahub.adapter.dataImporterDataObject = Class.create(pimcore.plugin.datahub.adapter.graphql, { 13 | 14 | createConfigPanel: function(data) { 15 | let fieldPanel = new pimcore.plugin.pimcoreDataImporterBundle.configuration.configItemDataObject(data, this); 16 | }, 17 | 18 | openConfiguration: function (id) { 19 | var existingPanel = Ext.getCmp("plugin_pimcore_datahub_configpanel_panel_" + id); 20 | if (existingPanel) { 21 | this.configPanel.editPanel.setActiveTab(existingPanel); 22 | return; 23 | } 24 | 25 | Ext.Ajax.request({ 26 | url: Routing.generate('pimcore_dataimporter_configdataobject_get'), 27 | params: { 28 | name: id 29 | }, 30 | success: function (response) { 31 | let data = Ext.decode(response.responseText); 32 | this.createConfigPanel(data); 33 | pimcore.layout.refresh(); 34 | }.bind(this) 35 | }); 36 | } 37 | }); 38 | -------------------------------------------------------------------------------- /src/Resources/public/js/pimcore/configuration/components/interpreter/json.js: -------------------------------------------------------------------------------- 1 | /** 2 | * This source file is available under the terms of the 3 | * Pimcore Open Core License (POCL) 4 | * Full copyright and license information is available in 5 | * LICENSE.md which is distributed with this source code. 6 | * 7 | * @copyright Copyright (c) Pimcore GmbH (http://www.pimcore.com) 8 | * @license Pimcore Open Core License (POCL) 9 | */ 10 | 11 | pimcore.registerNS("pimcore.plugin.pimcoreDataImporterBundle.configuration.components.interpreter.json"); 12 | pimcore.plugin.pimcoreDataImporterBundle.configuration.components.interpreter.json = Class.create(pimcore.plugin.pimcoreDataImporterBundle.configuration.components.abstractOptionType, { 13 | 14 | type: 'json', 15 | 16 | buildSettingsForm: function() { 17 | 18 | if(!this.form) { 19 | this.form = Ext.create('DataHub.DataImporter.StructuredValueForm', { 20 | defaults: { 21 | labelWidth: 200, 22 | width: 600 23 | }, 24 | border: false, 25 | items: [ 26 | { 27 | xtype: 'textfield', 28 | fieldLabel: t('plugin_pimcore_datahub_data_importer_configpanel_json_path'), 29 | name: this.dataNamePrefix + 'path', 30 | value: this.data.path || '', 31 | } 32 | ] 33 | }); 34 | } 35 | 36 | return this.form; 37 | } 38 | 39 | }); -------------------------------------------------------------------------------- /src/Resources/public/js/pimcore/configuration/components/mapping/operator/numeric.js: -------------------------------------------------------------------------------- 1 | /** 2 | * This source file is available under the terms of the 3 | * Pimcore Open Core License (POCL) 4 | * Full copyright and license information is available in 5 | * LICENSE.md which is distributed with this source code. 6 | * 7 | * @copyright Copyright (c) Pimcore GmbH (http://www.pimcore.com) 8 | * @license Pimcore Open Core License (POCL) 9 | */ 10 | 11 | pimcore.registerNS("pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.operator.numeric"); 12 | pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.operator.numeric = Class.create(pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.abstractOperator, { 13 | 14 | type: 'numeric', 15 | 16 | getMenuGroup: function() { 17 | return this.menuGroups.dataTypes; 18 | }, 19 | 20 | getIconClass: function() { 21 | return "pimcore_icon_data_group_numeric"; 22 | }, 23 | 24 | getFormItems: function() { 25 | return [ 26 | { 27 | xtype: 'checkbox', 28 | fieldLabel: t('plugin_pimcore_datahub_data_importer_configpanel_transformation_pipeline_numeric_return_null'), 29 | value: this.data.settings ? this.data.settings.returnNullIfEmpty : ' ', 30 | listeners: { 31 | change: this.inputChangePreviewUpdate.bind(this) 32 | }, 33 | name: 'settings.returnNullIfEmpty' 34 | } 35 | ]; 36 | } 37 | 38 | }); -------------------------------------------------------------------------------- /src/Migrations/Version20210305134111.php: -------------------------------------------------------------------------------- 1 | addSql(sprintf("INSERT IGNORE INTO users_permission_definitions (`key`, `category`) VALUES('%s', '%s');", Installer::DATAHUB_ADAPTER_PERMISSION, \Pimcore\Bundle\DataHubBundle\Installer::DATAHUB_PERMISSION_CATEGORY)); 37 | } 38 | 39 | public function down(Schema $schema): void 40 | { 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/DependencyInjection/CompilerPass/LoaderConfigurationFactoryPass.php: -------------------------------------------------------------------------------- 1 | findTaggedServiceIds(self::loader_tag); 27 | $loader = []; 28 | if (sizeof($taggedServices)) { 29 | foreach ($taggedServices as $id => $tags) { 30 | foreach ($tags as $attributes) { 31 | $loader[$attributes['type']] = new Reference($id); 32 | } 33 | } 34 | } 35 | 36 | $serviceLocator = $container->getDefinition(DataLoaderFactory::class); 37 | $serviceLocator->setArgument('$dataLoaderBluePrints', $loader); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/DataSource/Loader/AssetLoader.php: -------------------------------------------------------------------------------- 1 | assetPath); 33 | if (empty($asset)) { 34 | throw new InvalidConfigurationException("Asset {$this->assetPath} not found."); 35 | } 36 | 37 | $this->temporaryFile = $asset->getTemporaryFile(); 38 | 39 | return $this->temporaryFile; 40 | } 41 | 42 | public function setSettings(array $settings): void 43 | { 44 | if (empty($settings['assetPath'])) { 45 | throw new InvalidConfigurationException('Empty asset path.'); 46 | } 47 | 48 | $this->assetPath = $settings['assetPath']; 49 | } 50 | 51 | public function cleanup(): void 52 | { 53 | unlink($this->temporaryFile); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/Mapping/Operator/AbstractOperator.php: -------------------------------------------------------------------------------- 1 | applicationLogger = $applicationLogger; 37 | } 38 | 39 | /** 40 | * @param string $configName 41 | * 42 | * @return void 43 | */ 44 | public function setConfigName(string $configName) 45 | { 46 | $this->configName = $configName; 47 | } 48 | 49 | /** 50 | * @param mixed $inputData 51 | * 52 | * @return mixed 53 | */ 54 | public function generateResultPreview($inputData) 55 | { 56 | return $inputData; 57 | } 58 | 59 | public function setSettings(array $settings): void 60 | { 61 | //nothing to do 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/Resolver/Load/NotLoadStrategy.php: -------------------------------------------------------------------------------- 1 | findTaggedServiceIds(self::cleanup_tag); 27 | $cleanupStrategies = []; 28 | if (sizeof($taggedServices)) { 29 | foreach ($taggedServices as $id => $tags) { 30 | foreach ($tags as $attributes) { 31 | $cleanupStrategies[$attributes['type']] = new Reference($id); 32 | } 33 | } 34 | } 35 | 36 | $serviceLocator = $container->getDefinition(CleanupStrategyFactory::class); 37 | $serviceLocator->setArgument('$cleanupStrategies', $cleanupStrategies); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/DependencyInjection/CompilerPass/InterpreterConfigurationFactoryPass.php: -------------------------------------------------------------------------------- 1 | findTaggedServiceIds(self::interpreter_tag); 27 | $interpreters = []; 28 | if (sizeof($taggedServices)) { 29 | foreach ($taggedServices as $id => $tags) { 30 | foreach ($tags as $attributes) { 31 | $interpreters[$attributes['type']] = new Reference($id); 32 | } 33 | } 34 | } 35 | 36 | $serviceLocator = $container->getDefinition(InterpreterFactory::class); 37 | $serviceLocator->setArgument('$interpreterBluePrints', $interpreters); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/Mapping/Operator/OperatorInterface.php: -------------------------------------------------------------------------------- 1 | setTimestamp($config['general']['modificationDate']); 27 | 28 | if ($scheduleType === JobScheduler::NAME) { 29 | if (empty($config['executionConfig']['scheduledAt'])) { 30 | throw new InvalidScheduleException('No scheduled date/time'); 31 | } 32 | 33 | $scheduledAt = DateTime::createFromFormat('d-m-Y H:i', $config['executionConfig']['scheduledAt']); 34 | 35 | return new JobScheduler($scheduledAt, $modifiedAt); 36 | } 37 | 38 | if (empty($config['executionConfig']['cronDefinition'])) { 39 | throw new InvalidScheduleException('No cron definition provided'); 40 | } 41 | 42 | return new CronScheduler($config['executionConfig']['cronDefinition'], $modifiedAt); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/Event/DataObject/ProcessElementExceptionEvent.php: -------------------------------------------------------------------------------- 1 | message; 35 | } 36 | 37 | public function getException(): Throwable 38 | { 39 | return $this->exception; 40 | } 41 | 42 | /** 43 | * This is the last mapping configuration that was being processed when the exception was thrown. 44 | * No value will be set if all transformations were completed. 45 | * 46 | * @return null|MappingConfiguration 47 | */ 48 | public function getMappingConfiguration(): ?MappingConfiguration 49 | { 50 | return $this->mappingConfiguration; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /doc/03_Configuration/06_Mapping_Configuration/03_Data_Target/README.md: -------------------------------------------------------------------------------- 1 | # Data Target 2 | 3 | Data target definition assigns the result of the transformation pipeline to a data object field. The available data object 4 | fields for assignment depend on the transformation result of the [transformation pipeline](../02_Transformation_Pipeline.md). 5 | 6 | Three different data target types are available: 7 | 8 | ### Direct 9 | Assign data directly to fields defined directly in the data object class, object bricks or localized fields. The naming 10 | schema for Object brick fields is `..`. 11 | 12 |
13 | 14 | ![Data Target Direct](../../../img/data_target_direct.png) 15 | 16 | > The availability of target fields depends on the settings and the result type of the transformation pipeline. 17 | 18 | ### Classification Store 19 | Assign data to specific keys in a classification store. 20 | 21 |
22 | 23 | ![Data Target Classification Store](../../../img/data_target_classification_store.png) 24 | 25 | > The availability of target fields depends on the settings and the result type of the transformation pipeline. 26 | 27 | ### Classification Store Batch 28 | Assign multiple classification store attributes with one mapping. For this to work, the transformation result has to be 29 | an array with `-` as keys, and values as array values. 30 | 31 |
32 | 33 | ![Data Target Classification Store](../../../img/data_target_classification_store_batch.png) 34 | 35 | For more details and examples see [Classification Store Batch Details](./05_Classification_Store_Batch_Details.md). 36 | -------------------------------------------------------------------------------- /src/DataSource/Loader/DataLoaderFactory.php: -------------------------------------------------------------------------------- 1 | dataLoaderBluePrints = $dataLoaderBluePrints; 32 | } 33 | 34 | /** 35 | * @param array $configuration 36 | * 37 | * @return DataLoaderInterface 38 | * 39 | * @throws InvalidConfigurationException 40 | */ 41 | public function loadDataLoader(array $configuration) 42 | { 43 | if (empty($configuration['type']) || !array_key_exists($configuration['type'], $this->dataLoaderBluePrints)) { 44 | throw new InvalidConfigurationException('Unknown loader type `' . ($configuration['type'] ?? '') . '`'); 45 | } 46 | 47 | $dataLoader = clone $this->dataLoaderBluePrints[$configuration['type']]; 48 | $dataLoader->setSettings($configuration['settings'] ?? []); 49 | 50 | return $dataLoader; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/Installer.php: -------------------------------------------------------------------------------- 1 | get(\Pimcore\Bundle\ApplicationLoggerBundle\Installer::class); 36 | 37 | if ($appLoggerInstaller && !$appLoggerInstaller->isInstalled()) { 38 | $appLoggerInstaller->install(); 39 | } 40 | 41 | // create backend permission 42 | Permission\Definition::create(self::DATAHUB_ADAPTER_PERMISSION) 43 | ->setCategory(\Pimcore\Bundle\DataHubBundle\Installer::DATAHUB_PERMISSION_CATEGORY) 44 | ->save(); 45 | 46 | parent::install(); 47 | } 48 | 49 | public function getLastMigrationVersionClassName(): ?string 50 | { 51 | return Version20240715160305::class; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/Resources/public/css/icons.css: -------------------------------------------------------------------------------- 1 | 2 | .data_hub_cursor_pointer { 3 | cursor: pointer; 4 | } 5 | 6 | .data_hub_preview_panel_column_mapped { 7 | background-color: rgba(30, 179, 24, 0.2); 8 | } 9 | 10 | .data_hub_mapping_panel { 11 | border-bottom: 3px solid #d0d0d0; 12 | } 13 | .data_hub_mapping_panel .x-panel-header { 14 | background-color: #d0d0d0; 15 | } 16 | .data_hub_mapping_panel .x-data_hub_collapsed { 17 | background-color: #f6f6f6; 18 | } 19 | 20 | .plugin_pimcore_datahub_icon_mapping_remove .x-tool-img.x-tool-close { 21 | background-image: none; 22 | color: #F03C28; 23 | background-color: inherit; 24 | } 25 | 26 | .data_hub_cursor_pointer { 27 | cursor: pointer; 28 | } 29 | 30 | .data_hub_mapping_panel { 31 | border-bottom: 3px solid #d0d0d0; 32 | } 33 | .data_hub_mapping_panel .x-panel-header { 34 | background-color: #d0d0d0; 35 | } 36 | .data_hub_mapping_panel .x-data_hub_collapsed { 37 | background-color: #f6f6f6; 38 | } 39 | 40 | .plugin_pimcore_datahub_icon_dataImporterDataObject { 41 | background: url(/bundles/pimcoreadmin/img/flat-color-icons/feed_in.svg) center center no-repeat !important; 42 | } 43 | 44 | .plugin_pimcore_datahub_icon_next { 45 | background: url(/bundles/pimcoreadmin/img/flat-color-icons/next.svg) center center no-repeat !important; 46 | } 47 | .plugin_pimcore_datahub_icon_previous { 48 | background: url(/bundles/pimcoreadmin/img/flat-color-icons/previous.svg) center center no-repeat !important; 49 | } 50 | .plugin_pimcore_datahub_icon_wizard { 51 | background: url(/bundles/pimcoreadmin/img/flat-color-icons/idea.svg) center center no-repeat !important; 52 | } 53 | .plugin_pimcore_datahub_icon_collapse { 54 | background: url(/bundles/pimcoreadmin/img/flat-color-icons/menu.svg) center center no-repeat !important; 55 | } -------------------------------------------------------------------------------- /src/Resources/public/js/pimcore/configuration/components/mapping/operator/trim.js: -------------------------------------------------------------------------------- 1 | /** 2 | * This source file is available under the terms of the 3 | * Pimcore Open Core License (POCL) 4 | * Full copyright and license information is available in 5 | * LICENSE.md which is distributed with this source code. 6 | * 7 | * @copyright Copyright (c) Pimcore GmbH (http://www.pimcore.com) 8 | * @license Pimcore Open Core License (POCL) 9 | */ 10 | 11 | pimcore.registerNS("pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.operator.trim"); 12 | pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.operator.trim = Class.create(pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.abstractOperator, { 13 | 14 | type: 'trim', 15 | 16 | getMenuGroup: function() { 17 | return this.menuGroups.dataManipulation; 18 | }, 19 | 20 | getFormItems: function() { 21 | return [ 22 | { 23 | xtype: 'combo', 24 | fieldLabel: t('plugin_pimcore_datahub_data_importer_configpanel_transformation_pipeline_mode'), 25 | value: this.data.settings ? this.data.settings.mode : 'both', 26 | name: 'settings.mode', 27 | listeners: { 28 | change: this.inputChangePreviewUpdate.bind(this) 29 | }, 30 | store: [ 31 | ['left', t('plugin_pimcore_datahub_data_importer_configpanel_transformation_pipeline_left')], 32 | ['right', t('plugin_pimcore_datahub_data_importer_configpanel_transformation_pipeline_right')], 33 | ['both', t('plugin_pimcore_datahub_data_importer_configpanel_transformation_pipeline_both')] 34 | ] 35 | 36 | } 37 | ]; 38 | } 39 | 40 | }); -------------------------------------------------------------------------------- /src/Resources/public/js/pimcore/configuration/configEvents.js: -------------------------------------------------------------------------------- 1 | /** 2 | * This source file is available under the terms of the 3 | * Pimcore Open Core License (POCL) 4 | * Full copyright and license information is available in 5 | * LICENSE.md which is distributed with this source code. 6 | * 7 | * @copyright Copyright (c) Pimcore GmbH (http://www.pimcore.com) 8 | * @license Pimcore Open Core License (POCL) 9 | */ 10 | 11 | pimcore.plugin.pimcoreDataImporterBundle.configuration.events = { 12 | 13 | /** 14 | * Fired when data object class changed 15 | * 16 | * arguments 17 | * - combo 18 | * - newValue 19 | * - oldValue 20 | */ 21 | classChanged: 'class_changed', 22 | 23 | /** 24 | * Fired when data object class combo is initialized 25 | * 26 | * arguments 27 | * - combo 28 | * - newValue 29 | * - oldValue 30 | */ 31 | classInit: 'class_init', 32 | 33 | /** 34 | * Fired when transformation result preview is updated 35 | * 36 | * arguments 37 | * - transformationResultHandler (to load data from) 38 | * 39 | */ 40 | transformationResultPreviewUpdated: 'transformation_result_preview_updated', 41 | 42 | 43 | /** 44 | * Fired when transformation result type changed 45 | * 46 | * arguments 47 | * - newType 48 | */ 49 | transformationResultTypeChanged: 'transformation_result_type_changed', 50 | 51 | /** 52 | * Fired when loader type changed 53 | * 54 | * arguments 55 | * - newType 56 | */ 57 | loaderTypeChanged: 'loader_type_changed', 58 | 59 | 60 | /** 61 | * Fired when dirty state of config changed 62 | * 63 | * arguments 64 | * - dirty 65 | */ 66 | configDirtyChanged: 'config_dirty_changed', 67 | 68 | }; -------------------------------------------------------------------------------- /src/Migrations/Version20220304130000.php: -------------------------------------------------------------------------------- 1 | hasTable(QueueService::QUEUE_TABLE_NAME)) { 24 | $queueTable = $schema->getTable(QueueService::QUEUE_TABLE_NAME); 25 | $queueTable->addColumn('dispatched', 'bigint', ['notnull' => false, 'default' => null]); 26 | $queueTable->addColumn('workerId', 'string', ['notnull' => false, 'default' => null, 'length' => 13]); 27 | $queueTable->addIndex(['executionType', 'workerId'], 'bundle_index_queue_executiontype_workerId'); 28 | } 29 | } 30 | 31 | public function down(Schema $schema): void 32 | { 33 | if ($schema->hasTable(QueueService::QUEUE_TABLE_NAME)) { 34 | $queueTable = $schema->getTable(QueueService::QUEUE_TABLE_NAME); 35 | $queueTable->dropColumn('dispatched'); 36 | $queueTable->dropColumn('workerId'); 37 | $queueTable->dropIndex('bundle_index_queue_executiontype_workerId'); 38 | } 39 | } 40 | 41 | protected function getBundleName(): string 42 | { 43 | return 'PimcoreDataImporterBundle'; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/Resources/public/js/pimcore/configuration/components/mapping/operator/loadAsset.js: -------------------------------------------------------------------------------- 1 | /** 2 | * This source file is available under the terms of the 3 | * Pimcore Open Core License (POCL) 4 | * Full copyright and license information is available in 5 | * LICENSE.md which is distributed with this source code. 6 | * 7 | * @copyright Copyright (c) Pimcore GmbH (http://www.pimcore.com) 8 | * @license Pimcore Open Core License (POCL) 9 | */ 10 | 11 | pimcore.registerNS("pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.operator.loadAsset"); 12 | pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.operator.loadAsset = Class.create(pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.abstractOperator, { 13 | 14 | type: 'loadAsset', 15 | 16 | getMenuGroup: function() { 17 | return this.menuGroups.loadImport; 18 | }, 19 | 20 | getIconClass: function() { 21 | return "pimcore_icon_asset pimcore_icon_overlay_add"; 22 | }, 23 | 24 | getFormItems: function() { 25 | return [ 26 | { 27 | xtype: 'combo', 28 | fieldLabel: t('plugin_pimcore_datahub_data_importer_configpanel_transformation_pipeline_asset_load_strategy'), 29 | value: this.data.settings ? this.data.settings.loadStrategy : 'path', 30 | listeners: { 31 | change: this.inputChangePreviewUpdate.bind(this) 32 | }, 33 | name: 'settings.loadStrategy', 34 | store: [ 35 | ['path', t('plugin_pimcore_datahub_data_importer_configpanel_find_strategy_path')], 36 | ['id', t('plugin_pimcore_datahub_data_importer_configpanel_find_strategy_id')], 37 | ] 38 | } 39 | ]; 40 | } 41 | 42 | }); -------------------------------------------------------------------------------- /src/Mapping/Operator/Factory/Boolean.php: -------------------------------------------------------------------------------- 1 | 0 && is_numeric($inputData[0])) { 28 | return new RgbaColor(...$inputData); 29 | } 30 | } elseif (str_starts_with($inputData, '#')) { 31 | $color = new RgbaColor(); 32 | $color->setHex($inputData); 33 | 34 | return $color; 35 | } 36 | 37 | return new RgbaColor(); 38 | } 39 | 40 | /** 41 | * @param mixed $inputData 42 | * 43 | * @return mixed|string 44 | */ 45 | public function generateResultPreview($inputData) 46 | { 47 | if ($inputData instanceof RgbaColor) { 48 | return $inputData->__toString(); 49 | } 50 | 51 | return $inputData; 52 | } 53 | 54 | /** 55 | * @param string $inputType 56 | * @param int|null $index 57 | * 58 | * @return string 59 | */ 60 | public function evaluateReturnType(string $inputType, ?int $index = null): string 61 | { 62 | return TransformationDataTypeService::RGBA_COLOR; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/Resources/public/js/pimcore/configuration/components/resolver/load/id.js: -------------------------------------------------------------------------------- 1 | /** 2 | * This source file is available under the terms of the 3 | * Pimcore Open Core License (POCL) 4 | * Full copyright and license information is available in 5 | * LICENSE.md which is distributed with this source code. 6 | * 7 | * @copyright Copyright (c) Pimcore GmbH (http://www.pimcore.com) 8 | * @license Pimcore Open Core License (POCL) 9 | */ 10 | 11 | pimcore.registerNS('pimcore.plugin.pimcoreDataImporterBundle.configuration.components.resolver.load.id'); 12 | pimcore.plugin.pimcoreDataImporterBundle.configuration.components.resolver.load.id = Class.create(pimcore.plugin.pimcoreDataImporterBundle.configuration.components.abstractOptionType, { 13 | 14 | type: 'id', 15 | 16 | buildSettingsForm: function() { 17 | 18 | if(!this.form) { 19 | this.form = Ext.create('DataHub.DataImporter.StructuredValueForm', { 20 | defaults: { 21 | labelWidth: 200, 22 | width: 600, 23 | allowBlank: false, 24 | msgTarget: 'under' 25 | }, 26 | border: false, 27 | items: [ 28 | { 29 | xtype: 'combo', 30 | fieldLabel: t('plugin_pimcore_datahub_data_importer_configpanel_data_source_index'), 31 | name: this.dataNamePrefix + 'dataSourceIndex', 32 | value: this.data.dataSourceIndex, 33 | store: this.configItemRootContainer.columnHeaderStore, 34 | displayField: 'label', 35 | valueField: 'dataIndex', 36 | forceSelection: false, 37 | queryMode: 'local', 38 | triggerOnClick: false 39 | } 40 | ] 41 | }); 42 | } 43 | 44 | return this.form; 45 | } 46 | 47 | }); -------------------------------------------------------------------------------- /src/DataSource/Loader/UploadLoader.php: -------------------------------------------------------------------------------- 1 | pimcoreDataImporterUploadStorage = $pimcoreDataImporterUploadStorage; 38 | } 39 | 40 | public function loadData(): string 41 | { 42 | if ($this->pimcoreDataImporterUploadStorage->fileExists($this->uploadFilePath)) { 43 | $stream = $this->pimcoreDataImporterUploadStorage->readStream($this->uploadFilePath); 44 | $this->temporaryFile = self::getTemporaryFileFromStream($stream, true); 45 | 46 | return $this->temporaryFile; 47 | } 48 | 49 | throw new InvalidConfigurationException('No file uploaded for import.'); 50 | } 51 | 52 | public function setSettings(array $settings): void 53 | { 54 | $this->uploadFilePath = $settings['uploadFilePath']; 55 | } 56 | 57 | public function cleanup(): void 58 | { 59 | unlink($this->temporaryFile); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/Migrations/Version20211201173215.php: -------------------------------------------------------------------------------- 1 | hasTable(self::ORIGINAL_NAME)) { 49 | $this->addSql(sprintf('RENAME TABLE `%s` TO `%s`', self::ORIGINAL_NAME, self::TARGET_NAME)); 50 | } 51 | } 52 | 53 | public function down(Schema $schema): void 54 | { 55 | if ($schema->hasTable(self::TARGET_NAME)) { 56 | $this->addSql(sprintf('RENAME TABLE `%s` TO `%s`', self::TARGET_NAME, self::ORIGINAL_NAME)); 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/Resources/public/js/pimcore/configuration/components/mapping/operator/stringReplace.js: -------------------------------------------------------------------------------- 1 | /** 2 | * This source file is available under the terms of the 3 | * Pimcore Open Core License (POCL) 4 | * Full copyright and license information is available in 5 | * LICENSE.md which is distributed with this source code. 6 | * 7 | * @copyright Copyright (c) Pimcore GmbH (http://www.pimcore.com) 8 | * @license Pimcore Open Core License (POCL) 9 | */ 10 | 11 | pimcore.registerNS("pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.operator.stringReplace"); 12 | pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.operator.stringReplace = Class.create(pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.abstractOperator, { 13 | 14 | type: 'stringReplace', 15 | 16 | getMenuGroup: function() { 17 | return this.menuGroups.dataManipulation; 18 | }, 19 | 20 | getIconClass: function() { 21 | return 'pimcore_icon_operator_stringreplace'; 22 | }, 23 | 24 | getFormItems: function() { 25 | return [ 26 | { 27 | xtype: 'textfield', 28 | fieldLabel: t('plugin_pimcore_datahub_data_importer_configpanel_transformation_pipeline_search'), 29 | value: this.data.settings ? this.data.settings.search : '', 30 | name: 'settings.search', 31 | listeners: { 32 | change: this.inputChangePreviewUpdate.bind(this) 33 | } 34 | }, 35 | 36 | { 37 | xtype: 'textfield', 38 | fieldLabel: t('plugin_pimcore_datahub_data_importer_configpanel_transformation_pipeline_replace'), 39 | value: this.data.settings ? this.data.settings.replace : '', 40 | name: 'settings.replace', 41 | listeners: { 42 | change: this.inputChangePreviewUpdate.bind(this) 43 | } 44 | }, 45 | ]; 46 | } 47 | 48 | }); -------------------------------------------------------------------------------- /src/Resources/public/js/pimcore/configuration/components/mapping/operator/explode.js: -------------------------------------------------------------------------------- 1 | /** 2 | * This source file is available under the terms of the 3 | * Pimcore Open Core License (POCL) 4 | * Full copyright and license information is available in 5 | * LICENSE.md which is distributed with this source code. 6 | * 7 | * @copyright Copyright (c) Pimcore GmbH (http://www.pimcore.com) 8 | * @license Pimcore Open Core License (POCL) 9 | */ 10 | 11 | pimcore.registerNS("pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.operator.explode"); 12 | pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.operator.explode = Class.create(pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.abstractOperator, { 13 | 14 | type: 'explode', 15 | 16 | getMenuGroup: function() { 17 | return this.menuGroups.dataManipulation; 18 | }, 19 | 20 | getIconClass: function() { 21 | return "pimcore_icon_operator_splitter"; 22 | }, 23 | 24 | getFormItems: function() { 25 | return [ 26 | { 27 | xtype: 'textfield', 28 | fieldLabel: t('plugin_pimcore_datahub_data_importer_configpanel_transformation_pipeline_delimiter'), 29 | value: this.data.settings ? this.data.settings.delimiter : ' ', 30 | listeners: { 31 | change: this.inputChangePreviewUpdate.bind(this) 32 | }, 33 | name: 'settings.delimiter' 34 | }, 35 | 36 | { 37 | xtype: 'checkbox', 38 | fieldLabel: t('plugin_pimcore_datahub_data_importer_configpanel_transformation_pipeline_keep_sub_arrays'), 39 | value: this.data.settings ? this.data.settings.keepSubArrays : false, 40 | listeners: { 41 | change: this.inputChangePreviewUpdate.bind(this) 42 | }, 43 | name: 'settings.keepSubArrays' 44 | } 45 | ]; 46 | } 47 | 48 | }); -------------------------------------------------------------------------------- /src/Mapping/Operator/Simple/Combine.php: -------------------------------------------------------------------------------- 1 | glue = $settings['glue'] ?? ' '; 29 | } 30 | 31 | /** 32 | * @param mixed $inputData 33 | * @param bool $dryRun 34 | * 35 | * @return string 36 | */ 37 | public function process($inputData, bool $dryRun = false) 38 | { 39 | if (!is_array($inputData)) { 40 | $inputData = [$inputData]; 41 | } 42 | 43 | return implode($this->glue, $inputData); 44 | } 45 | 46 | /** 47 | * @param string $inputType 48 | * @param int|null $index 49 | * 50 | * @return string 51 | * 52 | * @throws InvalidConfigurationException 53 | */ 54 | public function evaluateReturnType(string $inputType, ?int $index = null): string 55 | { 56 | if ($inputType !== TransformationDataTypeService::DEFAULT_ARRAY) { 57 | throw new InvalidConfigurationException(sprintf("Unsupported input type '%s' for combine operator at transformation position %s", $inputType, $index)); 58 | } 59 | 60 | return TransformationDataTypeService::DEFAULT_TYPE; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/Mapping/Operator/GeopolyAbstractOperator.php: -------------------------------------------------------------------------------- 1 | $item) { 56 | if ($item instanceof GeoCoordinates) { 57 | $preview[$key] = 'Lat.: ' . $item->getLatitude() . ' Long.: ' . $item->getLongitude(); 58 | } 59 | } 60 | } 61 | 62 | return $preview; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/Resources/public/js/pimcore/configuration/components/resolver/publish/attributeBased.js: -------------------------------------------------------------------------------- 1 | /** 2 | * This source file is available under the terms of the 3 | * Pimcore Open Core License (POCL) 4 | * Full copyright and license information is available in 5 | * LICENSE.md which is distributed with this source code. 6 | * 7 | * @copyright Copyright (c) Pimcore GmbH (http://www.pimcore.com) 8 | * @license Pimcore Open Core License (POCL) 9 | */ 10 | 11 | pimcore.registerNS('pimcore.plugin.pimcoreDataImporterBundle.configuration.components.resolver.publish.attributeBased'); 12 | pimcore.plugin.pimcoreDataImporterBundle.configuration.components.resolver.publish.attributeBased = Class.create(pimcore.plugin.pimcoreDataImporterBundle.configuration.components.abstractOptionType, { 13 | 14 | type: 'attributeBased', 15 | 16 | buildSettingsForm: function() { 17 | 18 | if(!this.form) { 19 | this.form = Ext.create('DataHub.DataImporter.StructuredValueForm', { 20 | defaults: { 21 | labelWidth: 200, 22 | width: 600, 23 | allowBlank: false, 24 | msgTarget: 'under' 25 | }, 26 | border: false, 27 | items: [ 28 | { 29 | xtype: 'combo', 30 | fieldLabel: t('plugin_pimcore_datahub_data_importer_configpanel_data_source_index'), 31 | name: this.dataNamePrefix + 'dataSourceIndex', 32 | value: this.data.dataSourceIndex, 33 | store: this.configItemRootContainer.columnHeaderStore, 34 | displayField: 'label', 35 | valueField: 'dataIndex', 36 | forceSelection: false, 37 | queryMode: 'local', 38 | triggerOnClick: false 39 | } 40 | ] 41 | }); 42 | } 43 | 44 | return this.form; 45 | 46 | } 47 | 48 | }); -------------------------------------------------------------------------------- /src/Migrations/Version20251003105903.php: -------------------------------------------------------------------------------- 1 | hasTable($queueTableName)) { 32 | return; 33 | } 34 | 35 | $this->addSql(" 36 | ALTER TABLE $queueTableName 37 | MODIFY COLUMN `data` MEDIUMTEXT 38 | CHARACTER SET utf8mb4 39 | COLLATE utf8mb4_unicode_ci, 40 | ALGORITHM=COPY; 41 | "); 42 | } 43 | 44 | public function down(Schema $schema): void 45 | { 46 | $queueTableName = QueueService::QUEUE_TABLE_NAME; 47 | if (!$schema->hasTable($queueTableName)) { 48 | return; 49 | } 50 | 51 | $this->addSql(" 52 | ALTER TABLE $queueTableName 53 | MODIFY COLUMN `data` TEXT 54 | CHARACTER SET utf8mb4 55 | COLLATE utf8mb4_unicode_ci, 56 | ALGORITHM=COPY; 57 | "); 58 | } 59 | 60 | protected function getBundleName(): string 61 | { 62 | return 'PimcoreDataImporterBundle'; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/Resources/public/js/pimcore/configuration/components/interpreter/xml.js: -------------------------------------------------------------------------------- 1 | /** 2 | * This source file is available under the terms of the 3 | * Pimcore Open Core License (POCL) 4 | * Full copyright and license information is available in 5 | * LICENSE.md which is distributed with this source code. 6 | * 7 | * @copyright Copyright (c) Pimcore GmbH (http://www.pimcore.com) 8 | * @license Pimcore Open Core License (POCL) 9 | */ 10 | 11 | pimcore.registerNS('pimcore.plugin.pimcoreDataImporterBundle.configuration.components.interpreter.xml'); 12 | pimcore.plugin.pimcoreDataImporterBundle.configuration.components.interpreter.xml = Class.create(pimcore.plugin.pimcoreDataImporterBundle.configuration.components.abstractOptionType, { 13 | 14 | type: 'xml', 15 | 16 | buildSettingsForm: function() { 17 | 18 | if(!this.form) { 19 | this.form = Ext.create('DataHub.DataImporter.StructuredValueForm', { 20 | defaults: { 21 | labelWidth: 200, 22 | width: 600 23 | }, 24 | border: false, 25 | items: [ 26 | { 27 | xtype: 'textfield', 28 | fieldLabel: t('plugin_pimcore_datahub_data_importer_configpanel_xml_xpath'), 29 | name: this.dataNamePrefix + 'xpath', 30 | value: this.data.xpath || '/root/item', 31 | allowBlank: false, 32 | msgTarget: 'under' 33 | },{ 34 | xtype: 'textarea', 35 | fieldLabel: t('plugin_pimcore_datahub_data_importer_configpanel_xml_schema'), 36 | name: this.dataNamePrefix + 'schema', 37 | value: this.data.schema || '', 38 | grow: true, 39 | width: 900, 40 | scrollable: true 41 | } 42 | 43 | ] 44 | }); 45 | } 46 | 47 | return this.form; 48 | } 49 | 50 | }); -------------------------------------------------------------------------------- /src/Resolver/Factory/DataObjectFactory.php: -------------------------------------------------------------------------------- 1 | modelFactory = $modelFactory; 36 | } 37 | 38 | public function setSubType(string $subType): void 39 | { 40 | $this->subType = $subType; 41 | } 42 | 43 | /** 44 | * @throws InvalidConfigurationException 45 | * @throws Exception 46 | */ 47 | public function createNewElement(): ElementInterface 48 | { 49 | $class = ClassDefinition::getById($this->subType); 50 | if (empty($class)) { 51 | throw new InvalidConfigurationException("Class `{$this->subType}` not found."); 52 | } 53 | 54 | $className = '\\Pimcore\\Model\\DataObject\\' . ucfirst($class->getName()); 55 | $element = $this->modelFactory->build($className); 56 | 57 | if (!($element instanceof ElementInterface)) { 58 | throw new InvalidConfigurationException( 59 | "Object of class `{$this->subType}` could not be created." 60 | ); 61 | } 62 | 63 | $element->setKey(uniqid('import-', true)); 64 | 65 | return $element; 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/Mapping/Operator/Simple/FlattenArray.php: -------------------------------------------------------------------------------- 1 | hasTable(QueueService::QUEUE_TABLE_NAME)) { 24 | return; 25 | } 26 | 27 | $queueTable = $schema->getTable(QueueService::QUEUE_TABLE_NAME); 28 | 29 | if ($queueTable->hasColumn('userOwner')) { 30 | return; 31 | } 32 | 33 | $queueTable->addColumn('userOwner', 'integer', ['notnull' => true, 'default' => 0])->setUnsigned(true); 34 | 35 | if ($queueTable->hasIndex('bundle_index_queue_executiontype_userOwner')) { 36 | return; 37 | } 38 | 39 | $queueTable->addIndex(['userOwner'], 'bundle_index_queue_executiontype_userOwner'); 40 | } 41 | 42 | public function down(Schema $schema): void 43 | { 44 | if (!$schema->hasTable(QueueService::QUEUE_TABLE_NAME)) { 45 | return; 46 | } 47 | 48 | $queueTable = $schema->getTable(QueueService::QUEUE_TABLE_NAME); 49 | 50 | if (!$queueTable->hasColumn('userOwner')) { 51 | return; 52 | } 53 | 54 | $queueTable->dropColumn('userOwner'); 55 | 56 | if (!$queueTable->hasIndex('bundle_index_queue_executiontype_userOwner')) { 57 | return; 58 | } 59 | 60 | $queueTable->dropIndex('bundle_index_queue_executiontype_userOwner'); 61 | } 62 | 63 | protected function getBundleName(): string 64 | { 65 | return 'PimcoreDataImporterBundle'; 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/Resources/public/js/pimcore/configuration/components/loader/http.js: -------------------------------------------------------------------------------- 1 | /** 2 | * This source file is available under the terms of the 3 | * Pimcore Open Core License (POCL) 4 | * Full copyright and license information is available in 5 | * LICENSE.md which is distributed with this source code. 6 | * 7 | * @copyright Copyright (c) Pimcore GmbH (http://www.pimcore.com) 8 | * @license Pimcore Open Core License (POCL) 9 | */ 10 | 11 | pimcore.registerNS("pimcore.plugin.pimcoreDataImporterBundle.configuration.components.loader.http"); 12 | pimcore.plugin.pimcoreDataImporterBundle.configuration.components.loader.http = Class.create(pimcore.plugin.pimcoreDataImporterBundle.configuration.components.abstractOptionType, { 13 | 14 | type: 'http', 15 | 16 | buildSettingsForm: function() { 17 | 18 | if(!this.form) { 19 | this.form = Ext.create('DataHub.DataImporter.StructuredValueForm', { 20 | defaults: { 21 | labelWidth: 200, 22 | width: 600 23 | }, 24 | border: false, 25 | items: [ 26 | { 27 | xtype: 'combo', 28 | fieldLabel: t('plugin_pimcore_datahub_data_importer_configpanel_http_schema'), 29 | name: this.dataNamePrefix + 'schema', 30 | store: ['https://', 'http://'], 31 | forceSelection: true, 32 | value: this.data.schema, 33 | allowBlank: false, 34 | msgTarget: 'under', 35 | width: 330 36 | },{ 37 | xtype: 'textfield', 38 | fieldLabel: t('plugin_pimcore_datahub_data_importer_configpanel_http_url'), 39 | name: this.dataNamePrefix + 'url', 40 | value: this.data.url, 41 | allowBlank: false, 42 | msgTarget: 'under', 43 | width: 900 44 | 45 | } 46 | ] 47 | }); 48 | } 49 | 50 | return this.form; 51 | } 52 | 53 | }); -------------------------------------------------------------------------------- /src/Mapping/Operator/Factory/AsGeopoint.php: -------------------------------------------------------------------------------- 1 | getLongitude() . ' Long.: ' . $inputData->getLatitude(); 42 | } 43 | 44 | return $inputData; 45 | } 46 | 47 | /** 48 | * @param string $inputType 49 | * @param int|null $index 50 | * 51 | * @return string 52 | * 53 | * @throws InvalidConfigurationException 54 | */ 55 | public function evaluateReturnType(string $inputType, ?int $index = null): string 56 | { 57 | if ($inputType !== TransformationDataTypeService::DEFAULT_ARRAY) { 58 | throw new InvalidConfigurationException(sprintf("Unsupported input type '%s' for geoPoint operator at transformation position %s", $inputType, $index)); 59 | } 60 | 61 | return TransformationDataTypeService::GEOPOINT_VALUE; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/DependencyInjection/CompilerPass/MappingConfigurationFactoryPass.php: -------------------------------------------------------------------------------- 1 | findTaggedServiceIds(self::operator_tag); 29 | $operators = []; 30 | if (sizeof($taggedServices)) { 31 | foreach ($taggedServices as $id => $tags) { 32 | foreach ($tags as $attributes) { 33 | $operators[$attributes['type']] = new Reference($id); 34 | } 35 | } 36 | } 37 | 38 | $taggedServices = $container->findTaggedServiceIds(self::data_target_tag); 39 | $dataTargets = []; 40 | if (sizeof($taggedServices)) { 41 | foreach ($taggedServices as $id => $tags) { 42 | foreach ($tags as $attributes) { 43 | $dataTargets[$attributes['type']] = new Reference($id); 44 | } 45 | } 46 | } 47 | 48 | $serviceLocator = $container->getDefinition(MappingConfigurationFactory::class); 49 | $serviceLocator->setArgument('$operatorBluePrints', $operators); 50 | $serviceLocator->setArgument('$dataTargetBluePrints', $dataTargets); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/DataSource/Loader/PushLoader.php: -------------------------------------------------------------------------------- 1 | filesystem->mkdir($folder, 0775); 44 | 45 | $this->importFilePath = $folder . uniqid('push-import-'); 46 | 47 | $content = file_get_contents('php://input'); 48 | file_put_contents($this->importFilePath, $content); 49 | 50 | return $this->importFilePath; 51 | } 52 | 53 | public function cleanup(): void 54 | { 55 | unlink($this->importFilePath); 56 | } 57 | 58 | public function setSettings(array $settings): void 59 | { 60 | if (empty($settings['apiKey'])) { 61 | throw new InvalidConfigurationException('Empty API Key.'); 62 | } 63 | $this->apiKey = $settings['apiKey']; 64 | 65 | $this->ignoreNotEmptyQueue = $settings['ignoreNotEmptyQueue'] ?? false; 66 | } 67 | 68 | public function getApiKey(): string 69 | { 70 | return $this->apiKey; 71 | } 72 | 73 | public function isIgnoreNotEmptyQueue(): bool 74 | { 75 | return $this->ignoreNotEmptyQueue; 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/DataSource/Loader/HttpLoader.php: -------------------------------------------------------------------------------- 1 | filesystem->mkdir($folder, 0775); 44 | 45 | $this->importFilePath = $folder . uniqid('http-import-'); 46 | $fullUrl = $this->schema . $this->url; 47 | 48 | if (copy($fullUrl, $this->importFilePath)) { 49 | return $this->importFilePath; 50 | } else { 51 | throw new InvalidConfigurationException(sprintf('Could not copy from remote location `%s` to local tmp file `%s`', $fullUrl, $this->importFilePath)); 52 | } 53 | } 54 | 55 | public function cleanup(): void 56 | { 57 | unlink($this->importFilePath); 58 | } 59 | 60 | public function setSettings(array $settings): void 61 | { 62 | if (empty($settings['url'])) { 63 | throw new InvalidConfigurationException('Empty URL.'); 64 | } 65 | $this->url = $settings['url']; 66 | 67 | if (empty($settings['schema'])) { 68 | throw new InvalidConfigurationException('Empty Schema.'); 69 | } 70 | $this->schema = $settings['schema']; 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/Mapping/Operator/Simple/HtmlDecode.php: -------------------------------------------------------------------------------- 1 | configName = $configName; 45 | $this->rawData = $rawData; 46 | $this->dataObject = $dataObject; 47 | } 48 | 49 | public function getConfigName(): string 50 | { 51 | return $this->configName; 52 | } 53 | 54 | public function setConfigName(string $configName): self 55 | { 56 | $this->configName = $configName; 57 | 58 | return $this; 59 | } 60 | 61 | public function getRawData(): array 62 | { 63 | return $this->rawData; 64 | } 65 | 66 | public function setRawData(array $rawData): self 67 | { 68 | $this->rawData = $rawData; 69 | 70 | return $this; 71 | } 72 | 73 | public function getDataObject(): ElementInterface 74 | { 75 | return $this->dataObject; 76 | } 77 | 78 | public function setDataObject(ElementInterface $dataObject): self 79 | { 80 | $this->dataObject = $dataObject; 81 | 82 | return $this; 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /src/Mapping/Operator/Factory/AsCountries.php: -------------------------------------------------------------------------------- 1 | localeService->getDisplayRegions(); 31 | 32 | foreach ($inputData as &$input) { 33 | foreach ($countries as $countryCode => $country) { 34 | if (ltrim(rtrim($input)) == $country) { 35 | $input = $countryCode; 36 | 37 | break; 38 | } 39 | } 40 | } 41 | 42 | return $inputData; 43 | } 44 | 45 | /** 46 | * @param string $inputType 47 | * @param int|null $index 48 | * 49 | * @return string 50 | * 51 | * @throws InvalidConfigurationException 52 | */ 53 | public function evaluateReturnType(string $inputType, ?int $index = null): string 54 | { 55 | if ($inputType != TransformationDataTypeService::DEFAULT_ARRAY) { 56 | throw new InvalidConfigurationException(sprintf("Unsupported input type '%s' for as countries operator at transformation position %s", $inputType, $index)); 57 | } 58 | 59 | return TransformationDataTypeService::COUNTRY_ARRAY; 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /doc/06_Extending/01_Extend_Custom_Strategies.md: -------------------------------------------------------------------------------- 1 | # Extend via Custom Strategies 2 | 3 | The bundle architecture easily allows extension and customization of many 4 | parts of the importing process. 5 | 6 | Following strategies and configuration options can be extended all following the same schema: 7 | - Data Sources 8 | - File Formats 9 | - Resolver Loading Strategies 10 | - Element Location Strategies 11 | - Element Publishing Strategies 12 | - Cleanup Strategies 13 | - Transformation Pipeline Operators 14 | - Data Targets 15 | 16 | 17 | ### Extending Schema 18 | Extending one of the listed strategies and configuration options needs following steps. Also, have a look at the current 19 | implementations to see how things are working. 20 | 21 | #### 1) PHP Implementation 22 | PHP implementations always need to implement an interface (e.g. `DataLoaderInterface` for data sources). Sometimes there 23 | is also an abstract base class, that already implements certain functionality. Try to use them if possible 24 | (e.g. `AbstractInterpreter` for FileFormats). It is of course also possible to extend existing implementations. 25 | 26 | #### 2) Registering implementation as symfony service 27 | The php implementation needs to be registered as symfony service and tagged accordingly. 28 | Tag `name` defines the extension point (e.g. `pimcore.datahub.data_importer.loader` for data sources) and 29 | `type` defines the actual type of the extension. It must be unique and is also the link to values in the configuration file 30 | and the JavaScript implementation. 31 | 32 | ```yml 33 | Pimcore\Bundle\DataImporterBundle\DataSource\Loader\HttpLoader: 34 | tags: 35 | - { name: "pimcore.datahub.data_importer.loader", type: "http" } 36 | ``` 37 | 38 | #### 3) JavaScript Implementation 39 | Create a JavaScript class which meets following requirements: 40 | - Located in a certain namespace depending on the extension point 41 | (e.g. `pimcore.plugin.pimcoreDataImporterBundle.configuration.components.loader.*` for data sources). 42 | - Extend the `pimcore.plugin.pimcoreDataImporterBundle.configuration.components.abstractOptionType` class. 43 | - Define a `type` attribute that matches the `type` of the service definition, e.g. `type: 'http'` 44 | - Implement a function `buildSettingsForm` that creates and returns a form with all necessary setting field for the exension. 45 | -------------------------------------------------------------------------------- /src/Resources/public/js/pimcore/configuration/components/resolver/location/findOrCreateFolder.js: -------------------------------------------------------------------------------- 1 | /** 2 | * This source file is available under the terms of the 3 | * Pimcore Open Core License (POCL) 4 | * Full copyright and license information is available in 5 | * LICENSE.md which is distributed with this source code. 6 | * 7 | * @copyright Copyright (c) Pimcore GmbH (http://www.pimcore.com) 8 | * @license Pimcore Open Core License (POCL) 9 | */ 10 | 11 | pimcore.registerNS('pimcore.plugin.pimcoreDataImporterBundle.configuration.components.resolver.location.findOrCreateFolder'); 12 | pimcore.plugin.pimcoreDataImporterBundle.configuration.components.resolver.location.findOrCreateFolder = Class.create(pimcore.plugin.pimcoreDataImporterBundle.configuration.components.abstractOptionType, { 13 | 14 | type: 'findOrCreateFolder', 15 | 16 | buildSettingsForm: function() { 17 | 18 | if(!this.form) { 19 | 20 | this.form = Ext.create('DataHub.DataImporter.StructuredValueForm', { 21 | defaults: { 22 | labelWidth: 200, 23 | width: 600, 24 | allowBlank: false, 25 | msgTarget: 'under' 26 | }, 27 | border: false, 28 | items: [ 29 | { 30 | xtype: 'combo', 31 | fieldLabel: t('plugin_pimcore_datahub_data_importer_configpanel_data_source_index'), 32 | name: this.dataNamePrefix + 'dataSourceIndex', 33 | value: this.data.dataSourceIndex, 34 | store: this.configItemRootContainer.columnHeaderStore, 35 | displayField: 'label', 36 | valueField: 'dataIndex', 37 | forceSelection: false, 38 | queryMode: 'local', 39 | triggerOnClick: false 40 | },{ 41 | xtype: 'textfield', 42 | fieldLabel: t('plugin_pimcore_datahub_data_importer_configpanel_fallback_path'), 43 | name: this.dataNamePrefix + 'fallbackPath', 44 | value: this.data.fallbackPath 45 | } 46 | ] 47 | }); 48 | } 49 | 50 | return this.form; 51 | } 52 | 53 | }); -------------------------------------------------------------------------------- /src/Preview/PreviewService.php: -------------------------------------------------------------------------------- 1 | pimcoreDataImporterPreviewStorage = $pimcoreDataImporterPreviewStorage; 29 | } 30 | 31 | public function writePreviewFile(string $configName, string $sourcePath, User $user) 32 | { 33 | $target = $this->getPreviewFilePath($configName, $user); 34 | $this->pimcoreDataImporterPreviewStorage->write($target, file_get_contents($sourcePath)); 35 | } 36 | 37 | /** 38 | * @param string $configName 39 | * @param User $user 40 | * 41 | * @return string 42 | * 43 | * @throws \Exception 44 | */ 45 | protected function getPreviewFilePath(string $configName, User $user): string 46 | { 47 | $configuration = Configuration::getByName($configName); 48 | if (!$configuration) { 49 | throw new \Exception('Configuration ' . $configName . ' does not exist.'); 50 | } 51 | 52 | $filePath = $configuration->getName() . '/' . $user->getId() . '.import'; 53 | 54 | return $filePath; 55 | } 56 | 57 | public function getLocalPreviewFile(string $configName, User $user): ?string 58 | { 59 | $filePath = $this->getPreviewFilePath($configName, $user); 60 | 61 | if ($this->pimcoreDataImporterPreviewStorage->fileExists($filePath)) { 62 | $stream = $this->pimcoreDataImporterPreviewStorage->readStream($filePath); 63 | 64 | return self::getLocalFileFromStream($stream); 65 | } 66 | 67 | return null; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/Mapping/Operator/Factory/AsGeobounds.php: -------------------------------------------------------------------------------- 1 | getNorthEast() . ' SW: ' . $inputData->getSouthWest(); 46 | } 47 | 48 | return $inputData; 49 | } 50 | 51 | /** 52 | * @param string $inputType 53 | * @param int|null $index 54 | * 55 | * @return string 56 | * 57 | * @throws InvalidConfigurationException 58 | */ 59 | public function evaluateReturnType(string $inputType, ?int $index = null): string 60 | { 61 | if ($inputType !== TransformationDataTypeService::DEFAULT_ARRAY) { 62 | throw new InvalidConfigurationException(sprintf("Unsupported input type '%s' for geoBounds operator at transformation position %s", $inputType, $index)); 63 | } 64 | 65 | return TransformationDataTypeService::GEOBOUNDS_VALUE; 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/Mapping/Operator/Factory/ImageAdvanced.php: -------------------------------------------------------------------------------- 1 | getImage() ? $inputData->getImage()->getFullPath() : ''); 68 | } 69 | 70 | return $inputData; 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/Command/PrepareImportCommand.php: -------------------------------------------------------------------------------- 1 | importPreparationService = $importPreparationService; 37 | } 38 | 39 | protected function configure(): void 40 | { 41 | $this 42 | ->setName('datahub:data-importer:prepare-import') 43 | ->setDescription('Loads and interprets data source file and prepares queue items for import.') 44 | ->addArgument('config_name', InputArgument::REQUIRED | InputArgument::IS_ARRAY, 'Names of configs that should be considered.') 45 | ; 46 | } 47 | 48 | /** 49 | * @param InputInterface $input 50 | * @param OutputInterface $output 51 | * 52 | * @return int 53 | */ 54 | protected function execute(InputInterface $input, OutputInterface $output): int 55 | { 56 | $configNames = $input->getArgument('config_name'); 57 | 58 | if (empty($configNames)) { 59 | $output->writeln('No config given, nothing to do.'); 60 | } else { 61 | foreach ($configNames as $configName) { 62 | $output->writeln("Preparing import for config '$configName'"); 63 | $this->importPreparationService->prepareImport($configName); 64 | } 65 | } 66 | 67 | return 0; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/Mapping/MappingConfiguration.php: -------------------------------------------------------------------------------- 1 | label; 43 | } 44 | 45 | /** 46 | * @param string $label 47 | */ 48 | public function setLabel($label): void 49 | { 50 | $this->label = $label; 51 | } 52 | 53 | /** 54 | * @return mixed 55 | */ 56 | public function getDataSourceIndex() 57 | { 58 | return $this->dataSourceIndex; 59 | } 60 | 61 | /** 62 | * @param mixed $dataSourceIndex 63 | */ 64 | public function setDataSourceIndex($dataSourceIndex): void 65 | { 66 | $this->dataSourceIndex = $dataSourceIndex; 67 | } 68 | 69 | /** 70 | * @return OperatorInterface[] 71 | */ 72 | public function getTransformationPipeline(): array 73 | { 74 | return $this->transformationPipeline; 75 | } 76 | 77 | /** 78 | * @param OperatorInterface[] $transformationPipeline 79 | */ 80 | public function setTransformationPipeline($transformationPipeline): void 81 | { 82 | $this->transformationPipeline = $transformationPipeline; 83 | } 84 | 85 | public function getDataTarget(): DataTargetInterface 86 | { 87 | return $this->dataTarget; 88 | } 89 | 90 | /** 91 | * @param DataTargetInterface $dataTarget 92 | */ 93 | public function setDataTarget($dataTarget): void 94 | { 95 | $this->dataTarget = $dataTarget; 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /src/Command/ParallelProcessQueueCommand.php: -------------------------------------------------------------------------------- 1 | importProcessingService = $importProcessingService; 36 | $this->queueService = $queueService; 37 | } 38 | 39 | protected function configure(): void 40 | { 41 | parent::configure(); 42 | 43 | $this 44 | ->setName('datahub:data-importer:process-queue-parallel') 45 | ->setDescription('Processes all items of the queue that can be executed parallel.') 46 | ; 47 | } 48 | 49 | /** 50 | * Fetches the items that should be processed. 51 | * 52 | * Typically, you will fetch all the items of the database objects that 53 | * you want to process here. These will be passed to runSingleCommand(). 54 | * 55 | * This method is called exactly once in the master process. 56 | */ 57 | protected function doFetchItems(InputInterface $input, ?OutputInterface $output): array 58 | { 59 | return $this->queueService->getAllQueueEntryIds(ImportProcessingService::EXECUTION_TYPE_PARALLEL); 60 | } 61 | 62 | /** 63 | * Processes an item in the child process. 64 | */ 65 | protected function runSingleCommand(string $item, InputInterface $input, OutputInterface $output): void 66 | { 67 | $this->importProcessingService->processQueueItem((int) $item); 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/Migrations/Version20211110174732.php: -------------------------------------------------------------------------------- 1 | hasTable(self::DELTA_CACHE_TABLE)) { 37 | $this->addSql(sprintf('ALTER TABLE `%s` MODIFY `configName` VARCHAR(80)', self::DELTA_CACHE_TABLE)); 38 | } 39 | 40 | if ($schema->hasTable(self::LAST_CRON_TABLE)) { 41 | $this->addSql(sprintf('ALTER TABLE `%s` MODIFY `configName` VARCHAR(80)', self::LAST_CRON_TABLE)); 42 | } 43 | 44 | if ($schema->hasTable(self::IMPORTER_QUEUE_TABLE)) { 45 | $this->addSql(sprintf('ALTER TABLE `%s` MODIFY `configName` VARCHAR(80)', self::IMPORTER_QUEUE_TABLE)); 46 | } 47 | } 48 | 49 | /** 50 | * @param Schema $schema 51 | */ 52 | public function down(Schema $schema): void 53 | { 54 | if ($schema->hasTable(self::DELTA_CACHE_TABLE)) { 55 | $this->addSql(sprintf('ALTER TABLE `%s` MODIFY `configName` VARCHAR(50)', self::DELTA_CACHE_TABLE)); 56 | } 57 | 58 | if ($schema->hasTable(self::LAST_CRON_TABLE)) { 59 | $this->addSql(sprintf('ALTER TABLE `%s` MODIFY `configName` VARCHAR(50)', self::LAST_CRON_TABLE)); 60 | } 61 | 62 | if ($schema->hasTable(self::IMPORTER_QUEUE_TABLE)) { 63 | $this->addSql(sprintf('ALTER TABLE `%s` MODIFY `configName` VARCHAR(50)', self::IMPORTER_QUEUE_TABLE)); 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/Resources/public/js/pimcore/configuration/components/mapping/operator/staticText.js: -------------------------------------------------------------------------------- 1 | /** 2 | * This source file is available under the terms of the 3 | * Pimcore Open Core License (POCL) 4 | * Full copyright and license information is available in 5 | * LICENSE.md which is distributed with this source code. 6 | * 7 | * @copyright Copyright (c) Pimcore GmbH (http://www.pimcore.com) 8 | * @license Pimcore Open Core License (POCL) 9 | */ 10 | 11 | pimcore.registerNS("pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.operator.staticText"); 12 | pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.operator.staticText = Class.create(pimcore.plugin.pimcoreDataImporterBundle.configuration.components.mapping.abstractOperator, { 13 | 14 | type: 'staticText', 15 | 16 | getMenuGroup: function() { 17 | return this.menuGroups.dataManipulation; 18 | }, 19 | 20 | getFormItems: function() { 21 | return [ 22 | { 23 | xtype: 'combo', 24 | fieldLabel: t('plugin_pimcore_datahub_data_importer_configpanel_transformation_pipeline_static_text_mode'), 25 | value: this.data.settings ? this.data.settings.mode : 'append', 26 | name: 'settings.mode', 27 | listeners: { 28 | change: this.inputChangePreviewUpdate.bind(this) 29 | }, 30 | store: [ 31 | ['append', t('plugin_pimcore_datahub_data_importer_configpanel_transformation_pipeline_append')], 32 | ['prepend', t('plugin_pimcore_datahub_data_importer_configpanel_transformation_pipeline_prepend')] 33 | ] 34 | 35 | }, 36 | 37 | { 38 | xtype: 'textfield', 39 | fieldLabel: t('plugin_pimcore_datahub_data_importer_configpanel_transformation_pipeline_text'), 40 | value: this.data.settings ? this.data.settings.text : '', 41 | name: 'settings.text', 42 | listeners: { 43 | change: this.inputChangePreviewUpdate.bind(this) 44 | } 45 | }, 46 | 47 | { 48 | xtype: 'checkbox', 49 | fieldLabel: t('plugin_pimcore_datahub_data_importer_configpanel_transformation_pipeline_always_add'), 50 | value: this.data.settings ? this.data.settings.alwaysAdd : false, 51 | name: 'settings.alwaysAdd' 52 | } 53 | ]; 54 | } 55 | 56 | }); -------------------------------------------------------------------------------- /src/Mapping/Operator/Simple/StringReplace.php: -------------------------------------------------------------------------------- 1 | search = $settings['search'] ?? ''; 28 | $this->replace = $settings['replace'] ?? ''; 29 | } 30 | 31 | /** 32 | * @param mixed $inputData 33 | * @param bool $dryRun 34 | * 35 | * @return array|false|mixed|null 36 | */ 37 | public function process($inputData, bool $dryRun = false) 38 | { 39 | $returnScalar = false; 40 | if (!is_array($inputData)) { 41 | $returnScalar = true; 42 | $inputData = [$inputData]; 43 | } 44 | 45 | foreach ($inputData as &$data) { 46 | $data = str_replace($this->search, $this->replace, $data); 47 | } 48 | 49 | if ($returnScalar) { 50 | if (!empty($inputData)) { 51 | return reset($inputData); 52 | } 53 | 54 | return null; 55 | } else { 56 | return $inputData; 57 | } 58 | } 59 | 60 | /** 61 | * @param string $inputType 62 | * @param int|null $index 63 | * 64 | * @return string 65 | * 66 | * @throws InvalidConfigurationException 67 | */ 68 | public function evaluateReturnType(string $inputType, ?int $index = null): string 69 | { 70 | if (!in_array($inputType, [TransformationDataTypeService::DEFAULT_TYPE, TransformationDataTypeService::DEFAULT_ARRAY])) { 71 | throw new InvalidConfigurationException(sprintf("Unsupported input type '%s' for string replace operator at transformation position %s", $inputType, $index)); 72 | } 73 | 74 | return $inputType; 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/Resolver/Load/AttributeStrategy.php: -------------------------------------------------------------------------------- 1 | attributeName = $settings['attributeName']; 49 | $this->attributeLanguage = $settings['language'] ?? null; 50 | $this->includeUnpublished = $settings['includeUnpublished'] ?? false; 51 | } 52 | 53 | /** 54 | * @param string $identifier 55 | * 56 | * @return ElementInterface|null 57 | * 58 | * @throws InvalidConfigurationException 59 | */ 60 | public function loadElementByIdentifier($identifier): ?ElementInterface 61 | { 62 | return $this->dataObjectLoader->loadByAttribute($this->getClassName(), 63 | $this->attributeName, 64 | $identifier, 65 | $this->attributeLanguage, 66 | $this->includeUnpublished, 67 | 1); 68 | } 69 | 70 | public function loadFullIdentifierList(): array 71 | { 72 | $tableName = 'object_' . $this->dataObjectClassId; 73 | if ($this->attributeLanguage) { 74 | $tableName = 'object_localized_' . $this->dataObjectClassId . '_' . $this->attributeLanguage; 75 | } 76 | 77 | $sql = sprintf('SELECT `%s` FROM %s', $this->attributeName, $tableName); 78 | 79 | return $this->db->fetchFirstColumn($sql); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/Mapping/Operator/Factory/InputQuantityValue.php: -------------------------------------------------------------------------------- 1 | getValue() . 69 | ($inputData->getUnit() ? ' ['.$inputData->getUnit()->getAbbreviation().']' : ''); 70 | } 71 | 72 | return $inputData; 73 | } 74 | } 75 | --------------------------------------------------------------------------------