├── Configuration ├── Caches.yaml ├── Objects.yaml └── Settings.yaml ├── Classes ├── Exception │ ├── SiteNodeEmptyException.php │ ├── InvalidArgumentException.php │ └── ImportAlreadyExecutedException.php ├── DataType │ ├── Integer.php │ ├── DataTypeInterface.php │ ├── StringValue.php │ ├── Slug.php │ ├── DataType.php │ ├── Date.php │ ├── HtmlContent.php │ └── ExternalResource.php ├── Domain │ ├── Model │ │ ├── Event.php │ │ ├── ProviderPropertyValidity.php │ │ ├── RecordMapping.php │ │ ├── Import.php │ │ └── PresetPartDefinition.php │ ├── Repository │ │ ├── ImportRepository.php │ │ ├── EventRepository.php │ │ └── RecordMappingRepository.php │ └── Service │ │ └── ImportService.php ├── DataProvider │ ├── DataProviderInterface.php │ ├── AbstractDatabaseDataProvider.php │ ├── CsvDataProvider.php │ └── AbstractDataProvider.php ├── Service │ ├── ContextSwitcher.php │ ├── Vault.php │ ├── DimensionsImporter.php │ ├── ProcessedNodeService.php │ └── NodePropertyMapper.php ├── Importer │ ├── ImporterInterface.php │ ├── AbstractCommandBasedImporter.php │ └── AbstractImporter.php ├── Aspect │ └── EventLogAspect.php └── Command │ └── ImportCommandController.php ├── LICENSE.txt ├── Migrations ├── Mysql │ ├── Version20160523202439.php │ ├── Version20170718190551.php │ ├── Version20150319132414.php │ └── Version20150317235257.php └── Postgresql │ └── Version20170718190551.php ├── composer.json └── README.md /Configuration/Caches.yaml: -------------------------------------------------------------------------------- 1 | Ttree_ContentRepositoryImporter_Vault: 2 | frontend: Neos\Cache\Frontend\VariableFrontend 3 | backend: Neos\Cache\Backend\FileBackend 4 | -------------------------------------------------------------------------------- /Classes/Exception/SiteNodeEmptyException.php: -------------------------------------------------------------------------------- 1 | initializeValue($this->rawValue); 17 | return (integer)$this->value; 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /Classes/DataType/DataTypeInterface.php: -------------------------------------------------------------------------------- 1 | value = $value; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /Classes/DataType/Slug.php: -------------------------------------------------------------------------------- 1 | value = $slugify->slugify($value->getValue()); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /Classes/Domain/Model/Event.php: -------------------------------------------------------------------------------- 1 | externalIdentifier; 25 | } 26 | 27 | /** 28 | * @param string $externalIdentifier 29 | */ 30 | public function setExternalIdentifier($externalIdentifier) 31 | { 32 | $this->externalIdentifier = $externalIdentifier; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /Classes/DataProvider/DataProviderInterface.php: -------------------------------------------------------------------------------- 1 | node = $node; 19 | } 20 | 21 | /** 22 | * @param array $dimensions 23 | * @return NodeInterface|null 24 | */ 25 | public function to(array $dimensions) 26 | { 27 | return (new FlowQuery([$this->node]))->context([ 28 | 'dimensions' => $dimensions, 29 | 'targetDimensions' => array_map(function ($dimensionValues) { 30 | return array_shift($dimensionValues); 31 | }, $dimensions) 32 | ])->get(0); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /Classes/Domain/Repository/ImportRepository.php: -------------------------------------------------------------------------------- 1 | entityManager->getUnitOfWork()->getIdentityMap() as $className => $entities) { 27 | if ($className === $this->entityClassName) { 28 | foreach ($entities as $entityToPersist) { 29 | $this->entityManager->flush($entityToPersist); 30 | } 31 | break; 32 | } 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Neos project contributors 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Classes/Domain/Repository/EventRepository.php: -------------------------------------------------------------------------------- 1 | entityManager->getConnection(); 23 | 24 | $isMySQL = $connection->getDriver()->getName() === 'pdo_mysql'; 25 | if ($isMySQL) { 26 | $connection->query('SET FOREIGN_KEY_CHECKS=0'); 27 | } 28 | 29 | $connection->query("DELETE FROM neos_neos_eventlog_domain_model_event WHERE dtype = 'ttree_contentrepositoryimporter_event'"); 30 | 31 | if ($isMySQL) { 32 | $connection->query('SET FOREIGN_KEY_CHECKS=1'); 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /Classes/Domain/Model/ProviderPropertyValidity.php: -------------------------------------------------------------------------------- 1 | nodeOrTemplate = $nodeOrTemplate; 28 | } 29 | 30 | /** 31 | * @param string $propertyName 32 | * @return bool 33 | */ 34 | public function isValid($propertyName) 35 | { 36 | $availableProperties = $this->nodeOrTemplate->getNodeType()->getProperties(); 37 | return !(\in_array(substr($propertyName, 0, 1), ['_', '@']) || !isset($availableProperties[$propertyName])); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /Classes/Service/Vault.php: -------------------------------------------------------------------------------- 1 | preset = (string)$preset; 23 | } 24 | 25 | /** 26 | * @param string $key 27 | * @param mixed $value 28 | */ 29 | public function set($key, $value) 30 | { 31 | $this->storage->set(md5($this->preset . $key), $value, [$this->preset]); 32 | } 33 | 34 | /** 35 | * @param string $key 36 | * @return mixed 37 | */ 38 | public function get($key) 39 | { 40 | return $this->storage->get(md5($this->preset . $key)); 41 | } 42 | 43 | /** 44 | * @param string $key 45 | * @return bool 46 | */ 47 | public function has($key) 48 | { 49 | return $this->storage->has(md5($this->preset . $key)); 50 | } 51 | 52 | /** 53 | * @return void 54 | */ 55 | public function flush() 56 | { 57 | $this->storage->flushByTag($this->preset); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /Migrations/Mysql/Version20160523202439.php: -------------------------------------------------------------------------------- 1 | abortIf($this->connection->getDatabasePlatform()->getName() != 'mysql', 'Migration can only be executed safely on "mysql".'); 27 | 28 | $this->addSql('ALTER TABLE ttree_contentrepositoryimporter_domain_model_import ADD externalimportidentifier VARCHAR(255) DEFAULT NULL'); 29 | } 30 | 31 | /** 32 | * @param Schema $schema 33 | * @return void 34 | */ 35 | public function down(Schema $schema): void 36 | { 37 | $this->abortIf($this->connection->getDatabasePlatform()->getName() != 'mysql', 'Migration can only be executed safely on "mysql".'); 38 | 39 | $this->addSql('ALTER TABLE ttree_contentrepositoryimporter_domain_model_import DROP externalimportidentifier'); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /Classes/Importer/ImporterInterface.php: -------------------------------------------------------------------------------- 1 | abortIf($this->connection->getDatabasePlatform()->getName() != 'mysql', 'Migration can only be executed safely on "mysql".'); 28 | 29 | $this->addSql('ALTER TABLE ttree_contentrepositoryimporter_domain_model_import CHANGE `start` `starttime` DATETIME NOT NULL, CHANGE `end` `endtime` DATETIME DEFAULT NULL'); 30 | } 31 | 32 | /** 33 | * @param Schema $schema 34 | * @return void 35 | */ 36 | public function down(Schema $schema): void 37 | { 38 | $this->abortIf($this->connection->getDatabasePlatform()->getName() != 'mysql', 'Migration can only be executed safely on "mysql".'); 39 | 40 | $this->addSql('ALTER TABLE ttree_contentrepositoryimporter_domain_model_import CHANGE `starttime` `start` DATETIME NOT NULL, CHANGE `endtime` `end` DATETIME DEFAULT NULL'); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /Classes/DataProvider/AbstractDatabaseDataProvider.php: -------------------------------------------------------------------------------- 1 | 18 | */ 19 | protected $connections = []; 20 | 21 | /** 22 | * @return QueryBuilder 23 | * @throws Exception 24 | */ 25 | protected function createQuery(): QueryBuilder 26 | { 27 | $query = $this->getDatabaseConnection() 28 | ->createQueryBuilder(); 29 | 30 | if ($this->limit > 0) { 31 | $query 32 | ->setFirstResult($this->offset ?: 0) 33 | ->setMaxResults($this->limit); 34 | } 35 | 36 | return $query; 37 | } 38 | 39 | /** 40 | * @return Connection 41 | * @throws Exception 42 | */ 43 | protected function getDatabaseConnection(): Connection 44 | { 45 | $sourceName = isset($this->options['source']) ? $this->options['source'] : 'default'; 46 | 47 | if (isset($this->connections[$sourceName]) && $this->connections[$sourceName] instanceof Connection) { 48 | return $this->connections[$sourceName]; 49 | } 50 | 51 | $this->connections[$sourceName] = DriverManager::getConnection($this->settings['sources'][$sourceName], new Configuration()); 52 | return $this->connections[$sourceName]; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /Migrations/Mysql/Version20150319132414.php: -------------------------------------------------------------------------------- 1 | abortIf($this->connection->getDatabasePlatform()->getName() != "mysql"); 19 | 20 | $this->addSql("CREATE TABLE ttree_contentrepositoryimporter_domain_model_recordmapping (persistence_object_identifier VARCHAR(40) NOT NULL, creationdate DATETIME NOT NULL, modificationdate DATETIME DEFAULT NULL, importerclassname VARCHAR(255) NOT NULL, importerclassnamehash VARCHAR(32) NOT NULL, externalidentifier VARCHAR(255) NOT NULL, externalrelativeuri VARCHAR(255) DEFAULT NULL, nodeidentifier VARCHAR(255) NOT NULL, nodepath VARCHAR(4000) NOT NULL, nodepathhash VARCHAR(32) NOT NULL, INDEX importerclassnamehash_externalidentifier (importerclassnamehash, externalidentifier), INDEX nodepathhash (nodepathhash), INDEX nodeidentifier (nodeidentifier), UNIQUE INDEX UNIQ_8C1932AD8FC282F11EE98D63 (importerclassnamehash, externalidentifier), PRIMARY KEY(persistence_object_identifier))"); 21 | } 22 | 23 | /** 24 | * @param Schema $schema 25 | * @return void 26 | */ 27 | public function down(Schema $schema): void 28 | { 29 | $this->abortIf($this->connection->getDatabasePlatform()->getName() != "mysql"); 30 | 31 | $this->addSql("DROP TABLE ttree_contentrepositoryimporter_domain_model_recordmapping"); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /Classes/Domain/Repository/RecordMappingRepository.php: -------------------------------------------------------------------------------- 1 | createQuery(); 28 | 29 | $query->matching($query->logicalAnd( 30 | $query->equals('importerClassNameHash', md5($importerClassName)), 31 | $query->equals('externalIdentifier', $externalIdentifier) 32 | )); 33 | 34 | return $query->execute()->getFirst(); 35 | } 36 | 37 | /** 38 | * Persists all entities managed by the repository and all cascading dependencies 39 | * 40 | * @return void 41 | */ 42 | public function persistEntities() 43 | { 44 | foreach ($this->entityManager->getUnitOfWork()->getIdentityMap() as $className => $entities) { 45 | if ($className === $this->entityClassName) { 46 | foreach ($entities as $entityToPersist) { 47 | $this->entityManager->flush($entityToPersist); 48 | } 49 | break; 50 | } 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /Classes/DataType/DataType.php: -------------------------------------------------------------------------------- 1 | options = $this->configurationManager->getConfiguration( 39 | ConfigurationManager::CONFIGURATION_TYPE_SETTINGS, 40 | 'Ttree.ContentRepositoryImporter.dataTypeOptions.' . get_called_class() 41 | ) ?: []; 42 | } 43 | 44 | /** 45 | * @param string $value 46 | */ 47 | public function __construct($value) 48 | { 49 | $this->rawValue = $value; 50 | } 51 | 52 | /** 53 | * @return mixed 54 | */ 55 | public function getValue() 56 | { 57 | $this->initializeValue($this->rawValue); 58 | return $this->value; 59 | } 60 | 61 | /** 62 | * @param mixed $value 63 | * @return $this 64 | */ 65 | public static function create($value) 66 | { 67 | $class = get_called_class(); 68 | return new $class($value); 69 | } 70 | 71 | /** 72 | * @param mixed $value 73 | */ 74 | protected function initializeValue($value) 75 | { 76 | $this->value = $value; 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /Classes/DataType/Date.php: -------------------------------------------------------------------------------- 1 | value = clone $value; 17 | } elseif (is_numeric($value)) { 18 | // $value is UNIX timestamp 19 | if ($timestamp = (int)$value) { 20 | $this->value = new \DateTime(); 21 | $this->value->setTimestamp($timestamp); 22 | } else { 23 | $this->value = null; 24 | } 25 | } elseif (is_string($value)) { 26 | // fallback, try to parse $value as a date string 27 | $this->value = new \DateTime($value); 28 | } else { 29 | throw new \Exception(sprintf('Cannot convert %s to a DateTime object', $value)); 30 | } 31 | } 32 | 33 | /** 34 | * Initialize a date value from input 35 | * 36 | * Input value can be a date/time string, parsable by the PHP DateTime class, 37 | * or a numeric value (int or string), which then will be interpreted as a UNIX 38 | * timestamp in the local configured TZ. 39 | * 40 | */ 41 | public static function create($value) 42 | { 43 | return parent::create($value); 44 | } 45 | 46 | /** 47 | * Gets the DateTime value or null, if the datetime was initialized from null or 48 | * an empty timestamp. 49 | * 50 | * @return \DateTime | null 51 | */ 52 | public function getValue() 53 | { 54 | return parent::getValue(); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /Configuration/Settings.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | Ttree: 3 | ContentRepositoryImporter: 4 | 5 | sources: 6 | default: 7 | driver: 'pdo_mysql' 8 | dbname: '' 9 | host: '127.0.0.1' 10 | user: '' 11 | password: '' 12 | 13 | dimensionsImporter: 14 | presets: [] 15 | 16 | dataTypeOptions: 17 | 'Ttree\ContentRepositoryImporter\DataType\ExternalResource': 18 | downloadDirectory: '%FLOW_PATH_DATA%Persistent/Ttree.ContentRepositoryImporter/Downloads/' 19 | 20 | 'Ttree\ContentRepositoryImporter\DataType\HtmlContent': 21 | htmlPurifierOptions: 22 | 'HTML.AllowedElements': 'a,em,i,strong,b,blockquote,p,ul,ol,li' 23 | 'HTML.AllowedAttributes': 'a.href,a.title' 24 | 'HTML.TidyLevel': 'light' 25 | preProcessing: 26 | '#\[b\](.+)\[/b\]#i': '$1' 27 | '#\[i\](.+)\[/i\]#i': '$1' 28 | processingPerLine: 29 | '#^(.+)$#': '

$1

' 30 | postProcessing: 31 | '#^(?!.*?(?:<.*ul>|<.*li>|<.*ol>|<.*h.*>))(.+)$#uim': '

$1

' 32 | 33 | # presets: 34 | # 'base': 35 | # parts: 36 | # 'news': 37 | # label: 'News Import' 38 | # dataProviderClassName: 'Your\Package\Importer\DataProvider\NewsDataProvider' 39 | # importerClassName: 'Your\Package\Importer\Importer\NewsImporter' 40 | # 41 | # 'page': 42 | # label: 'Page Import' 43 | # dataProviderClassName: 'Your\Package\Importer\DataProvider\PageDataProvider' 44 | # importerClassName: 'Your\Package\Importer\Importer\PageImporter' 45 | # batchSize': 120 46 | # 47 | # 'pageContent': 48 | # label: 'Page Content Import' 49 | # dataProviderClassName: 'Your\Package\Importer\DataProvider\PageContentDataProvider' 50 | # importerClassName: 'Your\Package\Importer\Importer\PageContentImporter' 51 | # batchSize: 120 52 | -------------------------------------------------------------------------------- /Classes/Service/DimensionsImporter.php: -------------------------------------------------------------------------------- 1 | isValid($propertyName); 40 | }; 41 | $dimensionsData = $data['@dimensions']; 42 | $properties = \array_filter($data, $dataFilter, \ARRAY_FILTER_USE_KEY); 43 | 44 | $contextSwitcher = new ContextSwitcher($node); 45 | foreach (array_keys($dimensionsData) as $preset) { 46 | $dimensions = $this->settings[\str_replace('@', '', $preset)]; 47 | $nodeInContext = $contextSwitcher->to($dimensions); 48 | $localProperties = \array_filter($dimensionsData[$preset], $dataFilter, \ARRAY_FILTER_USE_KEY); 49 | $localProperties = Arrays::arrayMergeRecursiveOverrule($properties, $localProperties); 50 | $this->nodePropertyMapper->map($localProperties, $nodeInContext, $event); 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /Classes/Importer/AbstractCommandBasedImporter.php: -------------------------------------------------------------------------------- 1 | unsetAllNodeTemplateProperties($nodeTemplate); 29 | 30 | $externalIdentifier = $this->getExternalIdentifierFromRecordData($data); 31 | if (!isset($data['uriPathSegment'])) { 32 | $data['uriPathSegment'] = Slug::create($this->getLabelFromRecordData($data))->getValue(); 33 | } 34 | 35 | $this->nodeTemplate->setNodeType($this->nodeType); 36 | $this->nodeTemplate->setName($this->renderNodeName($externalIdentifier)); 37 | 38 | if (!isset($data['mode'])) { 39 | throw new \Exception(sprintf('Could not determine command mode from data record with external identifier %s. Please make sure that "mode" exists in that record.', $externalIdentifier), 1462985246103); 40 | } 41 | 42 | $commandMethodName = $data['mode'] . 'Command'; 43 | if (!method_exists($this, $commandMethodName)) { 44 | throw new \Exception(sprintf('Could not find a command method "%s" in %s for processing record with external identifier %s.', $commandMethodName, get_class($this), $externalIdentifier), 1462985425892); 45 | } 46 | 47 | $this->$commandMethodName($externalIdentifier, $data); 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /Classes/Service/ProcessedNodeService.php: -------------------------------------------------------------------------------- 1 | importService->addOrUpdateRecordMapping($this->buildImporterClassName($importerClassName, $presetPath), $externalIdentifier, $externalRelativeUri, $nodeIdentifier, $nodePath); 43 | } 44 | 45 | /** 46 | * @param string $importerClassName 47 | * @param string $externalIdentifier 48 | * @return RecordMapping 49 | */ 50 | public function get($importerClassName, $externalIdentifier, $presetPath) 51 | { 52 | return $this->recordMappingRepository->findOneByImporterClassNameAndExternalIdentifier($this->buildImporterClassName($importerClassName, $presetPath), $externalIdentifier); 53 | } 54 | 55 | protected function buildImporterClassName($importerClassName, $presetPath) 56 | { 57 | return $importerClassName . '@' . $presetPath; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /Migrations/Mysql/Version20150317235257.php: -------------------------------------------------------------------------------- 1 | abortIf($this->connection->getDatabasePlatform()->getName() != "mysql"); 19 | 20 | /** 21 | * We need to check what the name of table flow accounts table is. If you install flow, run migrations, then add usermananagement, 22 | * then run the UserManagement migrations, the name will will be neos_flow_... 23 | * However, if you install everything in one go and run migrations then, the order will be different because this migration 24 | * comes before the Flow migration where the table is renamed (Version20161124185047). So we need to check which of these two 25 | * tables exist and set the FK relation accordingly. 26 | **/ 27 | if ($this->sm->tablesExist('neos_neos_eventlog_domain_model_event')) { 28 | // "neos_" table is there - this means flow migrations have already been run. 29 | $this->addSql("ALTER TABLE neos_neos_eventlog_domain_model_event ADD externalidentifier VARCHAR(255) DEFAULT NULL"); 30 | } else if ($this->sm->tablesExist('typo3_neos_eventlog_domain_model_event')) { 31 | // Flow migrations have not been run fully yet, table still has the old name. 32 | $this->addSql("ALTER TABLE typo3_neos_eventlog_domain_model_event ADD externalidentifier VARCHAR(255) DEFAULT NULL"); 33 | } 34 | $this->addSql("CREATE TABLE ttree_contentrepositoryimporter_domain_model_import (persistence_object_identifier VARCHAR(40) NOT NULL, start DATETIME NOT NULL, end DATETIME DEFAULT NULL, PRIMARY KEY(persistence_object_identifier))"); 35 | } 36 | 37 | /** 38 | * @param Schema $schema 39 | * @return void 40 | */ 41 | public function down(Schema $schema): void 42 | { 43 | $this->abortIf($this->connection->getDatabasePlatform()->getName() != "mysql"); 44 | 45 | $this->addSql("ALTER TABLE neos_neos_eventlog_domain_model_event DROP externalidentifier"); 46 | $this->addSql("DROP TABLE ttree_contentrepositoryimporter_domain_model_import"); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /Classes/DataType/HtmlContent.php: -------------------------------------------------------------------------------- 1 | options, 'htmlPurifierOptions') ?: []; 23 | foreach ($options as $optionName => $optionValue) { 24 | $config->set($optionName, $optionValue); 25 | } 26 | $purifier = new \HTMLPurifier($config); 27 | $value = $purifier->purify($value); 28 | 29 | // Todo add options in data type settings 30 | $value = str_replace([ 31 | '', 33 | '
    ', 34 | '
', 35 | ' ' 36 | ], [ 37 | PHP_EOL . '' . PHP_EOL, 39 | PHP_EOL . '
    ' . PHP_EOL, 40 | PHP_EOL . '
' . PHP_EOL, 41 | ' ' 42 | ], $value); 43 | 44 | // Normalize tag 45 | $options = Arrays::getValueByPath($this->options, 'preProcessing') ?: []; 46 | if (count($options)) { 47 | $value = preg_replace(array_keys($options), array_values($options), $value); 48 | } 49 | 50 | // Normalize line break 51 | $value = preg_replace('/(^[\r\n]*|[\r\n]+)[\s\t]*[\r\n]+/', "\n", $value); 52 | 53 | // Process line per line 54 | $lines = preg_split("/\\r\\n|\\r|\\n/", $value); 55 | foreach ($lines as $key => $line) { 56 | $line = trim($line); 57 | $options = Arrays::getValueByPath($this->options, 'processingPerLine') ?: []; 58 | if (count($options)) { 59 | $lines[$key] = preg_replace(array_keys($options), array_values($options), $line); 60 | } 61 | } 62 | $value = implode(' ' . PHP_EOL, $lines); 63 | 64 | // Global post processing 65 | $options = Arrays::getValueByPath($this->options, 'postProcessing') ?: []; 66 | if (count($options)) { 67 | $value = preg_replace(array_keys($options), array_values($options), $value); 68 | } 69 | 70 | // Return everything on one line 71 | $value = str_replace(PHP_EOL, '', $value); 72 | 73 | // Remove duplicated space and trim 74 | $this->value = trim(preg_replace('/\s+/u', ' ', $value)); 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /Migrations/Postgresql/Version20170718190551.php: -------------------------------------------------------------------------------- 1 | abortIf($this->connection->getDatabasePlatform()->getName() != 'postgresql', 'Migration can only be executed safely on "postgresql".'); 28 | 29 | $this->addSql('CREATE TABLE ttree_contentrepositoryimporter_domain_model_import (persistence_object_identifier VARCHAR(40) NOT NULL, starttime TIMESTAMP(0) WITHOUT TIME ZONE NOT NULL, endtime TIMESTAMP(0) WITHOUT TIME ZONE DEFAULT NULL, externalimportidentifier VARCHAR(255) DEFAULT NULL, PRIMARY KEY(persistence_object_identifier))'); 30 | $this->addSql('CREATE TABLE ttree_contentrepositoryimporter_domain_model_recordmapping (persistence_object_identifier VARCHAR(40) NOT NULL, creationdate TIMESTAMP(0) WITHOUT TIME ZONE NOT NULL, modificationdate TIMESTAMP(0) WITHOUT TIME ZONE DEFAULT NULL, importerclassname VARCHAR(255) NOT NULL, importerclassnamehash VARCHAR(32) NOT NULL, externalidentifier VARCHAR(255) NOT NULL, externalrelativeuri VARCHAR(255) DEFAULT NULL, nodeidentifier VARCHAR(255) NOT NULL, nodepath VARCHAR(4000) NOT NULL, nodepathhash VARCHAR(32) NOT NULL, PRIMARY KEY(persistence_object_identifier))'); 31 | $this->addSql('CREATE INDEX nodepathhash ON ttree_contentrepositoryimporter_domain_model_recordmapping (nodepathhash)'); 32 | $this->addSql('CREATE INDEX nodeidentifier ON ttree_contentrepositoryimporter_domain_model_recordmapping (nodeidentifier)'); 33 | $this->addSql('CREATE UNIQUE INDEX UNIQ_8C1932AD8FC282F11EE98D63 ON ttree_contentrepositoryimporter_domain_model_recordmapping (importerclassnamehash, externalidentifier)'); 34 | $this->addSql('ALTER TABLE neos_neos_eventlog_domain_model_event ADD externalidentifier VARCHAR(255) DEFAULT NULL'); 35 | } 36 | 37 | /** 38 | * @param Schema $schema 39 | * @return void 40 | */ 41 | public function down(Schema $schema): void 42 | { 43 | $this->abortIf($this->connection->getDatabasePlatform()->getName() != 'postgresql', 'Migration can only be executed safely on "postgresql".'); 44 | 45 | $this->addSql('DROP TABLE ttree_contentrepositoryimporter_domain_model_import'); 46 | $this->addSql('DROP TABLE ttree_contentrepositoryimporter_domain_model_recordmapping'); 47 | $this->addSql('ALTER TABLE neos_neos_eventlog_domain_model_event DROP externalidentifier'); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /Classes/Service/NodePropertyMapper.php: -------------------------------------------------------------------------------- 1 | $propertyValue) { 40 | if (!$propertyValidity->isValid($propertyName)) { 41 | continue; 42 | } 43 | if ($nodeOrTemplate->getProperty($propertyName) != $propertyValue) { 44 | $nodeOrTemplate->setProperty($propertyName, $propertyValue); 45 | $nodeChanged = true; 46 | } 47 | } 48 | 49 | if (isset($data['__identifier']) && \is_string($data['__identifier']) && $nodeOrTemplate instanceof NodeTemplate) { 50 | $nodeOrTemplate->setIdentifier(trim($data['__identifier'])); 51 | } 52 | 53 | if ($nodeOrTemplate instanceof NodeInterface) { 54 | $path = $nodeOrTemplate->getContextPath(); 55 | if ($nodeChanged) { 56 | $this->importService->addEventMessage('Node:Processed:Updated', sprintf('Updating existing node "%s" %s (%s)', $nodeOrTemplate->getLabel(), $path, $nodeOrTemplate->getIdentifier()), LogLevel::INFO, $currentEvent); 57 | } else { 58 | $this->importService->addEventMessage('Node:Processed:Skipped', sprintf('Skipping unchanged node "%s" %s (%s)', $nodeOrTemplate->getLabel(), $path, $nodeOrTemplate->getIdentifier()), LogLevel::NOTICE, $currentEvent); 59 | } 60 | } 61 | 62 | return $nodeChanged; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /Classes/Aspect/EventLogAspect.php: -------------------------------------------------------------------------------- 1 | processRecord())") 50 | * @param JoinPointInterface $joinPoint 51 | * @throws \Neos\Flow\Exception 52 | */ 53 | public function addRecordStartedEvent(JoinPointInterface $joinPoint) 54 | { 55 | $data = $joinPoint->getMethodArgument('data'); 56 | $externalIdentifier = Arrays::getValueByPath($data, '__externalIdentifier'); 57 | $title = Arrays::getValueByPath($data, '__label'); 58 | /** @var ImporterInterface $importer */ 59 | $importer = $joinPoint->getProxy(); 60 | list($importerClassName) = $this->getImporterClassNames($importer); 61 | 62 | $data['__message'] = sprintf('%s: "%s" (%s)', $importerClassName, $title ?: '-- No label --', $externalIdentifier); 63 | $event = $importer->getImportService()->addEvent(sprintf('%s:Record:Started', $importerClassName), $externalIdentifier, $data); 64 | $importer->setCurrentEvent($event); 65 | } 66 | 67 | /** 68 | * Flush all event after the ImporterInterface::processRecord() 69 | * 70 | * As an import batch can be a long running process, this ensure that the EventLog is flushed after each record processing 71 | * 72 | * @Flow\After("within(Ttree\ContentRepositoryImporter\Importer\ImporterInterface) && method(.*->processRecord())") 73 | * @param JoinPointInterface $joinPoint 74 | */ 75 | public function flushEvents(JoinPointInterface $joinPoint) 76 | { 77 | try { 78 | $this->importService->persistEntities(); 79 | $this->nodeDataRepository->persistEntities(); 80 | } catch (Exception $exception) { 81 | $logMessage = $this->throwableStorage->logThrowable($exception); 82 | $this->logger->error($logMessage, LogEnvironment::fromMethodName(__METHOD__)); 83 | } 84 | } 85 | 86 | /** 87 | * Get the Importer and DataProvider class name 88 | * 89 | * @param ImporterInterface $importer 90 | * @return array 91 | */ 92 | protected function getImporterClassNames(ImporterInterface $importer) 93 | { 94 | $importerClassName = get_class($importer); 95 | $importerClassName = substr($importerClassName, strrpos($importerClassName, '\\') + 1); 96 | 97 | $dataProvider = $importer->getDataProvider(); 98 | $dataProviderClassName = get_class($dataProvider); 99 | $dataProviderClassName = substr($dataProviderClassName, strrpos($dataProviderClassName, '\\') + 1); 100 | 101 | return [$importerClassName, $dataProviderClassName]; 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /Classes/DataProvider/CsvDataProvider.php: -------------------------------------------------------------------------------- 1 | options['csvFilePath']) || !is_string($this->options['csvFilePath'])) { 42 | throw new InvalidArgumentException('Missing or invalid "csvFilePath" in preset part settings', 1429027715); 43 | } 44 | 45 | $this->csvFilePath = $this->options['csvFilePath']; 46 | if (!is_file($this->csvFilePath)) { 47 | throw new \Exception(sprintf('File "%s" not found', $this->csvFilePath), 1427882078); 48 | } 49 | 50 | if (isset($this->options['csvDelimiter'])) { 51 | $this->csvDelimiter = $this->options['csvDelimiter']; 52 | } 53 | 54 | if (isset($this->options['csvEnclosure'])) { 55 | $this->csvEnclosure = $this->options['csvEnclosure']; 56 | } 57 | 58 | $this->logger->debug(sprintf('%s will read from "%s", using %s as delimiter and %s as enclosure character.', get_class($this), $this->csvFilePath, $this->csvDelimiter, $this->csvEnclosure)); 59 | } 60 | 61 | /** 62 | * Fetch all the data from this Data Source. 63 | * 64 | * If offset and / or limit are set, only those records will be returned. 65 | * 66 | * @return array The records 67 | */ 68 | public function fetch() 69 | { 70 | static $currentLine = 0; 71 | $dataResult = array(); 72 | 73 | if (isset($this->options['skipHeader']) && $this->options['skipHeader'] === true) { 74 | $skipLines = 1; 75 | } elseif (isset($this->options['skipHeader']) && \is_numeric($this->options['skipHeader'])) { 76 | $skipLines = (int)$this->options['skipHeader']; 77 | } else { 78 | $skipLines = 0; 79 | } 80 | if (($handle = fopen($this->csvFilePath, 'r')) !== false) { 81 | while (($data = fgetcsv($handle, 65534, $this->csvDelimiter, $this->csvEnclosure)) !== false) { 82 | // skip header (maybe is better to set the first offset position instead) 83 | if ($currentLine < $skipLines) { 84 | $currentLine++; 85 | continue; 86 | } 87 | if ($currentLine >= $this->offset && $currentLine < ($this->offset + $this->limit)) { 88 | if (isset($data[0]) && $data[0] !== '') { 89 | $this->preProcessRecordData($data); 90 | $dataResult[] = $data; 91 | } 92 | } 93 | $currentLine++; 94 | } 95 | fclose($handle); 96 | } 97 | $this->logger->debug(sprintf('%s: read %s lines and found %s records.', $this->csvFilePath, $currentLine, count($dataResult))); 98 | return $dataResult; 99 | } 100 | 101 | /** 102 | * Can be use to process data in children class 103 | * 104 | * @param array $data 105 | */ 106 | protected function preProcessRecordData(&$data) 107 | { 108 | } 109 | 110 | } 111 | -------------------------------------------------------------------------------- /Classes/DataProvider/AbstractDataProvider.php: -------------------------------------------------------------------------------- 1 | options = $options; 84 | $this->presetName = $options['__presetName']; 85 | $this->partName = $options['__partName']; 86 | $this->vault = $vault; 87 | } 88 | 89 | /** 90 | * Factory method which returns an instance of the concrete implementation and passes the given options, offset 91 | * and limit to it. 92 | * 93 | * @param array $options Options for the Data Provider 94 | * @param integer $offset Record offset where the import should start 95 | * @param integer $limit Maximum number of records which should be imported 96 | * @param array $exceedingArguments Exceeding arguments of the command 97 | * @return DataProviderInterface 98 | */ 99 | public static function create(array $options = [], $offset = null, $limit = null, $exceedingArguments = null): DataProviderInterface 100 | { 101 | $dataProvider = new static($options, new Vault($options['__presetName'])); 102 | $dataProvider->setOffset($offset); 103 | $dataProvider->setLimit($limit); 104 | if ($exceedingArguments) { 105 | $dataProvider->setExceedingArguments($exceedingArguments); 106 | } 107 | 108 | return $dataProvider; 109 | } 110 | 111 | /** 112 | * Set the offset (record number) to start importing at 113 | * 114 | * @param int $offset 115 | */ 116 | public function setOffset($offset) 117 | { 118 | $this->offset = (int)$offset; 119 | } 120 | 121 | /** 122 | * Set the maximum number of records to import 123 | * 124 | * @param integer $limit 125 | */ 126 | public function setLimit($limit) 127 | { 128 | $this->limit = (integer)$limit; 129 | } 130 | 131 | /** 132 | * If a maximum number of records has been set 133 | * 134 | * @return boolean 135 | */ 136 | public function hasLimit() 137 | { 138 | return $this->limit > 0; 139 | } 140 | 141 | /** 142 | * @return array 143 | */ 144 | public function getExceedingArguments() 145 | { 146 | return $this->exceedingArguments; 147 | } 148 | 149 | /** 150 | * @param array $exceedingArguments 151 | */ 152 | public function setExceedingArguments($exceedingArguments) 153 | { 154 | $this->exceedingArguments = $exceedingArguments; 155 | } 156 | 157 | } 158 | -------------------------------------------------------------------------------- /Classes/Domain/Model/RecordMapping.php: -------------------------------------------------------------------------------- 1 | importerClassName = $importerClassName; 85 | $this->importerClassNameHash = md5($importerClassName); 86 | $this->externalIdentifier = $externalIdentifier; 87 | $this->externalRelativeUri = $externalRelativeUri; 88 | $this->nodeIdentifier = $nodeIdentifier; 89 | $this->nodePath = $nodePath; 90 | $this->nodePathHash = md5($nodePath); 91 | } 92 | 93 | /** 94 | * @return string 95 | */ 96 | public function getImporterClassName() 97 | { 98 | return $this->importerClassName; 99 | } 100 | 101 | /** 102 | * @return string 103 | */ 104 | public function getExternalIdentifier() 105 | { 106 | return $this->externalIdentifier; 107 | } 108 | 109 | /** 110 | * @return string 111 | */ 112 | public function getExternalRelativeUri() 113 | { 114 | return $this->externalRelativeUri; 115 | } 116 | 117 | /** 118 | * @param string $externalRelativeUri 119 | */ 120 | public function setExternalRelativeUri($externalRelativeUri) 121 | { 122 | $this->externalRelativeUri = $externalRelativeUri; 123 | } 124 | 125 | /** 126 | * @return string 127 | */ 128 | public function getNodeIdentifier() 129 | { 130 | return $this->nodeIdentifier; 131 | } 132 | 133 | /** 134 | * @param string $nodeIdentifier 135 | */ 136 | public function setNodeIdentifier($nodeIdentifier) 137 | { 138 | $this->nodeIdentifier = $nodeIdentifier; 139 | } 140 | 141 | /** 142 | * @return string 143 | */ 144 | public function getNodePath() 145 | { 146 | return $this->nodePath; 147 | } 148 | 149 | /** 150 | * @param string $nodePath 151 | */ 152 | public function setNodePath($nodePath) 153 | { 154 | $this->nodePath = $nodePath; 155 | $this->nodePathHash = md5($nodePath); 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "Helper package to import data in the Neos content repository", 3 | "name": "ttree/contentrepositoryimporter", 4 | "type": "neos-package", 5 | "license": "MIT", 6 | "require": { 7 | "neos/neos": "^7.0 || ^8.0", 8 | "ezyang/htmlpurifier": "^4.10.0", 9 | "cocur/slugify": "^2.5" 10 | }, 11 | "autoload": { 12 | "psr-4": { 13 | "Ttree\\ContentRepositoryImporter\\": "Classes" 14 | } 15 | }, 16 | "extra": { 17 | "branch-alias": { 18 | "dev-master": "4.1.x-dev" 19 | }, 20 | "applied-flow-migrations": [ 21 | "TYPO3.FLOW3-201201261636", 22 | "TYPO3.Fluid-201205031303", 23 | "TYPO3.FLOW3-201205292145", 24 | "TYPO3.FLOW3-201206271128", 25 | "TYPO3.FLOW3-201209201112", 26 | "TYPO3.Flow-201209251426", 27 | "TYPO3.Flow-201211151101", 28 | "TYPO3.Flow-201212051340", 29 | "TYPO3.TypoScript-130516234520", 30 | "TYPO3.TypoScript-130516235550", 31 | "TYPO3.TYPO3CR-130523180140", 32 | "TYPO3.Neos.NodeTypes-201309111655", 33 | "TYPO3.Flow-201310031523", 34 | "TYPO3.Flow-201405111147", 35 | "TYPO3.Neos-201407061038", 36 | "TYPO3.Neos-201409071922", 37 | "TYPO3.TYPO3CR-140911160326", 38 | "TYPO3.Neos-201410010000", 39 | "TYPO3.TYPO3CR-141101082142", 40 | "TYPO3.Neos-20141113115300", 41 | "TYPO3.Fluid-20141113120800", 42 | "TYPO3.Flow-20141113121400", 43 | "TYPO3.Fluid-20141121091700", 44 | "TYPO3.Neos-20141218134700", 45 | "TYPO3.Fluid-20150214130800", 46 | "TYPO3.Neos-20150303231600", 47 | "TYPO3.TYPO3CR-20150510103823", 48 | "TYPO3.Flow-20151113161300", 49 | "TYPO3.Form-20160601101500", 50 | "TYPO3.Flow-20161115140400", 51 | "TYPO3.Flow-20161115140430", 52 | "Neos.Flow-20161124204700", 53 | "Neos.Flow-20161124204701", 54 | "Neos.Twitter.Bootstrap-20161124204912", 55 | "Neos.Form-20161124205254", 56 | "Neos.Flow-20161124224015", 57 | "Neos.Party-20161124225257", 58 | "Neos.Eel-20161124230101", 59 | "Neos.Kickstart-20161124230102", 60 | "Neos.Setup-20161124230842", 61 | "Neos.Imagine-20161124231742", 62 | "Neos.Media-20161124233100", 63 | "Neos.NodeTypes-20161125002300", 64 | "Neos.SiteKickstarter-20161125002311", 65 | "Neos.Neos-20161125002322", 66 | "Neos.ContentRepository-20161125012000", 67 | "Neos.Fusion-20161125013710", 68 | "Neos.Setup-20161125014759", 69 | "Neos.SiteKickstarter-20161125095901", 70 | "Neos.Fusion-20161125104701", 71 | "Neos.NodeTypes-20161125104800", 72 | "Neos.Neos-20161125104802", 73 | "Neos.Kickstarter-20161125110814", 74 | "Neos.Neos-20161125122412", 75 | "Neos.Flow-20161125124112", 76 | "TYPO3.FluidAdaptor-20161130112935", 77 | "Neos.Fusion-20161201202543", 78 | "Neos.Neos-20161201222211", 79 | "Neos.Fusion-20161202215034", 80 | "Neos.Fusion-20161219092345", 81 | "Neos.ContentRepository-20161219093512", 82 | "Neos.Media-20161219094126", 83 | "Neos.Neos-20161219094403", 84 | "Neos.Neos-20161219122512", 85 | "Neos.Fusion-20161219130100", 86 | "Neos.Neos-20161220163741", 87 | "Neos.Neos-20170115114620", 88 | "Neos.Fusion-20170120013047", 89 | "Neos.Flow-20170125103800", 90 | "Neos.Seo-20170127154600", 91 | "Neos.Flow-20170127183102", 92 | "Neos.SwiftMailer-20161130105617", 93 | "Neos.ContentRepository.Search-20161210231100", 94 | "Neos.Flow-20180415105700", 95 | "Neos.Neos-20180907103800", 96 | "Neos.Neos.Ui-20190319094900", 97 | "Neos.Flow-20190425144900", 98 | "Neos.Flow-20190515215000", 99 | "Neos.Flow-20200813181400", 100 | "Neos.Flow-20201003165200", 101 | "Neos.Flow-20201109224100", 102 | "Neos.Flow-20201205172733", 103 | "Neos.Flow-20201207104500" 104 | ] 105 | } 106 | } -------------------------------------------------------------------------------- /Classes/Domain/Model/Import.php: -------------------------------------------------------------------------------- 1 | startTime = new \DateTimeImmutable(); 54 | $this->addImportStartedEvent(); 55 | } 56 | } 57 | 58 | /** 59 | * Sets an (external) identifier which allows for detecting already imported data sets. 60 | * 61 | * @param string $externalImportIdentifier 62 | */ 63 | public function setExternalImportIdentifier($externalImportIdentifier) 64 | { 65 | $this->externalImportIdentifier = $externalImportIdentifier; 66 | } 67 | 68 | /** 69 | * @return string 70 | */ 71 | public function getExternalImportIdentifier() 72 | { 73 | return $this->externalImportIdentifier; 74 | } 75 | 76 | /** 77 | * @param string $eventType 78 | * @param string $externalIdentifier 79 | * @param array $data 80 | * @param Event $parentEvent 81 | * @return Event 82 | */ 83 | public function addEvent($eventType, $externalIdentifier = null, array $data = null, Event $parentEvent = null) 84 | { 85 | if (is_array($data) && isset($data['__message'])) { 86 | $message = $parentEvent ? sprintf('- %s', $data['__message']) : $data['__message']; 87 | $severity = isset($data['__severity']) ? $data['__severity'] : LogLevel::INFO; 88 | $this->logger->log($severity, $message); 89 | } 90 | $event = new Event($eventType, $data, null, $parentEvent); 91 | $event->setExternalIdentifier($externalIdentifier); 92 | try { 93 | $this->eventEmittingService->add($event); 94 | } catch (\Exception $exception) { 95 | } 96 | 97 | return $event; 98 | } 99 | 100 | /** 101 | * Add a Import.Started event in the EventLog 102 | */ 103 | protected function addImportStartedEvent() 104 | { 105 | $event = new Event('Import.Started', array()); 106 | try { 107 | $this->eventEmittingService->add($event); 108 | } catch (\Exception $exception) { 109 | } 110 | } 111 | 112 | /** 113 | * Add a Import.Ended event in the EventLog 114 | */ 115 | protected function addImportEndedEvent() 116 | { 117 | try { 118 | $event = new Event('Import.Ended', array()); 119 | $this->eventEmittingService->add($event); 120 | } catch (\Exception $exception) { 121 | } 122 | } 123 | 124 | /** 125 | * @return \DateTime 126 | */ 127 | public function getStartTime() 128 | { 129 | return $this->startTime; 130 | } 131 | 132 | /** 133 | * @return \DateTime 134 | */ 135 | public function getEndTime() 136 | { 137 | return $this->endTime; 138 | } 139 | 140 | /** 141 | * @return integer 142 | */ 143 | public function getElapsedTime() 144 | { 145 | return (integer) $this->endTime->getTimestamp() - $this->startTime->getTimestamp(); 146 | } 147 | 148 | /** 149 | * @throws Exception 150 | */ 151 | public function end() 152 | { 153 | if ($this->endTime instanceof \DateTimeImmutable) { 154 | throw new Exception('This import has ended earlier', 1426763297); 155 | } 156 | $this->endTime = new \DateTimeImmutable(); 157 | $this->addImportEndedEvent(); 158 | } 159 | } 160 | -------------------------------------------------------------------------------- /Classes/Domain/Model/PresetPartDefinition.php: -------------------------------------------------------------------------------- 1 | currentPresetName = trim($setting['__currentPresetName']); 78 | if (!isset($setting['__currentPartName'])) { 79 | throw new InvalidArgumentException('Missing or invalid "__currentPartName" in preset part settings', 1426156155); 80 | } 81 | $this->currentPartName = trim($setting['__currentPartName']); 82 | if (!isset($setting['label']) || !is_string($setting['label'])) { 83 | throw new InvalidArgumentException('Missing or invalid "Label" in preset part settings', 1426156157); 84 | } 85 | $this->label = (string)$setting['label']; 86 | if (!isset($setting['dataProviderClassName']) || !is_string($setting['dataProviderClassName'])) { 87 | throw new InvalidArgumentException('Missing or invalid "dataProviderClassName" in preset part settings', 1426156158); 88 | } 89 | $this->dataProviderClassName = (string)$setting['dataProviderClassName']; 90 | if (!isset($setting['importerClassName']) || !is_string($setting['importerClassName'])) { 91 | throw new InvalidArgumentException('Missing or invalid "importerClassName" in preset part settings', 1426156159); 92 | } 93 | $this->importerClassName = (string)$setting['importerClassName']; 94 | $this->batchSize = isset($setting['batchSize']) ? (integer)$setting['batchSize'] : null; 95 | $this->offset = isset($setting['batchSize']) ? 0 : null; 96 | $this->dataProviderOptions = isset($setting['dataProviderOptions']) ? $setting['dataProviderOptions'] : []; 97 | $this->currentBatch = 1; 98 | $this->currentImportIdentifier = $currentImportIdentifier; 99 | $this->debug = (isset($setting['debug']) && $setting['debug'] === true) ? (boolean)$setting['debug'] : false; 100 | if ($this->debug === true) { 101 | $this->batchSize = 1; 102 | } 103 | } 104 | 105 | /** 106 | * Increment the batch number 107 | */ 108 | public function nextBatch() 109 | { 110 | ++$this->currentBatch; 111 | $this->offset += $this->batchSize; 112 | } 113 | 114 | /** 115 | * @return string 116 | */ 117 | public function getEventType() 118 | { 119 | return sprintf('Preset%s:%s', ucfirst($this->currentPresetName), ucfirst($this->currentPartName)); 120 | } 121 | 122 | /** 123 | * @return array 124 | */ 125 | public function getCommandArguments() 126 | { 127 | $arguments = [ 128 | 'presetName' => $this->currentPresetName, 129 | 'partName' => $this->currentPartName, 130 | 'currentImportIdentifier' => $this->currentImportIdentifier, 131 | 'dataProviderClassName' => $this->dataProviderClassName, 132 | 'importerClassName' => $this->importerClassName, 133 | 'currentBatch' => $this->currentBatch 134 | ]; 135 | if ($this->batchSize) { 136 | $arguments['batchSize'] = (integer)$this->batchSize; 137 | } else { 138 | $arguments['batchSize'] = 100000; 139 | } 140 | if ($this->offset) { 141 | $arguments['offset'] = (integer)$this->offset; 142 | } 143 | return $arguments; 144 | } 145 | 146 | /** 147 | * @return string 148 | */ 149 | public function getLabel() 150 | { 151 | return $this->label; 152 | } 153 | 154 | /** 155 | * @return boolean 156 | */ 157 | public function isDebug() 158 | { 159 | return $this->debug; 160 | } 161 | 162 | /** 163 | * @return string 164 | */ 165 | public function getCurrentImportIdentifier() 166 | { 167 | return $this->currentImportIdentifier; 168 | } 169 | 170 | /** 171 | * @return string 172 | */ 173 | public function getDataProviderClassName() 174 | { 175 | return $this->dataProviderClassName; 176 | } 177 | 178 | /** 179 | * @return string 180 | */ 181 | public function getImporterClassName() 182 | { 183 | return $this->importerClassName; 184 | } 185 | 186 | /** 187 | * @return int 188 | */ 189 | public function getCurrentBatch() 190 | { 191 | return $this->currentBatch; 192 | } 193 | 194 | /** 195 | * @return int 196 | */ 197 | public function getBatchSize() 198 | { 199 | return $this->batchSize; 200 | } 201 | 202 | /** 203 | * @return int 204 | */ 205 | public function getOffset() 206 | { 207 | return $this->offset; 208 | } 209 | } 210 | -------------------------------------------------------------------------------- /Classes/DataType/ExternalResource.php: -------------------------------------------------------------------------------- 1 | rawValue['forceDownload'] = true; 48 | } 49 | 50 | /** 51 | * Enable force download 52 | */ 53 | public function disableForceDownload() 54 | { 55 | $this->rawValue['forceDownload'] = false; 56 | } 57 | 58 | /** 59 | * @param string $value 60 | * @throws Exception 61 | * @throws \Neos\Flow\ResourceManagement\Exception 62 | * @throws \Neos\Flow\Utility\Exception 63 | */ 64 | protected function initializeValue($value) 65 | { 66 | if (!is_array($value)) { 67 | throw new Exception('Value must be an array, with source URI (sourceUri) and filename (filename)', 1425981082); 68 | } 69 | if (!isset($value['sourceUri'])) { 70 | throw new Exception('Missing source URI', 1425981083); 71 | } 72 | $sourceUri = trim($value['sourceUri']); 73 | if (!isset($value['filename'])) { 74 | throw new Exception('Missing filename URI', 1425981084); 75 | } 76 | $filename = trim($value['filename']); 77 | $fileExtension = strtolower(trim(pathinfo($filename, PATHINFO_EXTENSION))); 78 | $overrideFilename = isset($value['overrideFilename']) ? trim($value['overrideFilename']) : pathinfo($filename, PATHINFO_FILENAME); 79 | if ($fileExtension) { 80 | $overrideFilename = sprintf('%s.%s', $overrideFilename, $fileExtension); 81 | } 82 | 83 | if (!isset($this->options['downloadDirectory'])) { 84 | throw new Exception('Missing download directory data type option', 1425981085); 85 | } 86 | Files::createDirectoryRecursively($this->options['downloadDirectory']); 87 | 88 | $temporaryFilename = isset($value['temporaryPrefix']) ? trim(sprintf('%s-%s', $value['temporaryPrefix'], $overrideFilename)) : trim($overrideFilename); 89 | $temporaryFileAndPathname = sprintf('%s%s', $this->options['downloadDirectory'], $temporaryFilename); 90 | 91 | $username = isset($value['username']) ? $value['username'] : null; 92 | $password = isset($value['password']) ? $value['password'] : null; 93 | $this->download($sourceUri, $temporaryFileAndPathname, isset($value['forceDownload']) ? (boolean)$value['forceDownload'] : false, $username, $password); 94 | 95 | # Try to add file extenstion if missing 96 | if ($fileExtension === '') { 97 | $mimeTypeGuesser = new MimeTypeGuesser(); 98 | $mimeType = $mimeTypeGuesser->guess($temporaryFileAndPathname); 99 | $this->logger->debug(sprintf('Try to guess mime type for "%s" (%s), result: %s', $sourceUri, $filename, $mimeType)); 100 | $fileExtension = MediaTypes::getFilenameExtensionFromMediaType($mimeType); 101 | if ($fileExtension !== '') { 102 | $oldTemporaryDestination = $temporaryFileAndPathname; 103 | $temporaryFileAndPathname = sprintf('%s.%s', $temporaryFileAndPathname, $fileExtension); 104 | if (!is_file($temporaryFileAndPathname)) { 105 | copy($oldTemporaryDestination, $temporaryFileAndPathname); 106 | $this->logger->debug(sprintf('Rename "%s" to "%s"', $oldTemporaryDestination, $temporaryFileAndPathname)); 107 | } 108 | } 109 | } 110 | # Trim border 111 | if (isset($value['trimBorder']) && $value['trimBorder'] === true) { 112 | $this->trimImageBorder($temporaryFileAndPathname); 113 | } 114 | 115 | $sha1Hash = sha1_file($temporaryFileAndPathname); 116 | $resource = $this->resourceManager->getResourceBySha1($sha1Hash); 117 | if ($resource === null) { 118 | $this->logger->debug('Import new resource'); 119 | $resource = $this->resourceManager->importResource($temporaryFileAndPathname); 120 | $resource->setFilename(basename($temporaryFileAndPathname)); 121 | } else { 122 | $this->logger->debug('Use existing resource'); 123 | } 124 | 125 | $this->temporaryFileAndPathname = $temporaryFileAndPathname; 126 | 127 | $this->value = $resource; 128 | } 129 | 130 | /** 131 | * @param string $fileAndPathname 132 | */ 133 | protected function trimImageBorder($fileAndPathname) 134 | { 135 | $isProcessed = sprintf('%s.trimmed', $fileAndPathname); 136 | if (is_file($isProcessed)) { 137 | return; 138 | } 139 | $isImage = @getimagesize($fileAndPathname) ? true : false; 140 | if (!$isImage) { 141 | return; 142 | } 143 | $command = sprintf('convert "%s" -trim "%s" > /dev/null 2> /dev/null', $fileAndPathname, $fileAndPathname); 144 | exec($command, $output, $result); 145 | touch($isProcessed); 146 | } 147 | 148 | /** 149 | * @return string 150 | */ 151 | public function getTemporaryFileAndPathname() 152 | { 153 | return $this->temporaryFileAndPathname; 154 | } 155 | 156 | /** 157 | * @return string 158 | */ 159 | public function getSourceUri() 160 | { 161 | return $this->rawValue['sourceUri']; 162 | } 163 | 164 | /** 165 | * @param string $source 166 | * @param string $destination 167 | * @param boolean $force 168 | * @param string|null $username 169 | * @param string|null $password 170 | * @return boolean 171 | * @throws Exception 172 | */ 173 | protected function download($source, $destination, $force = false, $username = null, $password = null) 174 | { 175 | if ($force === false && is_file($destination)) { 176 | $this->logger->debug(sprintf('External resource "%s" skipped, local file "%s" exist', $source, $destination)); 177 | return true; 178 | } 179 | $fp = fopen($destination, 'w'); 180 | if (!$fp) { 181 | throw new Exception(sprintf('Unable to download the given file: %s', $source)); 182 | } 183 | 184 | $ch = curl_init(str_replace(" ", "%20", $source)); 185 | curl_setopt($ch, CURLOPT_TIMEOUT, 50); 186 | curl_setopt($ch, CURLOPT_FILE, $fp); 187 | curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); 188 | if ($username !== null && $password !== null) { 189 | curl_setopt($ch, CURLOPT_USERPWD, sprintf('%s:%s', $username, $password)); 190 | } 191 | curl_exec($ch); 192 | curl_close($ch); 193 | 194 | fclose($fp); 195 | 196 | $this->logger->debug(sprintf('External resource "%s" downloaded to "%s"', $source, $destination)); 197 | 198 | return true; 199 | } 200 | } 201 | -------------------------------------------------------------------------------- /Classes/Domain/Service/ImportService.php: -------------------------------------------------------------------------------- 1 | currentImport instanceof Import) { 57 | throw new Exception('Unable to resume, please stop the current import first', 1426638560); 58 | } 59 | $this->currentImport = $this->importRepository->findByIdentifier($identifier); 60 | } 61 | 62 | /** 63 | * Start a new Import 64 | * 65 | * @param string $identifier 66 | * @param boolean $force 67 | * @throws Exception 68 | * @throws IllegalObjectTypeException 69 | */ 70 | public function start($identifier = null, $force = false) 71 | { 72 | if ($this->currentImport instanceof Import) { 73 | throw new Exception('Unable to start a new import, please stop the current import first', 1426638560); 74 | } 75 | 76 | if ($identifier !== null) { 77 | $existingImport = $this->importRepository->findOneByExternalImportIdentifier($identifier); 78 | if (!$force && $existingImport instanceof Import) { 79 | throw new ImportAlreadyExecutedException(sprintf('An import referring to the external identifier "%s" has already been executed on %s.', $identifier, $existingImport->getStartTime()->format('d.m.Y h:m:s')), 1464028408403); 80 | } 81 | } 82 | 83 | $this->currentImport = new Import(); 84 | $this->currentImport->setExternalImportIdentifier($identifier); 85 | 86 | if ($force && isset($existingImport)) { 87 | $this->addEventMessage(sprintf('ImportService:start', 'Forcing re-import of data set with external identifier "%s".', $identifier), LogLevel::NOTICE); 88 | } 89 | 90 | $this->importRepository->add($this->currentImport); 91 | $this->persistenceManager->persistAll(); 92 | } 93 | 94 | /** 95 | * Stop and store the current Import 96 | */ 97 | public function stop() 98 | { 99 | if (!$this->currentImport instanceof Import) { 100 | throw new Exception('Unable to stop the current import, please start an import first', 1426638563); 101 | } 102 | $this->currentImport->end(); 103 | $this->importRepository->update($this->currentImport); 104 | $this->lastImport = clone $this->currentImport; 105 | unset($this->currentImport); 106 | } 107 | 108 | /** 109 | * @param string $importerClassName 110 | * @param string $externalIdentifier 111 | * @param string $externalRelativeUri 112 | * @param string $nodeIdentifier 113 | * @param string $nodePath 114 | * @throws Exception 115 | * @throws IllegalObjectTypeException 116 | */ 117 | public function addOrUpdateRecordMapping($importerClassName, $externalIdentifier, $externalRelativeUri, $nodeIdentifier, $nodePath) 118 | { 119 | $recordMapping = $this->recordMappingRepository->findOneByImporterClassNameAndExternalIdentifier($importerClassName, $externalIdentifier); 120 | if ($recordMapping === null) { 121 | $recordMapping = new RecordMapping($importerClassName, $externalIdentifier, $externalRelativeUri, $nodeIdentifier, $nodePath); 122 | $this->recordMappingRepository->add($recordMapping); 123 | } else { 124 | $recordMapping->setExternalRelativeUri($externalRelativeUri); 125 | $recordMapping->setNodeIdentifier($nodeIdentifier); 126 | $recordMapping->setNodePath($nodePath); 127 | $this->recordMappingRepository->update($recordMapping); 128 | } 129 | 130 | $this->recordMappingRepository->persistEntities(); 131 | $this->addEvent(sprintf('%s:Record:Ended', substr($importerClassName, strrpos($importerClassName, '\\') + 1)), $externalIdentifier, [ 132 | 'importerClassName' => $importerClassName, 133 | 'externalIdentifier' => $externalIdentifier, 134 | 'externalRelativeUri' => $externalRelativeUri, 135 | 'nodeIdentifier' => $nodeIdentifier, 136 | 'nodePath' => $nodePath 137 | ]); 138 | } 139 | 140 | /** 141 | * @param string $eventType 142 | * @param string $externalIdentifier 143 | * @param array $data 144 | * @param Event $parentEvent 145 | * @return Event 146 | * @throws Exception 147 | */ 148 | public function addEvent($eventType, $externalIdentifier = null, array $data = null, Event $parentEvent = null) 149 | { 150 | if (!$this->currentImport instanceof Import) { 151 | throw new Exception('Unable to add an event, please start an import first', 1426638562); 152 | } 153 | $event = $this->currentImport->addEvent($eventType, $externalIdentifier, $data ?: array(), $parentEvent); 154 | 155 | return $event; 156 | } 157 | 158 | /** 159 | * @param string $eventType 160 | * @param string $message 161 | * @param int $severity 162 | * @param Event $parentEvent 163 | * @return Event 164 | * @throws Exception 165 | */ 166 | public function addEventMessage($eventType, $message, $severity = LogLevel::INFO, Event $parentEvent = null) 167 | { 168 | if (!$this->currentImport instanceof Import) { 169 | throw new Exception('Unable to add an event, please start an import first', 1426638563); 170 | } 171 | $event = $this->currentImport->addEvent($eventType, null, [ 172 | '__message' => $message, 173 | '__severity' => $severity 174 | ], $parentEvent); 175 | 176 | return $event; 177 | } 178 | 179 | /** 180 | * Persist all pending entities 181 | */ 182 | public function persistEntities() 183 | { 184 | $this->importRepository->persistEntities(); 185 | } 186 | 187 | /** 188 | * @return Import 189 | * @throws Exception 190 | */ 191 | public function getCurrentImport() 192 | { 193 | if (!$this->currentImport instanceof Import) { 194 | throw new Exception('Unable to get current import, please start an import first', 1426638561); 195 | } 196 | return $this->currentImport; 197 | } 198 | 199 | /** 200 | * @return Import 201 | * @throws Exception 202 | */ 203 | public function getLastImport() 204 | { 205 | if (!$this->lastImport instanceof Import) { 206 | throw new Exception('Last import is not set', 1426638561); 207 | } 208 | return $this->lastImport; 209 | } 210 | 211 | /** 212 | * @return string 213 | * @throws Exception 214 | */ 215 | public function getCurrentImportIdentifier() 216 | { 217 | if (!$this->currentImport instanceof Import) { 218 | throw new Exception('Unable to get import identifier, please start an import first', 1426638561); 219 | } 220 | return $this->persistenceManager->getIdentifierByObject($this->currentImport); 221 | } 222 | } 223 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ContentRepositoryImporter 2 | ========================= 3 | 4 | This package contains generic utility to help importing data in the Neos Content Repository. 5 | 6 | What's included ? 7 | ----------------- 8 | 9 | * A command controller (CLI) to launch your import presets 10 | * Based on simple conventions 11 | * DataProvider: used to prepare and cleanup data from the external source 12 | * Importer: get the data from the DataProvider and push everything in the CR 13 | * DataType: Simple object used to cleanup value and share code between DataProvider 14 | * Split your import in multiple sub commands to avoid high memory usage 15 | * No big magic, you can always take control by overriding the default configuration and methods 16 | 17 | A basic DataProvider 18 | -------------------- 19 | 20 | Every data provider must extend the ``DataProvider`` abstract class or implement the 21 | interface ```DataProviderInterface```. Check the source code of the abstract data provider, there are some useful things 22 | to discover. 23 | 24 | It's important to update the ```count``` property when you process data from the external source. During the processing, 25 | you can decide to skip some data (invalid data, missing values, ...) so we can not use the SQL count feature. 26 | 27 | Try to do most of the data cleaning up in the data provider, so the data would arrive to the importer ready for insertion. 28 | Basically the array build by the provider should contains the data with the property name that match your node type property name. 29 | If you need to transport value that will not match the node properties, please prefix them with '_'. 30 | 31 | There is some magic value, those values MUST be on the first level of the array: 32 | 33 | - **__identifier** (optional) This UUID will be used in the imported node, you should use ```AbstractImporter::applyProperties``` to have this feature, used by default 34 | - **__externalIdentifier** (required) The external identifier of the data, this one is really important. The package keep track of imported data 35 | - **__label** (required) The label of this record used by the importer mainly for logging (this value is not imported, but useful to follow the process) 36 | if you run twice the same import, the imported node will be updated and not created. 37 | 38 | **Tips**: If the properties of your nodes are not at the first level of the array, you can override the method ```AbstractImporter::getPropertiesFromDataProviderPayload``` 39 | 40 | ### Output of the provider 41 | 42 | Your provider should output something like this: 43 | 44 | ``` 45 | [ 46 | '__label' => 'The external content lable, for internal use' 47 | '__externalIdentifier' => 'The external external identifier, for internal use' 48 | 'title' => 'My title' 49 | 'year' => 1999 50 | 'text' => '...' 51 | ] 52 | ``` 53 | 54 | **Tips**: If your provider does not return an array, you MUST registrer a TypeConverter to convert it to an array. The property mapper is 55 | used automatically by the Importer. 56 | 57 | ### Content Dimensions support 58 | 59 | If your data provider follow this convention, the importer can automatically create variants of your nodes: 60 | 61 | ``` 62 | [ 63 | '__label' => 'The external content lable, for internal use' 64 | '__externalIdentifier' => 'The external external identifier, for internal use' 65 | 'title' => 'My title' 66 | 'year' => 1999 67 | 'text' => '...', 68 | 69 | '@dimensions' => [ 70 | '@en' => [ 71 | '@strategy' => 'merge', 72 | 'title' => '...', 73 | ], 74 | '@fr' => [ 75 | '@strategy' => 'merge', 76 | 'title' => '...', 77 | ], 78 | ] 79 | ] 80 | ``` 81 | 82 | The ```@en``` is a preset name, you must configuration the presets on your ```Settings.yaml```: 83 | 84 | ``` 85 | Ttree: 86 | ContentRepositoryImporter: 87 | dimensionsImporter: 88 | presets: 89 | fr: 90 | language: ['fr', 'en', 'de'] 91 | en: 92 | language: ['en', 'de'] 93 | de: 94 | language: ['de'] 95 | ``` 96 | 97 | ### Share data between preset parts 98 | 99 | You can split your import in multiple parts. Each parts is executed in a separate request. Sometimes it's useful to share data between parts (ex. in the first 100 | part you import the taxonomy, and in the second parts you map documents with the taxonomy). Those solve this use case, we integrate a feature called **Vault**. The 101 | Vault is simply a cache accessible in the importer and data provider by calling ```$this->vault->set($key, $name)``` and ```$this->vault->get($key)```. The 102 | current preset is the namespace, so you can use simple keys like name, ids, ... 103 | 104 | The cache is flushed if you call ```flow import:init --preset your-preset```. 105 | 106 | ### Basic provider 107 | 108 | ```php 109 | class BasicDataProvider extends DataProvider { 110 | 111 | /** 112 | * @return array 113 | */ 114 | public function fetch() { 115 | $result = []; 116 | $query = $this->createQuery() 117 | ->select('*') 118 | ->from('demo_table', 'd') 119 | ->orderBy('d.name'); 120 | 121 | $statement = $query->execute(); 122 | while ($demoRecord = $statement->fetch()) { 123 | $result[] = [ 124 | '__externalIdentifier' => (integer)$demoRecord['id'], 125 | 'name' => String::create($demoRecord['name'])->getValue() 126 | ]; 127 | } 128 | 129 | $this->count = count($result); 130 | 131 | return $result; 132 | } 133 | 134 | } 135 | ``` 136 | 137 | A basic Importer 138 | ---------------- 139 | 140 | Every data importer must extend the ``AbstractImporter`` abstract class or implement the interface ```ImporterInterface```. 141 | 142 | In the `processRecord` method you handle the processing of every record, such as creating Content Repository node for each incoming data record. 143 | 144 | Do not forget to register the processed nodes with `registerNodeProcessing`. The method will handle feature like logging and tracking of imported node to decide if the local node need to be created or updated. 145 | 146 | ```php 147 | class ProductImporter extends AbstractImporter 148 | { 149 | 150 | /** 151 | * @var string 152 | */ 153 | protected $externalIdentifierDataKey = 'productNumber'; 154 | 155 | /** 156 | * @var string 157 | */ 158 | protected $labelDataKey = 'properties.name'; 159 | 160 | /** 161 | * @var string 162 | */ 163 | protected $nodeNamePrefix = 'product-'; 164 | 165 | /** 166 | * @var string 167 | */ 168 | protected $nodeTypeName = 'Acme.Demo:Product'; 169 | 170 | /** 171 | * Starts batch processing all commands 172 | * 173 | * @return void 174 | * @api 175 | */ 176 | public function process() 177 | { 178 | $this->initializeStorageNode('shop/products', 'products', 'Products', 'products'); 179 | $this->initializeNodeTemplates(); 180 | 181 | $nodeTemplate = new NodeTemplate(); 182 | $this->processBatch($nodeTemplate); 183 | } 184 | 185 | } 186 | 187 | ``` 188 | 189 | A basic preset 190 | -------------- 191 | 192 | You can configure an import preset in your ```Settings.yaml```. A preset is split in multiple parts. If you use the 193 | ```batchSize```, the current part will be executed by batch, by using a sub CLI request. This can solve memory or 194 | performance issue for big imports. 195 | 196 | ```yaml 197 | Ttree: 198 | ContentRepositoryImporter: 199 | sources: 200 | default: 201 | host: localhost 202 | driver: pdo_mysql 203 | dbname: database 204 | user: user 205 | password: password 206 | extraSourceDatabase: 207 | host: localhost 208 | driver: pdo_mysql 209 | dbname: database 210 | user: user 211 | password: password 212 | 213 | presets: 214 | 'base': 215 | parts: 216 | 'news': 217 | label: 'News Import' 218 | dataProviderClassName: 'Your\Package\Importer\DataProvider\NewsDataProvider' 219 | importerClassName: 'Your\Package\Importer\Importer\NewsImporter' 220 | 'page': 221 | label: 'Page Import' 222 | dataProviderClassName: 'Your\Package\Importer\DataProvider\PageDataProvider' 223 | dataProviderOptions: 224 | source: 'extraSourceDatabase' 225 | someOption: 'Some option that will be available in the options property of the data provider' 226 | importerClassName: 'Your\Package\Importer\Importer\PageImporter' 227 | importerOptions: 228 | siteNodePath: '/sites/my-site' 229 | someOption: 'Some option that will be available in the options property of the importer' 230 | batchSize': 120 231 | 232 | 'pageContent': 233 | label: 'Page Content Import' 234 | dataProviderClassName: 'Your\Package\Importer\DataProvider\PageContentDataProvider' 235 | importerClassName: 'Your\Package\Importer\Importer\PageContentImporter' 236 | batchSize: 120 237 | ``` 238 | 239 | Start your import process 240 | ------------------------- 241 | 242 | **Tips**: Do not forget to require this package from the package in which you do the importing, to ensure the correct loading order, so the settings would get overriden correctly. 243 | 244 | From the CLI: 245 | 246 | ``` 247 | flow import:batch --preset base 248 | ``` 249 | 250 | You can also filter the preset steps: 251 | 252 | ``` 253 | flow import:batch --preset base --parts page,pageContent 254 | ``` 255 | 256 | For testing purposes, or if you would like to override the value defined in your preset, you can also specify the number 257 | of records which should be imported at a time in an isolated sub-process: 258 | 259 | ``` 260 | flow import:batch --preset base --batch-size 50 261 | ``` 262 | 263 | Passing exceeding arguments to the DataProvider 264 | ----------------------------------------------- 265 | 266 | The import process supports passing unnamed exceeding arguments to the `DataProvider`. This can be useful if you e.g. want to 267 | allow importing only a single record 268 | 269 | ``` 270 | flow import:batch --preset base recordIdentifier:1234 271 | ``` 272 | 273 | Exceeding arguments will be available in the `DataProvider` through `$this->getExceedingArguments()`. You need to process 274 | this data yourself and apply it to your fetching logic. 275 | 276 | Command based importers 277 | ----------------------- 278 | 279 | Some data sources may consist of commands rather than data records. For example, a JSON file may contain `create`, 280 | `update` and `delete` instructions which reduce the guess-work on the importer's side, which records may be new, 281 | which should be updated and if the absence of a record means that the corresponding node should be deleted from 282 | the content repository. 283 | 284 | For these cases you can extend the `AbstractCommandBasedImporter`. If your data records contain a `mode` field, the 285 | importer will try to call a corresponding command method within the same class. 286 | 287 | Consider the following data source file as an example: 288 | 289 | ```json 290 | [ 291 | { 292 | "mode": "create", 293 | "mpn": "1081251137", 294 | "languageIdentifier": "de", 295 | "properties": { 296 | "label": "Coffee Machine", 297 | "price": "220000", 298 | "externalKey": "1081251137" 299 | } 300 | }, 301 | { 302 | "mode": "delete", 303 | "mpn": "591500202" 304 | } 305 | ] 306 | ``` 307 | 308 | A corresponding `ProductImporter` might look like this: 309 | 310 | ```php 311 | /** 312 | * Class ProductImporter 313 | */ 314 | class ProductImporter extends AbstractCommandBasedImporter 315 | { 316 | 317 | /** 318 | * @var string 319 | */ 320 | protected $storageNodeNodePath = 'products'; 321 | 322 | /** 323 | * @var string 324 | */ 325 | protected $storageNodeTitle = 'Products'; 326 | 327 | /** 328 | * @var string 329 | */ 330 | protected $externalIdentifierDataKey = 'mpn'; 331 | 332 | /** 333 | * @var string 334 | */ 335 | protected $labelDataKey = 'properties.Label'; 336 | 337 | /** 338 | * @var string 339 | */ 340 | protected $nodeNamePrefix = 'product-'; 341 | 342 | /** 343 | * @var string 344 | */ 345 | protected $nodeTypeName = 'Acme.MyShop:Product'; 346 | 347 | /** 348 | * Creates a new product 349 | * 350 | * @param string $externalIdentifier 351 | * @param array $data 352 | * @return void 353 | */ 354 | protected function createCommand($externalIdentifier, array $data) 355 | { 356 | $this->applyProperties($data['properties'], $this->nodeTemplate); 357 | 358 | $node = $this->storageNode->createNodeFromTemplate($this->nodeTemplate); 359 | $this->registerNodeProcessing($node, $externalIdentifier); 360 | } 361 | 362 | /** 363 | * Updates a product 364 | * 365 | * @param string $externalIdentifier 366 | * @param array $data 367 | * @return void 368 | */ 369 | protected function updateCommand($externalIdentifier, array $data) 370 | { 371 | $this->applyProperties($data['properties'], $this->nodeTemplate); 372 | 373 | $node = $this->storageNode->createNodeFromTemplate($this->nodeTemplate); 374 | $this->registerNodeProcessing($node, $externalIdentifier); 375 | } 376 | 377 | /** 378 | * Deletes a product 379 | * 380 | * @param string $externalIdentifier 381 | * @param array $data 382 | */ 383 | protected function deleteCommand($externalIdentifier, array $data) 384 | { 385 | // delete the product node 386 | } 387 | } 388 | ``` 389 | 390 | 391 | CSV Data Provider 392 | ----------------- 393 | 394 | This package comes with a basic data provider for CSV files which will suffice for many scenarios. The class name for 395 | this data provider is `Ttree\ContentRepositoryImporter\DataProvider\CsvDataProvider`. 396 | 397 | The following options can be passed to the data provider: 398 | 399 | - `csvFilePath`: the full path and filename leading to the file to import 400 | - `csvDelimiter`: the delimiter used in the CSV file (default: `,`) 401 | - `csvEnclosure`: the character which is used for enclosing the values (default: `"`) 402 | - `skipHeader`: if the first line in the CSV file should be ignored (default: false) 403 | 404 | Here is an example for a preset using the CSV Data Provider: 405 | 406 | ```yaml 407 | Ttree: 408 | ContentRepositoryImporter: 409 | presets: 410 | 'products': 411 | parts: 412 | 'products': 413 | label: 'Product Import' 414 | batchSize: 100 415 | dataProviderClassName: 'Ttree\ContentRepositoryImporter\DataProvider\CsvDataProvider' 416 | dataProviderOptions: 417 | csvFilePath: '/tmp/Products.csv' 418 | csvDelimiter: ';' 419 | csvEnclosure: '"' 420 | skipHeader: true 421 | importerClassName: 'Acme\MyProductImporter\Service\Import\ProductImporter' 422 | importerOptions: 423 | siteNodePath: '/sites/wwwacmecom' 424 | ``` 425 | 426 | 427 | Acknowledgments 428 | --------------- 429 | 430 | Development sponsored by [ttree ltd - neos solution provider](http://ttree.ch). 431 | 432 | We try our best to craft this package with a lots of love, we are open to sponsoring, support request, ... just contact us. 433 | 434 | License 435 | ------- 436 | 437 | Licensed under GPLv3+, see [LICENSE](LICENSE) 438 | -------------------------------------------------------------------------------- /Classes/Command/ImportCommandController.php: -------------------------------------------------------------------------------- 1 | settings = $settings; 105 | } 106 | 107 | /** 108 | * Reset the mapping between external identifier and local nodes 109 | * 110 | * @param string $preset 111 | * @param string $parts 112 | * @throws StopCommandException 113 | */ 114 | public function initCommand($preset, $parts = '') 115 | { 116 | $parts = Arrays::trimExplode(',', $parts); 117 | $presetSettings = $this->loadPreset($preset); 118 | array_walk($presetSettings['parts'], function ($partSetting, $partName) use ($preset, $parts) { 119 | $this->outputLine(); 120 | $this->outputPartTitle($partSetting, $partName); 121 | 122 | if ($parts !== array() && !in_array($partName, $parts)) { 123 | $this->outputLine('~ Skipped'); 124 | return; 125 | } 126 | 127 | if (!isset($partSetting['importerClassName'])) { 128 | $this->outputLine('Missing importerClassName in the current preset part (%s/%s), check your settings', [$preset, $partName]); 129 | return; 130 | } 131 | 132 | $identifier = $partSetting['importerClassName'] . '@' . $preset . '/' . $partName; 133 | /** @var RecordMapping $recordMapper */ 134 | foreach ($this->recordMapperRepository->findByImporterClassName($identifier) as $recordMapper) { 135 | $this->recordMapperRepository->remove($recordMapper); 136 | } 137 | }); 138 | $vault = new Vault($preset); 139 | $vault->flush(); 140 | } 141 | 142 | /** 143 | * Show the different pars of the preset 144 | * 145 | * @param string $preset 146 | * @throws StopCommandException 147 | */ 148 | public function showCommand($preset) 149 | { 150 | $presetSettings = $this->loadPreset($preset); 151 | array_walk($presetSettings['parts'], function ($partSetting, $partName) use ($preset) { 152 | $this->outputLine(); 153 | $this->outputPartTitle($partSetting, $partName); 154 | }); 155 | } 156 | 157 | /** 158 | * Run batch import 159 | * 160 | * This executes a batch import as configured in the settings for the specified preset. Optionally the "parts" can 161 | * be specified, separated by comma ",". 162 | * 163 | * Presets and parts need to be configured via settings first. Refer to the documentation for possible options and 164 | * example configurations. 165 | * 166 | * You may optionally specify an external import identifier which will be stored as meta data with the import run. 167 | * This identifier is used for checking if an import of the given data set (in general) has been executed earlier. 168 | * The external import identifier therefore does globally what the external record identifier does on a per record 169 | * basis. 170 | * 171 | * If an external import identifier was specified and an import using that identifier has been executed earlier, 172 | * this command will stop with a corresponding message. You can force running such an import by specifying the 173 | * --force flag. 174 | * 175 | * @param string $preset Name of the preset which holds the configuration for the import 176 | * @param string $parts Optional comma separated names of parts. If no parts are specified, all parts will be imported. 177 | * @param integer $batchSize Number of records to import at a time. If not specified, the batch size defined in the preset will be used. 178 | * @param string $identifier External identifier which is used for checking if an import of the same data has already been executed earlier. 179 | * @param boolean $force If set, an import will even be executed if it ran earlier with the same external import identifier. 180 | * @return void 181 | * @throws Exception 182 | * @throws StopCommandException 183 | * @throws IllegalObjectTypeException 184 | */ 185 | public function batchCommand($preset, $parts = '', $batchSize = null, $identifier = null, $force = false) 186 | { 187 | try { 188 | $this->importService->start($identifier, $force); 189 | } catch (ImportAlreadyExecutedException $e) { 190 | $this->outputLine($e->getMessage()); 191 | $this->outputLine('Import skipped. You can force running this import again by specifying --force.'); 192 | $this->quit(1); 193 | } 194 | 195 | $this->startTime = microtime(true); 196 | $parts = Arrays::trimExplode(',', $parts); 197 | 198 | $identifier = $this->importService->getCurrentImportIdentifier(); 199 | $this->outputLine('Start import with identifier %s', [$identifier]); 200 | 201 | $presetSettings = $this->loadPreset($preset); 202 | array_walk($presetSettings['parts'], function ($partSetting, $partName) use ($preset, $parts, $batchSize, $identifier) { 203 | $this->elapsedTime = 0; 204 | $this->batchCounter = 0; 205 | $this->outputLine(); 206 | $this->outputPartTitle($partSetting, $partName); 207 | 208 | $partSetting['__currentPresetName'] = $preset; 209 | $partSetting['__currentPartName'] = $partName; 210 | if ($batchSize !== null) { 211 | $partSetting['batchSize'] = $batchSize; 212 | } 213 | 214 | $partSetting = new PresetPartDefinition($partSetting, $identifier); 215 | if ($parts !== array() && !in_array($partName, $parts)) { 216 | $this->outputLine('~ Skipped'); 217 | return; 218 | } 219 | 220 | if ($partSetting->getBatchSize() && $partSetting->isDebug() === false) { 221 | while (($count = $this->executeCommand($partSetting)) > 0) { 222 | $partSetting->nextBatch(); 223 | } 224 | } else { 225 | $this->executeCommand($partSetting); 226 | } 227 | }); 228 | 229 | 230 | $this->importService->stop(); 231 | 232 | $import = $this->importService->getLastImport(); 233 | 234 | $this->outputLine(); 235 | $this->outputLine('Import finished'); 236 | $this->outputLine(sprintf('- Started %s', $import->getStartTime()->format(DATE_RFC2822))); 237 | $this->outputLine(sprintf('- Finished %s', $import->getEndTime()->format(DATE_RFC2822))); 238 | $this->outputLine(sprintf('- Runtime %d seconds', $import->getElapsedTime())); 239 | $this->outputLine(); 240 | $this->outputLine('See log for more details and possible errors.'); 241 | } 242 | 243 | /** 244 | * @param string $preset 245 | * @return array 246 | * @throws StopCommandException 247 | */ 248 | protected function loadPreset($preset) 249 | { 250 | $presetSettings = Arrays::getValueByPath($this->settings, ['presets', $preset]); 251 | if (!is_array($presetSettings)) { 252 | $this->outputLine(sprintf('Preset "%s" not found ...', $preset)); 253 | $this->quit(1); 254 | } 255 | 256 | $this->checkForPartsSettingsOrQuit($presetSettings, $preset); 257 | 258 | return $presetSettings; 259 | } 260 | 261 | /** 262 | * Execute a sub process which imports a batch as specified by the part definition. 263 | * 264 | * @param PresetPartDefinition $partSetting 265 | * @return integer The number of records which have been imported 266 | * @throws Exception 267 | * @throws StopCommandException 268 | */ 269 | protected function executeCommand(PresetPartDefinition $partSetting) 270 | { 271 | try { 272 | $this->importService->addEvent(sprintf('%s:Started', $partSetting->getEventType()), null, $partSetting->getCommandArguments()); 273 | $this->importService->persistEntities(); 274 | 275 | $startTime = microtime(true); 276 | 277 | ++$this->batchCounter; 278 | ob_start(NULL, 1<<20); 279 | $commandIdentifier = 'ttree.contentrepositoryimporter:import:executebatch'; 280 | $commandArguments = $partSetting->getCommandArguments(); 281 | $exceedingArguments = $this->request->getExceedingArguments(); 282 | if (!empty($exceedingArguments)) { 283 | $commandArguments['exceedingArguments'] = implode(',', $exceedingArguments); 284 | } 285 | $status = Scripts::executeCommand($commandIdentifier, $this->flowSettings, true, $commandArguments); 286 | if ($status !== true) { 287 | throw new Exception(vsprintf('Command: %s with parameters: %s', [$commandIdentifier, json_encode($partSetting->getCommandArguments())]), 1426767159); 288 | } 289 | $output = explode(PHP_EOL, ob_get_clean()); 290 | if (count($output) > 1) { 291 | $this->outputLine('+ Command "%s"', [$commandIdentifier]); 292 | $this->outputLine('+ with parameters:'); 293 | foreach ($partSetting->getCommandArguments() as $argumentName => $argumentValue) { 294 | $this->outputLine('+ %s: %s', [$argumentName, $argumentValue]); 295 | } 296 | foreach ($output as $line) { 297 | $line = trim($line); 298 | if ($line === '') { 299 | continue; 300 | } 301 | $this->outputLine('+ %s', [$line]); 302 | } 303 | } 304 | $count = (int)array_pop($output); 305 | $count = $count < 1 ? 0 : $count; 306 | 307 | $elapsedTime = (microtime(true) - $startTime) * 1000; 308 | $this->elapsedTime += $elapsedTime; 309 | $this->outputLine('+ #%d %d records in %dms, %d ms per record, %d ms per batch (avg)', [ 310 | $partSetting->getCurrentBatch(), 311 | $count, 312 | $elapsedTime, 313 | ($count > 0 ? $elapsedTime / $count : $elapsedTime), 314 | ($this->batchCounter > 0 ? $this->elapsedTime / $this->batchCounter : $this->elapsedTime) 315 | ]); 316 | $this->importService->addEvent(sprintf('%s:Ended', $partSetting->getEventType()), null, $partSetting->getCommandArguments()); 317 | $this->importService->persistEntities(); 318 | return $count; 319 | } catch (\Exception $exception) { 320 | $logMessage = $this->throwableStorage->logThrowable($exception); 321 | $this->logger->error($logMessage, LogEnvironment::fromMethodName(__METHOD__)); 322 | $this->outputLine('Error in parts "%s", please check your logs for more details', [$partSetting->getLabel()]); 323 | $this->outputLine('%s', [$exception->getMessage()]); 324 | $this->importService->addEvent(sprintf('%s:Failed', $partSetting->getEventType()), null, $partSetting->getCommandArguments()); 325 | $this->quit(1); 326 | } 327 | } 328 | 329 | /** 330 | * Import a single batch 331 | * 332 | * This internal command is called by executeCommand() and runs an isolated import for a batch as specified by 333 | * the command's arguments. 334 | * 335 | * @param string $presetName 336 | * @param string $partName 337 | * @param string $dataProviderClassName 338 | * @param string $importerClassName 339 | * @param string $currentImportIdentifier 340 | * @param integer $offset 341 | * @param integer $batchSize 342 | * @param array $exceedingArguments 343 | * @return void 344 | * @Flow\Internal 345 | */ 346 | public function executeBatchCommand($presetName, $partName, $dataProviderClassName, $importerClassName, $currentImportIdentifier, $offset = null, $batchSize = null, $exceedingArguments = null) 347 | { 348 | try { 349 | $vault = new Vault($presetName); 350 | 351 | $dataProviderOptions = Arrays::getValueByPath($this->settings, ['presets', $presetName, 'parts', $partName, 'dataProviderOptions']); 352 | $dataProviderOptions['__presetName'] = $presetName; 353 | $dataProviderOptions['__partName'] = $partName; 354 | 355 | /** @var DataProviderInterface $dataProvider */ 356 | $dataProvider = $dataProviderClassName::create(is_array($dataProviderOptions) ? $dataProviderOptions : [], $offset, $batchSize, $exceedingArguments); 357 | 358 | $importerOptions = Arrays::getValueByPath($this->settings, ['presets', $presetName, 'parts', $partName, 'importerOptions']); 359 | 360 | /** @var AbstractImporter $importer */ 361 | $importerOptions = is_array($importerOptions) ? $importerOptions : []; 362 | $importerOptions['__presetName'] = $presetName; 363 | $importerOptions['__partName'] = $partName; 364 | $importer = $this->objectManager->get($importerClassName, $importerOptions, $currentImportIdentifier, $vault); 365 | $importer->getImportService()->addEventMessage(sprintf('%s:Batch:Started', $importerClassName), sprintf('%s batch started (%s)', $importerClassName, $dataProviderClassName)); 366 | $importer->initialize($dataProvider); 367 | $importer->process(); 368 | $importer->getImportService()->addEventMessage(sprintf('%s:Batch:Ended', $importerClassName), sprintf('%s batch ended (%s)', $importerClassName, $dataProviderClassName)); 369 | $this->output((string)$importer->getProcessedRecords()); 370 | } catch (\Exception $exception) { 371 | $logMessage = $this->throwableStorage->logThrowable($exception); 372 | $this->logger->error($logMessage, LogEnvironment::fromMethodName(__METHOD__)); 373 | $this->outputLine('%s', [$exception->getMessage()]); 374 | $this->sendAndExit(1); 375 | } 376 | } 377 | 378 | /** 379 | * Clean up event log 380 | * 381 | * This command removes all Neos event log entries caused by the importer. 382 | * 383 | * @return void 384 | */ 385 | public function flushEventLogCommand() 386 | { 387 | $this->eventLogRepository->removeAll(); 388 | } 389 | 390 | /** 391 | * @param array $partSetting 392 | * @param string $partName 393 | */ 394 | protected function outputPartTitle(array $partSetting, $partName) 395 | { 396 | $this->outputFormatted(sprintf('+ %s (%s)', $partSetting['label'], $partName)); 397 | } 398 | 399 | /** 400 | * Checks if the preset settings contain a "parts" segment and quits if it does not. 401 | * 402 | * @param array $presetSettings 403 | * @param string $preset 404 | * @throws StopCommandException 405 | */ 406 | protected function checkForPartsSettingsOrQuit(array $presetSettings, $preset) 407 | { 408 | if (!isset($presetSettings['parts'])) { 409 | $this->outputLine('No "parts" array found for import preset "%s".', [ $preset ]); 410 | $this->outputLine(); 411 | $this->outputLine('Please note that the settings structure has changed slightly. Instead of just defining'); 412 | $this->outputLine('parts as a sub-array of the respective preset, you now need to define them in a sub-array'); 413 | $this->outputLine('called "parts".'); 414 | $this->outputLine(''); 415 | $this->outputLine('Ttree:'); 416 | $this->outputLine(' ContentRepositoryImporter:'); 417 | $this->outputLine(' presets:'); 418 | $this->outputLine(" '$preset':"); 419 | $this->outputLine(" parts:"); 420 | if (is_array($presetSettings) && count($presetSettings) > 0) { 421 | $firstPart = array_keys($presetSettings)[0]; 422 | $this->outputLine(" '$firstPart':"); 423 | } 424 | $this->outputLine(" ..."); 425 | $this->quit(1); 426 | } 427 | } 428 | 429 | } 430 | -------------------------------------------------------------------------------- /Classes/Importer/AbstractImporter.php: -------------------------------------------------------------------------------- 1 | options = $options; 231 | $this->presetName = $options['__presetName']; 232 | $this->partName = $options['__partName']; 233 | $this->vault = $vault; 234 | unset($this->options['__presetName'], $this->options['__partName']); 235 | $this->currentImportIdentifier = $currentImportIdentifier; 236 | } 237 | 238 | /** 239 | * Resume the current Import 240 | * 241 | * This is required because we use sub request in CLI controller 242 | */ 243 | public function initializeObject() 244 | { 245 | $this->importService->resume($this->currentImportIdentifier); 246 | } 247 | 248 | /** 249 | * @return DataProviderInterface 250 | */ 251 | public function getDataProvider() 252 | { 253 | return $this->dataProvider; 254 | } 255 | 256 | /** 257 | * @return ImportService 258 | */ 259 | public function getImportService() 260 | { 261 | return $this->importService; 262 | } 263 | 264 | /** 265 | * @param Event $event 266 | */ 267 | public function setCurrentEvent(Event $event) 268 | { 269 | $this->currentEvent = $event; 270 | } 271 | 272 | /** 273 | * @return Event 274 | */ 275 | public function getCurrentEvent() 276 | { 277 | return $this->currentEvent; 278 | } 279 | 280 | /** 281 | * @return integer 282 | */ 283 | public function getProcessedRecords() 284 | { 285 | return $this->processedRecords; 286 | } 287 | 288 | /** 289 | * Initialize import context 290 | * 291 | * @param DataProviderInterface $dataProvider 292 | * @throws Exception 293 | */ 294 | public function initialize(DataProviderInterface $dataProvider) 295 | { 296 | $this->dataProvider = $dataProvider; 297 | $contextConfiguration = ['workspaceName' => 'live', 'invisibleContentShown' => true]; 298 | $context = $this->contextFactory->create($contextConfiguration); 299 | $this->rootNode = $context->getRootNode(); 300 | 301 | $this->applyOption($this->storageNodeNodePath, 'storageNodeNodePath'); 302 | $this->applyOption($this->nodeTypeName, 'nodeTypeName'); 303 | 304 | if (isset($this->options['siteNodePath']) || isset($this->options['siteNodeIdentifier'])) { 305 | $siteNodePath = isset($this->options['siteNodePath']) ? trim($this->options['siteNodePath']) : null; 306 | $siteNodeIdentifier = isset($this->options['siteNodeIdentifier']) ? trim($this->options['siteNodeIdentifier']) : null; 307 | $this->siteNode = $this->rootNode->getNode($siteNodePath) ?: $context->getNodeByIdentifier($siteNodeIdentifier); 308 | if ($this->siteNode === null) { 309 | throw new Exception(sprintf('Site node not found (%s)', $siteNodePath ?: $siteNodeIdentifier), 1425077201); 310 | } 311 | } else { 312 | $this->log(get_class($this) . ': siteNodePath is not defined. Please make sure to set the target siteNodePath in your importer options.', LogLevel::WARNING); 313 | } 314 | } 315 | 316 | protected function applyOption(&$option, $optionName) 317 | { 318 | $option = isset($this->options[$optionName]) ? $this->options[$optionName] : $option; 319 | } 320 | 321 | /** 322 | * Starts batch processing all commands 323 | * 324 | * Override this method if you would like some other way of initialization. 325 | * 326 | * @return void 327 | * @api 328 | */ 329 | public function process() 330 | { 331 | $this->initializeStorageNode($this->storageNodeNodePath, $this->storageNodeTitle); 332 | $this->initializeNodeTemplates(); 333 | 334 | $nodeTemplate = new NodeTemplate(); 335 | $this->processBatch($nodeTemplate); 336 | } 337 | 338 | /** 339 | * Import data from the given data provider 340 | * 341 | * @param NodeTemplate $nodeTemplate 342 | * @throws Exception 343 | */ 344 | protected function processBatch(NodeTemplate $nodeTemplate = null) 345 | { 346 | $records = $this->dataProvider->fetch(); 347 | if (!\is_iterable($records)) { 348 | throw new Exception(sprintf('Expected records as an array while calling %s->fetch(), but returned %s instead.', get_class($this->dataProvider), gettype($records)), 1462960769826); 349 | } 350 | $records = $this->preProcessing($records); 351 | 352 | foreach ($records as $data) { 353 | if (!\is_array($data)) { 354 | $data = $this->propertyMapper->convert($data, 'array'); 355 | } 356 | $this->processRecord($nodeTemplate, $data); 357 | ++$this->processedRecords; 358 | } 359 | 360 | $this->postProcessing($records); 361 | } 362 | 363 | public function withStorageNode(NodeInterface $storageNode, \Closure $closure) 364 | { 365 | $previousStorageNode = $this->storageNode; 366 | try { 367 | $this->storageNode = $storageNode; 368 | $closure(); 369 | $this->storageNode = $previousStorageNode; 370 | } catch (\Exception $exception) { 371 | $this->storageNode = $previousStorageNode; 372 | throw $exception; 373 | } 374 | } 375 | 376 | /** 377 | * Processes a single record 378 | * 379 | * Override this method if you need a different approach. 380 | * 381 | * @param NodeTemplate $nodeTemplate 382 | * @param array $data 383 | * @return NodeInterface 384 | * @throws \Exception 385 | * @api 386 | */ 387 | public function processRecord(NodeTemplate $nodeTemplate, array $data) 388 | { 389 | $this->unsetAllNodeTemplateProperties($nodeTemplate); 390 | 391 | $externalIdentifier = $this->getExternalIdentifierFromRecordData($data); 392 | $nodeName = $this->renderNodeName($externalIdentifier); 393 | if (!isset($data['uriPathSegment'])) { 394 | $data['uriPathSegment'] = Slug::create($this->getLabelFromRecordData($data))->getValue(); 395 | } 396 | 397 | $recordMapping = $this->getNodeProcessing($externalIdentifier); 398 | if ($recordMapping !== null) { 399 | $node = $this->storageNode->getContext()->getNodeByIdentifier($recordMapping->getNodeIdentifier()); 400 | if ($node === null) { 401 | throw new \Exception(sprintf('Failed retrieving existing node for update. External identifier: %s Node identifier: %s. Maybe the record mapping in the database does not match the existing (imported) nodes anymore.', $externalIdentifier, $recordMapping->getNodeIdentifier()), 1462971366085); 402 | } 403 | $this->applyProperties($this->getPropertiesFromDataProviderPayload($data), $node); 404 | 405 | } else { 406 | $nodeTemplate->setNodeType($this->nodeType); 407 | $nodeTemplate->setName($nodeName); 408 | $this->applyProperties($this->getPropertiesFromDataProviderPayload($data), $nodeTemplate); 409 | 410 | $node = $this->createNodeFromTemplate($nodeTemplate, $data); 411 | $this->registerNodeProcessing($node, $externalIdentifier); 412 | } 413 | 414 | $this->dimensionsImporter->process($node, $this->getPropertiesFromDataProviderPayload($data), $this->currentEvent); 415 | 416 | return $node; 417 | } 418 | 419 | /** 420 | * @param NodeTemplate $templace 421 | * @param array $data 422 | * @return NodeInterface 423 | */ 424 | protected function createNodeFromTemplate(NodeTemplate $templace, array $data) 425 | { 426 | return $this->storageNode->createNodeFromTemplate($templace); 427 | } 428 | 429 | /** 430 | * @param NodeTemplate $nodeTemplate 431 | * @throws \Neos\ContentRepository\Exception\NodeException 432 | */ 433 | protected function unsetAllNodeTemplateProperties(NodeTemplate $nodeTemplate) 434 | { 435 | foreach ($nodeTemplate->getPropertyNames() as $propertyName) { 436 | if (!$nodeTemplate->hasProperty($propertyName)) { 437 | continue; 438 | } 439 | $nodeTemplate->removeProperty($propertyName); 440 | } 441 | } 442 | 443 | /** 444 | * @param array $data 445 | * @return array 446 | */ 447 | protected function getPropertiesFromDataProviderPayload(array $data) 448 | { 449 | return $data; 450 | } 451 | 452 | /** 453 | * Applies the given properties ($data) to the given Node or NodeTemplate 454 | * 455 | * @param array $data Property names and property values 456 | * @param NodeInterface|NodeTemplate $nodeOrTemplate The Node or Node Template 457 | * @return boolean True if an existing node has been modified, false if the new properties are the same like the old ones 458 | */ 459 | protected function applyProperties(array $data, $nodeOrTemplate) 460 | { 461 | return $this->nodePropertyMapper->map($data, $nodeOrTemplate, $this->currentEvent); 462 | } 463 | 464 | /** 465 | * Preprocess RAW data 466 | * 467 | * @param array|iterable $records 468 | * @return array 469 | */ 470 | protected function preProcessing($records) 471 | { 472 | return $records; 473 | } 474 | 475 | /** 476 | * Postprocessing 477 | * 478 | * @param array|iterable $records 479 | */ 480 | protected function postProcessing($records) 481 | { 482 | } 483 | 484 | /** 485 | * Checks if processing / import of the record specified by $externalIdentifier should be skipped. 486 | * 487 | * The following criteria for skipping the processing exist: 488 | * 489 | * 1) a record with the given external identifier already has been processed in the past 490 | * 2) a node with a node name equal to what a new node would have already exists 491 | * 492 | * These criteria can be enabled or disabled through $skipExistingNode and $skipAlreadyProcessed. 493 | * 494 | * @param string $externalIdentifier 495 | * @param string $nodeName 496 | * @param NodeInterface $storageNode 497 | * @param boolean $skipExistingNode 498 | * @param bool $skipAlreadyProcessed 499 | * @return bool 500 | * @throws Exception 501 | */ 502 | protected function skipNodeProcessing($externalIdentifier, $nodeName, NodeInterface $storageNode, $skipExistingNode = true, $skipAlreadyProcessed = true) 503 | { 504 | if ($skipAlreadyProcessed === true && $this->getNodeProcessing($externalIdentifier)) { 505 | $this->importService->addEventMessage('Node:Processed:Skipped', 'Skip already processed', LogLevel::NOTICE, $this->currentEvent); 506 | return true; 507 | } 508 | $node = $storageNode->getNode($nodeName); 509 | if ($skipExistingNode === true && $node instanceof NodeInterface) { 510 | $this->importService->addEventMessage('Node:Existing:Skipped', 'Skip existing node', LogLevel::WARNING, $this->currentEvent); 511 | $this->registerNodeProcessing($node, $externalIdentifier); 512 | return true; 513 | } 514 | 515 | return false; 516 | } 517 | 518 | /** 519 | * @param NodeInterface $node 520 | * @param string $externalIdentifier 521 | * @param string $externalRelativeUri 522 | */ 523 | protected function registerNodeProcessing(NodeInterface $node, $externalIdentifier, $externalRelativeUri = null) 524 | { 525 | $this->processedNodeService->set(get_called_class(), $externalIdentifier, $externalRelativeUri, $node->getIdentifier(), $node->getPath(), $this->presetPath()); 526 | } 527 | 528 | /** 529 | * @param string $externalIdentifier 530 | * @return RecordMapping 531 | */ 532 | protected function getNodeProcessing($externalIdentifier) 533 | { 534 | return $this->processedNodeService->get(get_called_class(), $externalIdentifier, $this->presetPath()); 535 | } 536 | 537 | /** 538 | * @return string 539 | */ 540 | protected function presetPath() 541 | { 542 | return $this->presetName . '/' . $this->partName; 543 | } 544 | 545 | /** 546 | * Create an entry in the event log 547 | * @param $message 548 | * @param int $severity 549 | * @throws Exception 550 | */ 551 | protected function log($message, $severity = LogLevel::INFO) 552 | { 553 | if (array_key_exists($severity, Logger::LOGLEVEL_MAPPING) === false) { 554 | throw new Exception('Invalid severity value', 1426868867); 555 | } 556 | $this->importService->addEventMessage(sprintf('Record:Import:Log:%s', $severity), $message, $severity, $this->currentEvent); 557 | } 558 | 559 | /** 560 | * Returns the external identifier of a record by looking it up in $data 561 | * 562 | * Either override this method for your own purpose or simply set $this->externalIdentifierDataKey 563 | * 564 | * @param array $data 565 | * @return string 566 | * @throws \Exception 567 | * @api 568 | */ 569 | protected function getExternalIdentifierFromRecordData(array $data) 570 | { 571 | $externalIdentifier = Arrays::getValueByPath($data, $this->externalIdentifierDataKey); 572 | if ($externalIdentifier === null) { 573 | throw new \Exception('Could not determine external identifier from record data. See ' . self::class . ' for more information.', 1462968317292); 574 | } 575 | return (string)$externalIdentifier; 576 | } 577 | 578 | /** 579 | * Render a valid node name for a new Node based on the current record 580 | * 581 | * @param string $externalIdentifier External identifier of the current record 582 | * @return string 583 | */ 584 | protected function renderNodeName($externalIdentifier) 585 | { 586 | return Slug::create(($this->nodeNamePrefix !== null ? $this->nodeNamePrefix : uniqid()) . $externalIdentifier)->getValue(); 587 | } 588 | 589 | /** 590 | * Returns a label for a record by looking it up in $data 591 | * 592 | * Either override this method for your own purpose or simply set $this->labelDataKey 593 | * 594 | * @param array $data 595 | * @return string 596 | * @throws \Exception 597 | * @api 598 | */ 599 | protected function getLabelFromRecordData(array $data) 600 | { 601 | $label = Arrays::getValueByPath($data, $this->labelDataKey); 602 | if ($label === null) { 603 | throw new \Exception('Could not determine label from record data (key: ' . $this->labelDataKey . '). See ' . self::class . ' for more information.', 1462968958372); 604 | } 605 | return (string)$label; 606 | } 607 | 608 | 609 | /** 610 | * Make sure that a (document) node exists which acts as a parent for nodes imported by this importer. 611 | * 612 | * The storage node is either created or just retrieved and finally stored in $this->storageNode. 613 | * 614 | * @param string $nodePathOrIdentifier A nodeIdentifier (prefixed with #) or an absolute or relative (to the site node) node path of the storage node 615 | * @param string $title Title for the storage node document 616 | * @return void 617 | * @throws Exception 618 | */ 619 | protected function initializeStorageNode($nodePathOrIdentifier, $title) 620 | { 621 | if (is_string($nodePathOrIdentifier) && $nodePathOrIdentifier[0] === '#') { 622 | $this->storageNode = $this->rootNode->getContext()->getNodeByIdentifier(\substr($nodePathOrIdentifier, 1)); 623 | } else { 624 | $this->storageNode = $this->getSiteNode()->getNode($nodePathOrIdentifier); 625 | 626 | preg_match('|([a-z0-9\-]+/)*([a-z0-9\-]+)$|', $nodePathOrIdentifier, $matches); 627 | $nodeName = $matches[2]; 628 | $uriPathSegment = Slug::create($title)->getValue(); 629 | 630 | $storageNodeTemplate = new NodeTemplate(); 631 | $storageNodeTemplate->setNodeType($this->nodeTypeManager->getNodeType($this->storageNodeTypeName)); 632 | 633 | if ($this->storageNode === null) { 634 | $storageNodeTemplate->setProperty('title', $title); 635 | $storageNodeTemplate->setProperty('uriPathSegment', $uriPathSegment); 636 | $storageNodeTemplate->setName($nodeName); 637 | $this->storageNode = $this->getSiteNode()->createNodeFromTemplate($storageNodeTemplate); 638 | } 639 | } 640 | 641 | if (!$this->storageNode instanceof NodeInterface) { 642 | throw new Exception('Storage node can not be empty', 1500558744); 643 | } 644 | } 645 | 646 | /** 647 | * @return NodeInterface 648 | * @throws SiteNodeEmptyException 649 | */ 650 | protected function getSiteNode() 651 | { 652 | if (!$this->siteNode instanceof NodeInterface) { 653 | throw new SiteNodeEmptyException(get_class($this) . ': siteNodePath is not defined. Please make sure to set the target siteNodePath in your importer options.'); 654 | } 655 | return $this->siteNode; 656 | } 657 | 658 | 659 | /** 660 | * Initializes the node template(s) used by this importer. 661 | * 662 | * Override this method if you need to work with other / multiple node types. 663 | * 664 | * @return void 665 | * @api 666 | */ 667 | protected function initializeNodeTemplates() 668 | { 669 | $this->nodeType = $this->nodeTypeManager->getNodeType($this->nodeTypeName); 670 | $this->nodeTemplate = new NodeTemplate(); 671 | $this->nodeTemplate->setNodeType($this->nodeType); 672 | } 673 | } 674 | --------------------------------------------------------------------------------