├── .gitignore ├── .scrutinizer.yml ├── .travis.yml ├── Command ├── GenerateImportValueObjectCommand.php ├── ImportCommand.php └── ListCommand.php ├── DependencyInjection ├── Configuration.php ├── MathielenImportEngineExtension.php └── StringOrFileList.php ├── Endpoint └── DoctrineEndpoint.php ├── Expression └── ExpressionLanguageProvider.php ├── Generator ├── ValueObject │ ├── FieldFormatGuess.php │ └── FieldFormatGuesser.php └── ValueObjectGenerator.php ├── MathielenImportEngineBundle.php ├── Resources ├── config │ └── services.xml └── skeleton │ └── valueobject.php.twig ├── Tests ├── Command │ └── ImportCommandTest.php ├── DependencyInjection │ ├── AbstractExtensionTest.php │ ├── AbstractTest.php │ ├── CompareTest.php │ ├── Fixtures │ │ ├── Xml │ │ │ ├── full.xml │ │ │ ├── medium.xml │ │ │ └── minimum.xml │ │ └── Yaml │ │ │ ├── full.yml │ │ │ ├── medium.yml │ │ │ └── minimum.yml │ ├── XmlExtensionTest.php │ └── YamlExtensionTest.php ├── MyImportedRow.php ├── UtilsTest.php └── bootstrap.php ├── Utils.php ├── composer.json ├── phpunit.xml.dist └── readme.md /.gitignore: -------------------------------------------------------------------------------- 1 | vendor 2 | composer.lock 3 | .idea 4 | .idea/* -------------------------------------------------------------------------------- /.scrutinizer.yml: -------------------------------------------------------------------------------- 1 | checks: 2 | php: 3 | code_rating: true 4 | duplication: true 5 | tools: 6 | external_code_coverage: true 7 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: php 2 | 3 | php: 4 | - 5.5 5 | - 5.6 6 | - 7.0 7 | 8 | before_script: 9 | - composer install --dev 10 | 11 | script: phpunit --coverage-text --coverage-clover=coverage.clover 12 | 13 | after_script: 14 | - wget https://scrutinizer-ci.com/ocular.phar 15 | - php ocular.phar code-coverage:upload --format=php-clover coverage.clover 16 | -------------------------------------------------------------------------------- /Command/GenerateImportValueObjectCommand.php: -------------------------------------------------------------------------------- 1 | addArgument('source_id', InputArgument::REQUIRED, "Id of the demo-source. Different StorageProviders need different id-styles.\n- file/directory: \"\"\n- doctrine: \"\"\n- service: \".[?arguments_like_url_query]\"") 31 | ->addArgument('name', InputArgument::REQUIRED, 'Classname of the valueobject that should be generated') 32 | ->addArgument('path', InputArgument::REQUIRED, 'Output directory for the class file') 33 | ->addOption('source_provider', null, InputOption::VALUE_OPTIONAL, 'Id of source provider. If not given it will be default', 'default') 34 | ->addOption('format', null, InputOption::VALUE_OPTIONAL, 'The format of the file (as a file extension). If not given it will be automatically determined.') 35 | ->addOption('skip-field-format-discovery', null, InputOption::VALUE_NONE, 'Do not scan source to determine the field-formats. Every fields will be assigned to the default-field-format') 36 | ->addOption('default-field-format', null, InputOption::VALUE_OPTIONAL, 'Default field format', 'string') 37 | ->setDescription('Generates a valueobject class file for use with the importengine.') 38 | ->setHelp(<<generate:import:valueobject command helps you generates new valueobjects 40 | for the mathielen/import-engine importer. 41 | 42 | What is a valueobject? 43 | A valueobject is a small object that represents a simple entity whose equality is not based 44 | on identity: i.e. two value objects are equal when they have the same value, not necessarily being the same object. 45 | 46 | Why do I need valueobjects for my importer? 47 | Here, the valueobject represents the current dataset that is processed by the importer (ie. a row of a file). 48 | Having a generated class that represents this dataset enables you to explicitly define validation rules and other 49 | related things. 50 | 51 | This command can help you generate the valueobject class based on a "demo-file" of your import. 52 | EOT 53 | ) 54 | ->setName('importengine:generate:valueobject') 55 | ; 56 | } 57 | 58 | public function execute(InputInterface $input, OutputInterface $output) 59 | { 60 | $sourceId = $input->getArgument('source_id'); 61 | $sourceProvider = $input->getOption('source_provider'); 62 | $clsName = $input->getArgument('name'); 63 | $path = $input->getArgument('path'); 64 | $format = $input->getOption('format'); 65 | $defaultFieldFormat = $input->getOption('default-field-format'); 66 | 67 | if (!is_dir($path) || !is_writable($path)) { 68 | throw new \RuntimeException(sprintf('The directory "%s" is not a directory or cannot be written to.', $path)); 69 | } 70 | 71 | /** @var StorageLocator $storageLocator */ 72 | $storageLocator = $this->getContainer()->get('mathielen_importengine.import.storagelocator'); 73 | $storageSelection = $storageLocator->selectStorage($sourceProvider, $sourceId); 74 | 75 | if (!empty($format)) { 76 | $storageSelection->addMetadata('format', FileExtensionDiscoverStrategy::fileExtensionToFormat($format)); 77 | } 78 | 79 | $storage = $storageLocator->getStorage($storageSelection); 80 | 81 | if (!$input->getOption('skip-field-format-discovery')) { 82 | $fieldDefinitions = $this->determineFieldDefinitions($storage, $defaultFieldFormat); 83 | } else { 84 | $fieldDefinitions = array_change_key_case(array_fill_keys($storage->getFields(), array('type' => $defaultFieldFormat)), CASE_LOWER); 85 | } 86 | 87 | $voGenerator = new ValueObjectGenerator(); 88 | $voGenerator->setSkeletonDirs($this->getContainer()->get('kernel')->locateResource('@MathielenImportEngineBundle/Resources/skeleton')); 89 | 90 | $filePath = $voGenerator->generate($fieldDefinitions, $clsName, $path); 91 | 92 | $output->writeln("Valueobject class file has been generated and saved to $filePath"); 93 | } 94 | 95 | private function determineFieldDefinitions(StorageInterface $storage, $defaultFieldFormat = 'string') 96 | { 97 | /** @var Importer $importer */ 98 | $importer = $this->getContainer()->get('mathielen_importengine.generator.valueobject.importer'); 99 | $import = Import::build($importer, $storage); 100 | 101 | /** @var ImportRunner $importRunner */ 102 | $importRunner = $this->getContainer()->get('mathielen_importengine.import.runner'); 103 | $importRunner->run($import); 104 | 105 | /** @var FieldFormatGuesser $fieldformatguesser */ 106 | $fieldformatguesser = $this->getContainer()->get('mathielen_importengine.generator.valueobject.fieldformatguesser'); 107 | 108 | return $fieldformatguesser->getFieldDefinitionGuess($defaultFieldFormat); 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /Command/ImportCommand.php: -------------------------------------------------------------------------------- 1 | importBuilder = $importBuilder; 52 | $this->importRunner = $importRunner; 53 | $this->eventDispatcher = $eventDispatcher; 54 | } 55 | 56 | protected function configure() 57 | { 58 | $this 59 | ->setDescription('Imports data with a definied importer') 60 | ->addArgument('source_id', InputArgument::OPTIONAL, "id of source. Different StorageProviders need different id data.\n- upload, directory: \"\"\n- doctrine: \"\"\n- service: \".[?arguments_like_url_query]\"") 61 | ->addArgument('source_provider', InputArgument::OPTIONAL, 'id of source provider', 'default') 62 | ->addOption('importer', 'i', InputOption::VALUE_REQUIRED, 'id/name of importer') 63 | ->addOption('context', 'c', InputOption::VALUE_REQUIRED, 'Supply optional context information to import. Supply key-value data in query style: key=value&otherkey=othervalue&...') 64 | ->addOption('limit', 'l', InputOption::VALUE_REQUIRED, 'Limit imported rows') 65 | ->addOption('dryrun', 'd', InputOption::VALUE_NONE, 'Do not import - Validation only') 66 | ->addOption('validate-and-run', null, InputOption::VALUE_NONE, 'Validate and run if no error') 67 | ; 68 | } 69 | 70 | protected function execute(InputInterface $input, OutputInterface $output) 71 | { 72 | $importerId = $input->getOption('importer'); 73 | $sourceProviderId = $input->getArgument('source_provider'); 74 | $sourceId = $input->getArgument('source_id'); 75 | $isDryrun = $input->getOption('dryrun'); 76 | $isValidateAndRun = $input->getOption('validate-and-run'); 77 | if ($isDryrun && $isValidateAndRun) { 78 | throw new \InvalidArgumentException("Cannot invoke with dryrun and validate-and-run"); 79 | } 80 | $runMode = $isDryrun ? 'dryrun' : ($isValidateAndRun ? 'validate_and_run' : 'run'); 81 | if ($context = $input->getOption('context')) { 82 | //parse key=value&key=value string to array 83 | if (strpos($context, '=') !== false) { 84 | parse_str($input->getOption('context'), $context); 85 | } 86 | } 87 | $limit = $input->getOption('limit'); 88 | 89 | if (empty($importerId) && empty($sourceId)) { 90 | throw new \InvalidArgumentException('There must be at least an importerId with a configured source-definition given or a sourceId which can be automatically recognized by pre-conditions.'); 91 | } 92 | 93 | $this->import($output, $importerId, $sourceProviderId, $sourceId, $context, $limit, $runMode); 94 | } 95 | 96 | protected function import(OutputInterface $output, $importerId, $sourceProviderId, $sourceId, $context = null, $limit = null, $runMode = 'run') 97 | { 98 | $output->writeln("Commencing import with mode $runMode using importer ".(empty($importerId) ? 'unknown' : "$importerId")." with source provider $sourceProviderId and source id $sourceId"); 99 | 100 | $sourceId = Utils::parseSourceId($sourceId); 101 | $progress = new ProgressBar($output); 102 | 103 | //set limit 104 | if ($limit) { 105 | $output->writeln("Limiting import to $limit rows."); 106 | 107 | $this->eventDispatcher->addListener(ImportConfigureEvent::AFTER_BUILD, function (ImportConfigureEvent $event) use ($limit) { 108 | $event->getImport()->importer()->filters()->add(new OffsetFilter(0, $limit)); 109 | }); 110 | } 111 | 112 | //show discovered importer id 113 | if (empty($importerId)) { 114 | $this->eventDispatcher->addListener(ImportRequestEvent::DISCOVERED, function (ImportRequestEvent $event) use ($output) { 115 | $importerId = $event->getImportRequest()->getImporterId(); 116 | $output->writeln("Importer discovered: $importerId"); 117 | }); 118 | } 119 | 120 | $importRequest = new ImportRequest($sourceId, $sourceProviderId, $importerId, Utils::whoAmI().'@CLI', $context); 121 | 122 | $import = $this->importBuilder->buildFromRequest($importRequest); 123 | 124 | //apply context info from commandline 125 | $importRun = $import->getRun(); 126 | 127 | //status callback 128 | $this->eventDispatcher->addListener(ImportItemEvent::AFTER_READ, function (ImportItemEvent $event) use ($output, &$progress) { 129 | /** @var ImportRun $importRun */ 130 | $importRun = $event->getContext()->getRun(); 131 | $stats = $importRun->getStatistics(); 132 | $processed = isset($stats['processed']) ? $stats['processed'] : 0; 133 | $max = $importRun->getInfo()['count']; 134 | 135 | if ($progress->getMaxSteps() != $max) { 136 | $progress = new ProgressBar($output, $max); 137 | $progress->start(); 138 | } 139 | 140 | $progress->setProgress($processed); 141 | }); 142 | 143 | if ($runMode === 'dryrun') { 144 | $this->importRunner->dryRun($import); 145 | } elseif ($runMode === 'validate_and_run') { 146 | $this->importRunner->dryRun($import); 147 | $this->importRunner->run($import); 148 | } else { 149 | $this->importRunner->run($import); 150 | } 151 | 152 | $progress->finish(); 153 | $output->writeln(''); 154 | $output->writeln('Import done'); 155 | $output->writeln(''); 156 | 157 | $this->writeStatistics($importRun->getStatistics(), new Table($output)); 158 | 159 | $this->writeValidationViolations( 160 | $import 161 | ->importer() 162 | ->validation() 163 | ->getViolations(), 164 | new Table($output)); 165 | 166 | $output->writeln(''); 167 | } 168 | 169 | protected function writeValidationViolations(array $violations, Table $table) 170 | { 171 | if (empty($violations)) { 172 | return; 173 | } 174 | $violations = $violations['source'] + $violations['target']; 175 | 176 | $table 177 | ->setHeaders(array('Constraint', 'Occurrences (lines)')) 178 | ; 179 | 180 | $tree = []; 181 | foreach ($violations as $line => $validations) { 182 | /** @var ConstraintViolation $validation */ 183 | foreach ($validations as $validation) { 184 | $key = $validation->__toString(); 185 | if (!isset($tree[$key])) { 186 | $tree[$key] = []; 187 | } 188 | $tree[$key][] = $line; 189 | } 190 | } 191 | 192 | $i = 0; 193 | foreach ($tree as $violation => $lines) { 194 | $table->addRow([$violation, implode(', ', Utils::numbersToRangeText($lines))]); 195 | ++$i; 196 | 197 | if ($i === self::MAX_VIOLATION_ERRORS) { 198 | $table->addRow(new TableSeparator()); 199 | $table->addRow(array(null, 'There are more errors...')); 200 | 201 | break; 202 | } 203 | } 204 | 205 | if ($i > 0) { 206 | $table->render(); 207 | } 208 | } 209 | 210 | protected function writeStatistics(array $statistics, Table $table) 211 | { 212 | $rows = []; 213 | foreach ($statistics as $k => $v) { 214 | $rows[] = [$k, $v]; 215 | } 216 | 217 | $table 218 | ->setHeaders(array('Statistics')) 219 | ->setRows($rows) 220 | ; 221 | $table->render(); 222 | } 223 | } 224 | -------------------------------------------------------------------------------- /Command/ListCommand.php: -------------------------------------------------------------------------------- 1 | importerRepository = $importerRepository; 25 | } 26 | 27 | protected function configure() 28 | { 29 | $this->setDescription('Lists all available importer'); 30 | } 31 | 32 | protected function execute(InputInterface $input, OutputInterface $output) 33 | { 34 | $table = new Table($output); 35 | $table->setHeaders(['id', 'auto-detectable', 'validation']); 36 | 37 | foreach ($this->importerRepository->getIds() as $importerId) { 38 | $importer = $this->importerRepository->get($importerId); 39 | 40 | $table->addRow([ 41 | $importerId, 42 | $this->importerRepository->hasPrecondition($importerId) ? 'Yes' : 'No', 43 | ($importer->validation() instanceof DummyValidation) ? 'No' : 'Yes', 44 | ]); 45 | } 46 | 47 | $table->render(); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /DependencyInjection/Configuration.php: -------------------------------------------------------------------------------- 1 | root('mathielen_import_engine') 19 | ->fixXmlConfig('importer') 20 | ->children() 21 | ->arrayNode('storageprovider') 22 | ->useAttributeAsKey('name') 23 | ->prototype('array') 24 | ->fixXmlConfig('service') //allows instead of 25 | ->fixXmlConfig('query', 'queries') //allows instead of 26 | ->children() 27 | ->enumNode('type') 28 | ->values($providerTypes) 29 | ->end() 30 | ->scalarNode('uri')->end() //file 31 | ->scalarNode('connection_factory')->end() //dbal & doctrine 32 | ->arrayNode('services') 33 | ->useAttributeAsKey('name') 34 | ->prototype('array') 35 | ->fixXmlConfig('method') //allows instead of 36 | ->beforeNormalization() 37 | ->ifArray() 38 | ->then(function ($v) { return isset($v['methods']) || isset($v['method']) ? $v : array('methods' => $v); }) 39 | ->end() 40 | ->children() 41 | ->arrayNode('methods') 42 | ->prototype('scalar')->end() 43 | ->end() 44 | ->end() 45 | ->end() 46 | ->end() 47 | ->arrayNode('queries') //dbal & doctrine 48 | ->beforeNormalization() 49 | ->ifString() 50 | ->then(function ($v) { return [$v]; }) 51 | ->end() 52 | ->useAttributeAsKey('name') 53 | ->prototype('scalar')->end() 54 | ->end() 55 | ->end() 56 | ->end() 57 | ->end() 58 | ->arrayNode('importers') 59 | ->requiresAtLeastOneElement() 60 | ->useAttributeAsKey('name') 61 | ->prototype('array') 62 | ->fixXmlConfig('mapping') //allows instead of 63 | ->children() 64 | ->arrayNode('context') 65 | ->beforeNormalization() 66 | ->ifString() 67 | ->then(function ($v) { return array($v); }) 68 | ->end() 69 | ->prototype('variable')->end() 70 | ->end() 71 | 72 | ->arrayNode('preconditions') 73 | ->fixXmlConfig('field') //allows instead of 74 | ->children() 75 | ->arrayNode('format') 76 | ->beforeNormalization() 77 | ->ifString() 78 | ->then(function ($v) { return array($v); }) 79 | ->end() 80 | ->prototype('enum') 81 | ->values($fileFormats) 82 | ->end() 83 | ->end() 84 | ->integerNode('fieldcount')->min(0)->end() 85 | ->arrayNode('filename') 86 | ->beforeNormalization() 87 | ->ifString() 88 | ->then(function ($v) { return array($v); }) 89 | ->end() 90 | ->prototype('scalar')->end() 91 | ->end() 92 | ->arrayNode('fieldset') 93 | ->prototype('scalar')->end() 94 | ->end() 95 | ->arrayNode('fields') 96 | ->prototype('scalar')->end() 97 | ->end() 98 | ->end() 99 | ->end() 100 | 101 | ->arrayNode('object_factory') 102 | ->children() 103 | ->enumNode('type') 104 | ->defaultValue('default') 105 | ->values(array('default', 'jms_serializer')) 106 | ->end() 107 | ->scalarNode('class') 108 | ->end() 109 | ->end() 110 | ->end() 111 | 112 | ->arrayNode('filters') 113 | ->prototype('scalar')->end() 114 | ->end() 115 | 116 | ->arrayNode('mappings') 117 | ->normalizeKeys(false) //do not change - to _ with field names 118 | ->useAttributeAsKey('from') 119 | ->prototype('array') 120 | ->beforeNormalization() 121 | ->ifString() 122 | ->then(function ($v) { return array('to' => $v); }) 123 | ->end() 124 | ->children() 125 | ->scalarNode('to')->end() 126 | ->scalarNode('converter')->end() 127 | ->end() 128 | ->end() 129 | ->end() 130 | 131 | ->arrayNode('source') 132 | ->children() 133 | ->enumNode('type') 134 | ->values($storageTypes) 135 | ->end() 136 | ->scalarNode('uri')->end() 137 | ->arrayNode('format') //file 138 | ->fixXmlConfig('argument') 139 | ->beforeNormalization() 140 | ->ifString() 141 | ->then(function ($v) { return array('type' => $v); }) 142 | ->end() 143 | ->children() 144 | ->scalarNode('type')->isRequired()->end() 145 | ->arrayNode('arguments') 146 | ->prototype('scalar')->end() 147 | ->end() 148 | ->end() 149 | ->end() 150 | ->scalarNode('service')->end() 151 | ->scalarNode('method')->end() 152 | ->end() 153 | ->end() 154 | 155 | ->arrayNode('validation') 156 | ->children() 157 | ->arrayNode('options') 158 | ->children() 159 | ->booleanNode('allowExtraFields')->end() 160 | ->booleanNode('allowMissingFields')->end() 161 | ->end() 162 | ->end() 163 | ->arrayNode('source') 164 | ->fixXmlConfig('constraint') //allows instead of 165 | ->beforeNormalization() 166 | ->ifArray() 167 | ->then(function ($v) { return isset($v['constraint']) || isset($v['constraints']) ? $v : array('constraints' => $v); }) 168 | ->end() 169 | ->children() 170 | ->arrayNode('constraints') 171 | ->useAttributeAsKey('field') 172 | ->prototype('scalar')->end() 173 | ->end() 174 | ->end() 175 | ->end() 176 | ->arrayNode('target') 177 | ->fixXmlConfig('constraint') //allows instead of 178 | ->beforeNormalization() 179 | ->ifArray() 180 | ->then(function ($v) { return isset($v['constraint']) || isset($v['constraints']) ? $v : array('constraints' => $v); }) 181 | ->end() 182 | ->children() 183 | ->arrayNode('constraints') 184 | ->useAttributeAsKey('field') 185 | ->prototype('scalar')->end() 186 | ->end() 187 | ->end() 188 | ->end() 189 | ->end() 190 | ->end() 191 | 192 | ->arrayNode('target') 193 | ->isRequired() 194 | ->beforeNormalization() 195 | ->always() 196 | ->then(function ($v) { return !isset($v['type']) ? ['type' => 'callable', 'callable' => $v] : $v; }) 197 | ->end() 198 | ->children() 199 | ->enumNode('type') 200 | ->values($storageTypes) 201 | ->end() 202 | ->arrayNode('format') //file 203 | ->fixXmlConfig('argument') 204 | ->beforeNormalization() 205 | ->ifString() 206 | ->then(function ($v) { return ['type' => $v]; }) 207 | ->end() 208 | ->children() 209 | ->scalarNode('type')->isRequired()->end() 210 | ->arrayNode('arguments') 211 | ->prototype('scalar')->end() 212 | ->end() 213 | ->end() 214 | ->end() 215 | ->scalarNode('uri')->end() //file 216 | ->variableNode('callable')->end() //callable 217 | ->variableNode('service')->end() //service 218 | ->variableNode('method')->end() //service 219 | ->scalarNode('entity')->end() //doctrine 220 | ->end() 221 | ->end() 222 | ->end() 223 | ->end() 224 | ->end() 225 | ->end(); 226 | 227 | return $treeBuilder; 228 | } 229 | } 230 | -------------------------------------------------------------------------------- /DependencyInjection/MathielenImportEngineExtension.php: -------------------------------------------------------------------------------- 1 | processConfiguration(new Configuration(), $configs); 20 | 21 | if (!empty($config['importers'])) { 22 | $loader = new XmlFileLoader($container, new FileLocator(__DIR__.'/../Resources/config')); 23 | $loader->load('services.xml'); 24 | 25 | $this->parseConfig($config, $container); 26 | } 27 | } 28 | 29 | private function parseConfig(array $config, ContainerBuilder $container) 30 | { 31 | $storageLocatorDef = $container->findDefinition('mathielen_importengine.import.storagelocator'); 32 | foreach ($config['storageprovider'] as $name => $sourceConfig) { 33 | $this->addStorageProviderDef($storageLocatorDef, $sourceConfig, $name); 34 | } 35 | 36 | $importerRepositoryDef = $container->findDefinition('mathielen_importengine.importer.repository'); 37 | foreach ($config['importers'] as $name => $importConfig) { 38 | $finderDef = null; 39 | if (isset($importConfig['preconditions'])) { 40 | $finderDef = $this->generateFinderDef($importConfig['preconditions']); 41 | } 42 | 43 | $objectFactoryDef = null; 44 | if (isset($importConfig['object_factory'])) { 45 | $objectFactoryDef = $this->generateObjectFactoryDef($importConfig['object_factory']); 46 | } 47 | 48 | $importerRepositoryDef->addMethodCall('register', array( 49 | $name, 50 | $this->generateImporterDef($importConfig, $objectFactoryDef), 51 | $finderDef, 52 | )); 53 | } 54 | } 55 | 56 | private function generateObjectFactoryDef(array $config) 57 | { 58 | if (!class_exists($config['class'])) { 59 | throw new InvalidConfigurationException("Object-Factory target-class '".$config['class']."' does not exist."); 60 | } 61 | 62 | if ($config['type'] == 'jms_serializer') { 63 | return new Definition('Mathielen\DataImport\Writer\ObjectWriter\JmsSerializerObjectFactory', array( 64 | $config['class'], 65 | new Reference('jms_serializer'), )); 66 | } 67 | 68 | return new Definition('Mathielen\DataImport\Writer\ObjectWriter\DefaultObjectFactory', array($config['class'])); 69 | } 70 | 71 | /** 72 | * @return \Symfony\Component\DependencyInjection\Definition 73 | */ 74 | private function generateFinderDef(array $finderConfig) 75 | { 76 | $finderDef = new Definition('Mathielen\ImportEngine\Importer\ImporterPrecondition'); 77 | 78 | if (isset($finderConfig['filename'])) { 79 | foreach ($finderConfig['filename'] as $conf) { 80 | $finderDef->addMethodCall('filename', array($conf)); 81 | } 82 | } 83 | 84 | if (isset($finderConfig['format'])) { 85 | foreach ($finderConfig['format'] as $conf) { 86 | $finderDef->addMethodCall('format', array($conf)); 87 | } 88 | } 89 | 90 | if (isset($finderConfig['fieldcount'])) { 91 | $finderDef->addMethodCall('fieldcount', array($finderConfig['fieldcount'])); 92 | } 93 | 94 | if (isset($finderConfig['fields'])) { 95 | foreach ($finderConfig['fields'] as $conf) { 96 | $finderDef->addMethodCall('field', array($conf)); 97 | } 98 | } 99 | 100 | if (isset($finderConfig['fieldset'])) { 101 | $finderDef->addMethodCall('fieldset', array($finderConfig['fieldset'])); 102 | } 103 | 104 | return $finderDef; 105 | } 106 | 107 | /** 108 | * @return \Symfony\Component\DependencyInjection\Definition 109 | */ 110 | private function generateImporterDef(array $importConfig, Definition $objectFactoryDef = null) 111 | { 112 | $importerDef = new Definition('Mathielen\ImportEngine\Importer\Importer', array( 113 | $this->getStorageDef($importConfig['target'], $objectFactoryDef), 114 | )); 115 | 116 | if (isset($importConfig['source'])) { 117 | $this->setSourceStorageDef($importConfig['source'], $importerDef); 118 | } 119 | 120 | //enable validation? 121 | if (isset($importConfig['validation']) && !empty($importConfig['validation'])) { 122 | $this->generateValidationDef($importConfig['validation'], $importerDef, $objectFactoryDef); 123 | } 124 | 125 | //add converters? 126 | if (isset($importConfig['mappings']) && !empty($importConfig['mappings'])) { 127 | $this->generateTransformerDef($importConfig['mappings'], $importerDef); 128 | } 129 | 130 | //add filters? 131 | if (isset($importConfig['filters']) && !empty($importConfig['filters'])) { 132 | $this->generateFiltersDef($importConfig['filters'], $importerDef); 133 | } 134 | 135 | //has static context? 136 | if (isset($importConfig['context'])) { 137 | $importerDef->addMethodCall('setContext', array( 138 | $importConfig['context'], 139 | )); 140 | } 141 | 142 | return $importerDef; 143 | } 144 | 145 | private function generateFiltersDef(array $filtersOptions, Definition $importerDef) 146 | { 147 | $filtersDef = new Definition('Mathielen\ImportEngine\Filter\Filters'); 148 | 149 | foreach ($filtersOptions as $filtersOption) { 150 | $filtersDef->addMethodCall('add', array( 151 | new Reference($filtersOption), 152 | )); 153 | } 154 | 155 | $importerDef->addMethodCall('filters', array( 156 | $filtersDef, 157 | )); 158 | } 159 | 160 | private function generateTransformerDef(array $mappingOptions, Definition $importerDef) 161 | { 162 | $mappingsDef = new Definition('Mathielen\ImportEngine\Mapping\Mappings'); 163 | 164 | //set converters 165 | foreach ($mappingOptions as $field => $fieldMapping) { 166 | $converter = null; 167 | if (isset($fieldMapping['converter'])) { 168 | $converter = $fieldMapping['converter']; 169 | } 170 | 171 | if (isset($fieldMapping['to'])) { 172 | $mappingsDef->addMethodCall('add', array( 173 | $field, 174 | $fieldMapping['to'], 175 | $converter, 176 | )); 177 | } elseif ($converter) { 178 | $mappingsDef->addMethodCall('setConverter', array( 179 | $converter, 180 | $field, 181 | )); 182 | } 183 | } 184 | 185 | $mappingFactoryDef = new Definition('Mathielen\ImportEngine\Mapping\DefaultMappingFactory', array( 186 | $mappingsDef, 187 | )); 188 | $converterProviderDef = new Definition('Mathielen\ImportEngine\Mapping\Converter\Provider\ContainerAwareConverterProvider', array( 189 | new Reference('service_container'), 190 | )); 191 | 192 | $transformerDef = new Definition('Mathielen\ImportEngine\Transformation\Transformation'); 193 | $transformerDef->addMethodCall('setMappingFactory', array( 194 | $mappingFactoryDef, 195 | )); 196 | $transformerDef->addMethodCall('setConverterProvider', array( 197 | $converterProviderDef, 198 | )); 199 | 200 | $importerDef->addMethodCall('transformation', array( 201 | $transformerDef, 202 | )); 203 | } 204 | 205 | private function generateValidatorDef(array $options) 206 | { 207 | //eventdispatcher aware source validatorfilter 208 | $validatorFilterDef = new Definition('Mathielen\DataImport\Filter\ValidatorFilter', array( 209 | new Reference('validator'), 210 | $options, 211 | new Reference('event_dispatcher'), 212 | )); 213 | 214 | return $validatorFilterDef; 215 | } 216 | 217 | private function generateValidationDef(array $validationConfig, Definition $importerDef, Definition $objectFactoryDef = null) 218 | { 219 | $validationDef = new Definition('Mathielen\ImportEngine\Validation\ValidatorValidation', array( 220 | new Reference('validator'), 221 | )); 222 | $importerDef->addMethodCall('validation', array( 223 | $validationDef, 224 | )); 225 | 226 | $validatorFilterDef = $this->generateValidatorDef( 227 | isset($validationConfig['options']) ? $validationConfig['options'] : array() 228 | ); 229 | 230 | if (isset($validationConfig['source'])) { 231 | $validationDef->addMethodCall('setSourceValidatorFilter', array( 232 | $validatorFilterDef, 233 | )); 234 | 235 | foreach ($validationConfig['source']['constraints'] as $field => $constraint) { 236 | $validationDef->addMethodCall('addSourceConstraint', array( 237 | $field, 238 | new Reference($constraint), 239 | )); 240 | } 241 | } 242 | 243 | //automatically apply class validation 244 | if (isset($validationConfig['target'])) { 245 | 246 | //using objects as result 247 | if ($objectFactoryDef) { 248 | 249 | //set eventdispatcher aware target CLASS-validatorfilter 250 | $validatorFilterDef = new Definition('Mathielen\DataImport\Filter\ClassValidatorFilter', array( 251 | new Reference('validator'), 252 | $objectFactoryDef, 253 | new Reference('event_dispatcher'), 254 | )); 255 | } else { 256 | foreach ($validationConfig['target']['constraints'] as $field => $constraint) { 257 | $validationDef->addMethodCall('addTargetConstraint', array( 258 | $field, 259 | new Reference($constraint), 260 | )); 261 | } 262 | } 263 | 264 | $validationDef->addMethodCall('setTargetValidatorFilter', array( 265 | $validatorFilterDef, 266 | )); 267 | } 268 | 269 | return $validationDef; 270 | } 271 | 272 | private function setSourceStorageDef(array $sourceConfig, Definition $importerDef) 273 | { 274 | $sDef = $this->getStorageDef($sourceConfig, $importerDef); 275 | $importerDef->addMethodCall('setSourceStorage', array( 276 | $sDef, 277 | )); 278 | } 279 | 280 | private function addStorageProviderDef(Definition $storageLocatorDef, $config, $id = 'default') 281 | { 282 | $formatDiscoverLocalFileStorageFactoryDef = new Definition('Mathielen\ImportEngine\Storage\Factory\FormatDiscoverLocalFileStorageFactory', array( 283 | new Definition('Mathielen\ImportEngine\Storage\Format\Discovery\MimeTypeDiscoverStrategy', array( 284 | array( 285 | 'text/plain' => new Definition('Mathielen\ImportEngine\Storage\Format\Factory\CsvAutoDelimiterFormatFactory'), 286 | 'text/csv' => new Definition('Mathielen\ImportEngine\Storage\Format\Factory\CsvAutoDelimiterFormatFactory'), 287 | ), 288 | )), 289 | new Reference('logger'), 290 | )); 291 | 292 | switch ($config['type']) { 293 | case 'directory': 294 | $spFinderDef = new Definition('Symfony\Component\Finder\Finder'); 295 | $spFinderDef->addMethodCall('in', array( 296 | $config['uri'], 297 | )); 298 | $spDef = new Definition('Mathielen\ImportEngine\Storage\Provider\FinderFileStorageProvider', array( 299 | $spFinderDef, 300 | $formatDiscoverLocalFileStorageFactoryDef, 301 | )); 302 | break; 303 | case 'upload': 304 | $spDef = new Definition('Mathielen\ImportEngine\Storage\Provider\UploadFileStorageProvider', array( 305 | $config['uri'], 306 | $formatDiscoverLocalFileStorageFactoryDef, 307 | )); 308 | break; 309 | case 'dbal': 310 | $listResolverDef = new Definition(StringOrFileList::class, array($config['queries'])); 311 | if (!isset($config['connection_factory'])) { 312 | $connectionFactoryDef = new Definition(DefaultConnectionFactory::class, array(array('default' => new Reference('doctrine.dbal.default_connection')))); 313 | } else { 314 | $connectionFactoryDef = new Reference($config['connection_factory']); 315 | } 316 | 317 | $spDef = new Definition('Mathielen\ImportEngine\Storage\Provider\DbalStorageProvider', array( 318 | $connectionFactoryDef, 319 | $listResolverDef, 320 | )); 321 | break; 322 | case 'doctrine': 323 | $listResolverDef = new Definition(StringOrFileList::class, array($config['queries'])); 324 | if (!isset($config['connection_factory'])) { 325 | $connectionFactoryDef = new Definition(DefaultConnectionFactory::class, array(array('default' => new Reference('doctrine.orm.entity_manager')))); 326 | } else { 327 | $connectionFactoryDef = new Reference($config['connection_factory']); 328 | } 329 | 330 | $spDef = new Definition('Mathielen\ImportEngine\Storage\Provider\DoctrineQueryStorageProvider', array( 331 | $connectionFactoryDef, 332 | $listResolverDef, 333 | )); 334 | break; 335 | case 'service': 336 | $spDef = new Definition('Mathielen\ImportEngine\Storage\Provider\ServiceStorageProvider', array( 337 | new Reference('service_container'), 338 | $config['services'], 339 | )); 340 | break; 341 | case 'file': 342 | $spDef = new Definition('Mathielen\ImportEngine\Storage\Provider\FileStorageProvider', array( 343 | $formatDiscoverLocalFileStorageFactoryDef, 344 | )); 345 | break; 346 | default: 347 | throw new InvalidConfigurationException('Unknown type for storage provider: '.$config['type']); 348 | } 349 | 350 | $storageLocatorDef->addMethodCall('register', array( 351 | $id, 352 | $spDef, 353 | )); 354 | } 355 | 356 | private function getStorageFileDefinitionFromUri($uri) 357 | { 358 | if (substr($uri, 0, 2) === '@=') { 359 | $uri = new Expression(substr($uri, 2)); 360 | } 361 | 362 | return new Definition('SplFileInfo', array( 363 | $uri, 364 | )); 365 | } 366 | 367 | /** 368 | * @return Definition 369 | */ 370 | private function getStorageDef(array $config, Definition $objectFactoryDef = null) 371 | { 372 | switch ($config['type']) { 373 | case 'file': 374 | $fileDef = $this->getStorageFileDefinitionFromUri($config['uri']); 375 | 376 | $format = $config['format']; 377 | $storageDef = new Definition('Mathielen\ImportEngine\Storage\LocalFileStorage', array( 378 | $fileDef, 379 | new Definition('Mathielen\ImportEngine\Storage\Format\\'.ucfirst($format['type']).'Format', $format['arguments']), 380 | )); 381 | 382 | break; 383 | case 'doctrine': 384 | $storageDef = new Definition('Mathielen\ImportEngine\Storage\DoctrineStorage', array( 385 | new Reference('doctrine.orm.entity_manager'), 386 | $config['entity'], 387 | )); 388 | 389 | break; 390 | //@deprecated 391 | case 'service': 392 | $storageDef = new Definition('Mathielen\ImportEngine\Storage\ServiceStorage', array( 393 | [new Reference($config['service']), $config['method']], //callable 394 | [], 395 | $objectFactoryDef, //from parameter array 396 | )); 397 | 398 | break; 399 | case 'callable': 400 | $config['callable'][0] = new Reference($config['callable'][0]); 401 | $storageDef = new Definition('Mathielen\ImportEngine\Storage\ServiceStorage', array( 402 | $config['callable'], 403 | [], 404 | $objectFactoryDef, //from parameter array 405 | )); 406 | 407 | break; 408 | default: 409 | throw new InvalidConfigurationException('Unknown type for storage: '.$config['type']); 410 | } 411 | 412 | return $storageDef; 413 | } 414 | 415 | public function getAlias() 416 | { 417 | return 'mathielen_import_engine'; 418 | } 419 | } 420 | -------------------------------------------------------------------------------- /DependencyInjection/StringOrFileList.php: -------------------------------------------------------------------------------- 1 | &$v) { 12 | if (filter_var($v, FILTER_VALIDATE_URL)) { 13 | //nothing 14 | } elseif (is_dir($v)) { 15 | $iterator = new RecursiveDirectoryIterator($v, \FilesystemIterator::KEY_AS_PATHNAME); 16 | $iterator = new \RecursiveIteratorIterator($iterator); 17 | 18 | /** @var \Symfony\Component\Finder\SplFileInfo $file */ 19 | foreach ($iterator as $file) { 20 | if (is_file($file)) { 21 | $listOrStringsOrFiles[$file->getRelativePathname()] = $file; 22 | } 23 | } 24 | 25 | unset($listOrStringsOrFiles[$k]); 26 | } elseif (is_file($v)) { 27 | $v = new \SplFileInfo($v); 28 | } else { 29 | throw new \RuntimeException("Unknown value type $v"); 30 | } 31 | } 32 | 33 | parent::__construct($listOrStringsOrFiles); 34 | } 35 | 36 | public function offsetGet($offset) 37 | { 38 | //offset could be myfile.sql?targetconnection 39 | $url = parse_url($offset); 40 | $offset = $url['path']; 41 | 42 | if (!parent::offsetExists($offset)) { 43 | return $this->checkStreamWrapper($offset); 44 | } 45 | 46 | $v = parent::offsetGet($offset); 47 | if ($v instanceof \SplFileInfo) { 48 | return file_get_contents($v); 49 | } 50 | 51 | return $v; 52 | } 53 | 54 | private function checkStreamWrapper($offset) 55 | { 56 | foreach ($this as $k => &$v) { 57 | if (filter_var($v, FILTER_VALIDATE_URL)) { 58 | $path = $v.'/'.$offset; 59 | 60 | if (file_exists($path)) { 61 | return file_get_contents($path); 62 | } 63 | } 64 | } 65 | 66 | throw new \InvalidArgumentException("Item with id '$offset' could not be found."); 67 | } 68 | } -------------------------------------------------------------------------------- /Endpoint/DoctrineEndpoint.php: -------------------------------------------------------------------------------- 1 | objectManager = $objectManager; 21 | $this->chunkSize = $chunkSize; 22 | } 23 | 24 | public function add($entity) 25 | { 26 | $this->objectManager->persist($entity); 27 | ++$this->currentChunkCount; 28 | 29 | if ($this->chunkCompleted()) { 30 | $this->objectManager->flush(); 31 | $this->currentChunkCount = 0; 32 | } 33 | } 34 | 35 | protected function chunkCompleted() 36 | { 37 | return $this->chunkSize && $this->currentChunkCount >= $this->chunkSize; 38 | } 39 | 40 | public function prepare(ImportProcessEvent $event) 41 | { 42 | $this->added = 0; 43 | } 44 | 45 | public function finish(ImportProcessEvent $event) 46 | { 47 | $this->objectManager->flush(); 48 | } 49 | 50 | public function rollback() 51 | { 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /Expression/ExpressionLanguageProvider.php: -------------------------------------------------------------------------------- 1 | hasBlankValues = true; 37 | } else { 38 | $type = $this->guessValueType($value); 39 | isset($this->typeDistribution[$type]) ? ++$this->typeDistribution[$type] : $this->typeDistribution[$type] = 1; 40 | } 41 | } 42 | 43 | public function guessFieldType($defaultFieldFormat) 44 | { 45 | $distributionCount = count($this->typeDistribution); 46 | 47 | if ($distributionCount == 1) { 48 | $types = array_keys($this->typeDistribution); 49 | $guessedType = $types[0]; 50 | } elseif ($distributionCount == 0) { 51 | $guessedType = $defaultFieldFormat; 52 | } else { 53 | arsort($this->typeDistribution); 54 | $guessedType = array_keys($this->typeDistribution)[0]; 55 | } 56 | 57 | return array( 58 | 'empty' => $this->hasBlankValues, 59 | 'type' => $guessedType, 60 | ); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /Generator/ValueObject/FieldFormatGuesser.php: -------------------------------------------------------------------------------- 1 | fields as $fieldName => $fieldGuess) { 16 | $fieldDefinition = $fieldGuess->guessFieldType($defaultFieldFormat); 17 | 18 | $sanitizedFieldName = self::strtocamelcase($fieldName); 19 | if ($sanitizedFieldName != $fieldName) { 20 | $fieldDefinition['serialized_name'] = $fieldName; 21 | $fieldName = $sanitizedFieldName; 22 | } 23 | 24 | $fieldDefinitions[$fieldName] = $fieldDefinition; 25 | } 26 | 27 | return $fieldDefinitions; 28 | } 29 | 30 | private static function strtocamelcase($str) 31 | { 32 | $str = iconv('utf-8', 'ascii//TRANSLIT', $str); 33 | 34 | return preg_replace_callback('#[^\w]+(.)#', 35 | create_function('$r', 'return strtoupper($r[1]);'), $str); 36 | } 37 | 38 | public function putRow(array $row) 39 | { 40 | foreach ($row as $k => $v) { 41 | $this->addGuess($k, $v); 42 | } 43 | } 44 | 45 | /** 46 | * @return FieldFormatGuess 47 | */ 48 | private function getOrCreateFieldGuess($fieldname) 49 | { 50 | $fieldname = strtolower($fieldname); 51 | if (!isset($this->fields[$fieldname])) { 52 | $this->fields[$fieldname] = new FieldFormatGuess(); 53 | } 54 | 55 | return $this->fields[$fieldname]; 56 | } 57 | 58 | private function addGuess($fieldname, $fieldvalue) 59 | { 60 | $fieldGuess = $this->getOrCreateFieldGuess($fieldname); 61 | $fieldGuess->addValue($fieldvalue); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /Generator/ValueObjectGenerator.php: -------------------------------------------------------------------------------- 1 | $namespace, 20 | 'class' => $class, 21 | 'field_definitions' => $fieldDefinitions, 22 | ); 23 | 24 | $this->renderFile('ValueObject.php.twig', $file, $parameters); 25 | 26 | return $file; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /MathielenImportEngineBundle.php: -------------------------------------------------------------------------------- 1 | addExpressionLanguageProvider(new ExpressionLanguageProvider()); 14 | 15 | return parent::build($container); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /Resources/config/services.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | Y-m-d 25 | 26 | 27 | Y-m-d 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | CASE_LOWER 37 | 38 | 39 | CASE_UPPER 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | putRow 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | -------------------------------------------------------------------------------- /Resources/skeleton/valueobject.php.twig: -------------------------------------------------------------------------------- 1 | getMockBuilder('Mathielen\ImportEngine\Import\ImportBuilder')->disableOriginalConstructor()->getMock(); 32 | 33 | $this->container = new ContainerBuilder(); 34 | $this->container->set('event_dispatcher', $this->createMock('Symfony\Component\EventDispatcher\EventDispatcherInterface')); 35 | $this->container->set('mathielen_importengine.import.builder', $ib); 36 | $this->container->set('mathielen_importengine.import.runner', $this->getMockBuilder('Mathielen\ImportEngine\Import\Run\ImportRunner')->disableOriginalConstructor()->getMock()); 37 | 38 | $this->command = new ImportCommand(); 39 | $this->command->setContainer($this->container); 40 | } 41 | 42 | public function testRunLimit() 43 | { 44 | $this->container->get('mathielen_importengine.import.builder') 45 | ->expects($this->once()) 46 | ->method('buildFromRequest') 47 | ->will($this->returnValue( 48 | new Import( 49 | new Importer( 50 | $this->createMock('Mathielen\ImportEngine\Storage\StorageInterface') 51 | ), 52 | $this->createMock('Mathielen\ImportEngine\Storage\StorageInterface'), 53 | new ImportRun(new ImportConfiguration()) 54 | ) 55 | )); 56 | 57 | $this->container->get('event_dispatcher') 58 | ->expects($this->exactly(2)) 59 | ->method('addListener') 60 | ->withConsecutive( 61 | array( 62 | ImportConfigureEvent::AFTER_BUILD, 63 | $this->anything(), 64 | ) 65 | ); 66 | 67 | $this->container->get('mathielen_importengine.import.runner') 68 | ->expects($this->once()) 69 | ->method('run'); 70 | 71 | $input = new ArrayInput(array('--limit' => 1, '--importer' => 'abc', 'source_id' => 'source_id'), $this->command->getDefinition()); 72 | $output = new TestOutput(); 73 | 74 | $this->command->run($input, $output); 75 | } 76 | 77 | public function testRunDryrun() 78 | { 79 | $this->container->get('mathielen_importengine.import.builder') 80 | ->expects($this->once()) 81 | ->method('buildFromRequest') 82 | ->will($this->returnValue( 83 | new Import( 84 | new Importer( 85 | $this->createMock('Mathielen\ImportEngine\Storage\StorageInterface') 86 | ), 87 | $this->createMock('Mathielen\ImportEngine\Storage\StorageInterface'), 88 | new ImportRun(new ImportConfiguration()) 89 | ) 90 | )); 91 | 92 | $this->container->get('mathielen_importengine.import.runner') 93 | ->expects($this->once()) 94 | ->method('dryrun'); 95 | 96 | $input = new ArrayInput(array('--dryrun' => true, '--importer' => 'abc', 'source_id' => 'source_id'), $this->command->getDefinition()); 97 | $output = new TestOutput(); 98 | 99 | $this->command->run($input, $output); 100 | } 101 | 102 | /** 103 | * @dataProvider getRunData 104 | */ 105 | public function testRun(array $input, $parsedSourceId) 106 | { 107 | $this->container->get('mathielen_importengine.import.builder') 108 | ->expects($this->once()) 109 | ->method('buildFromRequest') 110 | ->with(new ImportRequest($parsedSourceId, 'default', null, Utils::whoAmI().'@CLI')) 111 | ->will($this->returnValue( 112 | new Import( 113 | new Importer( 114 | $this->createMock('Mathielen\ImportEngine\Storage\StorageInterface') 115 | ), 116 | $this->createMock('Mathielen\ImportEngine\Storage\StorageInterface'), 117 | new ImportRun(new ImportConfiguration()) 118 | ) 119 | )); 120 | 121 | $this->container->get('mathielen_importengine.import.runner') 122 | ->expects($this->once()) 123 | ->method('run'); 124 | 125 | $input = new ArrayInput($input, $this->command->getDefinition()); 126 | $output = new TestOutput(); 127 | 128 | $this->command->run($input, $output); 129 | } 130 | 131 | public function getRunData() 132 | { 133 | return array( 134 | array(array('source_id' => 'source_id'), 'source_id'), 135 | array(array('source_id' => 'service.method?arg1=abc'), array('service' => 'service', 'method' => 'method', 'arguments' => array('arg1' => 'abc'))), 136 | ); 137 | } 138 | } 139 | 140 | class TestOutput extends Output 141 | { 142 | public $output = ''; 143 | 144 | public function clear() 145 | { 146 | $this->output = ''; 147 | } 148 | 149 | protected function doWrite($message, $newline) 150 | { 151 | $this->output .= $message.($newline ? "\n" : ''); 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /Tests/DependencyInjection/AbstractExtensionTest.php: -------------------------------------------------------------------------------- 1 | loadConfiguration($this->container, $resource); 15 | } 16 | 17 | $this->container->loadFromExtension($this->extension->getAlias()); 18 | $this->container->compile(); 19 | 20 | return $this->container; 21 | } 22 | 23 | public function testWithoutConfiguration() 24 | { 25 | $container = $this->getContainer(); 26 | $this->assertFalse($container->has('mathielen_importengine.import.storagelocator')); 27 | $this->assertFalse($container->has('mathielen_importengine.import.builder')); 28 | } 29 | 30 | public function testFullConfiguration() 31 | { 32 | $container = $this->getContainer('full'); 33 | $this->assertTrue($container->has('mathielen_importengine.import.storagelocator')); 34 | $this->assertTrue($container->has('mathielen_importengine.import.builder')); 35 | } 36 | 37 | public function testMediumConfiguration() 38 | { 39 | $container = $this->getContainer('medium'); 40 | $this->assertTrue($container->has('mathielen_importengine.import.storagelocator')); 41 | $this->assertTrue($container->has('mathielen_importengine.import.builder')); 42 | } 43 | 44 | public function testMinimumConfiguration() 45 | { 46 | $container = $this->getContainer('minimum'); 47 | $this->assertTrue($container->has('mathielen_importengine.import.storagelocator')); 48 | $this->assertTrue($container->has('mathielen_importengine.import.builder')); 49 | } 50 | 51 | public function testStorageProvidersAreProperlyRegisteredByTheirName() 52 | { 53 | $container = $this->getContainer('full'); 54 | 55 | $storageLocatorDef = $container->findDefinition('mathielen_importengine.import.storagelocator'); 56 | $methodCalls = $storageLocatorDef->getMethodCalls(); 57 | 58 | $registeredStorageProviderIds = []; 59 | foreach ($methodCalls as $methodCall) { 60 | $arguments = $methodCall[1]; 61 | 62 | $registeredStorageProviderIds[] = $arguments[0]; 63 | } 64 | 65 | $this->assertEquals(['upload', 'localdir', 'localfile', 'doctrine', 'services'], $registeredStorageProviderIds); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /Tests/DependencyInjection/AbstractTest.php: -------------------------------------------------------------------------------- 1 | extension = new MathielenImportEngineExtension(); 23 | 24 | $this->container = new ContainerBuilder(); 25 | $this->container->registerExtension($this->extension); 26 | $this->container->set('event_dispatcher', $this->createMock('Symfony\Component\EventDispatcher\EventDispatcherInterface')); 27 | $this->container->set('logger', $this->createMock('Psr\Log\LoggerInterface')); 28 | $this->container->set('import_service', new MyImportService()); //target service 29 | $this->container->set('jms_serializer', $this->createMock('JMS\Serializer\SerializerInterface')); 30 | $this->container->set('validator', $this->createMock('Symfony\Component\Validator\Validator\ValidatorInterface')); 31 | $this->container->set('doctrine.orm.entity_manager', $this->createMock('Doctrine\ORM\EntityManagerInterface')); 32 | $this->container->set('logger', $this->createMock('Psr\Log\LoggerInterface')); 33 | $this->container->set('some.converter.serviceid', new MyDummyService()); 34 | $this->container->set('some.other.converter.serviceid', new MyDummyService()); 35 | $this->container->set('email', new Email()); 36 | $this->container->set('url', new Url()); 37 | $this->container->set('notempty', new NotBlank()); 38 | } 39 | } 40 | 41 | class MyImportService 42 | { 43 | } 44 | 45 | class MyDummyService 46 | { 47 | } 48 | -------------------------------------------------------------------------------- /Tests/DependencyInjection/CompareTest.php: -------------------------------------------------------------------------------- 1 | setUp(); 14 | $container = $this->container; 15 | $loader = new XmlFileLoader($container, new FileLocator(__DIR__.'/Fixtures/Xml/')); 16 | $loader->load("$filename.xml"); 17 | 18 | $container->loadFromExtension('mathielen_import_engine'); 19 | $container->compile(); 20 | 21 | return $container->getDefinitions(); 22 | } 23 | 24 | private function getYamlDefinitions($filename) 25 | { 26 | $this->setUp(); 27 | $container = $this->container; 28 | $loader = new YamlFileLoader($container, new FileLocator(__DIR__.'/Fixtures/Yaml/')); 29 | $loader->load("$filename.yml"); 30 | 31 | $container->loadFromExtension('mathielen_import_engine'); 32 | $container->compile(); 33 | 34 | return $container->getDefinitions(); 35 | } 36 | 37 | public function testFullXmlAndYamlSame() 38 | { 39 | $this->assertEquals($this->getYamlDefinitions('full'), $this->getXmlDefinitions('full')); 40 | $this->assertEquals($this->getYamlDefinitions('medium'), $this->getXmlDefinitions('medium')); 41 | $this->assertEquals($this->getYamlDefinitions('minimum'), $this->getXmlDefinitions('minimum')); 42 | } 43 | } 44 | 45 | class MyImportedRow 46 | { 47 | } 48 | -------------------------------------------------------------------------------- /Tests/DependencyInjection/Fixtures/Xml/full.xml: -------------------------------------------------------------------------------- 1 | 6 | 7 | 8 | 9 | upload 10 | /tmp/uploaddir 11 | 12 | 13 | directory 14 | /tmp/cached 15 | 16 | 17 | file 18 | /tmp/cached/file 19 | 20 | 21 | doctrine 22 | SELECT id FROM Acme\DemoBundle\Entity\Person P WHERE P.age > 10 23 | Acme\DemoBundle\Entity\ImportData 24 | 25 | 26 | service 27 | 28 | exportMethod1 29 | exportMethod2 30 | 31 | 32 | 33 | 34 | 35 | 36 | value1 37 | 38 | value2.1 39 | value2.2.1 40 | value2.2.2 41 | 42 | 43 | 44 | excel 45 | csv 46 | 2 47 | header2 48 | header1 49 | header1 50 | header2 51 | regexp1 52 | regexp2 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | context 67 | 68 | excel 69 | regexp1 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | -------------------------------------------------------------------------------- /Tests/DependencyInjection/Fixtures/Xml/medium.xml: -------------------------------------------------------------------------------- 1 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | email 13 | url 14 | 15 | 16 | notempty 17 | notempty 18 | 19 | 20 | 21 | 22 | csv 23 | a 24 | b 25 | c 26 | false 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /Tests/DependencyInjection/Fixtures/Xml/minimum.xml: -------------------------------------------------------------------------------- 1 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /Tests/DependencyInjection/Fixtures/Yaml/full.yml: -------------------------------------------------------------------------------- 1 | mathielen_import_engine: 2 | storageprovider: 3 | upload: 4 | type: upload 5 | uri: /tmp/uploaddir 6 | localdir: 7 | type: directory 8 | uri: /tmp/cached 9 | localfile: 10 | type: file 11 | uri: /tmp/cached/file 12 | doctrine: 13 | type: doctrine 14 | queries: 15 | - SELECT id FROM Acme\DemoBundle\Entity\Person P WHERE P.age > 10 16 | - Acme\DemoBundle\Entity\ImportData 17 | services: 18 | type: service 19 | services: 20 | export_serviceA: [exportMethod1, exportMethod2] 21 | export_serviceB: ~ 22 | importers: 23 | maximum_importer: 24 | context: 25 | key1: value1 26 | key2: 27 | deep-key1: value2.1 28 | deep-key2: [value2.2.1, value2.2.2] 29 | preconditions: 30 | format: ['excel', 'csv'] 31 | fieldcount: 2 32 | fields: 33 | - 'header2' 34 | - 'header1' 35 | fieldset: 36 | - 'header1' 37 | - 'header2' 38 | filename: ['regexp1', 'regexp2'] 39 | object_factory: 40 | type: jms_serializer 41 | class: Mathielen\ImportEngineBundle\Tests\MyImportedRow 42 | mappings: 43 | header1: 44 | to: targetField1 45 | converter: some.converter.serviceid 46 | header2: 47 | converter: some.other.converter.serviceid 48 | header3: targetField3 49 | validation: 50 | options: 51 | allowExtraFields: false 52 | allowMissingFields: true 53 | target: ~ 54 | target: 55 | type: service 56 | service: import_service 57 | method: processImportRow 58 | doctrine_importer: 59 | context: context 60 | preconditions: 61 | format: excel 62 | filename: regexp1 63 | object_factory: 64 | type: default 65 | class: Mathielen\ImportEngineBundle\Tests\MyImportedRow 66 | source: 67 | type: file 68 | uri: /tmp/import.csv 69 | format: csv 70 | validation: 71 | target: ~ 72 | target: 73 | type: doctrine 74 | entity: Mathielen\ImportEngineBundle\Tests\MyImportedRow 75 | -------------------------------------------------------------------------------- /Tests/DependencyInjection/Fixtures/Yaml/medium.yml: -------------------------------------------------------------------------------- 1 | mathielen_import_engine: 2 | importers: 3 | minimum_importer: 4 | validation: 5 | source: 6 | header1: email 7 | header2: url 8 | target: 9 | header1: notempty 10 | header2: notempty 11 | target: 12 | type: file 13 | uri: /tmp/myfile.csv 14 | format: { type: csv, arguments: ['a', 'b', 'c', false] } 15 | -------------------------------------------------------------------------------- /Tests/DependencyInjection/Fixtures/Yaml/minimum.yml: -------------------------------------------------------------------------------- 1 | mathielen_import_engine: 2 | importers: 3 | minimum_importer: 4 | target: 5 | type: file 6 | uri: /tmp/myfile.csv 7 | format: csv 8 | -------------------------------------------------------------------------------- /Tests/DependencyInjection/XmlExtensionTest.php: -------------------------------------------------------------------------------- 1 | load($resource.'.xml'); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /Tests/DependencyInjection/YamlExtensionTest.php: -------------------------------------------------------------------------------- 1 | load($resource.'.yml'); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /Tests/MyImportedRow.php: -------------------------------------------------------------------------------- 1 | assertTrue(Utils::isCli()); 12 | } 13 | 14 | public function testWhoAmI() 15 | { 16 | $processUser = posix_getpwuid(posix_geteuid()); 17 | $actualUser = $processUser['name']; 18 | 19 | $this->assertEquals($actualUser, Utils::whoAmI()); 20 | } 21 | 22 | /** 23 | * @dataProvider getNumbersToRangeTextData 24 | */ 25 | public function testNumbersToRangeText(array $range, $expected) 26 | { 27 | $this->assertEquals($expected, Utils::numbersToRangeText($range)); 28 | } 29 | 30 | public function getNumbersToRangeTextData() 31 | { 32 | return [ 33 | [[], []], 34 | [[1], ['1']], 35 | [[1, 2, 3], ['1-3']], 36 | [[1, 2, 4, 5], ['1-2', '4-5']], 37 | [[1, 3, 5, 7], ['1', '3', '5', '7']], 38 | ]; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /Tests/bootstrap.php: -------------------------------------------------------------------------------- 1 | $service, 25 | 'method' => $method, 26 | 'arguments' => isset($parsedSourceId['query']) ? $parsedSourceId['query'] : null, 27 | ); 28 | } 29 | 30 | return $sourceId; 31 | } 32 | 33 | /** 34 | * @return bool 35 | */ 36 | public static function isWindows() 37 | { 38 | return strtoupper(substr(PHP_OS, 0, 3)) === 'WIN'; 39 | } 40 | 41 | /** 42 | * @return bool 43 | */ 44 | public static function isCli() 45 | { 46 | return php_sapi_name() == 'cli'; 47 | } 48 | 49 | /** 50 | * @return string 51 | */ 52 | public static function whoAmI() 53 | { 54 | if (self::isWindows()) { 55 | $user = getenv('username'); 56 | } else { 57 | $processUser = posix_getpwuid(posix_geteuid()); 58 | $user = $processUser['name']; 59 | } 60 | 61 | return $user; 62 | } 63 | 64 | /** 65 | * @return array 66 | */ 67 | public static function numbersToRangeText(array $numbers) 68 | { 69 | if (empty($numbers)) { 70 | return []; 71 | } 72 | 73 | $ranges = []; 74 | sort($numbers); 75 | 76 | $currentRange = []; 77 | foreach ($numbers as $number) { 78 | if (!empty($currentRange) && current($currentRange) !== $number - 1) { 79 | self::addRangeText($ranges, $currentRange); 80 | 81 | $currentRange = []; 82 | } 83 | 84 | $currentRange[] = $number; 85 | end($currentRange); 86 | } 87 | 88 | self::addRangeText($ranges, $currentRange); 89 | 90 | return $ranges; 91 | } 92 | 93 | private static function addRangeText(array &$ranges, array $currentRange) 94 | { 95 | $lastItem = current($currentRange); 96 | 97 | if (count($currentRange) === 1) { 98 | $ranges[] = $lastItem; 99 | } else { 100 | $firstItem = reset($currentRange); 101 | $ranges[] = $firstItem.'-'.$lastItem; 102 | } 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mathielen/import-engine-bundle", 3 | "type": "symfony-bundle", 4 | "description": "A generic import (and export) -engine that provides easy to use yet powerful features", 5 | "keywords": [ 6 | "csv", 7 | "import" 8 | ], 9 | "homepage": "https://github.com/mathielen/ImportEngineBundle", 10 | "license": "MIT", 11 | "authors": [ 12 | { 13 | "name": "Markus Thielen", 14 | "email": "markus@logicx.de" 15 | } 16 | ], 17 | "require": { 18 | "php": ">=5.5.0", 19 | "mathielen/import-engine": "dev-master", 20 | "symfony/framework-bundle": "*" 21 | }, 22 | "require-dev": { 23 | "symfony/console": "*", 24 | "symfony/validator": "*", 25 | "phpunit/phpunit": "*", 26 | "jms/serializer": "*", 27 | "doctrine/orm": "*" 28 | }, 29 | "suggest": { 30 | "symfony/console": "If you want to use the Commands", 31 | "sensio/generator-bundle": "If you want to use the GenerateImportValueObjectCommand" 32 | }, 33 | "autoload": { 34 | "psr-0": { 35 | "Mathielen\\ImportEngineBundle": "" 36 | } 37 | }, 38 | "target-dir": "Mathielen/ImportEngineBundle" 39 | } 40 | -------------------------------------------------------------------------------- /phpunit.xml.dist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 13 | 14 | 15 | ./Tests/ 16 | 17 | 18 | 19 | 20 | 21 | ./ 22 | 23 | ./Resources 24 | ./Tests 25 | ./vendor 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | Mathielen Import Engine Bundle 2 | ========================== 3 | 4 | [![Build Status](https://travis-ci.org/mathielen/ImportEngineBundle.png?branch=master)](https://travis-ci.org/mathielen/ImportEngineBundle) 5 | [![Scrutinizer Code Quality](https://scrutinizer-ci.com/g/mathielen/ImportEngineBundle/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/mathielen/ImportEngineBundle/?branch=master) 6 | [![Code Coverage](https://scrutinizer-ci.com/g/mathielen/ImportEngineBundle/badges/coverage.png?b=master)](https://scrutinizer-ci.com/g/mathielen/ImportEngineBundle/?branch=master) 7 | [![SensioLabsInsight](https://insight.sensiolabs.com/projects/16f2af0e-9318-47f7-bd12-d3f07caf1d21/mini.png)](https://insight.sensiolabs.com/projects/16f2af0e-9318-47f7-bd12-d3f07caf1d21) 8 | [![Latest Stable Version](https://poser.pugx.org/mathielen/import-engine-bundle/v/stable.png)](https://packagist.org/packages/mathielen/import-engine-bundle) 9 | 10 | 11 | Introduction 12 | ------------ 13 | This is a bundle for the [mathielen/import-engine library](https://github.com/mathielen/import-engine). 14 | It provides an easy way to configure a full-blown data importer for your symfony2 project. 15 | 16 | Installation 17 | ------------ 18 | This library is available on [Packagist](https://packagist.org/packages/mathielen/import-engine-bundle): 19 | 20 | To install it, run: 21 | 22 | ```bash 23 | $ composer require mathielen/import-engine-bundle 24 | ``` 25 | 26 | Then add the bundle to `app/AppKernel.php`: 27 | 28 | ```php 29 | public function registerBundles() 30 | { 31 | return array( 32 | ... 33 | new Mathielen\ImportEngineBundle\MathielenImportEngineBundle(), 34 | ... 35 | ); 36 | } 37 | ``` 38 | 39 | If you want to make use of excel files, please also make sure to include phpoffice/phpexcel in your project: 40 | 41 | ```bash 42 | $ composer require phpoffice/phpexcel 43 | ``` 44 | 45 | Configuration 46 | ------------ 47 | Add your importer configurations in your `app/config/config.yml`. 48 | 49 | Full example: 50 | ```yaml 51 | mathielen_import_engine: 52 | #configure storageproviders, that are used in all importers 53 | storageprovider: 54 | default: 55 | type: directory 56 | uri: /tmp/somedir 57 | upload: 58 | type: upload 59 | uri: "%kernel.root_dir%/Resources/import" 60 | doctrine: 61 | type: doctrine 62 | queries: #a list of DQL-Statements, Entity-Classnames, filenames or directories 63 | - SELECT id FROM Acme\DemoBundle\Entity\Person P WHERE P.age > 10 #dql statement 64 | - Acme\DemoBundle\Entity\ImportData #entity classname 65 | - %kernel.root_dir%/dql/mysql.dql #file with dql statement in it 66 | - %kernel.root_dir%/other-dql #directory 67 | dbal: 68 | type: dbal 69 | queries: %kernel.root_dir%/sql/ #same like doctrine 70 | services: 71 | type: service 72 | services: 73 | #the services export_serviceA and export_serviceB must be configured in DIC 74 | export_serviceA: [exportMethod1, exportMethod2] #restrict to specific methods of service 75 | export_serviceB: ~ #every method of service can be used 76 | 77 | #configure your Importers 78 | importers: 79 | your_importer_name: 80 | #some context information that is passed through the whole process 81 | context: 82 | key: value 83 | 84 | #automaticly recognize this importer by meeting of the conditions below 85 | preconditions: 86 | format: excel #format of data must be [csv, excel, xml] 87 | fieldcount: 2 #must have this number of fields 88 | fields: #these fields must exist (order is irrelevant) 89 | - 'header2' 90 | - 'header1' 91 | fieldset: #all fields must exist exactly this order 92 | - 'header1' 93 | - 'header2' 94 | filename: 'somefile.xls' #filename must match one of these regular expression(s) (can be a list) 95 | 96 | #use an object-factory to convert raw row-arrays to target objects 97 | object_factory: 98 | type: jms_serializer #[jms_serializer, default] 99 | class: Acme\DemoBundle\ValueObject\MyImportedRow 100 | 101 | #add mapping 102 | mappings: 103 | #simple a-to-b mapping 104 | source-field1: target-field1 105 | 106 | #convert the field (but dont map) 107 | source-field2: 108 | #converts excel's date-field to a Y-m-d string (you can use your own service-id here) 109 | converter: mathielen_importengine.converter.excel.genericdate 110 | 111 | #map and convert 112 | source-field3: 113 | to: target-field3 114 | converter: upperCase #use a converter that was registered with the converter-provider 115 | 116 | #validate imported data 117 | validation: 118 | source: #add constraints to source fields 119 | header1: email 120 | header2: notempty 121 | target: ~ #activate validation against generated object from object-factory (via annotations, xml) 122 | #or supply list of constraints like in source 123 | 124 | #target of import 125 | target: 126 | type: service #[service, doctrine, file] 127 | service: '@import_service' #service name in DIC 128 | method: processImportRow #method to invoke on service 129 | ``` 130 | 131 | Minimum example: 132 | ```yaml 133 | mathielen_import_engine: 134 | importers: 135 | minimum_importer: 136 | target: 137 | type: file 138 | uri: /tmp/myfile.csv 139 | format: csv 140 | 141 | another_minimum_importer: 142 | target: 143 | type: file 144 | uri: "@='%kernel.root_dir%/../output_'~date('Y-m-d')~'.csv'" #this uses symfony expression language 145 | #to create the filename. Just prefix your 146 | #expression with @= 147 | format: { type: csv, arguments: [','] } #delimiter is now ',' 148 | ``` 149 | 150 | Check out the Testsuite for more information. 151 | 152 | Usage 153 | ------------ 154 | 155 | ### On the command line 156 | 157 | #### Show your configured Import profiles 158 | ```bash 159 | $ app/console importengine:list 160 | ``` 161 | 162 | #### Let the framework discover which importer suites best (auto discovery) #### 163 | Uses the storageprovider "default" if not also given as argument. 164 | ```bash 165 | $ app/console importengine:import /tmp/somedir/myfile.csv 166 | ``` 167 | 168 | #### Import myfile.csv with "your_importer_name" importer #### 169 | Uses the storageprovider "default" if not also given as argument. 170 | ```bash 171 | $ app/console importengine:import -i your_importer_name /tmp/somedir/myfile.csv 172 | ``` 173 | 174 | #### Generate a [JMS Serializer](http://jmsyst.com/libs/serializer)-annotated ValueObject class for an arbitrary import source (ie. a file) 175 | ```bash 176 | $ app/console importengine:generate:valueobject data/myfile.csv Acme\\ValueObject\\MyFileRow src 177 | ``` 178 | 179 | ### Use the importer within a controller / service 180 | 181 | ```php 182 | namespace AppBundle\Controller; 183 | 184 | use Sensio\Bundle\FrameworkExtraBundle\Configuration\Route; 185 | use Sensio\Bundle\FrameworkExtraBundle\Configuration\Method; 186 | use Symfony\Bundle\FrameworkBundle\Controller\Controller; 187 | 188 | class DefaultController extends Controller 189 | { 190 | 191 | /** 192 | * Import a given file, that was POST'ed to the HTTP-Endpoint /app/import 193 | * * Using the default sorage provider 194 | * * The importer is auto-discovered with the format of the file 195 | * 196 | * @Route("/app/import", name="homepage") 197 | * @Method("POST") 198 | */ 199 | public function importAction(\Symfony\Component\HttpFoundation\Request $request) 200 | { 201 | //create the request for the import-engine 202 | $importRequest = new \Mathielen\ImportEngine\ValueObject\ImportRequest($request->files->getIterator()->current()); 203 | 204 | /** @var \Mathielen\ImportEngine\Import\ImportBuilder $importBuilder */ 205 | $importBuilder = $this->container->get('mathielen_importengine.import.builder'); 206 | $import = $importBuilder->build($importRequest); 207 | 208 | /** @var \Mathielen\ImportEngine\Import\Run\ImportRunner $importRunner */ 209 | $importRunner = $this->container->get('mathielen_importengine.import.runner'); 210 | $importRun = $importRunner->run($import); 211 | 212 | return $this->render('default/import.html.twig', $importRun->getStatistics()); 213 | } 214 | 215 | } 216 | ``` 217 | --------------------------------------------------------------------------------