├── src
    ├── Resources
    │   ├── doc
    │   │   └── schema.png
    │   └── config
    │   │   ├── exceptions_services.yaml
    │   │   ├── messenger_services.yaml
    │   │   └── services.yaml
    ├── Runner
    │   ├── PendingDataflowRunnerInterface.php
    │   ├── PendingDataflowRunner.php
    │   └── MessengerDataflowRunner.php
    ├── Event
    │   ├── CrEvent.php
    │   ├── Events.php
    │   └── ProcessingEvent.php
    ├── Processor
    │   ├── JobProcessorInterface.php
    │   └── JobProcessor.php
    ├── Exceptions
    │   ├── InterruptedProcessingException.php
    │   ├── UnsupportedItemTypeException.php
    │   └── UnknownDataflowTypeException.php
    ├── ExceptionsHandler
    │   ├── ExceptionHandlerInterface.php
    │   ├── NullExceptionHandler.php
    │   └── FilesystemExceptionHandler.php
    ├── DataflowType
    │   ├── AutoUpdateCountInterface.php
    │   ├── DataflowTypeInterface.php
    │   ├── Writer
    │   │   ├── DelegateWriterInterface.php
    │   │   ├── WriterInterface.php
    │   │   ├── PortWriterAdapter.php
    │   │   ├── CollectionWriter.php
    │   │   └── DelegatorWriter.php
    │   ├── Dataflow
    │   │   ├── DataflowInterface.php
    │   │   ├── Dataflow.php
    │   │   └── AMPAsyncDataflow.php
    │   ├── Result.php
    │   ├── AMPAsyncDataflowBuilder.php
    │   ├── AbstractDataflowType.php
    │   └── DataflowBuilder.php
    ├── MessengerMode
    │   ├── JobMessage.php
    │   └── JobMessageHandler.php
    ├── Manager
    │   ├── ScheduledDataflowManagerInterface.php
    │   └── ScheduledDataflowManager.php
    ├── Validator
    │   └── Constraints
    │   │   ├── Frequency.php
    │   │   └── FrequencyValidator.php
    ├── Factory
    │   └── ConnectionFactory.php
    ├── Registry
    │   ├── DataflowTypeRegistryInterface.php
    │   └── DataflowTypeRegistry.php
    ├── Logger
    │   ├── BufferHandler.php
    │   └── DelegatingLogger.php
    ├── DependencyInjection
    │   ├── Compiler
    │   │   ├── DataflowTypeCompilerPass.php
    │   │   ├── DefaultLoggerCompilerPass.php
    │   │   ├── ExceptionCompilerPass.php
    │   │   └── BusCompilerPass.php
    │   ├── CodeRhapsodieDataflowExtension.php
    │   └── Configuration.php
    ├── Repository
    │   ├── InitFromDbTrait.php
    │   ├── ScheduledDataflowRepository.php
    │   └── JobRepository.php
    ├── CodeRhapsodieDataflowBundle.php
    ├── Gateway
    │   └── JobGateway.php
    ├── Command
    │   ├── SchemaCommand.php
    │   ├── RunPendingDataflowsCommand.php
    │   ├── ScheduleListCommand.php
    │   ├── ChangeScheduleStatusCommand.php
    │   ├── ExecuteDataflowCommand.php
    │   ├── JobShowCommand.php
    │   ├── DatabaseSchemaCommand.php
    │   └── AddScheduledDataflowCommand.php
    ├── SchemaProvider
    │   └── DataflowSchemaProvider.php
    └── Entity
    │   ├── ScheduledDataflow.php
    │   └── Job.php
├── sonar-project.properties
├── UPGRADE.md
├── .gitignore
├── .github
    └── workflows
    │   ├── ci.yml
    │   ├── semicolons-kudos.yaml
    │   └── build.yml
├── Tests
    ├── bootstrap.php
    ├── DataflowType
    │   ├── Writer
    │   │   ├── PortWriterAdapterTest.php
    │   │   ├── CollectionWriterTest.php
    │   │   └── DelegatorWriterTest.php
    │   ├── Dataflow
    │   │   └── AMPAsyncDataflowTest.php
    │   └── AbstractDataflowTypeTest.php
    ├── MessengerMode
    │   └── JobMessageHandlerTest.php
    ├── Registry
    │   └── DataflowTypeRegistryTest.php
    ├── Runner
    │   ├── PendingDataflowRunnerTest.php
    │   └── MessengerDataflowRunnerTest.php
    ├── Validator
    │   └── Constraints
    │   │   └── FrequencyValidatorTest.php
    ├── Processor
    │   └── JobProcessorTest.php
    └── Manager
    │   └── ScheduledDataflowManagerTest.php
├── rector.php
├── phpunit.xml
├── LICENSE
├── .php-cs-fixer.dist.php
├── CONTRIBUTING.md
├── composer.json
├── CHANGELOG.md
└── README.md


/src/Resources/doc/schema.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/code-rhapsodie/dataflow-bundle/HEAD/src/Resources/doc/schema.png


--------------------------------------------------------------------------------
/sonar-project.properties:
--------------------------------------------------------------------------------
1 | sonar.projectKey=code-rhapsodie_dataflow-bundle_AYvYuaAwWE9sbcQmD1vw
2 | 
3 | sonar.sources=src
4 | sonar.tests=Tests


--------------------------------------------------------------------------------
/UPGRADE.md:
--------------------------------------------------------------------------------
1 | # Upgrade from v1.x to v2.0
2 | 
3 | [BC] `JobRepository` and `ScheduledDataflowRepository` are no longer a Doctrine ORM repository.
4 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | vendor
2 | composer.lock
3 | .phpunit.result.cache
4 | .php_cs.cache
5 | .php_cs
6 | .idea
7 | .phpunit.cache
8 | .php-version
9 | .php-cs-fixer.cache


--------------------------------------------------------------------------------
/src/Runner/PendingDataflowRunnerInterface.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\Runner;
 6 | 
 7 | interface PendingDataflowRunnerInterface
 8 | {
 9 |     public function runPendingDataflows(): void;
10 | }
11 | 


--------------------------------------------------------------------------------
/src/Event/CrEvent.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\Event;
 6 | 
 7 | use Symfony\Contracts\EventDispatcher\Event;
 8 | 
 9 | /**
10 |  * @codeCoverageIgnore
11 |  */
12 | abstract class CrEvent extends Event
13 | {
14 | }
15 | 


--------------------------------------------------------------------------------
/src/Processor/JobProcessorInterface.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\Processor;
 6 | 
 7 | use CodeRhapsodie\DataflowBundle\Entity\Job;
 8 | 
 9 | interface JobProcessorInterface
10 | {
11 |     public function process(Job $job): void;
12 | }
13 | 


--------------------------------------------------------------------------------
/src/Exceptions/InterruptedProcessingException.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\Exceptions;
 6 | 
 7 | /**
 8 |  * Exception thrown when a dataflow is interrupted by signal.
 9 |  */
10 | class InterruptedProcessingException extends \Exception
11 | {
12 | }
13 | 


--------------------------------------------------------------------------------
/src/Exceptions/UnsupportedItemTypeException.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\Exceptions;
 6 | 
 7 | /**
 8 |  * Exception thrown when a writer receives an item of an unsupported type.
 9 |  */
10 | class UnsupportedItemTypeException extends \Exception
11 | {
12 | }
13 | 


--------------------------------------------------------------------------------
/src/Event/Events.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\Event;
 6 | 
 7 | final class Events
 8 | {
 9 |     public const BEFORE_PROCESSING = 'coderhapsodie.dataflow.before_processing';
10 |     public const AFTER_PROCESSING = 'coderhapsodie.dataflow.after_processing';
11 | }
12 | 


--------------------------------------------------------------------------------
/src/ExceptionsHandler/ExceptionHandlerInterface.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\ExceptionsHandler;
 6 | 
 7 | interface ExceptionHandlerInterface
 8 | {
 9 |     public function save(?int $jobId, ?array $exceptions): void;
10 | 
11 |     public function find(int $jobId): ?array;
12 | }
13 | 


--------------------------------------------------------------------------------
/src/DataflowType/AutoUpdateCountInterface.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\DataflowType;
 6 | 
 7 | use CodeRhapsodie\DataflowBundle\Repository\JobRepository;
 8 | 
 9 | interface AutoUpdateCountInterface
10 | {
11 |     public function setRepository(JobRepository $repository): void;
12 | }
13 | 


--------------------------------------------------------------------------------
/src/MessengerMode/JobMessage.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\MessengerMode;
 6 | 
 7 | class JobMessage
 8 | {
 9 |     public function __construct(private int $jobId)
10 |     {
11 |     }
12 | 
13 |     public function getJobId(): int
14 |     {
15 |         return $this->jobId;
16 |     }
17 | }
18 | 


--------------------------------------------------------------------------------
/src/Resources/config/exceptions_services.yaml:
--------------------------------------------------------------------------------
1 | services:
2 |     CodeRhapsodie\DataflowBundle\ExceptionsHandler\ExceptionHandlerInterface: '@CodeRhapsodie\DataflowBundle\ExceptionsHandler\FilesystemExceptionHandler'
3 |     CodeRhapsodie\DataflowBundle\ExceptionsHandler\FilesystemExceptionHandler:
4 |         arguments:
5 |             $filesystem: ~ # Filled in compiler pass
6 | 


--------------------------------------------------------------------------------
/src/DataflowType/DataflowTypeInterface.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\DataflowType;
 6 | 
 7 | interface DataflowTypeInterface
 8 | {
 9 |     public function getLabel(): string;
10 | 
11 |     public function getAliases(): iterable;
12 | 
13 |     public function process(array $options, ?int $jobId = null): Result;
14 | }
15 | 


--------------------------------------------------------------------------------
/.github/workflows/ci.yml:
--------------------------------------------------------------------------------
 1 | name: CI
 2 | 
 3 | on: [push]
 4 | 
 5 | jobs:
 6 |   build-test:
 7 |     runs-on: ubuntu-latest
 8 | 
 9 |     steps:
10 |       - uses: actions/checkout@v4
11 |       - uses: php-actions/composer@v6 # or alternative dependency management
12 |       - uses: php-actions/phpunit@v4
13 |       - name: Run PHP CS Fixer
14 |         run: php vendor/bin/php-cs-fixer fix --dry-run --diff


--------------------------------------------------------------------------------
/Tests/bootstrap.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | // Skip autoloading if already done by phpunit alias (including from meta repo if this is vendor)
 4 | if (defined('PHPUNIT_COMPOSER_INSTALL')) {
 5 |     return;
 6 | }
 7 | $autoloadFile = __DIR__ . '/../vendor/autoload.php';
 8 | if (!file_exists($autoloadFile)) {
 9 |     throw new RuntimeException('Install dependencies to run test suite.');
10 | }
11 | require_once $autoloadFile;
12 | 


--------------------------------------------------------------------------------
/src/ExceptionsHandler/NullExceptionHandler.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\ExceptionsHandler;
 6 | 
 7 | class NullExceptionHandler implements ExceptionHandlerInterface
 8 | {
 9 |     public function save(?int $jobId, ?array $exceptions): void
10 |     {
11 |     }
12 | 
13 |     public function find(int $jobId): ?array
14 |     {
15 |         return null;
16 |     }
17 | }
18 | 


--------------------------------------------------------------------------------
/src/DataflowType/Writer/DelegateWriterInterface.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\DataflowType\Writer;
 6 | 
 7 | /**
 8 |  * A writer that can be used as a delegate of DelegatorWriter.
 9 |  */
10 | interface DelegateWriterInterface extends WriterInterface
11 | {
12 |     /**
13 |      * Returns true if the argument is of a supported type.
14 |      */
15 |     public function supports($item): bool;
16 | }
17 | 


--------------------------------------------------------------------------------
/src/DataflowType/Dataflow/DataflowInterface.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\DataflowType\Dataflow;
 6 | 
 7 | use CodeRhapsodie\DataflowBundle\DataflowType\Result;
 8 | 
 9 | /**
10 |  * Combines a reader, steps and writers as a data processing workflow.
11 |  */
12 | interface DataflowInterface
13 | {
14 |     /**
15 |      * Processes the data.
16 |      */
17 |     public function process(): Result;
18 | }
19 | 


--------------------------------------------------------------------------------
/src/Manager/ScheduledDataflowManagerInterface.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\Manager;
 6 | 
 7 | /**
 8 |  * Handles scheduled dataflows execution dates based on their frequency.
 9 |  */
10 | interface ScheduledDataflowManagerInterface
11 | {
12 |     /**
13 |      * Creates Job for each scheduled dataflow where next run date has been reached.
14 |      */
15 |     public function createJobsFromScheduledDataflows(): void;
16 | }
17 | 


--------------------------------------------------------------------------------
/src/Validator/Constraints/Frequency.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\Validator\Constraints;
 6 | 
 7 | use Symfony\Component\Validator\Constraint;
 8 | 
 9 | /**
10 |  * @Annotation
11 |  *
12 |  * @codeCoverageIgnore
13 |  */
14 | class Frequency extends Constraint
15 | {
16 |     public $message = 'The provided frequency "{{ string }}" must be a valid parameter for DateInterval::createFromDateString() and must not represent a negative value';
17 | }
18 | 


--------------------------------------------------------------------------------
/src/Event/ProcessingEvent.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\Event;
 6 | 
 7 | use CodeRhapsodie\DataflowBundle\Entity\Job;
 8 | 
 9 | /**
10 |  * Event used during the dataflow lifecycle.
11 |  *
12 |  * @codeCoverageIgnore
13 |  */
14 | class ProcessingEvent extends CrEvent
15 | {
16 |     /**
17 |      * ProcessingEvent constructor.
18 |      */
19 |     public function __construct(private Job $job)
20 |     {
21 |     }
22 | 
23 |     public function getJob(): Job
24 |     {
25 |         return $this->job;
26 |     }
27 | }
28 | 


--------------------------------------------------------------------------------
/src/DataflowType/Writer/WriterInterface.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\DataflowType\Writer;
 6 | 
 7 | /**
 8 |  * Represents a writer for dataflows.
 9 |  */
10 | interface WriterInterface
11 | {
12 |     /**
13 |      * Called before the dataflow is processed.
14 |      */
15 |     public function prepare();
16 | 
17 |     /**
18 |      * Write an item.
19 |      */
20 |     public function write(mixed $item);
21 | 
22 |     /**
23 |      * Called after the dataflow is processed.
24 |      */
25 |     public function finish();
26 | }
27 | 


--------------------------------------------------------------------------------
/src/DataflowType/Writer/PortWriterAdapter.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\DataflowType\Writer;
 6 | 
 7 | class PortWriterAdapter implements WriterInterface
 8 | {
 9 |     public function __construct(private \Port\Writer $writer)
10 |     {
11 |     }
12 | 
13 |     public function prepare()
14 |     {
15 |         $this->writer->prepare();
16 |     }
17 | 
18 |     public function write($item)
19 |     {
20 |         $this->writer->writeItem((array) $item);
21 |     }
22 | 
23 |     public function finish()
24 |     {
25 |         $this->writer->finish();
26 |     }
27 | }
28 | 


--------------------------------------------------------------------------------
/src/Exceptions/UnknownDataflowTypeException.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\Exceptions;
 6 | 
 7 | /**
 8 |  * Exception thrown when an unknown dataflow type is requested from the registry.
 9 |  */
10 | class UnknownDataflowTypeException extends \Exception
11 | {
12 |     public static function create(string $aliasOrFqcn, array $knownDataflowTypes): self
13 |     {
14 |         return new self(\sprintf(
15 |             'Unknown dataflow type FQCN or alias "%s". Registered dataflow types FQCN and aliases are %s.',
16 |             $aliasOrFqcn,
17 |             implode(', ', $knownDataflowTypes)
18 |         ));
19 |     }
20 | }
21 | 


--------------------------------------------------------------------------------
/src/MessengerMode/JobMessageHandler.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\MessengerMode;
 6 | 
 7 | use CodeRhapsodie\DataflowBundle\Processor\JobProcessorInterface;
 8 | use CodeRhapsodie\DataflowBundle\Repository\JobRepository;
 9 | use Symfony\Component\Messenger\Attribute\AsMessageHandler;
10 | 
11 | #[AsMessageHandler]
12 | class JobMessageHandler
13 | {
14 |     public function __construct(private JobRepository $repository, private JobProcessorInterface $processor)
15 |     {
16 |     }
17 | 
18 |     public function __invoke(JobMessage $message)
19 |     {
20 |         $this->processor->process($this->repository->find($message->getJobId()));
21 |     }
22 | }
23 | 


--------------------------------------------------------------------------------
/src/Runner/PendingDataflowRunner.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\Runner;
 6 | 
 7 | use CodeRhapsodie\DataflowBundle\Processor\JobProcessorInterface;
 8 | use CodeRhapsodie\DataflowBundle\Repository\JobRepository;
 9 | 
10 | class PendingDataflowRunner implements PendingDataflowRunnerInterface
11 | {
12 |     public function __construct(private JobRepository $repository, private JobProcessorInterface $processor)
13 |     {
14 |     }
15 | 
16 |     public function runPendingDataflows(): void
17 |     {
18 |         while (null !== ($job = $this->repository->findNextPendingDataflow())) {
19 |             $this->processor->process($job);
20 |         }
21 |     }
22 | }
23 | 


--------------------------------------------------------------------------------
/.github/workflows/semicolons-kudos.yaml:
--------------------------------------------------------------------------------
 1 | 
 2 | name: Kudos for Code
 3 | on:
 4 |   push:
 5 |     branches: ["master"]
 6 |   workflow_dispatch:
 7 | 
 8 | jobs:
 9 |   kudos:
10 |     name: Semicolons Kudos
11 |     permissions: write-all
12 |     runs-on: ubuntu-latest
13 |     steps:
14 |       - uses: actions/checkout@v4
15 |         with:
16 |           fetch-depth: 0
17 |       - uses: LoremLabs/kudos-for-code-action@latest
18 |         with:
19 |           search-dir: "."
20 |           destination: "artifact"
21 |           generate-nomerges: true
22 |           generate-validemails: true
23 |           generate-limitdepth: 0
24 |           generate-fromrepo: true
25 |           analyze-repo: false
26 |           skip-ids: ""
27 | 


--------------------------------------------------------------------------------
/src/Resources/config/messenger_services.yaml:
--------------------------------------------------------------------------------
 1 | services:
 2 |     CodeRhapsodie\DataflowBundle\Runner\PendingDataflowRunnerInterface: '@CodeRhapsodie\DataflowBundle\Runner\MessengerDataflowRunner'
 3 |     CodeRhapsodie\DataflowBundle\Runner\MessengerDataflowRunner:
 4 |         arguments:
 5 |             $repository: '@CodeRhapsodie\DataflowBundle\Repository\JobRepository'
 6 |             $bus: ~ # Filled in compiler pass
 7 | 
 8 |     CodeRhapsodie\DataflowBundle\MessengerMode\JobMessageHandler:
 9 |         arguments:
10 |             $repository: '@CodeRhapsodie\DataflowBundle\Repository\JobRepository'
11 |             $processor: '@CodeRhapsodie\DataflowBundle\Processor\JobProcessorInterface'
12 |         tags: ['messenger.message_handler']
13 | 


--------------------------------------------------------------------------------
/src/Factory/ConnectionFactory.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\Factory;
 6 | 
 7 | use Symfony\Component\DependencyInjection\Container;
 8 | 
 9 | /**
10 |  * Class ConnectionFactory.
11 |  *
12 |  * @codeCoverageIgnore
13 |  */
14 | class ConnectionFactory
15 | {
16 |     public function __construct(private Container $container, private string $connectionName)
17 |     {
18 |     }
19 | 
20 |     public function setConnectionName(string $connectionName)
21 |     {
22 |         $this->connectionName = $connectionName;
23 |     }
24 | 
25 |     public function getConnection(): \Doctrine\DBAL\Connection
26 |     {
27 |         return $this->container->get(\sprintf('doctrine.dbal.%s_connection', $this->connectionName));
28 |     }
29 | }
30 | 


--------------------------------------------------------------------------------
/rector.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | use Rector\CodeQuality\Rector\Class_\InlineConstructorDefaultToPropertyRector;
 6 | use Rector\Config\RectorConfig;
 7 | use Rector\Set\ValueObject\LevelSetList;
 8 | use Rector\Symfony\Set\SymfonySetList;
 9 | 
10 | return static function (RectorConfig $rectorConfig): void {
11 |     $rectorConfig->paths([
12 |         __DIR__ . '/src',
13 |         __DIR__ . '/Tests',
14 |     ]);
15 | 
16 |     // register a single rule
17 |     $rectorConfig->rule(InlineConstructorDefaultToPropertyRector::class);
18 | 
19 |     $rectorConfig->sets([
20 |         SymfonySetList::SYMFONY_70,
21 |         SymfonySetList::SYMFONY_CODE_QUALITY,
22 |         SymfonySetList::SYMFONY_CONSTRUCTOR_INJECTION,
23 |         LevelSetList::UP_TO_PHP_80,
24 |     ]);
25 | };
26 | 


--------------------------------------------------------------------------------
/phpunit.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <!-- http://www.phpunit.de/manual/current/en/appendixes.configuration.html -->
 3 | <phpunit xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" backupGlobals="false" bootstrap="Tests/bootstrap.php" colors="false" xsi:noNamespaceSchemaLocation="https://schema.phpunit.de/11.1/phpunit.xsd" cacheDirectory=".phpunit.cache" backupStaticProperties="false">
 4 |   <php>
 5 |     <ini name="error_reporting" value="-1"/>
 6 |   </php>
 7 |   <testsuites>
 8 |     <testsuite name="Dataflow tests suite">
 9 |       <directory suffix="Test.php">./Tests</directory>
10 |     </testsuite>
11 |   </testsuites>
12 |   <source>
13 |     <include>
14 |       <directory>./src/</directory>
15 |     </include>
16 |     <exclude>
17 |       <directory>Tests/</directory>
18 |       <directory>vendor/</directory>
19 |     </exclude>
20 |   </source>
21 | </phpunit>
22 | 


--------------------------------------------------------------------------------
/src/Registry/DataflowTypeRegistryInterface.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\Registry;
 6 | 
 7 | use CodeRhapsodie\DataflowBundle\DataflowType\DataflowTypeInterface;
 8 | 
 9 | /**
10 |  * Interfaces for dataflow types registries.
11 |  */
12 | interface DataflowTypeRegistryInterface
13 | {
14 |     /**
15 |      * Get a registered dataflow type from its FQCN or one of its aliases.
16 |      */
17 |     public function getDataflowType(string $fqcnOrAlias): DataflowTypeInterface;
18 | 
19 |     /**
20 |      * Get all registered dataflow types.
21 |      *
22 |      * @return iterable|DataflowTypeInterface[]
23 |      */
24 |     public function listDataflowTypes(): iterable;
25 | 
26 |     /**
27 |      * Registers a dataflow type.
28 |      */
29 |     public function registerDataflowType(DataflowTypeInterface $dataflowType): void;
30 | }
31 | 


--------------------------------------------------------------------------------
/.github/workflows/build.yml:
--------------------------------------------------------------------------------
 1 | name: Build
 2 | 
 3 | on:
 4 |   push:
 5 |     branches:
 6 |       - master
 7 | 
 8 | 
 9 | jobs:
10 |   build:
11 |     name: Build
12 |     runs-on: ubuntu-latest
13 |     permissions: read-all
14 |     steps:
15 |       - uses: actions/checkout@v2
16 |         with:
17 |           fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
18 |       - uses: sonarsource/sonarqube-scan-action@master
19 |         env:
20 |           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
21 |           SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}
22 |       # If you wish to fail your job when the Quality Gate is red, uncomment the
23 |       # following lines. This would typically be used to fail a deployment.
24 |       # - uses: sonarsource/sonarqube-quality-gate-action@master
25 |       #   timeout-minutes: 5
26 |       #   env:
27 |       #     SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}


--------------------------------------------------------------------------------
/src/Runner/MessengerDataflowRunner.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\Runner;
 6 | 
 7 | use CodeRhapsodie\DataflowBundle\Entity\Job;
 8 | use CodeRhapsodie\DataflowBundle\MessengerMode\JobMessage;
 9 | use CodeRhapsodie\DataflowBundle\Repository\JobRepository;
10 | use Symfony\Component\Messenger\MessageBusInterface;
11 | 
12 | class MessengerDataflowRunner implements PendingDataflowRunnerInterface
13 | {
14 |     public function __construct(private JobRepository $repository, private MessageBusInterface $bus)
15 |     {
16 |     }
17 | 
18 |     public function runPendingDataflows(): void
19 |     {
20 |         while (null !== ($job = $this->repository->findNextPendingDataflow())) {
21 |             $this->bus->dispatch(new JobMessage($job->getId()));
22 |             $job->setStatus(Job::STATUS_QUEUED);
23 |             $this->repository->save($job);
24 |         }
25 |     }
26 | }
27 | 


--------------------------------------------------------------------------------
/src/Logger/BufferHandler.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\Logger;
 6 | 
 7 | use Monolog\Formatter\FormatterInterface;
 8 | use Monolog\Formatter\LineFormatter;
 9 | use Monolog\Handler\AbstractProcessingHandler;
10 | use Monolog\LogRecord;
11 | 
12 | class BufferHandler extends AbstractProcessingHandler
13 | {
14 |     private const FORMAT = "[%datetime%] %level_name% when processing item %context.index%: %message% %context% %extra%\n";
15 | 
16 |     private array $buffer = [];
17 | 
18 |     public function clearBuffer(): array
19 |     {
20 |         $logs = $this->buffer;
21 |         $this->buffer = [];
22 | 
23 |         return $logs;
24 |     }
25 | 
26 |     protected function write(array|LogRecord $record): void
27 |     {
28 |         $this->buffer[] = $record['formatted'];
29 |     }
30 | 
31 |     protected function getDefaultFormatter(): FormatterInterface
32 |     {
33 |         return new LineFormatter(self::FORMAT);
34 |     }
35 | }
36 | 


--------------------------------------------------------------------------------
/src/Logger/DelegatingLogger.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\Logger;
 6 | 
 7 | use Psr\Log\AbstractLogger;
 8 | use Psr\Log\LoggerInterface;
 9 | 
10 | final class DelegatingLogger extends AbstractLogger
11 | {
12 |     /** @var LoggerInterface[] */
13 |     private ?array $loggers = null;
14 | 
15 |     public function __construct(iterable $loggers)
16 |     {
17 |         foreach ($loggers as $logger) {
18 |             if (!$logger instanceof LoggerInterface) {
19 |                 throw new \InvalidArgumentException(\sprintf('Only instances of %s should be passed to the constructor of %s. An instance of %s was passed instead.', LoggerInterface::class, self::class, $logger::class));
20 |             }
21 | 
22 |             $this->loggers[] = $logger;
23 |         }
24 |     }
25 | 
26 |     public function log($level, $message, array $context = []): void
27 |     {
28 |         foreach ($this->loggers as $logger) {
29 |             $logger->log($level, $message, $context);
30 |         }
31 |     }
32 | }
33 | 


--------------------------------------------------------------------------------
/src/DependencyInjection/Compiler/DataflowTypeCompilerPass.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\DependencyInjection\Compiler;
 6 | 
 7 | use CodeRhapsodie\DataflowBundle\Registry\DataflowTypeRegistry;
 8 | use Symfony\Component\DependencyInjection\Compiler\CompilerPassInterface;
 9 | use Symfony\Component\DependencyInjection\ContainerBuilder;
10 | use Symfony\Component\DependencyInjection\Reference;
11 | 
12 | /**
13 |  * Registers dataflow types in the registry.
14 |  *
15 |  * @codeCoverageIgnore
16 |  */
17 | class DataflowTypeCompilerPass implements CompilerPassInterface
18 | {
19 |     public function process(ContainerBuilder $container): void
20 |     {
21 |         if (!$container->has(DataflowTypeRegistry::class)) {
22 |             return;
23 |         }
24 | 
25 |         $registry = $container->findDefinition(DataflowTypeRegistry::class);
26 |         foreach ($container->findTaggedServiceIds('coderhapsodie.dataflow.type') as $id => $tags) {
27 |             $registry->addMethodCall('registerDataflowType', [new Reference($id)]);
28 |         }
29 |     }
30 | }
31 | 


--------------------------------------------------------------------------------
/Tests/DataflowType/Writer/PortWriterAdapterTest.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | namespace CodeRhapsodie\DataflowBundle\Tests\DataflowType\Writer;
 4 | 
 5 | use CodeRhapsodie\DataflowBundle\DataflowType\Writer\PortWriterAdapter;
 6 | use PHPUnit\Framework\TestCase;
 7 | 
 8 | class PortWriterAdapterTest extends TestCase
 9 | {
10 |     public function testAll()
11 |     {
12 |         $value = 'not an array';
13 | 
14 |         $writer = $this->getMockBuilder(\Port\Writer::class)
15 |             ->onlyMethods(['prepare', 'finish', 'writeItem'])
16 |             ->getMock()
17 |         ;
18 |         $writer
19 |             ->expects($this->once())
20 |             ->method('prepare')
21 |         ;
22 |         $writer
23 |             ->expects($this->once())
24 |             ->method('finish')
25 |         ;
26 |         $writer
27 |             ->expects($this->once())
28 |             ->method('writeItem')
29 |             ->with([$value])
30 |         ;
31 | 
32 |         $adapter = new PortWriterAdapter($writer);
33 |         $adapter->prepare();
34 |         $adapter->write($value);
35 |         $adapter->finish();
36 |     }
37 | }
38 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | Copyright (c) 2019 Code Rhapsodie
 2 | 
 3 | Permission is hereby granted, free of charge, to any person obtaining a copy
 4 | of this software and associated documentation files (the "Software"), to deal
 5 | in the Software without restriction, including without limitation the rights
 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 7 | copies of the Software, and to permit persons to whom the Software is furnished
 8 | to do so, subject to the following conditions:
 9 | 
10 | The above copyright notice and this permission notice shall be included in all
11 | copies or substantial portions of the Software.
12 | 
13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 | THE SOFTWARE.
20 | 


--------------------------------------------------------------------------------
/src/ExceptionsHandler/FilesystemExceptionHandler.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\ExceptionsHandler;
 6 | 
 7 | use League\Flysystem\Filesystem;
 8 | use League\Flysystem\FilesystemException;
 9 | 
10 | class FilesystemExceptionHandler implements ExceptionHandlerInterface
11 | {
12 |     public function __construct(private Filesystem $filesystem)
13 |     {
14 |     }
15 | 
16 |     public function save(?int $jobId, ?array $exceptions): void
17 |     {
18 |         if ($jobId === null || empty($exceptions)) {
19 |             return;
20 |         }
21 | 
22 |         $this->filesystem->write(\sprintf('dataflow-job-%s.log', $jobId), json_encode($exceptions));
23 |     }
24 | 
25 |     public function find(int $jobId): ?array
26 |     {
27 |         try {
28 |             if (!$this->filesystem->fileExists(\sprintf('dataflow-job-%s.log', $jobId))) {
29 |                 return [];
30 |             }
31 | 
32 |             return json_decode($this->filesystem->read(\sprintf('dataflow-job-%s.log', $jobId)), true);
33 |         } catch (FilesystemException) {
34 |             return [];
35 |         }
36 |     }
37 | }
38 | 


--------------------------------------------------------------------------------
/src/DependencyInjection/Compiler/DefaultLoggerCompilerPass.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\DependencyInjection\Compiler;
 6 | 
 7 | use CodeRhapsodie\DataflowBundle\Command\ExecuteDataflowCommand;
 8 | use CodeRhapsodie\DataflowBundle\Processor\JobProcessor;
 9 | use Symfony\Component\DependencyInjection\Compiler\CompilerPassInterface;
10 | use Symfony\Component\DependencyInjection\ContainerBuilder;
11 | use Symfony\Component\DependencyInjection\Reference;
12 | 
13 | class DefaultLoggerCompilerPass implements CompilerPassInterface
14 | {
15 |     public function process(ContainerBuilder $container): void
16 |     {
17 |         $defaultLogger = $container->getParameter('coderhapsodie.dataflow.default_logger');
18 |         if (!$container->has($defaultLogger)) {
19 |             return;
20 |         }
21 | 
22 |         foreach ([ExecuteDataflowCommand::class, JobProcessor::class] as $serviceId) {
23 |             if (!$container->has($serviceId)) {
24 |                 continue;
25 |             }
26 | 
27 |             $definition = $container->findDefinition($serviceId);
28 |             $definition->addMethodCall('setLogger', [new Reference($defaultLogger)]);
29 |         }
30 |     }
31 | }
32 | 


--------------------------------------------------------------------------------
/src/DataflowType/Writer/CollectionWriter.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\DataflowType\Writer;
 6 | 
 7 | use CodeRhapsodie\DataflowBundle\Exceptions\UnsupportedItemTypeException;
 8 | 
 9 | /**
10 |  * Delegates the writing of each item in a collection to an embedded writer.
11 |  */
12 | class CollectionWriter implements DelegateWriterInterface
13 | {
14 |     /**
15 |      * CollectionWriter constructor.
16 |      */
17 |     public function __construct(private WriterInterface $writer)
18 |     {
19 |     }
20 | 
21 |     public function prepare()
22 |     {
23 |         $this->writer->prepare();
24 |     }
25 | 
26 |     public function write($collection)
27 |     {
28 |         if (!is_iterable($collection)) {
29 |             throw new UnsupportedItemTypeException(\sprintf('Item to write was expected to be an iterable, received %s.', get_debug_type($collection)));
30 |         }
31 | 
32 |         foreach ($collection as $item) {
33 |             $this->writer->write($item);
34 |         }
35 |     }
36 | 
37 |     public function finish()
38 |     {
39 |         $this->writer->finish();
40 |     }
41 | 
42 |     public function supports($item): bool
43 |     {
44 |         return is_iterable($item);
45 |     }
46 | }
47 | 


--------------------------------------------------------------------------------
/src/DependencyInjection/Compiler/ExceptionCompilerPass.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\DependencyInjection\Compiler;
 6 | 
 7 | use CodeRhapsodie\DataflowBundle\ExceptionsHandler\FilesystemExceptionHandler;
 8 | use Symfony\Component\DependencyInjection\Compiler\CompilerPassInterface;
 9 | use Symfony\Component\DependencyInjection\ContainerBuilder;
10 | use Symfony\Component\DependencyInjection\Exception\InvalidArgumentException;
11 | use Symfony\Component\DependencyInjection\Reference;
12 | 
13 | class ExceptionCompilerPass implements CompilerPassInterface
14 | {
15 |     public function process(ContainerBuilder $container): void
16 |     {
17 |         if (!$container->hasParameter('coderhapsodie.dataflow.flysystem_service')) {
18 |             return;
19 |         }
20 | 
21 |         $flysystem = $container->getParameter('coderhapsodie.dataflow.flysystem_service');
22 |         if (!$container->has($flysystem)) {
23 |             throw new InvalidArgumentException(\sprintf('Service "%s" not found', $flysystem));
24 |         }
25 | 
26 |         $definition = $container->findDefinition(FilesystemExceptionHandler::class);
27 |         $definition->setArgument('$filesystem', new Reference($flysystem));
28 |     }
29 | }
30 | 


--------------------------------------------------------------------------------
/src/DependencyInjection/Compiler/BusCompilerPass.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\DependencyInjection\Compiler;
 6 | 
 7 | use CodeRhapsodie\DataflowBundle\Runner\MessengerDataflowRunner;
 8 | use Symfony\Component\DependencyInjection\Compiler\CompilerPassInterface;
 9 | use Symfony\Component\DependencyInjection\ContainerBuilder;
10 | use Symfony\Component\DependencyInjection\Exception\InvalidArgumentException;
11 | use Symfony\Component\DependencyInjection\Reference;
12 | 
13 | class BusCompilerPass implements CompilerPassInterface
14 | {
15 |     public function process(ContainerBuilder $container): void
16 |     {
17 |         if (!$container->hasParameter('coderhapsodie.dataflow.bus')) {
18 |             return;
19 |         }
20 | 
21 |         $bus = $container->getParameter('coderhapsodie.dataflow.bus');
22 |         if (!$container->has($bus)) {
23 |             throw new InvalidArgumentException(\sprintf('Service "%s" not found', $bus));
24 |         }
25 | 
26 |         if (!$container->has(MessengerDataflowRunner::class)) {
27 |             return;
28 |         }
29 | 
30 |         $definition = $container->findDefinition(MessengerDataflowRunner::class);
31 |         $definition->setArgument('$bus', new Reference($bus));
32 |     }
33 | }
34 | 


--------------------------------------------------------------------------------
/.php-cs-fixer.dist.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | $finder = PhpCsFixer\Finder::create()
 4 |     ->in('src')
 5 |     //->in('tests')
 6 |     ->files()->name('*.php');
 7 | 
 8 | $config = new PhpCsFixer\Config();
 9 | $config->setRules([
10 |     '@Symfony' => true,
11 |     '@Symfony:risky' => true,
12 |     '@PSR12' => true,
13 |     'array_syntax' => [
14 |         'syntax' => 'short',
15 |     ],
16 |     'combine_consecutive_unsets' => true,
17 |     'native_function_invocation' => [
18 |         'include' => [
19 |             '@compiler_optimized',
20 |         ],
21 |     ],
22 |     'no_extra_blank_lines' => [
23 |         'tokens' => [
24 |             'break',
25 |             'continue',
26 |             'extra',
27 |             'return',
28 |             'throw',
29 |             'use',
30 |             'parenthesis_brace_block',
31 |             'square_brace_block',
32 |             'curly_brace_block',
33 |         ],
34 |     ],
35 |     'ordered_class_elements' => true,
36 |     'ordered_imports' => true,
37 |     'yoda_style' => [
38 |         'equal' => false,
39 |         'identical' => false,
40 |         'less_and_greater' => false,
41 |         'always_move_variable' => false,
42 |     ],
43 | ])
44 |     ->setRiskyAllowed(true)
45 |     ->setFinder(
46 |         $finder
47 |     );
48 | 
49 | return $config;
50 | 


--------------------------------------------------------------------------------
/src/Repository/InitFromDbTrait.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\Repository;
 6 | 
 7 | /**
 8 |  * @codeCoverageIgnore
 9 |  */
10 | trait InitFromDbTrait
11 | {
12 |     abstract private function getFields(): array;
13 | 
14 |     private function initDateTime(array $datas): array
15 |     {
16 |         foreach ($this->getFields() as $key => $type) {
17 |             if ($type === 'datetime' && $datas[$key] !== null) {
18 |                 $datas[$key] = new \DateTime($datas[$key]);
19 |             }
20 |         }
21 | 
22 |         return $datas;
23 |     }
24 | 
25 |     private function initArray(array $datas): array
26 |     {
27 |         if (!\is_array($datas['options'])) {
28 |             $datas['options'] = $this->strToArray($datas['options']);
29 |         }
30 |         if (\array_key_exists('exceptions', $datas) && !\is_array($datas['exceptions'])) {
31 |             $datas['exceptions'] = $this->strToArray($datas['exceptions']);
32 |         }
33 | 
34 |         return $datas;
35 |     }
36 | 
37 |     private function strToArray($value): array
38 |     {
39 |         if ($value === null) {
40 |             return [];
41 |         }
42 | 
43 |         $array = json_decode($value, true, 512, \JSON_THROW_ON_ERROR);
44 | 
45 |         return ($array === false) ? [] : $array;
46 |     }
47 | }
48 | 


--------------------------------------------------------------------------------
/src/CodeRhapsodieDataflowBundle.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle;
 6 | 
 7 | use CodeRhapsodie\DataflowBundle\DependencyInjection\CodeRhapsodieDataflowExtension;
 8 | use CodeRhapsodie\DataflowBundle\DependencyInjection\Compiler\BusCompilerPass;
 9 | use CodeRhapsodie\DataflowBundle\DependencyInjection\Compiler\DataflowTypeCompilerPass;
10 | use CodeRhapsodie\DataflowBundle\DependencyInjection\Compiler\DefaultLoggerCompilerPass;
11 | use CodeRhapsodie\DataflowBundle\DependencyInjection\Compiler\ExceptionCompilerPass;
12 | use Symfony\Component\DependencyInjection\ContainerBuilder;
13 | use Symfony\Component\DependencyInjection\Extension\ExtensionInterface;
14 | use Symfony\Component\HttpKernel\Bundle\Bundle;
15 | 
16 | /**
17 |  * @codeCoverageIgnore
18 |  */
19 | class CodeRhapsodieDataflowBundle extends Bundle
20 | {
21 |     protected string $name = 'CodeRhapsodieDataflowBundle';
22 | 
23 |     public function getContainerExtension(): ?ExtensionInterface
24 |     {
25 |         return new CodeRhapsodieDataflowExtension();
26 |     }
27 | 
28 |     public function build(ContainerBuilder $container): void
29 |     {
30 |         $container
31 |             ->addCompilerPass(new DataflowTypeCompilerPass())
32 |             ->addCompilerPass(new DefaultLoggerCompilerPass())
33 |             ->addCompilerPass(new BusCompilerPass())
34 |             ->addCompilerPass(new ExceptionCompilerPass())
35 |         ;
36 |     }
37 | }
38 | 


--------------------------------------------------------------------------------
/Tests/DataflowType/Dataflow/AMPAsyncDataflowTest.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | namespace CodeRhapsodie\DataflowBundle\Tests\DataflowType\Dataflow;
 4 | 
 5 | use Amp\Delayed;
 6 | use CodeRhapsodie\DataflowBundle\DataflowType\Dataflow\AMPAsyncDataflow;
 7 | use CodeRhapsodie\DataflowBundle\DataflowType\Writer\WriterInterface;
 8 | use PHPUnit\Framework\TestCase;
 9 | 
10 | class AMPAsyncDataflowTest extends TestCase
11 | {
12 |     public function testProcess()
13 |     {
14 |         $reader = [1, 2, 3];
15 |         $result = [];
16 |         $dataflow = new AMPAsyncDataflow($reader, 'simple');
17 |         $dataflow->addStep(static fn($item) => $item + 1);
18 |         $dataflow->addStep(static function($item): \Generator {
19 |             yield new Delayed(10); //delay 10 milliseconds
20 |             return $item * 2;
21 |         });
22 |         $dataflow->addWriter(new class($result) implements WriterInterface {
23 |             private $buffer;
24 | 
25 |             public function __construct(&$buffer) {
26 |                 $this->buffer = &$buffer;
27 |             }
28 | 
29 |             public function prepare()
30 |             {
31 |             }
32 | 
33 |             public function write($item)
34 |             {
35 |                 $this->buffer[] = $item;
36 |             }
37 | 
38 |             public function finish()
39 |             {
40 |             }
41 |         });
42 |         $dataflow->process();
43 | 
44 |         self::assertSame([4, 6, 8], $result);
45 |     }
46 | }
47 | 


--------------------------------------------------------------------------------
/src/Validator/Constraints/FrequencyValidator.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\Validator\Constraints;
 6 | 
 7 | use Symfony\Component\Validator\Constraint;
 8 | use Symfony\Component\Validator\ConstraintValidator;
 9 | use Symfony\Component\Validator\Exception\UnexpectedTypeException;
10 | 
11 | class FrequencyValidator extends ConstraintValidator
12 | {
13 |     public function validate(mixed $value, Constraint $constraint)
14 |     {
15 |         if (!$constraint instanceof Frequency) {
16 |             throw new UnexpectedTypeException($constraint, Frequency::class);
17 |         }
18 | 
19 |         if ($value === null) {
20 |             return;
21 |         }
22 | 
23 |         try {
24 |             $interval = \DateInterval::createFromDateString($value);
25 |         } catch (\Exception) {
26 |             $interval = false;
27 |         }
28 | 
29 |         if (!$interval) {
30 |             $this->context->buildViolation($constraint->message)
31 |                 ->setParameter('{{ string }}', $value)
32 |                 ->addViolation()
33 |             ;
34 | 
35 |             return;
36 |         }
37 | 
38 |         $now = new \DateTime();
39 |         $dt = clone $now;
40 |         $dt->add($interval);
41 | 
42 |         if ($dt <= $now) {
43 |             $this->context->buildViolation($constraint->message)
44 |                 ->setParameter('{{ string }}', $value)
45 |                 ->addViolation()
46 |             ;
47 |         }
48 |     }
49 | }
50 | 


--------------------------------------------------------------------------------
/Tests/MessengerMode/JobMessageHandlerTest.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\Tests\MessengerMode;
 6 | 
 7 | use CodeRhapsodie\DataflowBundle\Entity\Job;
 8 | use CodeRhapsodie\DataflowBundle\MessengerMode\JobMessage;
 9 | use CodeRhapsodie\DataflowBundle\MessengerMode\JobMessageHandler;
10 | use CodeRhapsodie\DataflowBundle\Processor\JobProcessorInterface;
11 | use CodeRhapsodie\DataflowBundle\Repository\JobRepository;
12 | use PHPUnit\Framework\MockObject\MockObject;
13 | use PHPUnit\Framework\TestCase;
14 | 
15 | class JobMessageHandlerTest extends TestCase
16 | {
17 |     private JobRepository|MockObject $repository;
18 |     private JobProcessorInterface|MockObject $processor;
19 |     private JobMessageHandler $handler;
20 | 
21 |     protected function setUp(): void
22 |     {
23 |         $this->repository = $this->createMock(JobRepository::class);
24 |         $this->processor = $this->createMock(JobProcessorInterface::class);
25 | 
26 |         $this->handler = new JobMessageHandler($this->repository, $this->processor);
27 |     }
28 | 
29 |     public function testInvoke()
30 |     {
31 |         $message = new JobMessage($id = 32);
32 | 
33 |         $this->repository
34 |             ->expects($this->once())
35 |             ->method('find')
36 |             ->with($id)
37 |             ->willReturn($job = new Job())
38 |         ;
39 | 
40 |         $this->processor
41 |             ->expects($this->once())
42 |             ->method('process')
43 |             ->with($job)
44 |         ;
45 | 
46 |         ($this->handler)($message);
47 |     }
48 | }
49 | 


--------------------------------------------------------------------------------
/src/Registry/DataflowTypeRegistry.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\Registry;
 6 | 
 7 | use CodeRhapsodie\DataflowBundle\DataflowType\DataflowTypeInterface;
 8 | use CodeRhapsodie\DataflowBundle\Exceptions\UnknownDataflowTypeException;
 9 | 
10 | /**
11 |  * Array based dataflow types registry.
12 |  */
13 | class DataflowTypeRegistry implements DataflowTypeRegistryInterface
14 | {
15 |     /** @var array|DataflowTypeInterface[] */
16 |     private array $fqcnRegistry = [];
17 | 
18 |     /** @var array|DataflowTypeInterface[] */
19 |     private array $aliasesRegistry = [];
20 | 
21 |     public function getDataflowType(string $fqcnOrAlias): DataflowTypeInterface
22 |     {
23 |         if (isset($this->fqcnRegistry[$fqcnOrAlias])) {
24 |             return $this->fqcnRegistry[$fqcnOrAlias];
25 |         }
26 | 
27 |         if (isset($this->aliasesRegistry[$fqcnOrAlias])) {
28 |             return $this->aliasesRegistry[$fqcnOrAlias];
29 |         }
30 | 
31 |         throw UnknownDataflowTypeException::create($fqcnOrAlias, [...array_keys($this->fqcnRegistry), ...array_keys($this->aliasesRegistry)]);
32 |     }
33 | 
34 |     public function listDataflowTypes(): iterable
35 |     {
36 |         return $this->fqcnRegistry;
37 |     }
38 | 
39 |     public function registerDataflowType(DataflowTypeInterface $dataflowType): void
40 |     {
41 |         $this->fqcnRegistry[$dataflowType::class] = $dataflowType;
42 |         foreach ($dataflowType->getAliases() as $alias) {
43 |             $this->aliasesRegistry[$alias] = $dataflowType;
44 |         }
45 |     }
46 | }
47 | 


--------------------------------------------------------------------------------
/Tests/Registry/DataflowTypeRegistryTest.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | namespace CodeRhapsodie\DataflowBundle\Tests\Registry;
 4 | 
 5 | use CodeRhapsodie\DataflowBundle\Exceptions\UnknownDataflowTypeException;
 6 | use CodeRhapsodie\DataflowBundle\Registry\DataflowTypeRegistry;
 7 | use CodeRhapsodie\DataflowBundle\DataflowType\DataflowTypeInterface;
 8 | use PHPUnit\Framework\MockObject\MockObject;
 9 | use PHPUnit\Framework\TestCase;
10 | 
11 | class DataflowTypeRegistryTest extends TestCase
12 | {
13 |     private DataflowTypeRegistry $registry;
14 | 
15 |     protected function setUp(): void
16 |     {
17 |         $this->registry = new DataflowTypeRegistry();
18 |     }
19 | 
20 |     public function testEverything()
21 |     {
22 |         $alias1 = 'alias1';
23 |         $alias2 = 'alias2';
24 | 
25 |         /** @var MockObject|DataflowTypeInterface $type */
26 |         $type = $this->createMock(DataflowTypeInterface::class);
27 |         $type
28 |             ->expects($this->once())
29 |             ->method('getAliases')
30 |             ->willReturn([$alias1, $alias2])
31 |         ;
32 | 
33 |         $this->registry->registerDataflowType($type);
34 | 
35 |         $this->assertSame($type, $this->registry->getDataflowType($type::class));
36 |         $this->assertSame($type, $this->registry->getDataflowType($alias1));
37 |         $this->assertSame($type, $this->registry->getDataflowType($alias2));
38 |         $this->assertContains($type, $this->registry->listDataflowTypes());
39 |     }
40 | 
41 |     public function testUnknown()
42 |     {
43 |         $this->expectException(UnknownDataflowTypeException::class);
44 |         $this->registry->getDataflowType('unknown');
45 |     }
46 | }
47 | 


--------------------------------------------------------------------------------
/src/Gateway/JobGateway.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\Gateway;
 6 | 
 7 | use CodeRhapsodie\DataflowBundle\Entity\Job;
 8 | use CodeRhapsodie\DataflowBundle\ExceptionsHandler\ExceptionHandlerInterface;
 9 | use CodeRhapsodie\DataflowBundle\ExceptionsHandler\NullExceptionHandler;
10 | use CodeRhapsodie\DataflowBundle\Repository\JobRepository;
11 | 
12 | class JobGateway
13 | {
14 |     public function __construct(private JobRepository $repository, private ExceptionHandlerInterface $exceptionHandler)
15 |     {
16 |     }
17 | 
18 |     public function find(int $jobId): ?Job
19 |     {
20 |         $job = $this->repository->find($jobId);
21 | 
22 |         return $this->loadExceptions($job);
23 |     }
24 | 
25 |     public function save(Job $job): void
26 |     {
27 |         if (!$this->exceptionHandler instanceof NullExceptionHandler) {
28 |             $this->exceptionHandler->save($job->getId(), $job->getExceptions());
29 |             $job->setExceptions([]);
30 |         }
31 | 
32 |         $this->repository->save($job);
33 |     }
34 | 
35 |     public function findLastForDataflowId(int $scheduleId): ?Job
36 |     {
37 |         $job = $this->repository->findLastForDataflowId($scheduleId);
38 | 
39 |         return $this->loadExceptions($job);
40 |     }
41 | 
42 |     private function loadExceptions(?Job $job): ?Job
43 |     {
44 |         if ($job === null || $this->exceptionHandler instanceof NullExceptionHandler) {
45 |             return $job;
46 |         }
47 | 
48 |         $this->exceptionHandler->save($job->getId(), $job->getExceptions());
49 | 
50 |         return $job->setExceptions($this->exceptionHandler->find($job->getId()));
51 |     }
52 | }
53 | 


--------------------------------------------------------------------------------
/Tests/DataflowType/AbstractDataflowTypeTest.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | namespace CodeRhapsodie\DataflowBundle\Tests\DataflowType;
 4 | 
 5 | use CodeRhapsodie\DataflowBundle\DataflowType\AbstractDataflowType;
 6 | use CodeRhapsodie\DataflowBundle\DataflowType\DataflowBuilder;
 7 | use PHPUnit\Framework\TestCase;
 8 | use Symfony\Component\OptionsResolver\OptionsResolver;
 9 | 
10 | class AbstractDataflowTypeTest extends TestCase
11 | {
12 |     public function testProcess()
13 |     {
14 |         $label = 'Test label';
15 |         $options = ['testOption' => 'Test value'];
16 |         $values = [1, 2, 3];
17 |         $testCase = $this;
18 | 
19 |         $dataflowType = new class($label, $options, $values, $testCase) extends AbstractDataflowType
20 |         {
21 |             public function __construct(private string $label, private array $options, private array $values, private TestCase $testCase)
22 |             {
23 |             }
24 | 
25 |             public function getLabel(): string
26 |             {
27 |                 return $this->label;
28 |             }
29 | 
30 |             protected function configureOptions(OptionsResolver $optionsResolver): void
31 |             {
32 |                 $optionsResolver->setDefined('testOption');
33 |             }
34 | 
35 |             protected function buildDataflow(DataflowBuilder $builder, array $options): void
36 |             {
37 |                 $builder->setReader($this->values);
38 |                 $this->testCase->assertSame($this->options, $options);
39 |             }
40 |         };
41 | 
42 |         $result = $dataflowType->process($options);
43 |         $this->assertSame($label, $result->getName());
44 |         $this->assertSame(count($values), $result->getTotalProcessedCount());
45 |     }
46 | }
47 | 


--------------------------------------------------------------------------------
/Tests/Runner/PendingDataflowRunnerTest.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | namespace CodeRhapsodie\DataflowBundle\Tests\Runner;
 4 | 
 5 | use CodeRhapsodie\DataflowBundle\Entity\Job;
 6 | use CodeRhapsodie\DataflowBundle\Processor\JobProcessorInterface;
 7 | use CodeRhapsodie\DataflowBundle\Repository\JobRepository;
 8 | use CodeRhapsodie\DataflowBundle\Runner\PendingDataflowRunner;
 9 | use PHPUnit\Framework\MockObject\MockObject;
10 | use PHPUnit\Framework\TestCase;
11 | 
12 | class PendingDataflowRunnerTest extends TestCase
13 | {
14 |     private PendingDataflowRunner $runner;
15 |     private JobRepository|MockObject $repository;
16 |     private JobProcessorInterface|MockObject $processor;
17 | 
18 |     protected function setUp(): void
19 |     {
20 |         $this->repository = $this->createMock(JobRepository::class);
21 |         $this->processor = $this->createMock(JobProcessorInterface::class);
22 | 
23 |         $this->runner = new PendingDataflowRunner($this->repository, $this->processor);
24 |     }
25 | 
26 |     public function testRunPendingDataflows()
27 |     {
28 |         $job1 = new Job();
29 |         $job2 = new Job();
30 | 
31 |         $this->repository
32 |             ->expects($this->exactly(3))
33 |             ->method('findNextPendingDataflow')
34 |             ->willReturnOnConsecutiveCalls($job1, $job2, null)
35 |         ;
36 | 
37 |         $matcher = $this->exactly(2);
38 |         $this->processor
39 |             ->expects($matcher)
40 |             ->method('process')
41 |             ->with($this->callback(fn($arg) => match ($matcher->numberOfInvocations()) {
42 |                 1 => $arg === $job1,
43 |                 2 => $arg === $job2,
44 |                 default => false,
45 |             }))
46 |         ;
47 | 
48 |         $this->runner->runPendingDataflows();
49 |     }
50 | }
51 | 


--------------------------------------------------------------------------------
/src/DataflowType/Result.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\DataflowType;
 6 | 
 7 | /**
 8 |  * @codeCoverageIgnore
 9 |  */
10 | class Result
11 | {
12 |     private \DateInterval $elapsed;
13 | 
14 |     private int $errorCount = 0;
15 | 
16 |     private int $successCount = 0;
17 | 
18 |     private array $exceptions;
19 | 
20 |     public function __construct(private string $name, private \DateTimeInterface $startTime, private \DateTimeInterface $endTime, private int $totalProcessedCount, array $exceptions)
21 |     {
22 |         $this->elapsed = $startTime->diff($endTime);
23 |         $this->errorCount = \count($exceptions);
24 |         $this->successCount = $totalProcessedCount - $this->errorCount;
25 |         $this->exceptions = $exceptions;
26 |     }
27 | 
28 |     public function getName(): string
29 |     {
30 |         return $this->name;
31 |     }
32 | 
33 |     public function getStartTime(): \DateTimeInterface
34 |     {
35 |         return $this->startTime;
36 |     }
37 | 
38 |     public function getEndTime(): \DateTimeInterface
39 |     {
40 |         return $this->endTime;
41 |     }
42 | 
43 |     public function getElapsed(): \DateInterval
44 |     {
45 |         return $this->elapsed;
46 |     }
47 | 
48 |     public function getErrorCount(): int
49 |     {
50 |         return $this->errorCount;
51 |     }
52 | 
53 |     public function getSuccessCount(): int
54 |     {
55 |         return $this->successCount;
56 |     }
57 | 
58 |     public function getTotalProcessedCount(): int
59 |     {
60 |         return $this->totalProcessedCount;
61 |     }
62 | 
63 |     public function hasErrors(): bool
64 |     {
65 |         return $this->errorCount > 0;
66 |     }
67 | 
68 |     public function getExceptions(): array
69 |     {
70 |         return $this->exceptions;
71 |     }
72 | }
73 | 


--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
 1 | Contributing
 2 | ============
 3 | 
 4 | Thank you for contributing to this project!
 5 | 
 6 | Bug reports
 7 | -----------
 8 | 
 9 | If you find a bug, please submit an issue. Try to be as detailed as possible
10 | in your problem description to help us fix the bug.
11 | 
12 | Feature requests
13 | ----------------
14 | 
15 | If you wish to propose a feature, please submit an issue. Try to explain your
16 | use case as fully as possible to help us understand why you think the feature
17 | should be added.
18 | 
19 | Creating a pull request (PR)
20 | ----------------------------
21 | 
22 | First [fork the repository](https://help.github.com/articles/fork-a-repo/) on
23 | GitHub.
24 | 
25 | Then clone your fork:
26 | 
27 | ```bash
28 | $ git clone https://github.com/code-rhapsodie/dataflow-bundle.git
29 | $ git checkout -b bug-or-feature-description
30 | ```
31 | 
32 | And install the dependencies:
33 | 
34 | ```bash
35 | $ composer install
36 | ```
37 | 
38 | Write your code and add tests. Then run the tests:
39 | 
40 | ```bash
41 | $ vendor/bin/phpunit
42 | ```
43 | 
44 | Commit your changes and push them to GitHub:
45 | 
46 | ```bash
47 | $ git commit -m "Fix nasty bug"
48 | $ git push -u origin bug-or-feature-description
49 | ```
50 | 
51 | Then [create a pull request](https://help.github.com/articles/creating-a-pull-request/)
52 | on GitHub.
53 | 
54 | If you need to make some changes, commit and push them as you like. When asked
55 | to squash your commits, do so as follows:
56 | 
57 | ```bash
58 | git rebase -i
59 | git push origin bug-or-feature-description -f
60 | ```
61 | 
62 | Coding standard
63 | ---------------
64 | 
65 | This project follows the [Symfony](https://symfony.com/doc/current/contributing/code/standards.html) coding style.
66 | Please make sure your pull requests adhere to this standard.
67 | 
68 | To fix, execute this command after [download PHP CS Fixer](https://cs.symfony.com/):
69 | 
70 | ```shell script
71 | $ php php-cs-fixer.phar fix
72 | ```
73 | 


--------------------------------------------------------------------------------
/Tests/DataflowType/Writer/CollectionWriterTest.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | namespace CodeRhapsodie\DataflowBundle\Tests\DataflowType\Writer;
 4 | 
 5 | use CodeRhapsodie\DataflowBundle\DataflowType\Writer\CollectionWriter;
 6 | use CodeRhapsodie\DataflowBundle\DataflowType\Writer\WriterInterface;
 7 | use CodeRhapsodie\DataflowBundle\Exceptions\UnsupportedItemTypeException;
 8 | use PHPUnit\Framework\TestCase;
 9 | 
10 | class CollectionWriterTest extends TestCase
11 | {
12 |     public function testNotACollection()
13 |     {
14 |         $this->expectException(UnsupportedItemTypeException::class);
15 | 
16 |         $writer = new CollectionWriter($this->createMock(WriterInterface::class));
17 |         $writer->write('Not an iterable');
18 |     }
19 | 
20 |     public function testSupports()
21 |     {
22 |         $writer = new CollectionWriter($this->createMock(WriterInterface::class));
23 | 
24 |         $this->assertTrue($writer->supports([]));
25 |         $this->assertTrue($writer->supports(new \ArrayIterator([])));
26 |         $this->assertFalse($writer->supports(''));
27 |         $this->assertFalse($writer->supports(0));
28 |     }
29 | 
30 |     public function testAll()
31 |     {
32 |         $values = ['a', 'b', 'c'];
33 | 
34 |         $embeddedWriter = $this->createMock(WriterInterface::class);
35 |         $embeddedWriter
36 |             ->expects($this->once())
37 |             ->method('prepare')
38 |         ;
39 |         $embeddedWriter
40 |             ->expects($this->once())
41 |             ->method('finish')
42 |         ;
43 |         $matcher = $this->exactly(count($values));
44 |         $embeddedWriter
45 |             ->expects($matcher)
46 |             ->method('write')
47 |             ->with($this->callback(fn($arg) => $arg === $values[$matcher->numberOfInvocations() - 1]))
48 |         ;
49 | 
50 |         $writer = new CollectionWriter($embeddedWriter);
51 |         $writer->prepare();
52 |         $writer->write($values);
53 |         $writer->finish();
54 |     }
55 | }
56 | 


--------------------------------------------------------------------------------
/src/DependencyInjection/CodeRhapsodieDataflowExtension.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\DependencyInjection;
 6 | 
 7 | use CodeRhapsodie\DataflowBundle\DataflowType\DataflowTypeInterface;
 8 | use Symfony\Component\Config\FileLocator;
 9 | use Symfony\Component\DependencyInjection\ContainerBuilder;
10 | use Symfony\Component\DependencyInjection\Extension\Extension;
11 | use Symfony\Component\DependencyInjection\Loader\YamlFileLoader;
12 | 
13 | /**
14 |  * @codeCoverageIgnore
15 |  */
16 | class CodeRhapsodieDataflowExtension extends Extension
17 | {
18 |     public function load(array $configs, ContainerBuilder $container): void
19 |     {
20 |         $loader = new YamlFileLoader($container, new FileLocator(__DIR__.'/../Resources/config'));
21 |         $loader->load('services.yaml');
22 | 
23 |         $container
24 |             ->registerForAutoconfiguration(DataflowTypeInterface::class)
25 |             ->addTag('coderhapsodie.dataflow.type')
26 |         ;
27 |         $configuration = new Configuration();
28 |         $config = $this->processConfiguration($configuration, $configs);
29 | 
30 |         $container->setParameter('coderhapsodie.dataflow.dbal_default_connection', $config['dbal_default_connection']);
31 |         $container->setParameter('coderhapsodie.dataflow.default_logger', $config['default_logger']);
32 |         $container->setParameter('coderhapsodie.dataflow.exceptions_mode.type', $config['exceptions_mode']['type']);
33 | 
34 |         if ($config['exceptions_mode']['type'] === 'file') {
35 |             $container->setParameter('coderhapsodie.dataflow.flysystem_service', $config['exceptions_mode']['flysystem_service']);
36 |             $loader->load('exceptions_services.yaml');
37 |         }
38 | 
39 |         if ($config['messenger_mode']['enabled']) {
40 |             $container->setParameter('coderhapsodie.dataflow.bus', $config['messenger_mode']['bus']);
41 |             $loader->load('messenger_services.yaml');
42 |         }
43 |     }
44 | }
45 | 


--------------------------------------------------------------------------------
/src/DataflowType/AMPAsyncDataflowBuilder.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\DataflowType;
 6 | 
 7 | use CodeRhapsodie\DataflowBundle\DataflowType\Dataflow\AMPAsyncDataflow;
 8 | use CodeRhapsodie\DataflowBundle\DataflowType\Dataflow\DataflowInterface;
 9 | use CodeRhapsodie\DataflowBundle\DataflowType\Writer\WriterInterface;
10 | 
11 | class AMPAsyncDataflowBuilder extends DataflowBuilder
12 | {
13 |     private ?string $name = null;
14 |     private ?iterable $reader = null;
15 |     private array $steps = [];
16 | 
17 |     /** @var WriterInterface[] */
18 |     private array $writers = [];
19 | 
20 |     public function __construct(protected ?int $loopInterval = 0, protected ?int $emitInterval = 0)
21 |     {
22 |     }
23 | 
24 |     public function setName(string $name): self
25 |     {
26 |         $this->name = $name;
27 | 
28 |         return $this;
29 |     }
30 | 
31 |     public function setReader(iterable $reader): self
32 |     {
33 |         $this->reader = $reader;
34 | 
35 |         return $this;
36 |     }
37 | 
38 |     public function addStep(callable $step, int $priority = 0, int $scale = 1): self
39 |     {
40 |         $this->steps[$priority][] = ['step' => $step, 'scale' => $scale];
41 | 
42 |         return $this;
43 |     }
44 | 
45 |     public function addWriter(WriterInterface $writer): self
46 |     {
47 |         $this->writers[] = $writer;
48 | 
49 |         return $this;
50 |     }
51 | 
52 |     public function getDataflow(): DataflowInterface
53 |     {
54 |         $dataflow = new AMPAsyncDataflow($this->reader, $this->name, $this->loopInterval, $this->emitInterval);
55 | 
56 |         krsort($this->steps);
57 |         foreach ($this->steps as $stepArray) {
58 |             foreach ($stepArray as $step) {
59 |                 $dataflow->addStep($step['step'], $step['scale']);
60 |             }
61 |         }
62 | 
63 |         foreach ($this->writers as $writer) {
64 |             $dataflow->addWriter($writer);
65 |         }
66 | 
67 |         return $dataflow;
68 |     }
69 | }
70 | 


--------------------------------------------------------------------------------
/src/Command/SchemaCommand.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\Command;
 6 | 
 7 | use Symfony\Component\Console\Attribute\AsCommand;
 8 | use Symfony\Component\Console\Command\Command;
 9 | use Symfony\Component\Console\Input\ArrayInput;
10 | use Symfony\Component\Console\Input\InputInterface;
11 | use Symfony\Component\Console\Input\InputOption;
12 | use Symfony\Component\Console\Output\OutputInterface;
13 | use Symfony\Component\Console\Style\SymfonyStyle;
14 | 
15 | /**
16 |  * @codeCoverageIgnore
17 |  *
18 |  * @deprecated This command is deprecated and will be removed in 6.0, use this command "code-rhapsodie:dataflow:database-schema" instead.
19 |  */
20 | #[AsCommand('code-rhapsodie:dataflow:dump-schema', 'Generates schema create / update SQL queries')]
21 | class SchemaCommand extends Command
22 | {
23 |     protected function configure(): void
24 |     {
25 |         $this
26 |             ->setHelp('The <info>%command.name%</info> help you to generate SQL Query to create or update your database schema for this bundle')
27 |             ->addOption('update', null, InputOption::VALUE_NONE, 'Dump only the update SQL queries.')
28 |             ->addOption('connection', null, InputOption::VALUE_REQUIRED, 'Define the DBAL connection to use')
29 |         ;
30 |     }
31 | 
32 |     protected function execute(InputInterface $input, OutputInterface $output): int
33 |     {
34 |         $io = new SymfonyStyle($input, $output);
35 |         $io->warning('This command is deprecated and will be removed in 6.0, use this command "code-rhapsodie:dataflow:database-schema" instead.');
36 | 
37 |         $options = array_filter($input->getOptions());
38 | 
39 |         // add -- before each keys
40 |         $options = array_combine(
41 |             array_map(fn ($key) => '--'.$key, array_keys($options)),
42 |             array_values($options)
43 |         );
44 | 
45 |         $options['--dump-sql'] = true;
46 | 
47 |         $inputArray = new ArrayInput([
48 |             'command' => 'code-rhapsodie:dataflow:database-schema',
49 |             ...$options,
50 |         ]);
51 | 
52 |         return $this->getApplication()->doRun($inputArray, $output);
53 |     }
54 | }
55 | 


--------------------------------------------------------------------------------
/Tests/Validator/Constraints/FrequencyValidatorTest.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | namespace CodeRhapsodie\DataflowBundle\Tests\Validator\Constraints;
 4 | 
 5 | use CodeRhapsodie\DataflowBundle\Validator\Constraints\Frequency;
 6 | use CodeRhapsodie\DataflowBundle\Validator\Constraints\FrequencyValidator;
 7 | use PHPUnit\Framework\Attributes\DataProvider;
 8 | use Symfony\Component\Validator\ConstraintValidatorInterface;
 9 | use Symfony\Component\Validator\Test\ConstraintValidatorTestCase;
10 | 
11 | class FrequencyValidatorTest extends ConstraintValidatorTestCase
12 | {
13 |     protected function createValidator(): ConstraintValidatorInterface
14 |     {
15 |         return new FrequencyValidator();
16 |     }
17 | 
18 |     #[DataProvider('getValidValues')]
19 |     public function testValidValues($value)
20 |     {
21 |         $this->validator->validate($value, new Frequency());
22 | 
23 |         $this->assertNoViolation();
24 |     }
25 | 
26 |     public static function getValidValues()
27 |     {
28 |         return [
29 |             ['3 days'],
30 |             ['2 weeks'],
31 |             ['1 month'],
32 |             ['first sunday'],
33 |         ];
34 |     }
35 | 
36 |     public function testInvalidValue()
37 |     {
38 |         $constraint = new Frequency([
39 |             'message' => 'testMessage',
40 |         ]);
41 | 
42 |         $this->validator->validate('wrong value', $constraint);
43 | 
44 |         $this->buildViolation('testMessage')
45 |             ->setParameter('{{ string }}', 'wrong value')
46 |             ->assertRaised()
47 |         ;
48 |     }
49 | 
50 |     #[DataProvider('getNegativeValues')]
51 |     public function testNegativeIntervals($value)
52 |     {
53 |         $constraint = new Frequency([
54 |             'message' => 'testMessage',
55 |         ]);
56 | 
57 |         $this->validator->validate($value, $constraint);
58 | 
59 |         $this->buildViolation('testMessage')
60 |             ->setParameter('{{ string }}', $value)
61 |             ->assertRaised()
62 |         ;
63 |     }
64 | 
65 |     public static function getNegativeValues()
66 |     {
67 |         return [
68 |             ['now'],
69 |             ['-1 day'],
70 |             ['last month'],
71 |         ];
72 |     }
73 | }
74 | 


--------------------------------------------------------------------------------
/src/Manager/ScheduledDataflowManager.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\Manager;
 6 | 
 7 | use CodeRhapsodie\DataflowBundle\Entity\Job;
 8 | use CodeRhapsodie\DataflowBundle\Entity\ScheduledDataflow;
 9 | use CodeRhapsodie\DataflowBundle\Repository\JobRepository;
10 | use CodeRhapsodie\DataflowBundle\Repository\ScheduledDataflowRepository;
11 | use Doctrine\DBAL\Connection;
12 | 
13 | /**
14 |  * Handles scheduled dataflows execution dates based on their frequency.
15 |  */
16 | class ScheduledDataflowManager implements ScheduledDataflowManagerInterface
17 | {
18 |     public function __construct(private Connection $connection, private ScheduledDataflowRepository $scheduledDataflowRepository, private JobRepository $jobRepository)
19 |     {
20 |     }
21 | 
22 |     public function createJobsFromScheduledDataflows(): void
23 |     {
24 |         $this->connection->beginTransaction();
25 |         try {
26 |             foreach ($this->scheduledDataflowRepository->findReadyToRun() as $scheduled) {
27 |                 if ($this->jobRepository->findPendingForScheduledDataflow($scheduled) !== null) {
28 |                     continue;
29 |                 }
30 | 
31 |                 $this->createPendingForScheduled($scheduled);
32 |                 $this->updateScheduledDataflowNext($scheduled);
33 |             }
34 |         } catch (\Throwable $e) {
35 |             $this->connection->rollBack();
36 |             throw $e;
37 |         }
38 |         $this->connection->commit();
39 |     }
40 | 
41 |     private function updateScheduledDataflowNext(ScheduledDataflow $scheduled): void
42 |     {
43 |         $interval = \DateInterval::createFromDateString($scheduled->getFrequency());
44 |         $next = clone $scheduled->getNext();
45 |         $now = new \DateTime();
46 | 
47 |         while ($next < $now) {
48 |             $next->add($interval);
49 |         }
50 | 
51 |         $scheduled->setNext($next);
52 |         $this->scheduledDataflowRepository->save($scheduled);
53 |     }
54 | 
55 |     private function createPendingForScheduled(ScheduledDataflow $scheduled): void
56 |     {
57 |         $this->jobRepository->save(Job::createFromScheduledDataflow($scheduled));
58 |     }
59 | }
60 | 


--------------------------------------------------------------------------------
/src/DataflowType/Writer/DelegatorWriter.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\DataflowType\Writer;
 6 | 
 7 | use CodeRhapsodie\DataflowBundle\Exceptions\UnsupportedItemTypeException;
 8 | 
 9 | /**
10 |  * Writer that delegated the actual writing to other writers.
11 |  */
12 | class DelegatorWriter implements DelegateWriterInterface
13 | {
14 |     /** @var DelegateWriterInterface[] */
15 |     private array $delegates = [];
16 | 
17 |     /**
18 |      * DelegatorWriter constructor.
19 |      */
20 |     public function __construct()
21 |     {
22 |     }
23 | 
24 |     public function prepare()
25 |     {
26 |         foreach ($this->delegates as $delegate) {
27 |             $delegate->prepare();
28 |         }
29 |     }
30 | 
31 |     public function write($item)
32 |     {
33 |         foreach ($this->delegates as $delegate) {
34 |             if (!$delegate->supports($item)) {
35 |                 continue;
36 |             }
37 | 
38 |             $delegate->write($item);
39 | 
40 |             return;
41 |         }
42 | 
43 |         throw new UnsupportedItemTypeException(\sprintf('None of the registered delegate writers support the received item of type %s', get_debug_type($item)));
44 |     }
45 | 
46 |     public function finish()
47 |     {
48 |         foreach ($this->delegates as $delegate) {
49 |             $delegate->finish();
50 |         }
51 |     }
52 | 
53 |     public function supports($item): bool
54 |     {
55 |         foreach ($this->delegates as $delegate) {
56 |             if ($delegate->supports($item)) {
57 |                 return true;
58 |             }
59 |         }
60 | 
61 |         return false;
62 |     }
63 | 
64 |     /**
65 |      * Registers a collection of delegates.
66 |      *
67 |      * @param iterable|DelegateWriterInterface[] $delegates
68 |      */
69 |     public function addDelegates(iterable $delegates): void
70 |     {
71 |         foreach ($delegates as $delegate) {
72 |             $this->addDelegate($delegate);
73 |         }
74 |     }
75 | 
76 |     /**
77 |      * Registers one delegate.
78 |      */
79 |     public function addDelegate(DelegateWriterInterface $delegate): void
80 |     {
81 |         $this->delegates[] = $delegate;
82 |     }
83 | }
84 | 


--------------------------------------------------------------------------------
/src/Command/RunPendingDataflowsCommand.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\Command;
 6 | 
 7 | use CodeRhapsodie\DataflowBundle\Factory\ConnectionFactory;
 8 | use CodeRhapsodie\DataflowBundle\Manager\ScheduledDataflowManagerInterface;
 9 | use CodeRhapsodie\DataflowBundle\Runner\PendingDataflowRunnerInterface;
10 | use Symfony\Component\Console\Attribute\AsCommand;
11 | use Symfony\Component\Console\Command\Command;
12 | use Symfony\Component\Console\Command\LockableTrait;
13 | use Symfony\Component\Console\Input\InputInterface;
14 | use Symfony\Component\Console\Input\InputOption;
15 | use Symfony\Component\Console\Output\OutputInterface;
16 | 
17 | /**
18 |  * Runs dataflows according to user-defined schedule.
19 |  *
20 |  * @codeCoverageIgnore
21 |  */
22 | #[AsCommand('code-rhapsodie:dataflow:run-pending', 'Runs dataflows based on the scheduled defined in the UI.')]
23 | class RunPendingDataflowsCommand extends Command
24 | {
25 |     use LockableTrait;
26 | 
27 |     public function __construct(private ScheduledDataflowManagerInterface $manager, private PendingDataflowRunnerInterface $runner, private ConnectionFactory $connectionFactory)
28 |     {
29 |         parent::__construct();
30 |     }
31 | 
32 |     protected function configure(): void
33 |     {
34 |         $this
35 |             ->setHelp(
36 |                 <<<'EOF'
37 | The <info>%command.name%</info> command runs dataflows according to the schedule defined in the UI by the user.
38 | EOF
39 |             )
40 |             ->addOption('connection', null, InputOption::VALUE_REQUIRED, 'Define the DBAL connection to use');
41 |     }
42 | 
43 |     protected function execute(InputInterface $input, OutputInterface $output): int
44 |     {
45 |         if (!$this->lock()) {
46 |             $output->writeln('The command is already running in another process.');
47 | 
48 |             return 0;
49 |         }
50 | 
51 |         if ($input->getOption('connection') !== null) {
52 |             $this->connectionFactory->setConnectionName($input->getOption('connection'));
53 |         }
54 | 
55 |         $this->manager->createJobsFromScheduledDataflows();
56 |         $this->runner->runPendingDataflows();
57 | 
58 |         $this->release();
59 | 
60 |         return 0;
61 |     }
62 | }
63 | 


--------------------------------------------------------------------------------
/src/Command/ScheduleListCommand.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\Command;
 6 | 
 7 | use CodeRhapsodie\DataflowBundle\Factory\ConnectionFactory;
 8 | use CodeRhapsodie\DataflowBundle\Repository\ScheduledDataflowRepository;
 9 | use Symfony\Component\Console\Attribute\AsCommand;
10 | use Symfony\Component\Console\Command\Command;
11 | use Symfony\Component\Console\Input\InputInterface;
12 | use Symfony\Component\Console\Input\InputOption;
13 | use Symfony\Component\Console\Output\OutputInterface;
14 | use Symfony\Component\Console\Style\SymfonyStyle;
15 | 
16 | /**
17 |  * @codeCoverageIgnore
18 |  */
19 | #[AsCommand('code-rhapsodie:dataflow:schedule:list', 'List scheduled dataflows')]
20 | class ScheduleListCommand extends Command
21 | {
22 |     public function __construct(private ScheduledDataflowRepository $scheduledDataflowRepository, private ConnectionFactory $connectionFactory)
23 |     {
24 |         parent::__construct();
25 |     }
26 | 
27 |     protected function configure(): void
28 |     {
29 |         $this
30 |             ->setHelp('The <info>%command.name%</info> lists all scheduled dataflows.')
31 |             ->addOption('connection', null, InputOption::VALUE_REQUIRED, 'Define the DBAL connection to use');
32 |     }
33 | 
34 |     protected function execute(InputInterface $input, OutputInterface $output): int
35 |     {
36 |         if ($input->getOption('connection') !== null) {
37 |             $this->connectionFactory->setConnectionName($input->getOption('connection'));
38 |         }
39 |         $io = new SymfonyStyle($input, $output);
40 |         $display = [];
41 |         $schedules = $this->scheduledDataflowRepository->listAllOrderedByLabel();
42 |         foreach ($schedules as $schedule) {
43 |             $display[] = [
44 |                 $schedule['id'],
45 |                 $schedule['label'],
46 |                 $schedule['enabled'] ? 'yes' : 'no',
47 |                 $schedule['startTime'] ? (new \DateTime($schedule['startTime']))->format('Y-m-d H:i:s') : '-',
48 |                 $schedule['next'] ? (new \DateTime($schedule['next']))->format('Y-m-d H:i:s') : '-',
49 |             ];
50 |         }
51 | 
52 |         $io->table(['id', 'label', 'enabled?', 'last execution', 'next execution'], $display);
53 | 
54 |         return 0;
55 |     }
56 | }
57 | 


--------------------------------------------------------------------------------
/src/DataflowType/AbstractDataflowType.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\DataflowType;
 6 | 
 7 | use CodeRhapsodie\DataflowBundle\Repository\JobRepository;
 8 | use Psr\Log\LoggerAwareInterface;
 9 | use Psr\Log\LoggerAwareTrait;
10 | use Psr\Log\LoggerInterface;
11 | use Symfony\Component\OptionsResolver\OptionsResolver;
12 | 
13 | abstract class AbstractDataflowType implements DataflowTypeInterface, LoggerAwareInterface, AutoUpdateCountInterface
14 | {
15 |     use LoggerAwareTrait;
16 | 
17 |     private JobRepository $repository;
18 | 
19 |     private ?\DateTime $saveDate = null;
20 | 
21 |     /**
22 |      * @codeCoverageIgnore
23 |      */
24 |     public function getAliases(): iterable
25 |     {
26 |         return [];
27 |     }
28 | 
29 |     public function process(array $options, ?int $jobId = null): Result
30 |     {
31 |         $this->saveDate = new \DateTime('+1 minute');
32 | 
33 |         $optionsResolver = new OptionsResolver();
34 |         $this->configureOptions($optionsResolver);
35 |         $options = $optionsResolver->resolve($options);
36 | 
37 |         $builder = $this->createDataflowBuilder();
38 |         $builder->setName($this->getLabel());
39 |         $builder->addAfterItemProcessor(function (int|string $index, mixed $item, int $count) use ($jobId) {
40 |             if ($jobId === null || $this->saveDate > new \DateTime()) {
41 |                 return;
42 |             }
43 | 
44 |             $this->repository->updateCount($jobId, $count);
45 |             $this->saveDate = new \DateTime('+1 minute');
46 |         });
47 |         $this->buildDataflow($builder, $options);
48 |         $dataflow = $builder->getDataflow();
49 |         if ($dataflow instanceof LoggerAwareInterface && $this->logger instanceof LoggerInterface) {
50 |             $dataflow->setLogger($this->logger);
51 |         }
52 | 
53 |         return $dataflow->process();
54 |     }
55 | 
56 |     public function setRepository(JobRepository $repository): void
57 |     {
58 |         $this->repository = $repository;
59 |     }
60 | 
61 |     protected function createDataflowBuilder(): DataflowBuilder
62 |     {
63 |         return new DataflowBuilder();
64 |     }
65 | 
66 |     /**
67 |      * @codeCoverageIgnore
68 |      */
69 |     protected function configureOptions(OptionsResolver $optionsResolver): void
70 |     {
71 |     }
72 | 
73 |     abstract protected function buildDataflow(DataflowBuilder $builder, array $options): void;
74 | }
75 | 


--------------------------------------------------------------------------------
/src/DependencyInjection/Configuration.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\DependencyInjection;
 6 | 
 7 | use Symfony\Component\Config\Definition\Builder\TreeBuilder;
 8 | use Symfony\Component\Config\Definition\ConfigurationInterface;
 9 | use Symfony\Component\Messenger\MessageBusInterface;
10 | 
11 | class Configuration implements ConfigurationInterface
12 | {
13 |     public function getConfigTreeBuilder(): TreeBuilder
14 |     {
15 |         $treeBuilder = new TreeBuilder('code_rhapsodie_dataflow');
16 |         $rootNode = $treeBuilder->getRootNode();
17 | 
18 |         $rootNode
19 |             ->children()
20 |                 ->scalarNode('dbal_default_connection')
21 |                     ->defaultValue('default')
22 |                 ->end()
23 |                 ->scalarNode('default_logger')
24 |                     ->defaultValue('logger')
25 |                 ->end()
26 |                 ->arrayNode('messenger_mode')
27 |                     ->addDefaultsIfNotSet()
28 |                     ->children()
29 |                         ->booleanNode('enabled')
30 |                             ->defaultFalse()
31 |                         ->end()
32 |                         ->scalarNode('bus')
33 |                             ->defaultValue('messenger.default_bus')
34 |                         ->end()
35 |                     ->end()
36 |                     ->validate()
37 |                         ->ifTrue(static fn ($v): bool => $v['enabled'] && !interface_exists(MessageBusInterface::class))
38 |                         ->thenInvalid('You need "symfony/messenger" in order to use Dataflow messenger mode.')
39 |                     ->end()
40 |                 ->end()
41 |                 ->arrayNode('exceptions_mode')
42 |                     ->addDefaultsIfNotSet()
43 |                     ->children()
44 |                         ->scalarNode('type')
45 |                             ->defaultValue('database')
46 |                         ->end()
47 |                         ->scalarNode('flysystem_service')
48 |                         ->end()
49 |                     ->end()
50 |                     ->validate()
51 |                         ->ifTrue(static fn ($v): bool => $v['type'] === 'file' && !class_exists('\League\Flysystem\Filesystem'))
52 |                         ->thenInvalid('You need "league/flysystem" to use Dataflow file exception mode.')
53 |                     ->end()
54 |                 ->end()
55 |             ->end()
56 |         ;
57 | 
58 |         return $treeBuilder;
59 |     }
60 | }
61 | 


--------------------------------------------------------------------------------
/src/DataflowType/DataflowBuilder.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\DataflowType;
 6 | 
 7 | use CodeRhapsodie\DataflowBundle\DataflowType\Dataflow\Dataflow;
 8 | use CodeRhapsodie\DataflowBundle\DataflowType\Dataflow\DataflowInterface;
 9 | use CodeRhapsodie\DataflowBundle\DataflowType\Writer\WriterInterface;
10 | 
11 | class DataflowBuilder
12 | {
13 |     private ?string $name = null;
14 |     private ?iterable $reader = null;
15 |     private array $steps = [];
16 | 
17 |     /** @var WriterInterface[] */
18 |     private array $writers = [];
19 | 
20 |     private ?\Closure $customExceptionIndex = null;
21 |     /**
22 |      * @var \Closure[]
23 |      */
24 |     private array $afterItemProcessors = [];
25 | 
26 |     public function setName(string $name): self
27 |     {
28 |         $this->name = $name;
29 | 
30 |         return $this;
31 |     }
32 | 
33 |     public function setReader(iterable $reader): self
34 |     {
35 |         $this->reader = $reader;
36 | 
37 |         return $this;
38 |     }
39 | 
40 |     public function addStep(callable $step, int $priority = 0): self
41 |     {
42 |         $this->steps[$priority][] = $step;
43 | 
44 |         return $this;
45 |     }
46 | 
47 |     public function addWriter(WriterInterface $writer): self
48 |     {
49 |         $this->writers[] = $writer;
50 | 
51 |         return $this;
52 |     }
53 | 
54 |     public function setCustomExceptionIndex(callable $callable): self
55 |     {
56 |         $this->customExceptionIndex = \Closure::fromCallable($callable);
57 | 
58 |         return $this;
59 |     }
60 | 
61 |     public function addAfterItemProcessor(callable $callable): self
62 |     {
63 |         $this->afterItemProcessors[] = \Closure::fromCallable($callable);
64 | 
65 |         return $this;
66 |     }
67 | 
68 |     public function getDataflow(): DataflowInterface
69 |     {
70 |         $dataflow = new Dataflow($this->reader, $this->name);
71 | 
72 |         krsort($this->steps);
73 |         foreach ($this->steps as $stepArray) {
74 |             foreach ($stepArray as $step) {
75 |                 $dataflow->addStep($step);
76 |             }
77 |         }
78 | 
79 |         foreach ($this->writers as $writer) {
80 |             $dataflow->addWriter($writer);
81 |         }
82 | 
83 |         if (\is_callable($this->customExceptionIndex)) {
84 |             $dataflow->setCustomExceptionIndex($this->customExceptionIndex);
85 |         }
86 | 
87 |         $dataflow->setAfterItemProcessors($this->afterItemProcessors);
88 | 
89 |         return $dataflow;
90 |     }
91 | }
92 | 


--------------------------------------------------------------------------------
/src/SchemaProvider/DataflowSchemaProvider.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\SchemaProvider;
 6 | 
 7 | use CodeRhapsodie\DataflowBundle\Repository\JobRepository;
 8 | use CodeRhapsodie\DataflowBundle\Repository\ScheduledDataflowRepository;
 9 | use Doctrine\DBAL\Schema\Schema;
10 | 
11 | /**
12 |  * Class JobSchemaProvider.
13 |  *
14 |  * @codeCoverageIgnore
15 |  */
16 | class DataflowSchemaProvider
17 | {
18 |     public function createSchema()
19 |     {
20 |         $schema = new Schema();
21 |         $tableJob = $schema->createTable(JobRepository::TABLE_NAME);
22 |         $tableJob->addColumn('id', 'integer', [
23 |             'autoincrement' => true,
24 |         ]);
25 |         $tableJob->setPrimaryKey(['id']);
26 | 
27 |         $tableJob->addColumn('scheduled_dataflow_id', 'integer', ['notnull' => false]);
28 |         $tableJob->addColumn('status', 'integer', ['notnull' => true]);
29 |         $tableJob->addColumn('label', 'string', ['notnull' => true, 'length' => 255]);
30 |         $tableJob->addColumn('dataflow_type', 'string', ['notnull' => true, 'length' => 255]);
31 |         $tableJob->addColumn('options', 'json', ['notnull' => true]);
32 |         $tableJob->addColumn('requested_date', 'datetime', ['notnull' => false]);
33 |         $tableJob->addColumn('count', 'integer', ['notnull' => false]);
34 |         $tableJob->addColumn('exceptions', 'json', ['notnull' => false]);
35 |         $tableJob->addColumn('start_time', 'datetime', ['notnull' => false]);
36 |         $tableJob->addColumn('end_time', 'datetime', ['notnull' => false]);
37 | 
38 |         $tableSchedule = $schema->createTable(ScheduledDataflowRepository::TABLE_NAME);
39 |         $tableSchedule->addColumn('id', 'integer', [
40 |             'autoincrement' => true,
41 |         ]);
42 |         $tableSchedule->setPrimaryKey(['id']);
43 |         $tableSchedule->addColumn('label', 'string', ['notnull' => true, 'length' => 255]);
44 |         $tableSchedule->addColumn('dataflow_type', 'string', ['notnull' => true, 'length' => 255]);
45 |         $tableSchedule->addColumn('options', 'json', ['notnull' => true]);
46 |         $tableSchedule->addColumn('frequency', 'string', ['notnull' => true, 'length' => 255]);
47 |         $tableSchedule->addColumn('next', 'datetime', ['notnull' => false]);
48 |         $tableSchedule->addColumn('enabled', 'boolean', ['notnull' => true]);
49 | 
50 |         $tableJob->addForeignKeyConstraint($tableSchedule->getName(), ['scheduled_dataflow_id'], ['id']);
51 |         $tableJob->addIndex(['status'], 'idx_status');
52 | 
53 |         return $schema;
54 |     }
55 | }
56 | 


--------------------------------------------------------------------------------
/Tests/Runner/MessengerDataflowRunnerTest.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | namespace CodeRhapsodie\DataflowBundle\Tests\Runner;
 4 | 
 5 | use CodeRhapsodie\DataflowBundle\Entity\Job;
 6 | use CodeRhapsodie\DataflowBundle\MessengerMode\JobMessage;
 7 | use CodeRhapsodie\DataflowBundle\Repository\JobRepository;
 8 | use CodeRhapsodie\DataflowBundle\Runner\MessengerDataflowRunner;
 9 | use PHPUnit\Framework\MockObject\MockObject;
10 | use PHPUnit\Framework\TestCase;
11 | use Symfony\Component\Messenger\Envelope;
12 | use Symfony\Component\Messenger\MessageBusInterface;
13 | 
14 | class MessengerDataflowRunnerTest extends TestCase
15 | {
16 |     private MessengerDataflowRunner $runner;
17 |     private JobRepository|MockObject $repository;
18 |     private MessageBusInterface|MockObject $bus;
19 | 
20 |     protected function setUp(): void
21 |     {
22 |         $this->repository = $this->createMock(JobRepository::class);
23 |         $this->bus = $this->createMock(MessageBusInterface::class);
24 | 
25 |         $this->runner = new MessengerDataflowRunner($this->repository, $this->bus);
26 |     }
27 | 
28 |     public function testRunPendingDataflows()
29 |     {
30 |         $job1 = (new Job())->setId($id1 = 10);
31 |         $job2 = (new Job())->setId($id2 = 20);
32 | 
33 |         $this->repository
34 |             ->expects($this->exactly(3))
35 |             ->method('findNextPendingDataflow')
36 |             ->willReturnOnConsecutiveCalls($job1, $job2, null)
37 |         ;
38 |         $matcher = $this->exactly(2);
39 |         $this->repository
40 |             ->expects($matcher)
41 |             ->method('save')
42 |             ->with($this->callback(fn($arg) => match ($matcher->numberOfInvocations()) {
43 |                 1 => $arg === $job1,
44 |                 2 => $arg === $job2,
45 |                 default => false,
46 |             }))
47 |         ;
48 | 
49 |         $matcher = $this->exactly(2);
50 |         $this->bus
51 |             ->expects($matcher)
52 |             ->method('dispatch')
53 |             ->with($this->callback(fn($arg) => match ($matcher->numberOfInvocations()) {
54 |                 1 => $arg instanceof JobMessage && $arg->getJobId() === $id1,
55 |                 2 => $arg instanceof JobMessage && $arg->getJobId() === $id2,
56 |                 default => false,
57 |             }))
58 |             ->willReturnOnConsecutiveCalls(
59 |                 new Envelope(new JobMessage($id1)),
60 |                 new Envelope(new JobMessage($id2))
61 |             )
62 |         ;
63 | 
64 |         $this->runner->runPendingDataflows();
65 | 
66 |         $this->assertSame(Job::STATUS_QUEUED, $job1->getStatus());
67 |         $this->assertSame(Job::STATUS_QUEUED, $job2->getStatus());
68 |     }
69 | }
70 | 


--------------------------------------------------------------------------------
/composer.json:
--------------------------------------------------------------------------------
 1 | {
 2 |     "name": "code-rhapsodie/dataflow-bundle",
 3 |     "description": "Data processing framework inspired by PortPHP",
 4 |     "type": "symfony-bundle",
 5 |     "keywords": [
 6 |         "dataflow",
 7 |         "import",
 8 |         "export",
 9 |         "data processing"
10 |     ],
11 |     "license": "MIT",
12 |     "authors": [
13 |         {
14 |             "name": "Jérémy J.",
15 |             "email": "jeremy@code-rhapsodie.fr",
16 |             "role": "Developer"
17 |         },
18 |         {
19 |             "name": "Jean-Baptiste Nahan",
20 |             "email": "jean-baptiste@code-rhapsodie.fr",
21 |             "role": "Developer"
22 |         },
23 |         {
24 |             "name": "Manuel Farrica",
25 |             "email": "manuel@code-rhapsodie.fr",
26 |             "role": "Developer"
27 |         },
28 |         {
29 |             "name": "Code Rhapsodie",
30 |             "homepage": "https://www.code-rhapsodie.fr/"
31 |         }
32 |     ],
33 |     "autoload": {
34 |         "psr-4": {
35 |             "CodeRhapsodie\\DataflowBundle\\": "src/"
36 |         }
37 |     },
38 |     "autoload-dev": {
39 |         "psr-4": {
40 |             "CodeRhapsodie\\DataflowBundle\\Tests\\": "Tests/"
41 |         }
42 |     },
43 |     "require": {
44 |         "php": "^8.0",
45 |         "ext-json": "*",
46 |         "doctrine/dbal": "^3.0||^4.0",
47 |         "doctrine/doctrine-bundle": "^2.0",
48 |         "monolog/monolog": "^2.0||^3.0",
49 |         "psr/log": "^1.1||^2.0||^3.0",
50 |         "symfony/config": "^7.0",
51 |         "symfony/console": "^7.0",
52 |         "symfony/dependency-injection": "^7.0",
53 |         "symfony/event-dispatcher": "^7.0",
54 |         "symfony/http-kernel": "^7.0",
55 |         "symfony/lock": "^7.0",
56 |         "symfony/monolog-bridge": "^7.0",
57 |         "symfony/options-resolver": "^7.0",
58 |         "symfony/validator": "^7.0",
59 |         "symfony/yaml": "^7.0"
60 |     },
61 |     "require-dev": {
62 |         "amphp/amp": "^2.5",
63 |         "friendsofphp/php-cs-fixer": "^3.75",
64 |         "phpunit/phpunit": "^11",
65 |         "portphp/portphp": "^1.9",
66 |         "rector/rector": "^2.0",
67 |         "symfony/messenger": "^7.0"
68 |     },
69 |     "suggest": {
70 |         "amphp/amp": "Provide asynchronous steps for your dataflows",
71 |         "portphp/portphp": "Provides generic readers, steps and writers for your dataflows.",
72 |         "symfony/messenger": "Allows messenger mode, i.e. letting workers run jobs"
73 |     },
74 |     "config": {
75 |         "sort-packages": true
76 |     },
77 |     "extra": {
78 |         "branch-alias": {
79 |             "dev-master": "2.1.x-dev",
80 |             "dev-v2.0.x": "2.0.x-dev",
81 |             "dev-v1.x": "1.x-dev"
82 |         }
83 |     }
84 | }
85 | 


--------------------------------------------------------------------------------
/CHANGELOG.md:
--------------------------------------------------------------------------------
  1 | # Version 5.4.1
  2 | * Fix File Exceptions integration
  3 | 
  4 | # Version 5.4.0
  5 | * Add possibility to save exceptions in file
  6 | 
  7 | # Version 5.3.1
  8 | * Fix interface naming
  9 | 
 10 | # Version 5.3.0
 11 | * Added auto update count processed item while running job
 12 | 
 13 | # Version 5.2.0
 14 | * Added custom index for job status
 15 | 
 16 | # Version 5.1.0
 17 | * Refactor SchemaDump command
 18 | 
 19 | # Version 5.0.1
 20 | * Fix compatibility with doctrine 4
 21 | 
 22 | # Version 5.0.0
 23 | * Initiate Kudos on dataflow-bundle
 24 | * Added Symfony 7 support
 25 | * Removed Symfony 6 compatibility
 26 | * Removed Symfony 5 compatibility
 27 | * Removed Symfony 4 compatibility
 28 | * Removed Symfony 3 compatibility
 29 | * Changed README.md
 30 | * Added CI
 31 | 
 32 | # Version 4.1.3
 33 | * Fix log exception argument typing
 34 | 
 35 | # Version 4.1.2
 36 | * Fix DBAL 2.12 compatibility break
 37 | 
 38 | # Version 4.1.0
 39 | 
 40 | * Added custom index for exception log
 41 | 
 42 | # Version 4.0.0
 43 | 
 44 | * Added Symfony 6 support
 45 | * PHP minimum requirements bumped to 8.0
 46 | 
 47 | # Version 3.1.0
 48 | 
 49 | * Added optional "messenger mode", to delegate jobs execution to workers from the Symfony messenger component
 50 | * Added support for asynchronous steps execution, using the AMPHP library (contribution from [matyo91](https://github.com/matyo91))
 51 | 
 52 | # Version 3.0.0
 53 | 
 54 | * Added PHP 8 support
 55 | * PHP minimum requirements bumped to 7.3
 56 | * Added Doctrine DBAL 3 support
 57 | * Doctrine DBAL minimum requirements bumped to 2.12
 58 | 
 59 | # Version 2.2.0
 60 | 
 61 | * Improve logging Dataflow job
 62 | 
 63 | # Version 2.1.1
 64 | 
 65 | * Fixed some Symfony 5 compatibility issues
 66 | 
 67 | # Version 2.1.0
 68 | 
 69 | * Added CollectionWriter and DelegatorWriter
 70 | * Adding Symfony 5.0 compatibility
 71 | * Save all exceptions caught in the log for `code-rhapsodie:dataflow:execute`
 72 | * Added more output when errors occured during `code-rhapsodie:dataflow:execute`
 73 | 
 74 | # Version 2.0.2
 75 | 
 76 | * Fixed the connection proxy class created by the factory
 77 | 
 78 | # Version 2.0.1
 79 | 
 80 | * Fixed next execution time not increasing for scheduled dataflows
 81 | 
 82 | # Version 2.0.0
 83 | 
 84 | * Add Doctrine DBAL multi-connection support
 85 | * Add configuration to define the default Doctrine DBAL connection
 86 | * Remove Doctrine ORM
 87 | * Rewrite repositories
 88 | 
 89 | # Version 1.0.1
 90 | 
 91 | * Fix lost dependency
 92 | * Fix schedule removing
 93 | 
 94 | # Version 1.0.0
 95 | 
 96 | Initial version
 97 | 
 98 | * Define and configure a Dataflow
 99 | * Run the Job scheduled
100 | * Run one Dataflow from the command line
101 | * Define the schedule for a Dataflow from the command line
102 | * Enable/Disable a scheduled Dataflow from the command line
103 | * Display the list of scheduled Dataflow from the command line
104 | * Display the result for the last Job for a Dataflow from the command line
105 | 


--------------------------------------------------------------------------------
/src/Processor/JobProcessor.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\Processor;
 6 | 
 7 | use CodeRhapsodie\DataflowBundle\DataflowType\AutoUpdateCountInterface;
 8 | use CodeRhapsodie\DataflowBundle\DataflowType\Result;
 9 | use CodeRhapsodie\DataflowBundle\Entity\Job;
10 | use CodeRhapsodie\DataflowBundle\Event\Events;
11 | use CodeRhapsodie\DataflowBundle\Event\ProcessingEvent;
12 | use CodeRhapsodie\DataflowBundle\Gateway\JobGateway;
13 | use CodeRhapsodie\DataflowBundle\Logger\BufferHandler;
14 | use CodeRhapsodie\DataflowBundle\Logger\DelegatingLogger;
15 | use CodeRhapsodie\DataflowBundle\Registry\DataflowTypeRegistryInterface;
16 | use CodeRhapsodie\DataflowBundle\Repository\JobRepository;
17 | use Monolog\Logger;
18 | use Psr\Log\LoggerAwareInterface;
19 | use Psr\Log\LoggerAwareTrait;
20 | use Symfony\Component\EventDispatcher\EventDispatcherInterface;
21 | 
22 | class JobProcessor implements JobProcessorInterface, LoggerAwareInterface
23 | {
24 |     use LoggerAwareTrait;
25 | 
26 |     public function __construct(
27 |         private JobRepository $repository,
28 |         private DataflowTypeRegistryInterface $registry,
29 |         private EventDispatcherInterface $dispatcher,
30 |         private JobGateway $jobGateway,
31 |     ) {
32 |     }
33 | 
34 |     public function process(Job $job): void
35 |     {
36 |         $this->beforeProcessing($job);
37 | 
38 |         $dataflowType = $this->registry->getDataflowType($job->getDataflowType());
39 |         if ($dataflowType instanceof AutoUpdateCountInterface) {
40 |             $dataflowType->setRepository($this->repository);
41 |         }
42 | 
43 |         $loggers = [new Logger('dataflow_internal', [$bufferHandler = new BufferHandler()])];
44 |         if (isset($this->logger)) {
45 |             $loggers[] = $this->logger;
46 |         }
47 |         $logger = new DelegatingLogger($loggers);
48 | 
49 |         if ($dataflowType instanceof LoggerAwareInterface) {
50 |             $dataflowType->setLogger($logger);
51 |         }
52 | 
53 |         $result = $dataflowType->process($job->getOptions(), $job->getId());
54 | 
55 |         if (!$dataflowType instanceof LoggerAwareInterface) {
56 |             foreach ($result->getExceptions() as $index => $e) {
57 |                 $logger->error($e, ['index' => $index]);
58 |             }
59 |         }
60 | 
61 |         $this->afterProcessing($job, $result, $bufferHandler);
62 |     }
63 | 
64 |     private function beforeProcessing(Job $job): void
65 |     {
66 |         $this->dispatcher->dispatch(new ProcessingEvent($job), Events::BEFORE_PROCESSING);
67 | 
68 |         $job
69 |             ->setStatus(Job::STATUS_RUNNING)
70 |             ->setStartTime(new \DateTime())
71 |         ;
72 |         $this->jobGateway->save($job);
73 |     }
74 | 
75 |     private function afterProcessing(Job $job, Result $result, BufferHandler $bufferLogger): void
76 |     {
77 |         $job
78 |             ->setEndTime($result->getEndTime())
79 |             ->setStatus(Job::STATUS_COMPLETED)
80 |             ->setCount($result->getSuccessCount())
81 |             ->setExceptions($bufferLogger->clearBuffer())
82 |         ;
83 | 
84 |         $this->jobGateway->save($job);
85 | 
86 |         $this->dispatcher->dispatch(new ProcessingEvent($job), Events::AFTER_PROCESSING);
87 |     }
88 | }
89 | 


--------------------------------------------------------------------------------
/src/Command/ChangeScheduleStatusCommand.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\Command;
 6 | 
 7 | use CodeRhapsodie\DataflowBundle\Entity\ScheduledDataflow;
 8 | use CodeRhapsodie\DataflowBundle\Factory\ConnectionFactory;
 9 | use CodeRhapsodie\DataflowBundle\Repository\ScheduledDataflowRepository;
10 | use Symfony\Component\Console\Attribute\AsCommand;
11 | use Symfony\Component\Console\Command\Command;
12 | use Symfony\Component\Console\Input\InputArgument;
13 | use Symfony\Component\Console\Input\InputInterface;
14 | use Symfony\Component\Console\Input\InputOption;
15 | use Symfony\Component\Console\Output\OutputInterface;
16 | use Symfony\Component\Console\Style\SymfonyStyle;
17 | 
18 | /**
19 |  * @codeCoverageIgnore
20 |  */
21 | #[AsCommand('code-rhapsodie:dataflow:schedule:change-status', 'Change schedule status')]
22 | class ChangeScheduleStatusCommand extends Command
23 | {
24 |     public function __construct(private ScheduledDataflowRepository $scheduledDataflowRepository, private ConnectionFactory $connectionFactory)
25 |     {
26 |         parent::__construct();
27 |     }
28 | 
29 |     protected function configure(): void
30 |     {
31 |         $this
32 |             ->setHelp('The <info>%command.name%</info> command able you to change schedule status.')
33 |             ->addArgument('schedule-id', InputArgument::REQUIRED, 'Id of the schedule')
34 |             ->addOption('enable', null, InputOption::VALUE_NONE, 'Enable the schedule')
35 |             ->addOption('disable', null, InputOption::VALUE_NONE, 'Disable the schedule')
36 |             ->addOption('connection', null, InputOption::VALUE_REQUIRED, 'Define the DBAL connection to use');
37 |     }
38 | 
39 |     protected function execute(InputInterface $input, OutputInterface $output): int
40 |     {
41 |         if ($input->getOption('connection') !== null) {
42 |             $this->connectionFactory->setConnectionName($input->getOption('connection'));
43 |         }
44 |         $io = new SymfonyStyle($input, $output);
45 |         /** @var ScheduledDataflow|null $schedule */
46 |         $schedule = $this->scheduledDataflowRepository->find((int) $input->getArgument('schedule-id'));
47 | 
48 |         if (!$schedule) {
49 |             $io->error(\sprintf('Cannot find scheduled dataflow with id "%d".', $input->getArgument('schedule-id')));
50 | 
51 |             return 1;
52 |         }
53 | 
54 |         if ($input->getOption('enable') && $input->getOption('disable')) {
55 |             $io->error('You cannot pass enable and disable options in the same time.');
56 | 
57 |             return 2;
58 |         }
59 |         if (!$input->getOption('enable') && !$input->getOption('disable')) {
60 |             $io->error('You must pass enable or disable option.');
61 | 
62 |             return 3;
63 |         }
64 | 
65 |         try {
66 |             $schedule->setEnabled($input->getOption('enable'));
67 |             $this->scheduledDataflowRepository->save($schedule);
68 |             $io->success(\sprintf('Schedule with id "%s" has been successfully updated.', $schedule->getId()));
69 |         } catch (\Exception $e) {
70 |             $io->error(\sprintf('An error occured when changing schedule status : "%s".', $e->getMessage()));
71 | 
72 |             return 4;
73 |         }
74 | 
75 |         return 0;
76 |     }
77 | }
78 | 


--------------------------------------------------------------------------------
/src/Command/ExecuteDataflowCommand.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | declare(strict_types=1);
 4 | 
 5 | namespace CodeRhapsodie\DataflowBundle\Command;
 6 | 
 7 | use CodeRhapsodie\DataflowBundle\DataflowType\AutoUpdateCountInterface;
 8 | use CodeRhapsodie\DataflowBundle\Factory\ConnectionFactory;
 9 | use CodeRhapsodie\DataflowBundle\Registry\DataflowTypeRegistryInterface;
10 | use CodeRhapsodie\DataflowBundle\Repository\JobRepository;
11 | use Psr\Log\LoggerAwareInterface;
12 | use Psr\Log\LoggerAwareTrait;
13 | use Symfony\Component\Console\Attribute\AsCommand;
14 | use Symfony\Component\Console\Command\Command;
15 | use Symfony\Component\Console\Input\InputArgument;
16 | use Symfony\Component\Console\Input\InputInterface;
17 | use Symfony\Component\Console\Input\InputOption;
18 | use Symfony\Component\Console\Output\OutputInterface;
19 | use Symfony\Component\Console\Style\SymfonyStyle;
20 | 
21 | /**
22 |  * Runs one dataflow.
23 |  *
24 |  * @codeCoverageIgnore
25 |  */
26 | #[AsCommand('code-rhapsodie:dataflow:execute', 'Runs one dataflow type with provided options')]
27 | class ExecuteDataflowCommand extends Command implements LoggerAwareInterface
28 | {
29 |     use LoggerAwareTrait;
30 | 
31 |     public function __construct(private DataflowTypeRegistryInterface $registry, private ConnectionFactory $connectionFactory, private JobRepository $jobRepository)
32 |     {
33 |         parent::__construct();
34 |     }
35 | 
36 |     protected function configure(): void
37 |     {
38 |         $this
39 |             ->setHelp(
40 |                 <<<'EOF'
41 | The <info>%command.name%</info> command runs one dataflow with the provided options.
42 | 
43 |   <info>php %command.full_name% App\Dataflow\MyDataflow '{"option1": "value1", "option2": "value2"}'</info>
44 | EOF
45 |             )
46 |             ->addArgument('fqcn', InputArgument::REQUIRED, 'FQCN or alias of the dataflow type')
47 |             ->addArgument('options', InputArgument::OPTIONAL, 'Options for the dataflow type as a json string', '[]')
48 |             ->addOption('connection', null, InputOption::VALUE_REQUIRED, 'Define the DBAL connection to use');
49 |     }
50 | 
51 |     protected function execute(InputInterface $input, OutputInterface $output): int
52 |     {
53 |         if ($input->getOption('connection') !== null) {
54 |             $this->connectionFactory->setConnectionName($input->getOption('connection'));
55 |         }
56 |         $fqcnOrAlias = $input->getArgument('fqcn');
57 |         $options = json_decode($input->getArgument('options'), true, 512, \JSON_THROW_ON_ERROR);
58 |         $io = new SymfonyStyle($input, $output);
59 | 
60 |         $dataflowType = $this->registry->getDataflowType($fqcnOrAlias);
61 |         if ($dataflowType instanceof AutoUpdateCountInterface) {
62 |             $dataflowType->setRepository($this->jobRepository);
63 |         }
64 | 
65 |         if ($dataflowType instanceof LoggerAwareInterface && isset($this->logger)) {
66 |             $dataflowType->setLogger($this->logger);
67 |         }
68 | 
69 |         $result = $dataflowType->process($options);
70 | 
71 |         $io->writeln('Executed: '.$result->getName());
72 |         $io->writeln('Start time: '.$result->getStartTime()->format('Y/m/d H:i:s'));
73 |         $io->writeln('End time: '.$result->getEndTime()->format('Y/m/d H:i:s'));
74 |         $io->writeln('Success: '.$result->getSuccessCount());
75 | 
76 |         if ($result->hasErrors()) {
77 |             $io->error("Errors: {$result->getErrorCount()}\nExceptions traces are available in the logs.");
78 | 
79 |             return 1;
80 |         }
81 | 
82 |         return 0;
83 |     }
84 | }
85 | 


--------------------------------------------------------------------------------
/Tests/Processor/JobProcessorTest.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | namespace CodeRhapsodie\DataflowBundle\Tests\Processor;
 4 | 
 5 | use CodeRhapsodie\DataflowBundle\DataflowType\DataflowTypeInterface;
 6 | use CodeRhapsodie\DataflowBundle\DataflowType\Result;
 7 | use CodeRhapsodie\DataflowBundle\Entity\Job;
 8 | use CodeRhapsodie\DataflowBundle\Event\Events;
 9 | use CodeRhapsodie\DataflowBundle\Event\ProcessingEvent;
10 | use CodeRhapsodie\DataflowBundle\Gateway\JobGateway;
11 | use CodeRhapsodie\DataflowBundle\Processor\JobProcessor;
12 | use CodeRhapsodie\DataflowBundle\Registry\DataflowTypeRegistryInterface;
13 | use CodeRhapsodie\DataflowBundle\Repository\JobRepository;
14 | use PHPUnit\Framework\MockObject\MockObject;
15 | use PHPUnit\Framework\TestCase;
16 | use Symfony\Component\EventDispatcher\EventDispatcherInterface;
17 | 
18 | class JobProcessorTest extends TestCase
19 | {
20 |     private JobProcessor $processor;
21 |     private JobRepository|MockObject $repository;
22 |     private DataflowTypeRegistryInterface|MockObject $registry;
23 |     private EventDispatcherInterface|MockObject $dispatcher;
24 |     private JobGateway|MockObject $jobGateway;
25 | 
26 |     protected function setUp(): void
27 |     {
28 |         $this->repository = $this->createMock(JobRepository::class);
29 |         $this->registry = $this->createMock(DataflowTypeRegistryInterface::class);
30 |         $this->dispatcher = $this->createMock(EventDispatcherInterface::class);
31 |         $this->jobGateway = $this->createMock(JobGateway::class);
32 | 
33 |         $this->processor = new JobProcessor($this->repository, $this->registry, $this->dispatcher, $this->jobGateway);
34 |     }
35 | 
36 |     public function testProcess()
37 |     {
38 |         $now = new \DateTimeImmutable();
39 |         $job = (new Job())
40 |             ->setStatus(Job::STATUS_PENDING)
41 |             ->setDataflowType($type = 'type')
42 |             ->setOptions($options = ['option1' => 'value1'])
43 |         ;
44 | 
45 |         $matcher = $this->exactly(2);
46 |         $this->dispatcher
47 |             ->expects($matcher)
48 |             ->method('dispatch')
49 |             ->with(
50 |                 $this->callback(fn($arg) => $arg instanceof ProcessingEvent && $arg->getJob() === $job),
51 |                 $this->callback(fn($arg) => match ($matcher->numberOfInvocations()) {
52 |                     1 => $arg === Events::BEFORE_PROCESSING,
53 |                     2 => $arg === Events::AFTER_PROCESSING,
54 |                     default => false,
55 |                 })
56 |             );
57 | 
58 |         $dataflowType = $this->createMock(DataflowTypeInterface::class);
59 | 
60 |         $this->registry
61 |             ->expects($this->once())
62 |             ->method('getDataflowType')
63 |             ->with($type)
64 |             ->willReturn($dataflowType)
65 |         ;
66 | 
67 |         $bag = [new \Exception('message1')];
68 | 
69 |         $result = new Result('name', new \DateTimeImmutable(), $end = new \DateTimeImmutable(), $count = 10, $bag);
70 | 
71 |         $dataflowType
72 |             ->expects($this->once())
73 |             ->method('process')
74 |             ->with($options)
75 |             ->willReturn($result)
76 |         ;
77 | 
78 |         $this->jobGateway
79 |             ->expects($this->exactly(2))
80 |             ->method('save')
81 |         ;
82 | 
83 |         $this->processor->process($job);
84 | 
85 |         $this->assertGreaterThanOrEqual($now, $job->getStartTime());
86 |         $this->assertSame(Job::STATUS_COMPLETED, $job->getStatus());
87 |         $this->assertSame($end, $job->getEndTime());
88 |         $this->assertSame($count - count($bag), $job->getCount());
89 |     }
90 | }
91 | 


--------------------------------------------------------------------------------
/src/DataflowType/Dataflow/Dataflow.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | 
  3 | declare(strict_types=1);
  4 | 
  5 | namespace CodeRhapsodie\DataflowBundle\DataflowType\Dataflow;
  6 | 
  7 | use CodeRhapsodie\DataflowBundle\DataflowType\Result;
  8 | use CodeRhapsodie\DataflowBundle\DataflowType\Writer\WriterInterface;
  9 | use Psr\Log\LoggerAwareInterface;
 10 | use Psr\Log\LoggerAwareTrait;
 11 | 
 12 | class Dataflow implements DataflowInterface, LoggerAwareInterface
 13 | {
 14 |     use LoggerAwareTrait;
 15 | 
 16 |     /** @var callable[] */
 17 |     private array $steps = [];
 18 | 
 19 |     /** @var WriterInterface[] */
 20 |     private array $writers = [];
 21 | 
 22 |     private ?\Closure $customExceptionIndex = null;
 23 | 
 24 |     private ?\DateTimeInterface $dateTime = null;
 25 | 
 26 |     /**
 27 |      * @var \Closure[]
 28 |      */
 29 |     private array $afterItemProcessors = [];
 30 | 
 31 |     public function __construct(private iterable $reader, private ?string $name)
 32 |     {
 33 |     }
 34 | 
 35 |     /**
 36 |      * @return $this
 37 |      */
 38 |     public function addStep(callable $step): self
 39 |     {
 40 |         $this->steps[] = $step;
 41 | 
 42 |         return $this;
 43 |     }
 44 | 
 45 |     /**
 46 |      * @return $this
 47 |      */
 48 |     public function addWriter(WriterInterface $writer): self
 49 |     {
 50 |         $this->writers[] = $writer;
 51 | 
 52 |         return $this;
 53 |     }
 54 | 
 55 |     /**
 56 |      * @return $this
 57 |      */
 58 |     public function setCustomExceptionIndex(callable $callable): self
 59 |     {
 60 |         $this->customExceptionIndex = \Closure::fromCallable($callable);
 61 | 
 62 |         return $this;
 63 |     }
 64 | 
 65 |     /**
 66 |      * @param array<callable> $processors
 67 |      */
 68 |     public function setAfterItemProcessors(array $processors): self
 69 |     {
 70 |         $this->afterItemProcessors = array_map(fn (callable $callable) => \Closure::fromCallable($callable), $processors);
 71 | 
 72 |         return $this;
 73 |     }
 74 | 
 75 |     public function process(): Result
 76 |     {
 77 |         $count = 0;
 78 |         $exceptions = [];
 79 |         $startTime = new \DateTime();
 80 | 
 81 |         try {
 82 |             foreach ($this->writers as $writer) {
 83 |                 $writer->prepare();
 84 |             }
 85 | 
 86 |             foreach ($this->reader as $index => $item) {
 87 |                 try {
 88 |                     $this->processItem($item);
 89 |                 } catch (\Throwable $e) {
 90 |                     $exceptionIndex = $index;
 91 |                     try {
 92 |                         if (\is_callable($this->customExceptionIndex)) {
 93 |                             $exceptionIndex = (string) ($this->customExceptionIndex)($item, $index);
 94 |                         }
 95 |                     } catch (\Throwable $e2) {
 96 |                         $exceptions[$index] = $e2;
 97 |                         $this->logException($e2, $index);
 98 |                     }
 99 |                     $exceptions[$exceptionIndex] = $e;
100 |                     $this->logException($e, $exceptionIndex);
101 |                 }
102 | 
103 |                 ++$count;
104 | 
105 |                 foreach ($this->afterItemProcessors as $afterItemProcessor) {
106 |                     $afterItemProcessor($index, $item, $count);
107 |                 }
108 |             }
109 | 
110 |             foreach ($this->writers as $writer) {
111 |                 $writer->finish();
112 |             }
113 |         } catch (\Throwable $e) {
114 |             $exceptions[] = $e;
115 |             $this->logException($e);
116 |         }
117 | 
118 |         return new Result($this->name, $startTime, new \DateTime(), $count, $exceptions);
119 |     }
120 | 
121 |     private function processItem(mixed $item): void
122 |     {
123 |         foreach ($this->steps as $step) {
124 |             $item = \call_user_func($step, $item);
125 | 
126 |             if ($item === false) {
127 |                 return;
128 |             }
129 |         }
130 | 
131 |         foreach ($this->writers as $writer) {
132 |             $writer->write($item);
133 |         }
134 |     }
135 | 
136 |     private function logException(\Throwable $e, string|int|null $index = null): void
137 |     {
138 |         if (!isset($this->logger)) {
139 |             return;
140 |         }
141 | 
142 |         $this->logger->error($e, ['exception' => $e, 'index' => $index]);
143 |     }
144 | }
145 | 


--------------------------------------------------------------------------------
/src/Command/JobShowCommand.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | 
  3 | declare(strict_types=1);
  4 | 
  5 | namespace CodeRhapsodie\DataflowBundle\Command;
  6 | 
  7 | use CodeRhapsodie\DataflowBundle\Entity\Job;
  8 | use CodeRhapsodie\DataflowBundle\Factory\ConnectionFactory;
  9 | use CodeRhapsodie\DataflowBundle\Gateway\JobGateway;
 10 | use Symfony\Component\Console\Attribute\AsCommand;
 11 | use Symfony\Component\Console\Command\Command;
 12 | use Symfony\Component\Console\Input\InputInterface;
 13 | use Symfony\Component\Console\Input\InputOption;
 14 | use Symfony\Component\Console\Output\OutputInterface;
 15 | use Symfony\Component\Console\Style\SymfonyStyle;
 16 | 
 17 | /**
 18 |  * @codeCoverageIgnore
 19 |  */
 20 | #[AsCommand('code-rhapsodie:dataflow:job:show', 'Display job details for schedule or specific job')]
 21 | class JobShowCommand extends Command
 22 | {
 23 |     private const STATUS_MAPPING = [
 24 |         Job::STATUS_PENDING => 'Pending',
 25 |         Job::STATUS_RUNNING => 'Running',
 26 |         Job::STATUS_COMPLETED => 'Completed',
 27 |     ];
 28 | 
 29 |     public function __construct(private JobGateway $jobGateway, private ConnectionFactory $connectionFactory)
 30 |     {
 31 |         parent::__construct();
 32 |     }
 33 | 
 34 |     protected function configure(): void
 35 |     {
 36 |         $this
 37 |             ->setHelp('The <info>%command.name%</info> display job details for schedule or specific job.')
 38 |             ->addOption('job-id', null, InputOption::VALUE_REQUIRED, 'Id of the job to get details')
 39 |             ->addOption('schedule-id', null, InputOption::VALUE_REQUIRED, 'Id of schedule for last execution details')
 40 |             ->addOption('details', null, InputOption::VALUE_NONE, 'Display full details')
 41 |             ->addOption('connection', null, InputOption::VALUE_REQUIRED, 'Define the DBAL connection to use');
 42 |     }
 43 | 
 44 |     protected function execute(InputInterface $input, OutputInterface $output): int
 45 |     {
 46 |         if ($input->getOption('connection') !== null) {
 47 |             $this->connectionFactory->setConnectionName($input->getOption('connection'));
 48 |         }
 49 | 
 50 |         $io = new SymfonyStyle($input, $output);
 51 | 
 52 |         $jobId = (int) $input->getOption('job-id');
 53 |         $scheduleId = (int) $input->getOption('schedule-id');
 54 |         if ($jobId && $scheduleId) {
 55 |             $io->error('You must use `job-id` OR `schedule-id` option, not the 2 in the same time.');
 56 | 
 57 |             return 1;
 58 |         }
 59 | 
 60 |         if ($scheduleId) {
 61 |             $job = $this->jobGateway->findLastForDataflowId($scheduleId);
 62 |         } elseif ($jobId) {
 63 |             $job = $this->jobGateway->find($jobId);
 64 |         } else {
 65 |             $io->error('You must pass `job-id` or `schedule-id` option.');
 66 | 
 67 |             return 2;
 68 |         }
 69 | 
 70 |         if ($job === null) {
 71 |             $io->error('Cannot find job :/');
 72 | 
 73 |             return 3;
 74 |         }
 75 | 
 76 |         /** @var Job $job */
 77 |         $display = [
 78 |             ['Job id', $job->getId()],
 79 |             ['Label', $job->getLabel()],
 80 |             ['Requested at', $job->getRequestedDate()->format('Y-m-d H:i:s')],
 81 |             ['Started at', $job->getStartTime() ? $job->getStartTime()->format('Y-m-d H:i:s') : '-'],
 82 |             ['Ended at', $job->getEndTime() ? $job->getEndTime()->format('Y-m-d H:i:s') : '-'],
 83 |             ['Object number', $job->getCount()],
 84 |             ['Errors', \count((array) $job->getExceptions())],
 85 |             ['Status', $this->translateStatus($job->getStatus())],
 86 |         ];
 87 |         if ($input->getOption('details')) {
 88 |             $display[] = ['Type', $job->getDataflowType()];
 89 |             $display[] = ['Options', json_encode($job->getOptions(), \JSON_THROW_ON_ERROR)];
 90 |             $io->section('Summary');
 91 |         }
 92 | 
 93 |         $io->table(['Field', 'Value'], $display);
 94 |         if ($input->getOption('details')) {
 95 |             $io->section('Exceptions');
 96 |             $exceptions = array_map(fn (string $exception) => substr($exception, 0, 900).'…', $job->getExceptions());
 97 | 
 98 |             $io->write($exceptions);
 99 |         }
100 | 
101 |         return 0;
102 |     }
103 | 
104 |     private function translateStatus(int $status): string
105 |     {
106 |         return self::STATUS_MAPPING[$status] ?? 'Unknown status';
107 |     }
108 | }
109 | 


--------------------------------------------------------------------------------
/src/Command/DatabaseSchemaCommand.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | 
  3 | declare(strict_types=1);
  4 | 
  5 | namespace CodeRhapsodie\DataflowBundle\Command;
  6 | 
  7 | use CodeRhapsodie\DataflowBundle\Factory\ConnectionFactory;
  8 | use CodeRhapsodie\DataflowBundle\Repository\JobRepository;
  9 | use CodeRhapsodie\DataflowBundle\Repository\ScheduledDataflowRepository;
 10 | use CodeRhapsodie\DataflowBundle\SchemaProvider\DataflowSchemaProvider;
 11 | use Doctrine\DBAL\Schema\Comparator;
 12 | use Doctrine\DBAL\Schema\Schema;
 13 | use Doctrine\DBAL\Schema\Table;
 14 | use Symfony\Component\Console\Attribute\AsCommand;
 15 | use Symfony\Component\Console\Command\Command;
 16 | use Symfony\Component\Console\Input\InputInterface;
 17 | use Symfony\Component\Console\Input\InputOption;
 18 | use Symfony\Component\Console\Output\OutputInterface;
 19 | use Symfony\Component\Console\Question\ConfirmationQuestion;
 20 | use Symfony\Component\Console\Style\SymfonyStyle;
 21 | 
 22 | #[AsCommand(name: 'code-rhapsodie:dataflow:database-schema', description: 'Generates schema create / update SQL queries')]
 23 | class DatabaseSchemaCommand extends Command
 24 | {
 25 |     public function __construct(private ConnectionFactory $connectionFactory)
 26 |     {
 27 |         parent::__construct();
 28 |     }
 29 | 
 30 |     protected function configure(): void
 31 |     {
 32 |         $this
 33 |             ->setHelp('The <info>%command.name%</info> help you to generate SQL Query to create or update your database schema for this bundle')
 34 |             ->addOption('dump-sql', null, InputOption::VALUE_NONE, 'Dump only the update SQL queries.')
 35 |             ->addOption('update', null, InputOption::VALUE_NONE, 'Dump/execute only the update SQL queries.')
 36 |             ->addOption('connection', null, InputOption::VALUE_REQUIRED, 'Define the DBAL connection to use');
 37 |     }
 38 | 
 39 |     protected function execute(InputInterface $input, OutputInterface $output): int
 40 |     {
 41 |         $io = new SymfonyStyle($input, $output);
 42 | 
 43 |         if ($input->getOption('connection') !== null) {
 44 |             $this->connectionFactory->setConnectionName($input->getOption('connection'));
 45 |         }
 46 | 
 47 |         $connection = $this->connectionFactory->getConnection();
 48 | 
 49 |         $schemaProvider = new DataflowSchemaProvider();
 50 |         $schema = $schemaProvider->createSchema();
 51 | 
 52 |         $sqls = $schema->toSql($connection->getDatabasePlatform());
 53 | 
 54 |         if ($input->getOption('update')) {
 55 |             $sm = $connection->createSchemaManager();
 56 | 
 57 |             $tableArray = [JobRepository::TABLE_NAME, ScheduledDataflowRepository::TABLE_NAME];
 58 |             $tables = [];
 59 |             foreach ($sm->listTables() as $table) {
 60 |                 /** @var Table $table */
 61 |                 if (\in_array($table->getName(), $tableArray)) {
 62 |                     $tables[] = $table;
 63 |                 }
 64 |             }
 65 | 
 66 |             $namespaces = [];
 67 | 
 68 |             if ($connection->getDatabasePlatform()->supportsSchemas()) {
 69 |                 $namespaces = $sm->listSchemaNames();
 70 |             }
 71 | 
 72 |             $sequences = [];
 73 | 
 74 |             if ($connection->getDatabasePlatform()->supportsSequences()) {
 75 |                 $sequences = $sm->listSequences();
 76 |             }
 77 | 
 78 |             $oldSchema = new Schema($tables, $sequences, $sm->createSchemaConfig(), $namespaces);
 79 | 
 80 |             $sqls = $connection->getDatabasePlatform()->getAlterSchemaSQL((new Comparator($connection->getDatabasePlatform()))->compareSchemas($oldSchema, $schema));
 81 | 
 82 |             if (empty($sqls)) {
 83 |                 $io->info('There is no update SQL queries.');
 84 |             }
 85 |         }
 86 | 
 87 |         if ($input->getOption('dump-sql')) {
 88 |             $io->text('Execute these SQL Queries on your database:');
 89 |             foreach ($sqls as $sql) {
 90 |                 $io->text($sql.';');
 91 |             }
 92 | 
 93 |             return Command::SUCCESS;
 94 |         }
 95 | 
 96 |         if (!$io->askQuestion(new ConfirmationQuestion('Are you sure to update database ?', true))) {
 97 |             $io->text('Execution canceled.');
 98 | 
 99 |             return Command::SUCCESS;
100 |         }
101 | 
102 |         foreach ($sqls as $sql) {
103 |             $connection->executeQuery($sql);
104 |         }
105 | 
106 |         $io->success(\sprintf('%d queries executed.', \count($sqls)));
107 | 
108 |         return parent::SUCCESS;
109 |     }
110 | }
111 | 


--------------------------------------------------------------------------------
/src/Entity/ScheduledDataflow.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | 
  3 | declare(strict_types=1);
  4 | 
  5 | namespace CodeRhapsodie\DataflowBundle\Entity;
  6 | 
  7 | use CodeRhapsodie\DataflowBundle\Validator\Constraints\Frequency;
  8 | use Symfony\Component\Validator\Constraints as Asserts;
  9 | 
 10 | /**
 11 |  * Schedule for a regular execution of a dataflow.
 12 |  *
 13 |  * @codeCoverageIgnore
 14 |  */
 15 | class ScheduledDataflow
 16 | {
 17 |     public const AVAILABLE_FREQUENCIES = [
 18 |         '1 hour',
 19 |         '1 day',
 20 |         '1 week',
 21 |         '1 month',
 22 |     ];
 23 | 
 24 |     private const KEYS = ['id', 'label', 'dataflow_type', 'options', 'frequency', 'next', 'enabled'];
 25 | 
 26 |     private ?int $id = null;
 27 | 
 28 |     #[Asserts\NotBlank]
 29 |     #[Asserts\Length(min: 1, max: 255)]
 30 |     #[Asserts\Regex('#^[[:alnum:] ]+\z#u')]
 31 |     private ?string $label = null;
 32 | 
 33 |     #[Asserts\NotBlank]
 34 |     #[Asserts\Length(min: 1, max: 255)]
 35 |     #[Asserts\Regex('#^[[:alnum:]\\\]+\z#u')]
 36 |     private ?string $dataflowType = null;
 37 | 
 38 |     private ?array $options = null;
 39 | 
 40 |     /**
 41 |      * @Frequency()
 42 |      */
 43 |     #[Asserts\NotBlank]
 44 |     private ?string $frequency = null;
 45 | 
 46 |     private ?\DateTimeInterface $next = null;
 47 | 
 48 |     private ?bool $enabled = null;
 49 | 
 50 |     public static function createFromArray(array $datas)
 51 |     {
 52 |         $lost = array_diff(static::KEYS, array_keys($datas));
 53 |         if (\count($lost) > 0) {
 54 |             throw new \LogicException('The first argument of '.__METHOD__.'  must be contains: "'.implode(', ', $lost).'"');
 55 |         }
 56 | 
 57 |         $scheduledDataflow = new self();
 58 |         $scheduledDataflow->id = $datas['id'] === null ? null : (int) $datas['id'];
 59 | 
 60 |         $scheduledDataflow->setLabel($datas['label']);
 61 |         $scheduledDataflow->setDataflowType($datas['dataflow_type']);
 62 |         $scheduledDataflow->setOptions($datas['options']);
 63 |         $scheduledDataflow->setFrequency($datas['frequency']);
 64 |         $scheduledDataflow->setNext($datas['next']);
 65 |         $scheduledDataflow->setEnabled($datas['enabled'] === null ? null : (bool) $datas['enabled']);
 66 | 
 67 |         return $scheduledDataflow;
 68 |     }
 69 | 
 70 |     public function toArray(): array
 71 |     {
 72 |         return [
 73 |             'id' => $this->getId(),
 74 |             'label' => $this->getLabel(),
 75 |             'dataflow_type' => $this->getDataflowType(),
 76 |             'options' => $this->getOptions(),
 77 |             'frequency' => $this->getFrequency(),
 78 |             'next' => $this->getNext(),
 79 |             'enabled' => $this->getEnabled(),
 80 |         ];
 81 |     }
 82 | 
 83 |     public function setId(int $id): self
 84 |     {
 85 |         $this->id = $id;
 86 | 
 87 |         return $this;
 88 |     }
 89 | 
 90 |     public function getId(): ?int
 91 |     {
 92 |         return $this->id;
 93 |     }
 94 | 
 95 |     public function getLabel(): ?string
 96 |     {
 97 |         return $this->label;
 98 |     }
 99 | 
100 |     public function setLabel(?string $label): self
101 |     {
102 |         $this->label = $label;
103 | 
104 |         return $this;
105 |     }
106 | 
107 |     public function getDataflowType(): ?string
108 |     {
109 |         return $this->dataflowType;
110 |     }
111 | 
112 |     public function setDataflowType(?string $dataflowType): self
113 |     {
114 |         $this->dataflowType = $dataflowType;
115 | 
116 |         return $this;
117 |     }
118 | 
119 |     public function getOptions(): ?array
120 |     {
121 |         return $this->options;
122 |     }
123 | 
124 |     public function setOptions(?array $options): self
125 |     {
126 |         $this->options = $options;
127 | 
128 |         return $this;
129 |     }
130 | 
131 |     public function getFrequency(): ?string
132 |     {
133 |         return $this->frequency;
134 |     }
135 | 
136 |     public function setFrequency(?string $frequency): self
137 |     {
138 |         $this->frequency = $frequency;
139 | 
140 |         return $this;
141 |     }
142 | 
143 |     public function getNext(): ?\DateTimeInterface
144 |     {
145 |         return $this->next;
146 |     }
147 | 
148 |     public function setNext(?\DateTimeInterface $next): self
149 |     {
150 |         $this->next = $next;
151 | 
152 |         return $this;
153 |     }
154 | 
155 |     public function getEnabled(): ?bool
156 |     {
157 |         return $this->enabled;
158 |     }
159 | 
160 |     public function setEnabled(?bool $enabled): self
161 |     {
162 |         $this->enabled = $enabled;
163 | 
164 |         return $this;
165 |     }
166 | }
167 | 


--------------------------------------------------------------------------------
/Tests/Manager/ScheduledDataflowManagerTest.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | 
  3 | namespace CodeRhapsodie\DataflowBundle\Tests\Manager;
  4 | 
  5 | use CodeRhapsodie\DataflowBundle\Entity\Job;
  6 | use CodeRhapsodie\DataflowBundle\Entity\ScheduledDataflow;
  7 | use CodeRhapsodie\DataflowBundle\Manager\ScheduledDataflowManager;
  8 | use CodeRhapsodie\DataflowBundle\Repository\JobRepository;
  9 | use CodeRhapsodie\DataflowBundle\Repository\ScheduledDataflowRepository;
 10 | use Doctrine\DBAL\Connection;
 11 | use PHPUnit\Framework\MockObject\MockObject;
 12 | use PHPUnit\Framework\TestCase;
 13 | 
 14 | class ScheduledDataflowManagerTest extends TestCase
 15 | {
 16 |     private ScheduledDataflowManager $manager;
 17 |     private Connection|MockObject $connection;
 18 |     private ScheduledDataflowRepository|MockObject $scheduledDataflowRepository;
 19 |     private JobRepository|MockObject $jobRepository;
 20 | 
 21 |     protected function setUp(): void
 22 |     {
 23 |         $this->connection = $this->createMock(Connection::class);
 24 |         $this->scheduledDataflowRepository = $this->createMock(ScheduledDataflowRepository::class);
 25 |         $this->jobRepository = $this->createMock(JobRepository::class);
 26 | 
 27 |         $this->manager = new ScheduledDataflowManager($this->connection, $this->scheduledDataflowRepository, $this->jobRepository);
 28 |     }
 29 | 
 30 |     public function testCreateJobsFromScheduledDataflows()
 31 |     {
 32 |         $scheduled1 = new ScheduledDataflow();
 33 |         $scheduled2 = (new ScheduledDataflow())
 34 |             ->setId(-1)
 35 |             ->setDataflowType($type = 'testType')
 36 |             ->setOptions($options = ['opt' => 'val'])
 37 |             ->setNext($next = new \DateTime())
 38 |             ->setLabel($label = 'testLabel')
 39 |             ->setFrequency($frequency = '1 year')
 40 |         ;
 41 | 
 42 |         $this->scheduledDataflowRepository
 43 |             ->expects($this->once())
 44 |             ->method('findReadyToRun')
 45 |             ->willReturn([$scheduled1, $scheduled2])
 46 |         ;
 47 | 
 48 |         $matcher = $this->exactly(2);
 49 |         $this->jobRepository
 50 |             ->expects($matcher)
 51 |             ->method('findPendingForScheduledDataflow')
 52 |             ->with($this->callback(fn($arg) => match ($matcher->numberOfInvocations()) {
 53 |                 1 => $arg === $scheduled1,
 54 |                 2 => $arg === $scheduled2,
 55 |                 default => false,
 56 |             }))
 57 |             ->willReturnOnConsecutiveCalls(new Job(), null)
 58 |         ;
 59 | 
 60 |         $this->connection
 61 |             ->expects($this->once())
 62 |             ->method('beginTransaction')
 63 |         ;
 64 |         $this->jobRepository
 65 |             ->expects($this->once())
 66 |             ->method('save')
 67 |             ->with(
 68 |                 $this->callback(fn(Job $job) => $job->getStatus() === Job::STATUS_PENDING
 69 |                 && $job->getDataflowType() === $type
 70 |                 && $job->getOptions() === $options
 71 |                 && $job->getRequestedDate() == $next
 72 |                 && $job->getLabel() === $label
 73 |                 && $job->getScheduledDataflowId() === $scheduled2->getId())
 74 |             )
 75 |         ;
 76 | 
 77 |         $this->scheduledDataflowRepository
 78 |             ->expects($this->once())
 79 |             ->method('save')
 80 |             ->with($scheduled2)
 81 |         ;
 82 | 
 83 |         $this->connection
 84 |             ->expects($this->once())
 85 |             ->method('commit')
 86 |         ;
 87 | 
 88 |         $this->manager->createJobsFromScheduledDataflows();
 89 | 
 90 |         $this->assertEquals($next->add(\DateInterval::createFromDateString($frequency)), $scheduled2->getNext());
 91 |     }
 92 | 
 93 |     public function testCreateJobsFromScheduledDataflowsWithError()
 94 |     {
 95 |         $scheduled1 = new ScheduledDataflow();
 96 | 
 97 |         $this->scheduledDataflowRepository
 98 |             ->expects($this->once())
 99 |             ->method('findReadyToRun')
100 |             ->willReturn([$scheduled1])
101 |         ;
102 | 
103 |         $this->jobRepository
104 |             ->expects($this->exactly(1))
105 |             ->method('findPendingForScheduledDataflow')
106 |             ->with($scheduled1)
107 |             ->willThrowException(new \Exception())
108 |         ;
109 | 
110 |         $this->connection
111 |             ->expects($this->once())
112 |             ->method('beginTransaction')
113 |         ;
114 |         $this->jobRepository
115 |             ->expects($this->never())
116 |             ->method('save')
117 |         ;
118 | 
119 |         $this->connection
120 |             ->expects($this->never())
121 |             ->method('commit')
122 |         ;
123 |         $this->connection
124 |             ->expects($this->once())
125 |             ->method('rollBack')
126 |         ;
127 | 
128 |         $this->expectException(\Exception::class);
129 | 
130 |         $this->manager->createJobsFromScheduledDataflows();
131 |     }
132 | }
133 | 


--------------------------------------------------------------------------------
/src/Repository/ScheduledDataflowRepository.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | 
  3 | declare(strict_types=1);
  4 | 
  5 | namespace CodeRhapsodie\DataflowBundle\Repository;
  6 | 
  7 | use CodeRhapsodie\DataflowBundle\Entity\ScheduledDataflow;
  8 | use Doctrine\DBAL\Connection;
  9 | use Doctrine\DBAL\ParameterType;
 10 | use Doctrine\DBAL\Query\QueryBuilder;
 11 | 
 12 | /**
 13 |  * Repository for the ScheduledDataflow entity.
 14 |  *
 15 |  * @codeCoverageIgnore
 16 |  */
 17 | class ScheduledDataflowRepository
 18 | {
 19 |     use InitFromDbTrait;
 20 | 
 21 |     public const TABLE_NAME = 'cr_dataflow_scheduled';
 22 | 
 23 |     /**
 24 |      * JobRepository constructor.
 25 |      */
 26 |     public function __construct(private Connection $connection)
 27 |     {
 28 |     }
 29 | 
 30 |     /**
 31 |      * Finds all enabled scheduled dataflows with a passed next run date.
 32 |      *
 33 |      * @return ScheduledDataflow[]
 34 |      */
 35 |     public function findReadyToRun(): iterable
 36 |     {
 37 |         $qb = $this->createQueryBuilder();
 38 |         $qb->andWhere($qb->expr()->lte('next', $qb->createNamedParameter(new \DateTime(), 'datetime')))
 39 |             ->andWhere($qb->expr()->eq('enabled', $qb->createNamedParameter(1, ParameterType::INTEGER)))
 40 |             ->orderBy('next', 'ASC')
 41 |         ;
 42 | 
 43 |         $stmt = $qb->executeQuery();
 44 |         if ($stmt->rowCount() === 0) {
 45 |             return [];
 46 |         }
 47 |         while (false !== ($row = $stmt->fetchAssociative())) {
 48 |             yield ScheduledDataflow::createFromArray($this->initDateTime($this->initArray($row)));
 49 |         }
 50 |     }
 51 | 
 52 |     public function find(int $scheduleId): ?ScheduledDataflow
 53 |     {
 54 |         $qb = $this->createQueryBuilder();
 55 |         $qb->andWhere($qb->expr()->eq('id', $qb->createNamedParameter($scheduleId, ParameterType::INTEGER)))
 56 |             ->setMaxResults(1)
 57 |         ;
 58 | 
 59 |         return $this->returnFirstOrNull($qb);
 60 |     }
 61 | 
 62 |     public function findAllOrderedByLabel(): iterable
 63 |     {
 64 |         $qb = $this->createQueryBuilder();
 65 |         $qb->orderBy('label', 'ASC');
 66 | 
 67 |         $stmt = $qb->executeQuery();
 68 |         if ($stmt->rowCount() === 0) {
 69 |             return [];
 70 |         }
 71 |         while (false !== ($row = $stmt->fetchAssociative())) {
 72 |             yield ScheduledDataflow::createFromArray($this->initDateTime($this->initArray($row)));
 73 |         }
 74 |     }
 75 | 
 76 |     public function listAllOrderedByLabel(): array
 77 |     {
 78 |         $query = $this->connection->createQueryBuilder()
 79 |             ->from(static::TABLE_NAME, 'w')
 80 |             ->select('w.id', 'w.label', 'w.enabled', 'w.next', 'max(j.start_time) as startTime')
 81 |             ->leftJoin('w', JobRepository::TABLE_NAME, 'j', 'j.scheduled_dataflow_id = w.id')
 82 |             ->orderBy('w.label', 'ASC')
 83 |             ->groupBy('w.id');
 84 | 
 85 |         return $query->executeQuery()->fetchAllAssociative();
 86 |     }
 87 | 
 88 |     public function save(ScheduledDataflow $scheduledDataflow)
 89 |     {
 90 |         $datas = $scheduledDataflow->toArray();
 91 |         unset($datas['id']);
 92 | 
 93 |         if (\is_array($datas['options'])) {
 94 |             $datas['options'] = json_encode($datas['options'], \JSON_THROW_ON_ERROR);
 95 |         }
 96 | 
 97 |         if ($scheduledDataflow->getId() === null) {
 98 |             $this->connection->insert(static::TABLE_NAME, $datas, $this->getFields());
 99 |             $scheduledDataflow->setId((int) $this->connection->lastInsertId());
100 | 
101 |             return;
102 |         }
103 |         $this->connection->update(static::TABLE_NAME, $datas, ['id' => $scheduledDataflow->getId()], $this->getFields());
104 |     }
105 | 
106 |     public function delete(int $id): void
107 |     {
108 |         $this->connection->beginTransaction();
109 |         try {
110 |             $this->connection->delete(JobRepository::TABLE_NAME, ['scheduled_dataflow_id' => $id]);
111 |             $this->connection->delete(static::TABLE_NAME, ['id' => $id]);
112 |         } catch (\Throwable $e) {
113 |             $this->connection->rollBack();
114 |             throw $e;
115 |         }
116 | 
117 |         $this->connection->commit();
118 |     }
119 | 
120 |     public function createQueryBuilder($alias = null): QueryBuilder
121 |     {
122 |         $qb = $this->connection->createQueryBuilder();
123 |         $qb->select('*')
124 |             ->from(static::TABLE_NAME, $alias);
125 | 
126 |         return $qb;
127 |     }
128 | 
129 |     private function returnFirstOrNull(QueryBuilder $qb): ?ScheduledDataflow
130 |     {
131 |         $stmt = $qb->executeQuery();
132 |         if ($stmt->rowCount() === 0) {
133 |             return null;
134 |         }
135 | 
136 |         return ScheduledDataflow::createFromArray($this->initDateTime($this->initArray($stmt->fetchAssociative())));
137 |     }
138 | 
139 |     private function getFields(): array
140 |     {
141 |         return [
142 |             'id' => ParameterType::INTEGER,
143 |             'label' => ParameterType::STRING,
144 |             'dataflow_type' => ParameterType::STRING,
145 |             'options' => ParameterType::STRING,
146 |             'frequency' => ParameterType::STRING,
147 |             'next' => 'datetime',
148 |             'enabled' => ParameterType::BOOLEAN,
149 |         ];
150 |     }
151 | }
152 | 


--------------------------------------------------------------------------------
/Tests/DataflowType/Writer/DelegatorWriterTest.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | 
  3 | namespace CodeRhapsodie\DataflowBundle\Tests\DataflowType\Writer;
  4 | 
  5 | use CodeRhapsodie\DataflowBundle\DataflowType\Writer\DelegateWriterInterface;
  6 | use CodeRhapsodie\DataflowBundle\DataflowType\Writer\DelegatorWriter;
  7 | use CodeRhapsodie\DataflowBundle\Exceptions\UnsupportedItemTypeException;
  8 | use PHPUnit\Framework\MockObject\MockObject;
  9 | use PHPUnit\Framework\TestCase;
 10 | 
 11 | class DelegatorWriterTest extends TestCase
 12 | {
 13 |     private DelegatorWriter $delegatorWriter;
 14 |     private DelegateWriterInterface|MockObject $delegateInt;
 15 |     private DelegateWriterInterface|MockObject $delegateString;
 16 |     private DelegateWriterInterface|MockObject $delegateArray;
 17 | 
 18 |     protected function setUp(): void
 19 |     {
 20 |         $this->delegateInt = $this->createMock(DelegateWriterInterface::class);
 21 |         $this->delegateInt->method('supports')->willReturnCallback(fn($argument) => is_int($argument));
 22 | 
 23 |         $this->delegateString = $this->createMock(DelegateWriterInterface::class);
 24 |         $this->delegateString->method('supports')->willReturnCallback(fn($argument) => is_string($argument));
 25 | 
 26 |         $this->delegateArray = $this->createMock(DelegateWriterInterface::class);
 27 |         $this->delegateArray->method('supports')->willReturnCallback(fn($argument) => is_array($argument));
 28 | 
 29 |         $this->delegatorWriter = new DelegatorWriter();
 30 |         $this->delegatorWriter->addDelegates([
 31 |             $this->delegateInt,
 32 |             $this->delegateString,
 33 |             $this->delegateArray,
 34 |         ]);
 35 |     }
 36 | 
 37 |     public function testUnsupported()
 38 |     {
 39 |         $this->expectException(UnsupportedItemTypeException::class);
 40 | 
 41 |         $this->delegatorWriter->write(new \stdClass());
 42 |     }
 43 | 
 44 |     public function testStopAtFirstSupportingDelegate()
 45 |     {
 46 |         $value = 0;
 47 | 
 48 |         $this->delegateInt->expects($this->once())->method('supports');
 49 |         $this->delegateInt
 50 |             ->expects($this->once())
 51 |             ->method('write')
 52 |             ->with($value)
 53 |         ;
 54 |         $this->delegateString->expects($this->never())->method('supports');
 55 |         $this->delegateArray->expects($this->never())->method('supports');
 56 |         $this->delegateString->expects($this->never())->method('write');
 57 |         $this->delegateArray->expects($this->never())->method('write');
 58 | 
 59 |         $this->delegatorWriter->write($value);
 60 |     }
 61 | 
 62 |     public function testNotSupported()
 63 |     {
 64 |         $value = new \stdClass();
 65 | 
 66 |         $this->delegateInt
 67 |             ->expects($this->once())
 68 |             ->method('supports')
 69 |             ->with($value)
 70 |         ;
 71 |         $this->delegateString
 72 |             ->expects($this->once())
 73 |             ->method('supports')
 74 |             ->with($value)
 75 |         ;
 76 |         $this->delegateArray
 77 |             ->expects($this->once())
 78 |             ->method('supports')
 79 |             ->with($value)
 80 |         ;
 81 | 
 82 |         $this->assertFalse($this->delegatorWriter->supports($value));
 83 |     }
 84 | 
 85 |     public function testSupported()
 86 |     {
 87 |         $value = '';
 88 | 
 89 |         $this->delegateInt
 90 |             ->expects($this->once())
 91 |             ->method('supports')
 92 |             ->with($value)
 93 |         ;
 94 |         $this->delegateString
 95 |             ->expects($this->once())
 96 |             ->method('supports')
 97 |             ->with($value)
 98 |         ;
 99 |         $this->delegateArray
100 |             ->expects($this->never())
101 |             ->method('supports')
102 |         ;
103 | 
104 |         $this->assertTrue($this->delegatorWriter->supports($value));
105 |     }
106 | 
107 |     public function testAll()
108 |     {
109 |         $value = ['a'];
110 | 
111 |         $this->delegateInt
112 |             ->expects($this->once())
113 |             ->method('supports')
114 |             ->with($value)
115 |         ;
116 |         $this->delegateString
117 |             ->expects($this->once())
118 |             ->method('supports')
119 |             ->with($value)
120 |         ;
121 |         $this->delegateArray
122 |             ->expects($this->once())
123 |             ->method('supports')
124 |             ->with($value)
125 |         ;
126 | 
127 |         $this->delegateInt->expects($this->once())->method('prepare');
128 |         $this->delegateString->expects($this->once())->method('prepare');
129 |         $this->delegateArray->expects($this->once())->method('prepare');
130 | 
131 |         $this->delegateInt->expects($this->once())->method('finish');
132 |         $this->delegateString->expects($this->once())->method('finish');
133 |         $this->delegateArray->expects($this->once())->method('finish');
134 | 
135 |         $this->delegateInt->expects($this->never())->method('write');
136 |         $this->delegateString->expects($this->never())->method('write');
137 |         $this->delegateArray
138 |             ->expects($this->once())
139 |             ->method('write')
140 |             ->with($value)
141 |         ;
142 | 
143 |         $this->delegatorWriter->prepare();
144 |         $this->delegatorWriter->write($value);
145 |         $this->delegatorWriter->finish();
146 |     }
147 | }
148 | 


--------------------------------------------------------------------------------
/src/Command/AddScheduledDataflowCommand.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | 
  3 | declare(strict_types=1);
  4 | 
  5 | namespace CodeRhapsodie\DataflowBundle\Command;
  6 | 
  7 | use CodeRhapsodie\DataflowBundle\Entity\ScheduledDataflow;
  8 | use CodeRhapsodie\DataflowBundle\Factory\ConnectionFactory;
  9 | use CodeRhapsodie\DataflowBundle\Registry\DataflowTypeRegistryInterface;
 10 | use CodeRhapsodie\DataflowBundle\Repository\ScheduledDataflowRepository;
 11 | use Symfony\Component\Console\Attribute\AsCommand;
 12 | use Symfony\Component\Console\Command\Command;
 13 | use Symfony\Component\Console\Input\InputInterface;
 14 | use Symfony\Component\Console\Input\InputOption;
 15 | use Symfony\Component\Console\Output\OutputInterface;
 16 | use Symfony\Component\Console\Style\SymfonyStyle;
 17 | use Symfony\Component\Validator\Validator\ValidatorInterface;
 18 | 
 19 | /**
 20 |  * @codeCoverageIgnore
 21 |  */
 22 | #[AsCommand('code-rhapsodie:dataflow:schedule:add', 'Create a scheduled dataflow')]
 23 | class AddScheduledDataflowCommand extends Command
 24 | {
 25 |     public function __construct(private DataflowTypeRegistryInterface $registry, private ScheduledDataflowRepository $scheduledDataflowRepository, private ValidatorInterface $validator, private ConnectionFactory $connectionFactory)
 26 |     {
 27 |         parent::__construct();
 28 |     }
 29 | 
 30 |     protected function configure(): void
 31 |     {
 32 |         $this
 33 |             ->setHelp('The <info>%command.name%</info> allows you to create a new scheduled dataflow.')
 34 |             ->addOption('label', null, InputOption::VALUE_REQUIRED, 'Label of the scheduled dataflow')
 35 |             ->addOption('type', null, InputOption::VALUE_REQUIRED, 'Type of the scheduled dataflow (FQCN)')
 36 |             ->addOption(
 37 |                 'options',
 38 |                 null,
 39 |                 InputOption::VALUE_OPTIONAL,
 40 |                 'Options of the scheduled dataflow (ex: {"option1": "value1", "option2": "value2"})'
 41 |             )
 42 |             ->addOption('frequency', null, InputOption::VALUE_REQUIRED, 'Frequency of the scheduled dataflow')
 43 |             ->addOption('first_run', null, InputOption::VALUE_REQUIRED, 'Date for the first run of the scheduled dataflow (Y-m-d H:i:s)')
 44 |             ->addOption('enabled', null, InputOption::VALUE_REQUIRED, 'State of the scheduled dataflow')
 45 |             ->addOption('connection', null, InputOption::VALUE_REQUIRED, 'Define the DBAL connection to use');
 46 |     }
 47 | 
 48 |     protected function execute(InputInterface $input, OutputInterface $output): int
 49 |     {
 50 |         if ($input->getOption('connection') !== null) {
 51 |             $this->connectionFactory->setConnectionName($input->getOption('connection'));
 52 |         }
 53 |         $choices = [];
 54 |         $typeMapping = [];
 55 |         foreach ($this->registry->listDataflowTypes() as $fqcn => $dataflowType) {
 56 |             $choices[] = $dataflowType->getLabel();
 57 |             $typeMapping[$dataflowType->getLabel()] = $fqcn;
 58 |         }
 59 | 
 60 |         $io = new SymfonyStyle($input, $output);
 61 |         $label = $input->getOption('label');
 62 |         if (!$label) {
 63 |             $label = $io->ask('What is the scheduled dataflow label?');
 64 |         }
 65 |         $type = $input->getOption('type');
 66 |         if (!$type) {
 67 |             $selectedType = $io->choice('What is the scheduled dataflow type?', $choices);
 68 |             $type = $typeMapping[$selectedType];
 69 |         }
 70 |         $options = $input->getOption('options');
 71 |         if (!$options) {
 72 |             $options = $io->ask(
 73 |                 'What are the launch options for the scheduled dataflow? (ex: {"option1": "value1", "option2": "value2"})',
 74 |                 json_encode([])
 75 |             );
 76 |         }
 77 |         $frequency = $input->getOption('frequency');
 78 |         if (!$frequency) {
 79 |             $frequency = $io->choice(
 80 |                 'What is the frequency for the scheduled dataflow?',
 81 |                 ScheduledDataflow::AVAILABLE_FREQUENCIES
 82 |             );
 83 |         }
 84 |         $firstRun = $input->getOption('first_run');
 85 |         if (!$firstRun) {
 86 |             $firstRun = $io->ask('When is the first execution of the scheduled dataflow (format: Y-m-d H:i:s)?');
 87 |         }
 88 |         $enabled = $input->getOption('enabled');
 89 |         if (!$enabled) {
 90 |             $enabled = $io->confirm('Enable the scheduled dataflow?');
 91 |         }
 92 | 
 93 |         $newScheduledDataflow = ScheduledDataflow::createFromArray([
 94 |             'id' => null,
 95 |             'label' => $label,
 96 |             'dataflow_type' => $type,
 97 |             'options' => json_decode($options, true, 512, \JSON_THROW_ON_ERROR),
 98 |             'frequency' => $frequency,
 99 |             'next' => new \DateTime($firstRun),
100 |             'enabled' => $enabled,
101 |         ]);
102 | 
103 |         $errors = $this->validator->validate($newScheduledDataflow);
104 |         if (\count($errors) > 0) {
105 |             $io->error((string) $errors);
106 | 
107 |             return 2;
108 |         }
109 | 
110 |         $this->scheduledDataflowRepository->save($newScheduledDataflow);
111 |         $io->success(\sprintf(
112 |             'New scheduled dataflow "%s" (id:%d) was created successfully.',
113 |             $newScheduledDataflow->getLabel(),
114 |             $newScheduledDataflow->getId()
115 |         ));
116 | 
117 |         return 0;
118 |     }
119 | }
120 | 


--------------------------------------------------------------------------------
/src/Resources/config/services.yaml:
--------------------------------------------------------------------------------
  1 | services:
  2 |     _defaults:
  3 |         public: false
  4 | 
  5 |     CodeRhapsodie\DataflowBundle\Registry\DataflowTypeRegistryInterface: '@CodeRhapsodie\DataflowBundle\Registry\DataflowTypeRegistry'
  6 |     CodeRhapsodie\DataflowBundle\Registry\DataflowTypeRegistry:
  7 | 
  8 |     CodeRhapsodie\DataflowBundle\Command\AddScheduledDataflowCommand:
  9 |         arguments:
 10 |             $registry: '@CodeRhapsodie\DataflowBundle\Registry\DataflowTypeRegistryInterface'
 11 |             $scheduledDataflowRepository: '@CodeRhapsodie\DataflowBundle\Repository\ScheduledDataflowRepository'
 12 |             $validator: '@validator'
 13 |             $connectionFactory: '@CodeRhapsodie\DataflowBundle\Factory\ConnectionFactory'
 14 |         tags: ['console.command']
 15 | 
 16 |     CodeRhapsodie\DataflowBundle\Command\ChangeScheduleStatusCommand:
 17 |         arguments:
 18 |             $scheduledDataflowRepository: '@CodeRhapsodie\DataflowBundle\Repository\ScheduledDataflowRepository'
 19 |             $connectionFactory: '@CodeRhapsodie\DataflowBundle\Factory\ConnectionFactory'
 20 |         tags: ['console.command']
 21 | 
 22 |     CodeRhapsodie\DataflowBundle\Command\ExecuteDataflowCommand:
 23 |         arguments:
 24 |             $registry: '@CodeRhapsodie\DataflowBundle\Registry\DataflowTypeRegistryInterface'
 25 |             $connectionFactory: '@CodeRhapsodie\DataflowBundle\Factory\ConnectionFactory'
 26 |             $jobRepository: '@CodeRhapsodie\DataflowBundle\Repository\JobRepository'
 27 |         tags: ['console.command']
 28 | 
 29 |     CodeRhapsodie\DataflowBundle\Command\JobShowCommand:
 30 |         arguments:
 31 |             $jobGateway: '@CodeRhapsodie\DataflowBundle\Gateway\JobGateway'
 32 |             $connectionFactory: '@CodeRhapsodie\DataflowBundle\Factory\ConnectionFactory'
 33 |         tags: ['console.command']
 34 | 
 35 |     CodeRhapsodie\DataflowBundle\Command\RunPendingDataflowsCommand:
 36 |         arguments:
 37 |             $manager: '@CodeRhapsodie\DataflowBundle\Manager\ScheduledDataflowManagerInterface'
 38 |             $runner: '@CodeRhapsodie\DataflowBundle\Runner\PendingDataflowRunnerInterface'
 39 |             $connectionFactory: '@CodeRhapsodie\DataflowBundle\Factory\ConnectionFactory'
 40 |         tags: ['console.command']
 41 | 
 42 |     CodeRhapsodie\DataflowBundle\Command\ScheduleListCommand:
 43 |         arguments:
 44 |             $scheduledDataflowRepository: '@CodeRhapsodie\DataflowBundle\Repository\ScheduledDataflowRepository'
 45 |             $connectionFactory: '@CodeRhapsodie\DataflowBundle\Factory\ConnectionFactory'
 46 |         tags: ['console.command']
 47 | 
 48 |     CodeRhapsodie\DataflowBundle\Command\SchemaCommand:
 49 |         deprecated:
 50 |             package: 'code-rhapsodie/dataflow-bundle'
 51 |             version: '5.0'
 52 |         tags: ['console.command']
 53 | 
 54 |     CodeRhapsodie\DataflowBundle\Command\DatabaseSchemaCommand:
 55 |         arguments:
 56 |             $connectionFactory: '@CodeRhapsodie\DataflowBundle\Factory\ConnectionFactory'
 57 |         tags: [ 'console.command' ]
 58 | 
 59 |     CodeRhapsodie\DataflowBundle\Repository\ScheduledDataflowRepository:
 60 |         lazy: true
 61 |         arguments: ['@coderhapsodie.dataflow.connection']
 62 | 
 63 |     CodeRhapsodie\DataflowBundle\Repository\JobRepository:
 64 |         lazy: true
 65 |         arguments: ['@coderhapsodie.dataflow.connection']
 66 | 
 67 |     coderhapsodie.dataflow.connection: "@coderhapsodie.dataflow.connection.internal"
 68 | 
 69 |     coderhapsodie.dataflow.connection.internal:
 70 |         lazy: true
 71 |         class: Doctrine\DBAL\Connection
 72 |         factory: ['@CodeRhapsodie\DataflowBundle\Factory\ConnectionFactory', 'getConnection']
 73 | 
 74 |     CodeRhapsodie\DataflowBundle\Factory\ConnectionFactory:
 75 |         arguments: ['@service_container', '%coderhapsodie.dataflow.dbal_default_connection%']
 76 | 
 77 |     CodeRhapsodie\DataflowBundle\Manager\ScheduledDataflowManagerInterface: '@CodeRhapsodie\DataflowBundle\Manager\ScheduledDataflowManager'
 78 |     CodeRhapsodie\DataflowBundle\Manager\ScheduledDataflowManager:
 79 |         arguments:
 80 |             $connection: '@coderhapsodie.dataflow.connection'
 81 |             $scheduledDataflowRepository: '@CodeRhapsodie\DataflowBundle\Repository\ScheduledDataflowRepository'
 82 |             $jobRepository: '@CodeRhapsodie\DataflowBundle\Repository\JobRepository'
 83 | 
 84 |     CodeRhapsodie\DataflowBundle\Runner\PendingDataflowRunnerInterface: '@CodeRhapsodie\DataflowBundle\Runner\PendingDataflowRunner'
 85 |     CodeRhapsodie\DataflowBundle\Runner\PendingDataflowRunner:
 86 |         arguments:
 87 |             $repository: '@CodeRhapsodie\DataflowBundle\Repository\JobRepository'
 88 |             $processor: '@CodeRhapsodie\DataflowBundle\Processor\JobProcessorInterface'
 89 | 
 90 |     CodeRhapsodie\DataflowBundle\Processor\JobProcessorInterface: '@CodeRhapsodie\DataflowBundle\Processor\JobProcessor'
 91 |     CodeRhapsodie\DataflowBundle\Processor\JobProcessor:
 92 |         arguments:
 93 |             $repository: '@CodeRhapsodie\DataflowBundle\Repository\JobRepository'
 94 |             $registry: '@CodeRhapsodie\DataflowBundle\Registry\DataflowTypeRegistryInterface'
 95 |             $dispatcher: '@event_dispatcher'
 96 |             $jobGateway: '@CodeRhapsodie\DataflowBundle\Gateway\JobGateway'
 97 |     CodeRhapsodie\DataflowBundle\ExceptionsHandler\ExceptionHandlerInterface: '@CodeRhapsodie\DataflowBundle\ExceptionsHandler\NullExceptionHandler'
 98 |     CodeRhapsodie\DataflowBundle\ExceptionsHandler\NullExceptionHandler:
 99 |     CodeRhapsodie\DataflowBundle\Gateway\JobGateway:
100 |         arguments:
101 |             $repository: '@CodeRhapsodie\DataflowBundle\Repository\JobRepository'
102 |             $exceptionHandler: '@CodeRhapsodie\DataflowBundle\ExceptionsHandler\ExceptionHandlerInterface'
103 | 


--------------------------------------------------------------------------------
/src/DataflowType/Dataflow/AMPAsyncDataflow.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | 
  3 | declare(strict_types=1);
  4 | 
  5 | namespace CodeRhapsodie\DataflowBundle\DataflowType\Dataflow;
  6 | 
  7 | use function Amp\coroutine;
  8 | 
  9 | use Amp\Deferred;
 10 | use Amp\Delayed;
 11 | use Amp\Loop;
 12 | use Amp\Producer;
 13 | use Amp\Promise;
 14 | 
 15 | use function Amp\Promise\wait;
 16 | 
 17 | use CodeRhapsodie\DataflowBundle\DataflowType\Result;
 18 | use CodeRhapsodie\DataflowBundle\DataflowType\Writer\WriterInterface;
 19 | use Psr\Log\LoggerAwareInterface;
 20 | use Psr\Log\LoggerAwareTrait;
 21 | 
 22 | class AMPAsyncDataflow implements DataflowInterface, LoggerAwareInterface
 23 | {
 24 |     use LoggerAwareTrait;
 25 | 
 26 |     /** @var callable[] */
 27 |     private array $steps = [];
 28 | 
 29 |     /** @var WriterInterface[] */
 30 |     private array $writers = [];
 31 | 
 32 |     private array $states = [];
 33 | 
 34 |     private array $stepsJobs = [];
 35 | 
 36 |     public function __construct(private iterable $reader, private ?string $name, private ?int $loopInterval = 0, private ?int $emitInterval = 0)
 37 |     {
 38 |         if (!\function_exists('Amp\\Promise\\wait')) {
 39 |             throw new \RuntimeException('Amp is not loaded. Suggest install it with composer require amphp/amp');
 40 |         }
 41 |     }
 42 | 
 43 |     /**
 44 |      * @param int $scale
 45 |      *
 46 |      * @return $this
 47 |      */
 48 |     public function addStep(callable $step, $scale = 1): self
 49 |     {
 50 |         $this->steps[] = [$step, $scale];
 51 | 
 52 |         return $this;
 53 |     }
 54 | 
 55 |     /**
 56 |      * @return $this
 57 |      */
 58 |     public function addWriter(WriterInterface $writer): self
 59 |     {
 60 |         $this->writers[] = $writer;
 61 | 
 62 |         return $this;
 63 |     }
 64 | 
 65 |     public function process(): Result
 66 |     {
 67 |         $count = 0;
 68 |         $exceptions = [];
 69 |         $startTime = new \DateTime();
 70 | 
 71 |         try {
 72 |             foreach ($this->writers as $writer) {
 73 |                 $writer->prepare();
 74 |             }
 75 | 
 76 |             $deferred = new Deferred();
 77 |             $resolved = false; // missing $deferred->isResolved() in version 2.5
 78 |             $producer = new Producer(function (callable $emit) {
 79 |                 foreach ($this->reader as $index => $item) {
 80 |                     yield new Delayed($this->emitInterval);
 81 |                     yield $emit([$index, $item]);
 82 |                 }
 83 |             });
 84 | 
 85 |             $watcherId = Loop::repeat($this->loopInterval, function () use ($deferred, &$resolved, $producer, &$count, &$exceptions) {
 86 |                 if (yield $producer->advance()) {
 87 |                     $it = $producer->getCurrent();
 88 |                     [$index, $item] = $it;
 89 |                     $this->states[$index] = [$index, 0, $item];
 90 |                 } elseif (!$resolved && \count($this->states) === 0) {
 91 |                     $resolved = true;
 92 |                     $deferred->resolve();
 93 |                 }
 94 | 
 95 |                 foreach ($this->states as $state) {
 96 |                     $this->processState($state, $count, $exceptions);
 97 |                 }
 98 |             });
 99 | 
100 |             wait($deferred->promise());
101 |             Loop::cancel($watcherId);
102 | 
103 |             foreach ($this->writers as $writer) {
104 |                 $writer->finish();
105 |             }
106 |         } catch (\Throwable $e) {
107 |             $exceptions[] = $e;
108 |             $this->logException($e);
109 |         }
110 | 
111 |         return new Result($this->name, $startTime, new \DateTime(), $count, $exceptions);
112 |     }
113 | 
114 |     /**
115 |      * @param int   $count      internal count reference
116 |      * @param array $exceptions internal exceptions
117 |      */
118 |     private function processState(mixed $state, int &$count, array &$exceptions): void
119 |     {
120 |         [$readIndex, $stepIndex, $item] = $state;
121 |         if ($stepIndex < \count($this->steps)) {
122 |             if (!isset($this->stepsJobs[$stepIndex])) {
123 |                 $this->stepsJobs[$stepIndex] = [];
124 |             }
125 |             [$step, $scale] = $this->steps[$stepIndex];
126 |             if ((is_countable($this->stepsJobs[$stepIndex]) ? \count($this->stepsJobs[$stepIndex]) : 0) < $scale && !isset($this->stepsJobs[$stepIndex][$readIndex])) {
127 |                 $this->stepsJobs[$stepIndex][$readIndex] = true;
128 |                 /** @var Promise<void> $promise */
129 |                 $promise = coroutine($step)($item);
130 |                 $promise->onResolve(function (?\Throwable $exception = null, $newItem = null) use ($stepIndex, $readIndex, &$exceptions) {
131 |                     if ($exception) {
132 |                         $exceptions[$stepIndex] = $exception;
133 |                         $this->logException($exception, (string) $stepIndex);
134 |                     } elseif ($newItem === false) {
135 |                         unset($this->states[$readIndex]);
136 |                     } else {
137 |                         $this->states[$readIndex] = [$readIndex, $stepIndex + 1, $newItem];
138 |                     }
139 | 
140 |                     unset($this->stepsJobs[$stepIndex][$readIndex]);
141 |                 });
142 |             }
143 |         } else {
144 |             unset($this->states[$readIndex]);
145 | 
146 |             foreach ($this->writers as $writer) {
147 |                 $writer->write($item);
148 |             }
149 | 
150 |             ++$count;
151 |         }
152 |     }
153 | 
154 |     private function logException(\Throwable $e, ?string $index = null): void
155 |     {
156 |         if (!isset($this->logger)) {
157 |             return;
158 |         }
159 | 
160 |         $this->logger->error($e, ['exception' => $e, 'index' => $index]);
161 |     }
162 | }
163 | 


--------------------------------------------------------------------------------
/src/Repository/JobRepository.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | 
  3 | declare(strict_types=1);
  4 | 
  5 | namespace CodeRhapsodie\DataflowBundle\Repository;
  6 | 
  7 | use CodeRhapsodie\DataflowBundle\Entity\Job;
  8 | use CodeRhapsodie\DataflowBundle\Entity\ScheduledDataflow;
  9 | use Doctrine\DBAL\Connection;
 10 | use Doctrine\DBAL\ParameterType;
 11 | use Doctrine\DBAL\Query\QueryBuilder;
 12 | 
 13 | /**
 14 |  * Repository.
 15 |  *
 16 |  * @codeCoverageIgnore
 17 |  */
 18 | class JobRepository
 19 | {
 20 |     use InitFromDbTrait;
 21 | 
 22 |     public const TABLE_NAME = 'cr_dataflow_job';
 23 | 
 24 |     /**
 25 |      * JobRepository constructor.
 26 |      */
 27 |     public function __construct(private Connection $connection)
 28 |     {
 29 |     }
 30 | 
 31 |     public function find(int $jobId)
 32 |     {
 33 |         $qb = $this->createQueryBuilder();
 34 |         $qb
 35 |             ->andWhere($qb->expr()->eq('id', $qb->createNamedParameter($jobId, ParameterType::INTEGER)))
 36 |         ;
 37 | 
 38 |         return $this->returnFirstOrNull($qb);
 39 |     }
 40 | 
 41 |     public function findOneshotDataflows(): iterable
 42 |     {
 43 |         $qb = $this->createQueryBuilder();
 44 |         $qb
 45 |             ->andWhere($qb->expr()->isNull('scheduled_dataflow_id'))
 46 |             ->andWhere($qb->expr()->eq('status', $qb->createNamedParameter(Job::STATUS_PENDING, ParameterType::INTEGER)));
 47 |         $stmt = $qb->executeQuery();
 48 |         if ($stmt->rowCount() === 0) {
 49 |             return [];
 50 |         }
 51 |         while (false !== ($row = $stmt->fetchAssociative())) {
 52 |             yield Job::createFromArray($this->initDateTime($this->initArray($row)));
 53 |         }
 54 |     }
 55 | 
 56 |     public function findPendingForScheduledDataflow(ScheduledDataflow $scheduled): ?Job
 57 |     {
 58 |         $qb = $this->createQueryBuilder();
 59 |         $qb
 60 |             ->andWhere($qb->expr()->eq('scheduled_dataflow_id', $qb->createNamedParameter($scheduled->getId(), ParameterType::INTEGER)))
 61 |             ->andWhere($qb->expr()->eq('status', $qb->createNamedParameter(Job::STATUS_PENDING, ParameterType::INTEGER)));
 62 | 
 63 |         return $this->returnFirstOrNull($qb);
 64 |     }
 65 | 
 66 |     public function findNextPendingDataflow(): ?Job
 67 |     {
 68 |         $qb = $this->createQueryBuilder();
 69 |         $qb->andWhere($qb->expr()->lte('requested_date', $qb->createNamedParameter(new \DateTime(), 'datetime')))
 70 |             ->andWhere($qb->expr()->eq('status', $qb->createNamedParameter(Job::STATUS_PENDING, ParameterType::INTEGER)))
 71 |             ->orderBy('requested_date', 'ASC')
 72 |             ->setMaxResults(1)
 73 |         ;
 74 | 
 75 |         return $this->returnFirstOrNull($qb);
 76 |     }
 77 | 
 78 |     public function findLastForDataflowId(int $dataflowId): ?Job
 79 |     {
 80 |         $qb = $this->createQueryBuilder();
 81 |         $qb->andWhere($qb->expr()->eq('scheduled_dataflow_id', $qb->createNamedParameter($dataflowId, ParameterType::INTEGER)))
 82 |             ->orderBy('requested_date', 'DESC')
 83 |             ->setMaxResults(1)
 84 |         ;
 85 | 
 86 |         return $this->returnFirstOrNull($qb);
 87 |     }
 88 | 
 89 |     public function findLatests(): iterable
 90 |     {
 91 |         $qb = $this->createQueryBuilder();
 92 |         $qb
 93 |             ->orderBy('requested_date', 'DESC')
 94 |             ->setMaxResults(20);
 95 |         $stmt = $qb->executeQuery();
 96 |         if ($stmt->rowCount() === 0) {
 97 |             return [];
 98 |         }
 99 |         while (false !== ($row = $stmt->fetchAssociative())) {
100 |             yield Job::createFromArray($row);
101 |         }
102 |     }
103 | 
104 |     public function findForScheduled(int $id): iterable
105 |     {
106 |         $qb = $this->createQueryBuilder();
107 |         $qb->andWhere($qb->expr()->eq('scheduled_dataflow_id', $qb->createNamedParameter($id, ParameterType::INTEGER)))
108 |             ->orderBy('requested_date', 'DESC')
109 |             ->setMaxResults(20);
110 |         $stmt = $qb->executeQuery();
111 |         if ($stmt->rowCount() === 0) {
112 |             return [];
113 |         }
114 |         while (false !== ($row = $stmt->fetchAssociative())) {
115 |             yield Job::createFromArray($row);
116 |         }
117 |     }
118 | 
119 |     public function save(Job $job)
120 |     {
121 |         $datas = $job->toArray();
122 |         unset($datas['id']);
123 | 
124 |         if (\is_array($datas['options'])) {
125 |             $datas['options'] = json_encode($datas['options'], \JSON_THROW_ON_ERROR);
126 |         }
127 |         if (\is_array($datas['exceptions'])) {
128 |             $datas['exceptions'] = json_encode($datas['exceptions'], \JSON_THROW_ON_ERROR);
129 |         }
130 | 
131 |         if ($job->getId() === null) {
132 |             $this->connection->insert(static::TABLE_NAME, $datas, $this->getFields());
133 |             $job->setId((int) $this->connection->lastInsertId());
134 | 
135 |             return;
136 |         }
137 |         $this->connection->update(static::TABLE_NAME, $datas, ['id' => $job->getId()], $this->getFields());
138 |     }
139 | 
140 |     public function updateCount(int $jobId, int $count): void
141 |     {
142 |         $this->connection->update(static::TABLE_NAME, ['count' => $count], ['id' => $jobId]);
143 |     }
144 | 
145 |     public function createQueryBuilder($alias = null): QueryBuilder
146 |     {
147 |         $qb = $this->connection->createQueryBuilder();
148 |         $qb->select('*')
149 |             ->from(static::TABLE_NAME, $alias);
150 | 
151 |         return $qb;
152 |     }
153 | 
154 |     private function returnFirstOrNull(QueryBuilder $qb): ?Job
155 |     {
156 |         $stmt = $qb->executeQuery();
157 |         if ($stmt->rowCount() === 0) {
158 |             return null;
159 |         }
160 | 
161 |         return Job::createFromArray($this->initDateTime($this->initArray($stmt->fetchAssociative())));
162 |     }
163 | 
164 |     private function getFields(): array
165 |     {
166 |         return [
167 |             'id' => ParameterType::INTEGER,
168 |             'status' => ParameterType::INTEGER,
169 |             'label' => ParameterType::STRING,
170 |             'dataflow_type' => ParameterType::STRING,
171 |             'options' => ParameterType::STRING,
172 |             'requested_date' => 'datetime',
173 |             'scheduled_dataflow_id' => ParameterType::INTEGER,
174 |             'count' => ParameterType::INTEGER,
175 |             'exceptions' => ParameterType::STRING,
176 |             'start_time' => 'datetime',
177 |             'end_time' => 'datetime',
178 |         ];
179 |     }
180 | }
181 | 


--------------------------------------------------------------------------------
/src/Entity/Job.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | 
  3 | declare(strict_types=1);
  4 | 
  5 | namespace CodeRhapsodie\DataflowBundle\Entity;
  6 | 
  7 | use Symfony\Component\Validator\Constraints as Asserts;
  8 | 
  9 | /**
 10 |  * Dataflow execution status.
 11 |  *
 12 |  * @codeCoverageIgnore
 13 |  */
 14 | class Job
 15 | {
 16 |     public const STATUS_PENDING = 0;
 17 |     public const STATUS_RUNNING = 1;
 18 |     public const STATUS_COMPLETED = 2;
 19 |     public const STATUS_QUEUED = 3;
 20 | 
 21 |     private const KEYS = [
 22 |         'id',
 23 |         'status',
 24 |         'label',
 25 |         'dataflow_type',
 26 |         'options',
 27 |         'requested_date',
 28 |         'scheduled_dataflow_id',
 29 |         'count',
 30 |         'exceptions',
 31 |         'start_time',
 32 |         'end_time',
 33 |     ];
 34 | 
 35 |     private ?int $id = null;
 36 | 
 37 |     #[Asserts\Range(min: 0, max: 2)]
 38 |     private int $status = self::STATUS_PENDING;
 39 | 
 40 |     #[Asserts\NotBlank]
 41 |     #[Asserts\Length(min: 1, max: 255)]
 42 |     #[Asserts\Regex('#^[[:alnum:] ]+\z#u')]
 43 |     private ?string $label = null;
 44 | 
 45 |     #[Asserts\NotBlank]
 46 |     #[Asserts\Length(min: 1, max: 255)]
 47 |     #[Asserts\Regex('#^[[:alnum:]\\\]+\z#u')]
 48 |     private ?string $dataflowType = null;
 49 | 
 50 |     private ?array $options = null;
 51 | 
 52 |     private ?\DateTimeInterface $requestedDate = null;
 53 | 
 54 |     private ?int $scheduledDataflowId = null;
 55 | 
 56 |     private ?int $count = 0;
 57 | 
 58 |     private ?array $exceptions = null;
 59 | 
 60 |     private ?\DateTimeInterface $startTime = null;
 61 | 
 62 |     private ?\DateTimeInterface $endTime = null;
 63 | 
 64 |     public static function createFromScheduledDataflow(ScheduledDataflow $scheduled): self
 65 |     {
 66 |         return (new static())
 67 |             ->setStatus(static::STATUS_PENDING)
 68 |             ->setDataflowType($scheduled->getDataflowType())
 69 |             ->setOptions($scheduled->getOptions())
 70 |             ->setRequestedDate(clone $scheduled->getNext())
 71 |             ->setLabel($scheduled->getLabel())
 72 |             ->setScheduledDataflowId($scheduled->getId());
 73 |     }
 74 | 
 75 |     public static function createFromArray(array $datas)
 76 |     {
 77 |         $lost = array_diff(static::KEYS, array_keys($datas));
 78 |         if (\count($lost) > 0) {
 79 |             throw new \LogicException('The first argument of '.__METHOD__.'  must be contains: "'.implode(', ', $lost).'"');
 80 |         }
 81 | 
 82 |         $job = new self();
 83 |         $job->id = $datas['id'] === null ? null : (int) $datas['id'];
 84 |         $job->setStatus($datas['status'] === null ? null : (int) $datas['status']);
 85 |         $job->setLabel($datas['label']);
 86 |         $job->setDataflowType($datas['dataflow_type']);
 87 |         $job->setOptions($datas['options']);
 88 |         $job->setRequestedDate($datas['requested_date']);
 89 |         $job->setScheduledDataflowId($datas['scheduled_dataflow_id'] === null ? null : (int) $datas['scheduled_dataflow_id']);
 90 |         $job->setCount($datas['count'] === null ? null : (int) $datas['count']);
 91 |         $job->setExceptions($datas['exceptions']);
 92 |         $job->setStartTime($datas['start_time']);
 93 |         $job->setEndTime($datas['end_time']);
 94 | 
 95 |         return $job;
 96 |     }
 97 | 
 98 |     public function toArray(): array
 99 |     {
100 |         return [
101 |             'id' => $this->getId(),
102 |             'status' => $this->getStatus(),
103 |             'label' => $this->getLabel(),
104 |             'dataflow_type' => $this->getDataflowType(),
105 |             'options' => $this->getOptions(),
106 |             'requested_date' => $this->getRequestedDate(),
107 |             'scheduled_dataflow_id' => $this->getScheduledDataflowId(),
108 |             'count' => $this->getCount(),
109 |             'exceptions' => $this->getExceptions(),
110 |             'start_time' => $this->getStartTime(),
111 |             'end_time' => $this->getEndTime(),
112 |         ];
113 |     }
114 | 
115 |     public function setId(int $id): self
116 |     {
117 |         $this->id = $id;
118 | 
119 |         return $this;
120 |     }
121 | 
122 |     public function getId(): ?int
123 |     {
124 |         return $this->id;
125 |     }
126 | 
127 |     public function getStatus(): int
128 |     {
129 |         return $this->status;
130 |     }
131 | 
132 |     public function setStatus(int $status): self
133 |     {
134 |         $this->status = $status;
135 | 
136 |         return $this;
137 |     }
138 | 
139 |     public function getLabel(): ?string
140 |     {
141 |         return $this->label;
142 |     }
143 | 
144 |     public function setLabel(?string $label): self
145 |     {
146 |         $this->label = $label;
147 | 
148 |         return $this;
149 |     }
150 | 
151 |     public function getDataflowType(): ?string
152 |     {
153 |         return $this->dataflowType;
154 |     }
155 | 
156 |     public function setDataflowType(?string $dataflowType): self
157 |     {
158 |         $this->dataflowType = $dataflowType;
159 | 
160 |         return $this;
161 |     }
162 | 
163 |     public function getOptions(): ?array
164 |     {
165 |         return $this->options;
166 |     }
167 | 
168 |     public function setOptions(?array $options): self
169 |     {
170 |         $this->options = $options;
171 | 
172 |         return $this;
173 |     }
174 | 
175 |     public function getRequestedDate(): ?\DateTimeInterface
176 |     {
177 |         return $this->requestedDate;
178 |     }
179 | 
180 |     public function setRequestedDate(?\DateTimeInterface $requestedDate): self
181 |     {
182 |         $this->requestedDate = $requestedDate;
183 | 
184 |         return $this;
185 |     }
186 | 
187 |     public function getScheduledDataflowId(): ?int
188 |     {
189 |         return $this->scheduledDataflowId;
190 |     }
191 | 
192 |     public function setScheduledDataflowId(?int $scheduledDataflowId): self
193 |     {
194 |         $this->scheduledDataflowId = $scheduledDataflowId;
195 | 
196 |         return $this;
197 |     }
198 | 
199 |     public function getCount(): ?int
200 |     {
201 |         return $this->count;
202 |     }
203 | 
204 |     public function setCount(?int $count): self
205 |     {
206 |         $this->count = $count;
207 | 
208 |         return $this;
209 |     }
210 | 
211 |     public function getExceptions(): ?array
212 |     {
213 |         return $this->exceptions;
214 |     }
215 | 
216 |     public function setExceptions(?array $exceptions): self
217 |     {
218 |         $this->exceptions = $exceptions;
219 | 
220 |         return $this;
221 |     }
222 | 
223 |     public function getStartTime(): ?\DateTimeInterface
224 |     {
225 |         return $this->startTime;
226 |     }
227 | 
228 |     public function setStartTime(?\DateTimeInterface $startTime): self
229 |     {
230 |         $this->startTime = $startTime;
231 | 
232 |         return $this;
233 |     }
234 | 
235 |     public function getEndTime(): ?\DateTimeInterface
236 |     {
237 |         return $this->endTime;
238 |     }
239 | 
240 |     public function setEndTime(?\DateTimeInterface $endTime): self
241 |     {
242 |         $this->endTime = $endTime;
243 | 
244 |         return $this;
245 |     }
246 | }
247 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # Code Rhapsodie Dataflow Bundle
  2 | 
  3 | DataflowBundle is a bundle for Symfony 3.4+
  4 | providing an easy way to create import / export dataflow.
  5 | 
  6 | | Dataflow | Symfony                  | Support |
  7 | |----------|--------------------------|---------|
  8 | | 5.x      | 7.x                      | yes     |
  9 | | 4.x      | 3.4 \| 4.x \| 5.x \| 6.x | yes     |
 10 | | 3.x      | 3.4 \| 4.x \| 5.x        | no      |
 11 | | 2.x      | 3.4 \| 4.x               | no      |
 12 | | 1.x      | 3.4 \| 4.x               | no      |
 13 | 
 14 | Dataflow uses a linear generic workflow in three parts:
 15 | 
 16 | * one reader
 17 | * any number of steps that can be synchronous or asynchronous
 18 | * one or more writers
 19 | 
 20 | The reader can read data from anywhere and return data row by row. Each step processes the current row data.
 21 | The steps are executed in the order in which they are added.
 22 | And, one or more writers save the row anywhere you want.
 23 | 
 24 | As the following schema shows, you can define more than one dataflow:
 25 | 
 26 | ![Dataflow schema](src/Resources/doc/schema.png)
 27 | 
 28 | # Features
 29 | 
 30 | * Define and configure a Dataflow
 31 | * Run the Job scheduled
 32 | * Run one Dataflow from the command line
 33 | * Define the schedule for a Dataflow from the command line
 34 | * Enable/Disable a scheduled Dataflow from the command line
 35 | * Display the list of scheduled Dataflow from the command line
 36 | * Display the result for the last Job for a Dataflow from the command line
 37 | * Work with multiple Doctrine DBAL connections
 38 | 
 39 | ## Installation
 40 | 
 41 | Security notice: Symfony 4.x is not supported before 4.1.12, see https://github.com/advisories/GHSA-pgwj-prpq-jpc2
 42 | 
 43 | ### Add the dependency
 44 | 
 45 | To install this bundle, run this command :
 46 | 
 47 | ```shell script
 48 | $ composer require code-rhapsodie/dataflow-bundle
 49 | ```
 50 | 
 51 | #### Suggest
 52 | 
 53 | You can use the generic readers, writers and steps from [PortPHP](https://github.com/portphp/portphp).
 54 | 
 55 | For the writers, you must use the adapter `CodeRhapsodie\DataflowBundle\DataflowType\Writer\PortWriterAdapter` like
 56 | this:
 57 | 
 58 | ```php
 59 | <?php
 60 | // ...
 61 | $streamWriter = new \Port\Writer\StreamMergeWriter();
 62 | 
 63 | $builder->addWriter(new \CodeRhapsodie\DataflowBundle\DataflowType\Writer\PortWriterAdapter($streamWriter));
 64 | // ...
 65 | ```
 66 | 
 67 | ### Register the bundle
 68 | 
 69 | Add `CodeRhapsodie\DataflowBundle\CodeRhapsodieDataflowBundle::class => ['all' => true],
 70 | ` in the `config/bundles.php` file.
 71 | 
 72 | Like this:
 73 | 
 74 | ```php
 75 | <?php
 76 | 
 77 | return [
 78 |      // ...
 79 |     CodeRhapsodie\DataflowBundle\CodeRhapsodieDataflowBundle::class => ['all' => true],
 80 |     // ...
 81 | ];
 82 | ```
 83 | 
 84 | ### Update the database
 85 | 
 86 | This bundle uses Doctrine DBAL to store Dataflow schedule into the database table (`cr_dataflow_scheduled`)
 87 | and jobs (`cr_dataflow_job`).
 88 | 
 89 | If you use [Doctrine Migration Bundle](https://symfony.com/doc/master/bundles/DoctrineMigrationsBundle/index.html)
 90 | or [Phinx](https://phinx.org/)
 91 | or [Kaliop Migration Bundle](https://github.com/kaliop-uk/ezmigrationbundle) or whatever,
 92 | you can add a new migration with the generated SQL query from this command:
 93 | 
 94 | ```shell script
 95 | $ bin/console code-rhapsodie:dataflow:dump-schema
 96 | ```
 97 | 
 98 | If you have already the tables, you can add a new migration with the generated update SQL query from this command:
 99 | 
100 | ```shell script
101 | $ bin/console code-rhapsodie:dataflow:dump-schema --update
102 | ```
103 | 
104 | ## Configuration
105 | 
106 | By default, the Doctrine DBAL connection used is `default`. You can configure the default connection.
107 | Add this configuration into your Symfony configuration:
108 | 
109 | ```yaml
110 | code_rhapsodie_dataflow:
111 |   dbal_default_connection: test #Name of the default connection used by Dataflow bundle
112 | ```
113 | 
114 | By default, the `logger` service will be used to log all exceptions and custom messages.
115 | If you want to use another logger, like a specific Monolog handler, Add this configuration:
116 | 
117 | ```yaml
118 | code_rhapsodie_dataflow:
119 |   default_logger: monolog.logger.custom #Service ID of the logger you want Dataflow to use
120 | ```
121 | 
122 | ### Messenger mode
123 | 
124 | Dataflow can delegate the execution of its jobs to the Symfony messenger component, if available.
125 | This allows jobs to be executed concurrently by workers instead of sequentially.
126 | 
127 | To enable messenger mode:
128 | 
129 | ```yaml
130 | code_rhapsodie_dataflow:
131 |   messenger_mode:
132 |     enabled: true
133 |     # bus: 'messenger.default_bus' #Service ID of the bus you want Dataflow to use, if not the default one
134 | ```
135 | 
136 | You also need to route Dataflow messages to the proper transport:
137 | 
138 | ```yaml
139 | # config/packages/messenger.yaml
140 | framework:
141 |   messenger:
142 |     transports:
143 |       async: '%env(MESSENGER_TRANSPORT_DSN)%'
144 | 
145 |     routing:
146 |       CodeRhapsodie\DataflowBundle\MessengerMode\JobMessage: async
147 | ```
148 | 
149 | ### Exceptions mode
150 | Dataflow can save exceptions in any filesystem you want. 
151 | This allows dataflow to save exceptions in filesystem instead of the database
152 | You have to install `league/flysystem`.
153 | 
154 | To enable exceptions mode:
155 | 
156 | ```yaml
157 | code_rhapsodie_dataflow:
158 |   exceptions_mode:
159 |     type: 'file'
160 |     flysystem_service: 'app.filesystem' #The name of the \League\Flysystem\Filesystem service
161 | ```
162 | 
163 | ## Define a dataflow type
164 | 
165 | This bundle uses a fixed and simple workflow structure in order to let you focus on the data processing logic part of
166 | your dataflow.
167 | 
168 | A dataflow type defines the different parts of your dataflow. A dataflow is made of:
169 | 
170 | - exactly one *Reader*
171 | - any number of *Steps*
172 | - one or more *Writers*
173 | 
174 | Dataflow types can be configured with options.
175 | 
176 | A dataflow type must implement `CodeRhapsodie\DataflowBundle\DataflowType\DataflowTypeInterface`.
177 | 
178 | To help with creating your dataflow types, an abstract
179 | class `CodeRhapsodie\DataflowBundle\DataflowType\AbstractDataflowType`
180 | is provided, allowing you to define your dataflow through a handy
181 | builder `CodeRhapsodie\DataflowBundle\DataflowType\DataflowBuilder`.
182 | 
183 | This is an example to define one class DataflowType:
184 | 
185 | ```php
186 | <?php
187 | namespace CodeRhapsodie\DataflowExemple\DataflowType;
188 | 
189 | use CodeRhapsodie\DataflowBundle\DataflowType\AbstractDataflowType;
190 | use CodeRhapsodie\DataflowBundle\DataflowType\DataflowBuilder;
191 | use CodeRhapsodie\DataflowExemple\Reader\FileReader;
192 | use CodeRhapsodie\DataflowExemple\Writer\FileWriter;
193 | 
194 | class MyFirstDataflowType extends AbstractDataflowType
195 | {
196 |     private $myReader;
197 | 
198 |     private $myWriter;
199 | 
200 |     public function __construct(FileReader $myReader, FileWriter $myWriter)
201 |     {
202 |         $this->myReader = $myReader;
203 |         $this->myWriter = $myWriter;
204 |     }
205 | 
206 |     protected function buildDataflow(DataflowBuilder $builder, array $options): void
207 |     {
208 |         $this->myWriter->setDestinationFilePath($options['to-file']);
209 | 
210 |         $builder
211 |             ->setReader($this->myReader->read($options['from-file']))
212 |             ->addStep(function ($data) use ($options) {
213 |                 // TODO : Write your code here...
214 |                 return $data;
215 |             })
216 |             ->addWriter($this->myWriter)
217 |         ;
218 |     }
219 | 
220 |     protected function configureOptions(OptionsResolver $optionsResolver): void
221 |     {
222 |         $optionsResolver->setDefaults(['to-file' => '/tmp/dataflow.csv', 'from-file' => null]);
223 |         $optionsResolver->setRequired('from-file');
224 |     }
225 | 
226 |     public function getLabel(): string
227 |     {
228 |         return 'My First Dataflow';
229 |     }
230 | 
231 |     public function getAliases(): iterable
232 |     {
233 |         return ['mfd'];
234 |     }
235 | }
236 | 
237 | ```
238 | 
239 | Dataflow types must be tagged with `coderhapsodie.dataflow.type`.
240 | 
241 | If you're using Symfony auto-configuration for your services, this tag will be automatically added to all services
242 | implementing `DataflowTypeInterface`.
243 | 
244 | Otherwise, manually add the tag `coderhapsodie.dataflow.type` in your dataflow type service configuration:
245 | 
246 | ```yaml
247 | ```yaml
248 | CodeRhapsodie\DataflowExemple\DataflowType\MyFirstDataflowType:
249 |   tags:
250 |     - { name: coderhapsodie.dataflow.type }
251 | ```
252 | 
253 | ### Use options for your dataflow type
254 | 
255 | The `AbstractDataflowType` can help you define options for your Dataflow type.
256 | 
257 | Add this method in your DataflowType class:
258 | 
259 | ```php
260 | <?php
261 | // ...
262 | use Symfony\Component\OptionsResolver\OptionsResolver;
263 | 
264 | class MyFirstDataflowType extends AbstractDataflowType
265 | {
266 |     // ...
267 |     protected function configureOptions(OptionsResolver $optionsResolver): void
268 |     {
269 |         $optionsResolver->setDefaults(['to-file' => '/tmp/dataflow.csv', 'from-file' => null]);
270 |         $optionsResolver->setRequired('from-file');
271 |     }
272 | 
273 | }
274 | ```
275 | 
276 | With this configuration, the option `fileName` is required. For an advanced usage of the option resolver, read
277 | the [Symfony documentation](https://symfony.com/doc/current/components/options_resolver.html).
278 | 
279 | For asynchronous management, `AbstractDataflowType` come with two default options :
280 | 
281 | - loopInterval : default to 0. Update this interval if you wish customise the `tick` loop duration.
282 | - emitInterval : default to 0. Update this interval to have a control when reader must emit new data in the flow
283 |   pipeline.
284 | 
285 | ### Logging
286 | 
287 | All exceptions will be caught and written in the logger.
288 | If you want to add custom messages in the log, you can inject the logger in your readers / steps / writers.
289 | If your DataflowType class extends `AbstractDataflowType`, the logger is accessible as `$this->logger`.
290 | 
291 | ```php
292 | <?php
293 | // ...
294 | use Symfony\Component\OptionsResolver\OptionsResolver;
295 | 
296 | class MyDataflowType extends AbstractDataflowType
297 | {
298 |     // ...
299 |     protected function buildDataflow(DataflowBuilder $builder, array $options): void
300 |     {
301 |         $this->myWriter->setLogger($this->logger);
302 |     }
303 | 
304 | }
305 | ```
306 | 
307 | When using the `code-rhapsodie:dataflow:run-pending` command, this logger will also be used to save the log in the
308 | corresponding job in the database.
309 | 
310 | ### Check if your DataflowType is ready
311 | 
312 | Execute this command to check if your DataflowType is correctly registered:
313 | 
314 | ```shell script
315 | $ bin/console debug:container --tag coderhapsodie.dataflow.type
316 | ```
317 | 
318 | The result is like this:
319 | 
320 | ```
321 | Symfony Container Public and Private Services Tagged with "coderhapsodie.dataflow.type" Tag
322 | ===========================================================================================
323 | 
324 |  ---------------------------------------------------------------- ---------------------------------------------------------------- 
325 |   Service ID                                                       Class name                                                      
326 |  ---------------------------------------------------------------- ---------------------------------------------------------------- 
327 |   CodeRhapsodie\DataflowExemple\DataflowType\MyFirstDataflowType   CodeRhapsodie\DataflowExemple\DataflowType\MyFirstDataflowType  
328 |  ---------------------------------------------------------------- ---------------------------------------------------------------- 
329 | 
330 | ```
331 | 
332 | ### Readers
333 | 
334 | *Readers* provide the dataflow with elements to import / export. Usually, elements are read from an external resource (
335 | file, database, webservice, etc).
336 | 
337 | A *Reader* can be any `iterable`.
338 | 
339 | The only constraint on the returned elements typing is that they cannot be `false`.
340 | 
341 | The reader can be a generator like this example :
342 | 
343 | ```php
344 | <?php
345 | 
346 | namespace CodeRhapsodie\DataflowExemple\Reader;
347 | 
348 | class FileReader
349 | {
350 |     public function read(string $filename): iterable
351 |     {
352 |         if (!$filename) {
353 |             throw new \Exception("The file name is not defined. Define it with 'setFilename' method");
354 |         }
355 | 
356 |         if (!$fh = fopen($filename, 'r')) {
357 |             throw new \Exception("Unable to open file '".$filename."' for read.");
358 |         }
359 | 
360 |         while (false !== ($read = fgets($fh))) {
361 |             yield explode('|', trim($read));
362 |         }
363 |     }
364 | }
365 | ```
366 | 
367 | You can set up this reader as follows:
368 | 
369 | ```php
370 | $builder->setReader(($this->myReader)())
371 | ``` 
372 | 
373 | ### Steps
374 | 
375 | *Steps* are operations performed on the elements before they are handled by the *Writers*. Usually, steps are either:
376 | 
377 | - converters, that alter the element
378 | - filters, that conditionally prevent further operations on the element
379 | - generators, that can include asynchronous operations
380 | 
381 | A *Step* can be any callable, taking the element as its argument, and returning either:
382 | 
383 | - the element, possibly altered
384 | - `false`, if no further operations should be performed on this element
385 | 
386 | A few examples:
387 | 
388 | ```php
389 | <?php
390 | //[...]
391 | $builder->addStep(function ($item) {
392 |     // Titles are changed to all caps before export
393 |     $item['title'] = strtoupper($item['title']);
394 | 
395 |     return $item;
396 | });
397 | 
398 | // asynchronous step with 2 scale factor
399 | $builder->addStep(function ($item): \Generator {
400 |     yield new \Amp\Delayed(1000); // asynchronous processing for 1 second long
401 | 
402 |     // Titles are changed to all caps before export
403 |     $item['title'] = strtolower($item['title']);
404 | 
405 |     return $item;
406 | }, 2);
407 | 
408 | $builder->addStep(function ($item) {
409 |     // Private items are not exported
410 |     if ($item['private']) {
411 |         return false;
412 |     }
413 | 
414 |     return $item;
415 | });
416 | //[...]
417 | ```
418 | 
419 | Note : you can ensure writing order for asynchronous operations if all steps are scaled at 1 factor.
420 | 
421 | ### Writers
422 | 
423 | *Writers* perform the actual import / export operations.
424 | 
425 | A *Writer* must implement `CodeRhapsodie\DataflowBundle\DataflowType\Writer\WriterInterface`.
426 | As this interface is not compatible with `Port\Writer`, the
427 | adapter `CodeRhapsodie\DataflowBundle\DataflowType\Writer\PortWriterAdapter` is provided.
428 | 
429 | This example show how to use the predefined PhpPort Writer :
430 | 
431 | ```php
432 | $builder->addWriter(new PortWriterAdapter(new \Port\FileWriter()));
433 | ```
434 | 
435 | Or your own Writer:
436 | 
437 | ```php
438 | <?php
439 | namespace CodeRhapsodie\DataflowExemple\Writer;
440 | 
441 | use CodeRhapsodie\DataFlowBundle\DataflowType\Writer\WriterInterface;
442 | 
443 | class FileWriter implements WriterInterface
444 | {
445 |     private $fh;
446 | 
447 |     /** @var string */
448 |     private $path;
449 | 
450 |     public function setDestinationFilePath(string $path) {
451 |         $this->path = $path;
452 |     }
453 | 
454 |     public function prepare()
455 |     {
456 |         if (null === $this->path) {
457 |             throw new \Exception('Define the destination file name before use');
458 |         }
459 |         if (!$this->fh = fopen($this->path, 'w')) {
460 |             throw new \Exception('Unable to open in write mode the output file.');
461 |         }
462 |     }
463 | 
464 |     public function write($item)
465 |     {
466 |         fputcsv($this->fh, $item);
467 |     }
468 | 
469 |     public function finish()
470 |     {
471 |         fclose($this->fh);
472 |     }
473 | }
474 | ```
475 | 
476 | #### CollectionWriter
477 | 
478 | If you want to write multiple items from a single item read, you can use the generic `CollectionWriter`. This writer
479 | will iterate over any `iterable` it receives, and pass each item from that collection to your own writer that handles
480 | single items.
481 | 
482 | ```php
483 | $builder->addWriter(new CollectionWriter($mySingleItemWriter));
484 | ```
485 | 
486 | #### DelegatorWriter
487 | 
488 | If you want to call different writers depending on what item is read, you can use the generic `DelegatorWriter`.
489 | 
490 | As an example, let's suppose our items are arrays with the first entry being either `product` or `order`. We want to use
491 | a different writer based on that value.
492 | 
493 | First, create your writers implementing `DelegateWriterInterface` (this interface extends `WriterInterface` so your
494 | writers can still be used without the `DelegatorWriter`).
495 | 
496 | ```php
497 | <?php
498 | namespace CodeRhapsodie\DataflowExemple\Writer;
499 | 
500 | use CodeRhapsodie\DataFlowBundle\DataflowType\Writer\WriterInterface;
501 | 
502 | class ProductWriter implements DelegateWriterInterface
503 | {
504 |     public function supports($item): bool
505 |     {
506 |         return 'product' === reset($item);
507 |     }
508 | 
509 |     public function prepare()
510 |     {
511 |     }
512 | 
513 |     public function write($item)
514 |     {
515 |         // Process your product
516 |     }
517 | 
518 |     public function finish()
519 |     {
520 |     }
521 | }
522 | ```
523 | 
524 | ```php
525 | <?php
526 | namespace CodeRhapsodie\DataflowExemple\Writer;
527 | 
528 | use CodeRhapsodie\DataFlowBundle\DataflowType\Writer\WriterInterface;
529 | 
530 | class OrderWriter implements DelegateWriterInterface
531 | {
532 |     public function supports($item): bool
533 |     {
534 |         return 'order' === reset($item);
535 |     }
536 | 
537 |     public function prepare()
538 |     {
539 |     }
540 | 
541 |     public function write($item)
542 |     {
543 |         // Process your order
544 |     }
545 | 
546 |     public function finish()
547 |     {
548 |     }
549 | }
550 | ```
551 | 
552 | Then, configure your `DelegatorWriter` and add it to your dataflow type.
553 | 
554 | ```php
555 |     protected function buildDataflow(DataflowBuilder $builder, array $options): void
556 |     {
557 |         // Snip add reader and steps
558 | 
559 |         $delegatorWriter = new DelegatorWriter();
560 |         $delegatorWriter->addDelegate(new ProductWriter());
561 |         $delegatorWriter->addDelegate(new OrderWriter());
562 | 
563 |         $builder->addWriter($delegatorWriter);
564 |     }
565 | ```
566 | 
567 | During execution, the `DelegatorWriter` will simply pass each item received to its first delegate (in the order those
568 | were added) that supports it. If no delegate supports an item, an exception will be thrown.
569 | 
570 | ## Queue
571 | 
572 | All pending dataflow job processes are stored in a queue into the database.
573 | 
574 | Add this command into your crontab for execute all queued jobs:
575 | 
576 | ```shell script
577 | $ SYMFONY_ENV=prod php bin/console code-rhapsodie:dataflow:run-pending
578 | ```
579 | 
580 | ## Commands
581 | 
582 | Several commands are provided to manage schedules and run jobs.
583 | 
584 | `code-rhapsodie:dataflow:run-pending` Executes job in the queue according to their schedule.
585 | 
586 | When messenger mode is enabled, jobs will still be created according to their schedule, but execution will be handled by
587 | the messenger component instead.
588 | 
589 | `code-rhapsodie:dataflow:schedule:list` Display the list of dataflows scheduled.
590 | 
591 | `code-rhapsodie:dataflow:schedule:change-status` Enable or disable a scheduled dataflow
592 | 
593 | `code-rhapsodie:dataflow:schedule:add` Add the schedule for a dataflow.
594 | 
595 | `code-rhapsodie:dataflow:job:show` Display the last result of a job.
596 | 
597 | `code-rhapsodie:dataflow:execute` Let you execute one dataflow job.
598 | 
599 | `code-rhapsodie:dataflow:dump-schema` Generates schema create / update SQL queries
600 | 
601 | ### Work with many databases
602 | 
603 | All commands have a `--connection` option to define what Doctrine DBAL connection to use during execution.
604 | 
605 | Example:
606 | 
607 | This command uses the `default` DBAL connection to generate all schema update queries.
608 | 
609 | ```shell script
610 | $ bin/console code-rhapsodie:dataflow:dump-schema --update --connection=default
611 | ```
612 | 
613 | To execute all pending job for a specific connection use:
614 | 
615 | ```shell script
616 | # Run for dataflow DBAL connection
617 | $ bin/console code-rhapsodie:dataflow:run-pending --connection=dataflow
618 | # Run for default DBAL connection
619 | $ bin/console code-rhapsodie:dataflow:run-pending --connection=default
620 | ```
621 | 
622 | # Issues and feature requests
623 | 
624 | Please report issues and request features at https://github.com/code-rhapsodie/dataflow-bundle/issues.
625 | 
626 | Please note that only the last release of the 4.x and the 5.x versions of this bundle are actively supported.
627 | 
628 | # Contributing
629 | 
630 | Contributions are very welcome. Please see [CONTRIBUTING.md](CONTRIBUTING.md) for
631 | details. Thanks to [everyone who has contributed](https://github.com/code-rhapsodie/dataflow-bundle/graphs/contributors)
632 | already.
633 | 
634 | # License
635 | 
636 | This package is licensed under the [MIT license](LICENSE).
637 | 


--------------------------------------------------------------------------------