├── .gitignore
├── .scrutinizer.yml
├── .travis.yml
├── Beanstalkd
├── Job.php
└── JobManager.php
├── CHANGELOG.md
├── Command
├── CountCommand.php
├── CreateJobCommand.php
├── PruneCommand.php
├── ResetCommand.php
└── RunCommand.php
├── Controller
├── ControllerTrait.php
├── QueueController.php
└── TrendsController.php
├── DEVELOPMENT.md
├── DependencyInjection
├── Compiler
│ ├── BeanstalkdCompilerPass.php
│ ├── GridCompilerPass.php
│ ├── RabbitMQCompilerPass.php
│ ├── RedisCompilerPass.php
│ ├── WorkerCompilerPass.php
│ └── WorkerCompilerTrait.php
├── Configuration.php
├── DtcQueueExtension.php
├── RabbitMQConfiguration.php
└── RedisConfiguration.php
├── Doctrine
├── BaseDoctrineJobManager.php
├── DoctrineJobManager.php
├── DoctrineJobTimingManager.php
├── DoctrineRunManager.php
├── DtcQueueListener.php
├── ProgressCallbackTrait.php
└── StalledTrait.php
├── Document
├── BaseJob.php
├── BaseRun.php
├── Job.php
├── JobArchive.php
├── JobTiming.php
├── Run.php
└── RunArchive.php
├── DtcQueueBundle.php
├── Entity
├── BaseJob.php
├── BaseRun.php
├── Job.php
├── JobArchive.php
├── JobTiming.php
├── Run.php
└── RunArchive.php
├── EventDispatcher
├── Event.php
├── EventDispatcher.php
└── EventSubscriberInterface.php
├── Exception
├── ArgumentsNotSetException.php
├── ClassNotFoundException.php
├── ClassNotSubclassException.php
├── DuplicateWorkerException.php
├── PriorityException.php
├── UnsupportedException.php
└── WorkerNotRegisteredException.php
├── LICENSE
├── Manager
├── AbstractJobManager.php
├── ArchivableJobManager.php
├── JobIdTrait.php
├── JobManagerInterface.php
├── JobTimingManager.php
├── PriorityJobManager.php
├── RetryableJobManager.php
├── RunManager.php
├── SaveableTrait.php
├── StallableJobManager.php
├── VerifyTrait.php
└── WorkerManager.php
├── Model
├── BaseJob.php
├── BaseStatistics.php
├── Job.php
├── JobTiming.php
├── MessageableJob.php
├── MessageableJobWithId.php
├── MicrotimeTrait.php
├── RetryableJob.php
├── Run.php
├── StallableJob.php
└── Worker.php
├── ODM
├── CommonTrait.php
├── JobManager.php
├── JobTimingManager.php
├── LiveJobsGridSource.php
├── RunManager.php
└── StubLiveJobsGridSource.php
├── ORM
├── CommonTrait.php
├── JobManager.php
├── JobTimingManager.php
├── LiveJobsGridSource.php
├── RunManager.php
└── StubLiveJobsGridSource.php
├── README.md
├── RabbitMQ
├── Job.php
└── JobManager.php
├── Redis
├── BaseJobManager.php
├── Job.php
├── JobManager.php
├── PhpRedis.php
├── Predis.php
├── RedisInterface.php
└── StatusTrait.php
├── Resources
├── config
│ ├── dtc_grid.yaml
│ ├── dtc_queue.yaml
│ ├── queue.yml
│ └── routing.yml
├── doc
│ ├── create-job.md
│ ├── full-configuration.md
│ ├── images
│ │ └── trends-example.png
│ ├── symfony2-3.md
│ ├── symfony4-5.md
│ └── troubleshooting.md
├── docker
│ └── php
│ │ └── ubuntu
│ │ └── Dockerfile
├── public
│ └── js
│ │ ├── jobs.js
│ │ └── trends.js
└── views
│ ├── Queue
│ ├── grid.html.twig
│ ├── jobs.html.twig
│ ├── jobs_running.html.twig
│ ├── macros.html.twig
│ ├── nav.html.twig
│ ├── status.html.twig
│ ├── trends.html.twig
│ └── workers.html.twig
│ └── layout.html.twig
├── Run
└── Loop.php
├── Tests
├── Beanstalkd
│ ├── JobManagerTest.php
│ ├── JobManagerWithTubeTest.php
│ └── JobTest.php
├── Command
│ ├── CommandTrait.php
│ ├── CountCommandTest.php
│ ├── CreateJobCommandTest.php
│ ├── PruneCommandTest.php
│ ├── ResetCommandTest.php
│ └── RunCommandTest.php
├── Controller
│ ├── ControllerTrait.php
│ ├── QueueControllerTest.php
│ ├── TrendsControllerTest.php
│ └── test.yml
├── DependencyInjection
│ ├── Compiler
│ │ ├── BeanstalkdCompilerPassTest.php
│ │ ├── RabbitMQCompilerPassTest.php
│ │ ├── RedisCompilerPassTest.php
│ │ └── WorkerCompilerPassTest.php
│ └── DtcQueueExtensionTest.php
├── Doctrine
│ ├── BaseLiveJobGridSourceTest.php
│ ├── DoctrineJobManagerTest.php
│ └── JobTestTrait.php
├── Document
│ ├── JobArchiveTest.php
│ ├── JobTest.php
│ ├── JobTimingTest.php
│ ├── RunArchiveTest.php
│ └── RunTest.php
├── Entity
│ ├── JobArchiveTest.php
│ ├── JobTest.php
│ ├── JobTimingTest.php
│ ├── RunArchiveTest.php
│ └── RunTest.php
├── EventDispatcher
│ ├── EventDispatcherTest.php
│ ├── EventTest.php
│ └── StubEventSubscriber.php
├── FibonacciWorker.php
├── GetterSetterTrait.php
├── Manager
│ ├── AutoRetryTrait.php
│ ├── BaseJobManagerTest.php
│ ├── JobTimingManagerTest.php
│ ├── PriorityTestTrait.php
│ ├── RetryableTrait.php
│ ├── RunManagerTest.php
│ ├── SaveJobTrait.php
│ ├── SaveableTraitTest.php
│ ├── StaticJobManagerTest.php
│ └── WorkerManagerTest.php
├── Model
│ ├── JobTest.php
│ ├── JobTimingTest.php
│ ├── RunTest.php
│ └── WorkerTest.php
├── ODM
│ ├── JobManagerTest.php
│ ├── LiveJobsGridSourceTest.php
│ └── RunManagerTest.php
├── ORM
│ ├── ContainerExtended.php
│ ├── JobManagerTest.php
│ ├── LiveJobsGridSourceTest.php
│ └── RunManagerTest.php
├── RabbitMQ
│ ├── JobManagerTest.php
│ └── JobTest.php
├── RecordingTrait.php
├── Redis
│ ├── JobManagerNoPriorityTest.php
│ ├── JobManagerTest.php
│ ├── PhpRedisJobManagerTest.php
│ └── PredisPhpRedisTest.php
├── Run
│ └── LoopTest.php
├── StaticJobManager.php
├── StubJobManager.php
├── StubJobTimingManager.php
├── StubRunManager.php
├── Util
│ └── UtilTest.php
└── bootstrap.php
├── UPGRADING-2.0.md
├── UPGRADING-3.0.md
├── UPGRADING-4.0.md
├── UPGRADING-5.0.md
├── UPGRADING-6.0.md
├── UPGRADING-7.0.md
├── Util
└── Util.php
├── composer.json
├── phpunit.xml.dist
└── sonar-project.properties
/.gitignore:
--------------------------------------------------------------------------------
1 | # composer
2 | composer.phar
3 | vendor
4 | composer.lock
5 |
6 | # bin
7 | bin
8 |
9 | # PHP CS Fixer
10 | .php_cs.cache
11 |
12 | # Test temporary files
13 | Tests/Controller/DtcGridBundle/
14 |
--------------------------------------------------------------------------------
/.scrutinizer.yml:
--------------------------------------------------------------------------------
1 | filter:
2 | excluded_paths:
3 | - "Tests/"
4 | tools:
5 | external_code_coverage:
6 | timeout: 900
7 |
8 |
9 | build:
10 | dependencies:
11 | override:
12 | - true
13 | nodes:
14 | analysis:
15 | tests:
16 | override:
17 | - php-scrutinizer-run
18 |
--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 | language: php
2 | dist: xenial
3 | os: linux
4 |
5 | php:
6 | - 7.3
7 | - 7.4
8 | - 8.0
9 |
10 | services:
11 | - mongodb
12 | - mysql
13 | - redis
14 | - postgresql
15 |
16 | addons:
17 | apt:
18 | packages:
19 | - rabbitmq-server
20 | sonarcloud:
21 | organization: "mmucklo-github"
22 |
23 | # beanstalkd setup from https://github.com/assaf/ironium/blob/220c112fd92ffea144b954ae4697c6b5cabe7016/.travis.yml
24 | # (MIT Licensed - see LICENSE for MIT License information, Copyright (c) 2014 Assaf Arkin)
25 | before_install:
26 | - sudo apt-get update
27 | - sudo apt-get install -y beanstalkd
28 | - echo "START=yes" | sudo tee -a /etc/default/beanstalkd > /dev/null
29 | - sudo service beanstalkd restart
30 | - mysql -e 'CREATE DATABASE queue_test;'
31 | before_script:
32 | - composer self-update
33 | - (echo 'no' | pecl install redis) || true
34 | - (php -m | grep 'redis') || echo "extension = redis.so" > ~/.phpenv/versions/$(phpenv version-name)/etc/conf.d/redis.ini
35 | - pecl install mongodb; echo 'extension = mongodb.so' >> ~/.phpenv/versions/$(phpenv version-name)/etc/conf.d/travis.ini
36 | - composer config "platform.ext-mongo" "1.6.16"
37 | - COMPOSER_MEMORY_LIMIT=-1 composer require alcaeus/mongo-php-adapter
38 | - echo 'xdebug.mode = coverage' >> ~/.phpenv/versions/$(phpenv version-name)/etc/conf.d/travis.ini
39 | - COMPOSER_MEMORY_LIMIT=-1 composer install
40 |
41 | script:
42 | - REDIS_HOST=localhost BEANSTALKD_HOST=localhost BEANSTALKD_PORT=11300 MONGODB_HOST=localhost RABBIT_MQ_HOST=localhost MYSQL_HOST=localhost MYSQL_USER=root MYSQL_DATABASE=queue_test php -d memory_limit=-1 bin/phpunit --coverage-clover=coverage.clover --log-junit=phpunit.result.xml && touch build_passed
43 | - if [ -f build_passed ]; then bin/ocular code-coverage:upload --format=php-clover coverage.clover; fi
44 | - echo $(ls -l)
45 | - echo $(pwd)
46 | - echo $(ls -l phpunit.result.xml)
47 | - (whichsonar=$(which sonar-scanner) && if [ -n "$whichsonar" -a "$TRAVIS_BRANCH" = "master" ]; then sonar-scanner; fi) || true
48 |
--------------------------------------------------------------------------------
/Beanstalkd/Job.php:
--------------------------------------------------------------------------------
1 | beanJob;
19 | }
20 |
21 | /**
22 | * @param mixed $beanJob
23 | */
24 | public function setBeanJob($beanJob)
25 | {
26 | $this->beanJob = $beanJob;
27 | }
28 |
29 | public function getTtr()
30 | {
31 | return $this->ttr;
32 | }
33 |
34 | /**
35 | * @param $ttr
36 | */
37 | public function setTtr($ttr)
38 | {
39 | $this->ttr = $ttr;
40 | }
41 | }
42 |
--------------------------------------------------------------------------------
/Command/CountCommand.php:
--------------------------------------------------------------------------------
1 | jobManager = $jobManager;
18 | }
19 |
20 | protected function configure(): void
21 | {
22 | $this
23 | ->setName('dtc:queue:count')
24 | ->setDescription('Display job queue status.');
25 | }
26 |
27 | protected function execute(InputInterface $input, OutputInterface $output): int
28 | {
29 | $waitingCount = $this->jobManager->getWaitingJobCount();
30 | $status = $this->jobManager->getStatus();
31 | $firstJob = key($status);
32 | if ($firstJob) {
33 | $jobKeys = array_keys($status);
34 | $maxLength = max(array_map(function ($item) {
35 | return strlen($item ?: '');
36 | }, $jobKeys));
37 | $formatLen = $this->determineFormatLength($maxLength);
38 | $format = '%-'.$formatLen.'s';
39 | $headingArgs = ['Job name'];
40 | $initialKeys = array_keys($status[$firstJob]);
41 | $this->formatHeadings($initialKeys, $headingArgs, $format);
42 | array_unshift($headingArgs, $format);
43 | $msg = call_user_func_array('sprintf', $headingArgs);
44 | $output->writeln($msg);
45 | $this->outputStatus($output, $status, $initialKeys, $format);
46 | }
47 | $output->writeln("Total waiting jobs: {$waitingCount}");
48 |
49 | return $this::SUCCESS;
50 | }
51 |
52 | /**
53 | * @param string $format
54 | */
55 | private function outputStatus(OutputInterface $output, array $status, array $initialKeys, $format)
56 | {
57 | foreach ($status as $func => $info) {
58 | $lineArgs = [$format, $func];
59 | foreach ($initialKeys as $statusKey) {
60 | $lineArgs[] = $info[$statusKey];
61 | }
62 | $msg = call_user_func_array('sprintf', $lineArgs);
63 | $output->writeln($msg);
64 | }
65 | }
66 |
67 | /**
68 | * @param string $format
69 | */
70 | private function formatHeadings(array $initialKeys, array &$headingArgs, &$format)
71 | {
72 | foreach ($initialKeys as $statusName) {
73 | $headingStr = ucwords(str_replace('_', ' ', $statusName));
74 | $format .= ' %'.(1 + strlen($headingStr)).'s';
75 | $headingArgs[] = $headingStr;
76 | }
77 | }
78 |
79 | /**
80 | * @param int $maxLength
81 | *
82 | * @return int
83 | */
84 | private function determineFormatLength($maxLength)
85 | {
86 | $formatLen = $maxLength > 50 ? 50 : $maxLength;
87 | $formatMinLen = strlen('Job name') + 1;
88 | $formatLen = $formatLen < $formatMinLen ? $formatMinLen : $formatLen;
89 |
90 | return $formatLen;
91 | }
92 | }
93 |
--------------------------------------------------------------------------------
/Command/ResetCommand.php:
--------------------------------------------------------------------------------
1 | setName('dtc:queue:reset')
19 | ->setDescription('Reset jobs with exception or stalled status');
20 | }
21 |
22 | public function setJobManager($jobManager)
23 | {
24 | $this->jobManager = $jobManager;
25 | }
26 |
27 | protected function execute(InputInterface $input, OutputInterface $output): int
28 | {
29 | $countException = $this->jobManager->resetExceptionJobs();
30 | $countStalled = $this->jobManager->resetStalledJobs();
31 | $output->writeln("$countException job(s) in status 'exception' reset");
32 | $output->writeln("$countStalled job(s) stalled (in status 'running') reset");
33 |
34 | return $this::SUCCESS;
35 | }
36 | }
37 |
--------------------------------------------------------------------------------
/Controller/ControllerTrait.php:
--------------------------------------------------------------------------------
1 | container->has('templating')) {
13 | return new Response($this->container->get('templating')->render($template, $params));
14 | } elseif ($this->container->has('twig')) {
15 | return new Response($this->container->get('twig')->render($template, $params));
16 | }
17 | throw new \Exception('Need Twig Bundle or Templating component installed');
18 | }
19 |
20 | protected function validateJobTimingManager()
21 | {
22 | if ($this->container->hasParameter('dtc_queue.manager.job_timing') &&
23 | $this->container->getParameter('dtc_queue.manager.job_timing')) {
24 | $this->validateManagerType('dtc_queue.manager.job_timing');
25 | } elseif ($this->container->hasParameter('dtc_queue.manager.run') &&
26 | $this->container->hasParameter('dtc_queue.manager.run')) {
27 | $this->validateManagerType('dtc_queue.manager.run');
28 | } else {
29 | $this->validateManagerType('dtc_queue.manager.job');
30 | }
31 | }
32 |
33 | protected function validateRunManager()
34 | {
35 | if ($this->container->hasParameter('dtc_queue.manager.job_timing') &&
36 | $this->container->getParameter('dtc_queue.manager.job_timing')) {
37 | $this->validateManagerType('dtc_queue.manager.run');
38 | } else {
39 | $this->validateManagerType('dtc_queue.manager.job');
40 | }
41 | }
42 |
43 | /**
44 | * @param string $type
45 | */
46 | protected function validateManagerType($type)
47 | {
48 | $managerType = $this->container->getParameter($type);
49 | if ('mongodb' !== $managerType && 'orm' != $managerType && 'odm' != $managerType) {
50 | throw new UnsupportedException("Unsupported manager type: $managerType");
51 | }
52 | }
53 |
54 | protected function addCssJs(array &$params)
55 | {
56 | $params['css'] = $this->container->getParameter('dtc_grid.theme.css');
57 | $params['js'] = $this->container->getParameter('dtc_grid.theme.js');
58 | $jQuery = $this->container->getParameter('dtc_grid.jquery');
59 | array_unshift($params['js'], $jQuery['url']);
60 | $params['chartjs'] = $this->container->getParameter('dtc_queue.admin.chartjs');
61 | }
62 | }
63 |
--------------------------------------------------------------------------------
/DependencyInjection/Compiler/BeanstalkdCompilerPass.php:
--------------------------------------------------------------------------------
1 | hasParameter('dtc_queue.beanstalkd.host') &&
15 | $container->getParameter('dtc_queue.beanstalkd.host')) {
16 | $definition = new Definition(
17 | 'Pheanstalk\\Pheanstalk',
18 | [
19 | new Definition(
20 | 'Pheanstalk\\Connection',
21 | [
22 | new Definition(
23 | 'Pheanstalk\\SocketFactory',
24 | [$container->getParameter('dtc_queue.beanstalkd.host'), $container->getParameter('dtc_queue.beanstalkd.port')]
25 | ),
26 | ]
27 | ),
28 | ]
29 | );
30 | $container->setDefinition('dtc_queue.beanstalkd', $definition);
31 | $definition = $container->getDefinition('dtc_queue.manager.job.beanstalkd');
32 | $definition->addMethodCall('setBeanstalkd', [new Reference('dtc_queue.beanstalkd')]);
33 | if ($container->hasParameter('dtc_queue.beanstalkd.tube')) {
34 | $definition->addMethodCall('setTube', [$container->getParameter('dtc_queue.beanstalkd.tube')]);
35 | }
36 | }
37 | }
38 | }
39 |
--------------------------------------------------------------------------------
/DependencyInjection/Compiler/GridCompilerPass.php:
--------------------------------------------------------------------------------
1 | getDefinition('dtc_queue.grid_source.jobs_waiting.odm')->setClass('Dtc\QueueBundle\ODM\StubLiveJobsGridSource');
16 | $container->getDefinition('dtc_queue.grid_source.jobs_waiting.orm')->setClass('Dtc\QueueBundle\ORM\StubLiveJobsGridSource');
17 | $container->getDefinition('dtc_queue.grid_source.jobs_running.odm')->setClass('Dtc\QueueBundle\ODM\StubLiveJobsGridSource');
18 | $container->getDefinition('dtc_queue.grid_source.jobs_running.orm')->setClass('Dtc\QueueBundle\ORM\StubLiveJobsGridSource');
19 |
20 | return;
21 | }
22 | $container->getDefinition('dtc_queue.grid_source.jobs_waiting.odm')->setClass('Dtc\QueueBundle\ODM\LiveJobsGridSource');
23 | $container->getDefinition('dtc_queue.grid_source.jobs_waiting.orm')->setClass('Dtc\QueueBundle\ORM\LiveJobsGridSource');
24 | $container->getDefinition('dtc_queue.grid_source.jobs_running.odm')->setClass('Dtc\QueueBundle\ODM\LiveJobsGridSource');
25 | $container->getDefinition('dtc_queue.grid_source.jobs_running.orm')->setClass('Dtc\QueueBundle\ORM\LiveJobsGridSource');
26 | $defaultManagerType = $container->getParameter('dtc_queue.manager.job');
27 | $runManagerType = $container->getParameter($this->getRunManagerType($container));
28 | if ('orm' === $defaultManagerType || 'orm' === $runManagerType || 'odm' === $defaultManagerType || 'odm' === $runManagerType) {
29 | $filename = __DIR__.DIRECTORY_SEPARATOR.'..'.DIRECTORY_SEPARATOR.'..'.DIRECTORY_SEPARATOR.'Resources'.DIRECTORY_SEPARATOR.'config'.DIRECTORY_SEPARATOR.'dtc_grid.yaml';
30 | $cacheDir = $container->getParameter('kernel.cache_dir');
31 | if (class_exists('Dtc\GridBundle\Util\ColumnUtil')) {
32 | \Dtc\GridBundle\Util\ColumnUtil::cacheClassesFromFile($cacheDir, $filename);
33 | }
34 | }
35 |
36 | $this->addLiveJobs($container);
37 | }
38 |
39 | /**
40 | * @throws
41 | */
42 | protected function addLiveJobs(ContainerBuilder $container)
43 | {
44 | $jobReflection = new \ReflectionClass($container->getParameter('dtc_queue.class.job'));
45 |
46 | // Custom grid sources for waiting and running jobs.
47 | if ($jobReflection->isSubclassOf(\Dtc\QueueBundle\Document\BaseJob::class)) {
48 | \Dtc\GridBundle\DependencyInjection\Compiler\GridSourceCompilerPass::addGridSource($container, 'dtc_queue.grid_source.jobs_waiting.odm');
49 | \Dtc\GridBundle\DependencyInjection\Compiler\GridSourceCompilerPass::addGridSource($container, 'dtc_queue.grid_source.jobs_running.odm');
50 | }
51 | if ($jobReflection->isSubclassOf(\Dtc\QueueBundle\Entity\BaseJob::class)) {
52 | \Dtc\GridBundle\DependencyInjection\Compiler\GridSourceCompilerPass::addGridSource($container, 'dtc_queue.grid_source.jobs_waiting.orm');
53 | \Dtc\GridBundle\DependencyInjection\Compiler\GridSourceCompilerPass::addGridSource($container, 'dtc_queue.grid_source.jobs_running.orm');
54 | }
55 | }
56 | }
57 |
--------------------------------------------------------------------------------
/DependencyInjection/Compiler/WorkerCompilerTrait.php:
--------------------------------------------------------------------------------
1 | hasParameter('dtc_queue.manager.run') &&
13 | $container->getParameter('dtc_queue.manager.run')) {
14 | $managerType = 'dtc_queue.manager.run';
15 | }
16 |
17 | return $managerType;
18 | }
19 | }
20 |
--------------------------------------------------------------------------------
/Doctrine/BaseDoctrineJobManager.php:
--------------------------------------------------------------------------------
1 | objectManager = $objectManager;
36 | parent::__construct($runManager, $jobTimingManager, $jobClass, $jobArchiveClass);
37 | }
38 |
39 | protected function getFetchCount($totalCount)
40 | {
41 | $fetchCount = intval($totalCount / 10);
42 | if ($fetchCount < self::FETCH_COUNT_MIN) {
43 | $fetchCount = self::FETCH_COUNT_MIN;
44 | }
45 | if ($fetchCount > self::FETCH_COUNT_MAX) {
46 | $fetchCount = self::FETCH_COUNT_MAX;
47 | }
48 |
49 | return $fetchCount;
50 | }
51 |
52 | protected function getSaveCount($totalCount)
53 | {
54 | $saveCount = intval($totalCount / 10);
55 | if ($saveCount < self::SAVE_COUNT_MIN) {
56 | $saveCount = self::SAVE_COUNT_MIN;
57 | }
58 | if ($saveCount > self::SAVE_COUNT_MAX) {
59 | $saveCount = self::SAVE_COUNT_MAX;
60 | }
61 |
62 | return $saveCount;
63 | }
64 |
65 | /**
66 | * @return ObjectManager
67 | */
68 | public function getObjectManager()
69 | {
70 | return $this->objectManager;
71 | }
72 |
73 | /**
74 | * @return \Doctrine\Persistence\ObjectRepository
75 | */
76 | public function getRepository()
77 | {
78 | return $this->getObjectManager()->getRepository($this->getJobClass());
79 | }
80 |
81 | protected function flush()
82 | {
83 | $this->getObjectManager()->flush();
84 | }
85 |
86 | public function deleteJob(\Dtc\QueueBundle\Model\Job $job)
87 | {
88 | $this->persist($job, 'remove');
89 | }
90 |
91 | abstract protected function persist($object, $action = 'persist');
92 |
93 | protected function addWorkerNameMethod(array &$criterion, $workerName = null, $method = null)
94 | {
95 | if (null !== $workerName) {
96 | $criterion['workerName'] = $workerName;
97 | }
98 | if (null !== $method) {
99 | $criterion['method'] = $method;
100 | }
101 | }
102 | }
103 |
--------------------------------------------------------------------------------
/Doctrine/DoctrineJobTimingManager.php:
--------------------------------------------------------------------------------
1 | objectManager = $objectManager;
16 | parent::__construct($jobTimingClass, $recordTimings);
17 | }
18 |
19 | /**
20 | * @return \Doctrine\Persistence\ObjectManager
21 | */
22 | public function getObjectManager()
23 | {
24 | return $this->objectManager;
25 | }
26 |
27 | public function performRecording($status, \DateTime $finishedAt = null)
28 | {
29 | if (null === $finishedAt) {
30 | $finishedAt = \Dtc\QueueBundle\Util\Util::getMicrotimeDateTime();
31 | }
32 |
33 | /** @var JobTiming $jobTiming */
34 | $jobTiming = new $this->jobTimingClass();
35 | $jobTiming->setFinishedAt($finishedAt);
36 | $jobTiming->setStatus($status);
37 | $objectManager = $this->getObjectManager();
38 | $objectManager->persist($jobTiming);
39 | $objectManager->flush();
40 | }
41 |
42 | abstract protected function removeOlderThan($objectName, $field, \DateTime $olderThan);
43 |
44 | abstract protected function persist($object, $action = 'persist');
45 |
46 | public function pruneJobTimings(\DateTime $olderThan)
47 | {
48 | return $this->removeOlderThan($this->getJobTimingClass(), 'finishedAt', $olderThan);
49 | }
50 | }
51 |
--------------------------------------------------------------------------------
/Doctrine/DoctrineRunManager.php:
--------------------------------------------------------------------------------
1 | runArchiveClass = $runArchiveClass;
19 | $this->objectManager = $objectManager;
20 | parent::__construct($runClass);
21 | }
22 |
23 | /**
24 | * @return \Doctrine\Persistence\ObjectManager
25 | */
26 | public function getObjectManager()
27 | {
28 | return $this->objectManager;
29 | }
30 |
31 | /**
32 | * @return \Doctrine\Persistence\ObjectRepository
33 | */
34 | public function getRepository()
35 | {
36 | return $this->getObjectManager()->getRepository($this->getRunClass());
37 | }
38 |
39 | /**
40 | * @return string|null
41 | */
42 | public function getRunArchiveClass()
43 | {
44 | return $this->runArchiveClass;
45 | }
46 |
47 | /**
48 | * @return int
49 | *
50 | * @throws \Exception
51 | */
52 | public function pruneStalledRuns()
53 | {
54 | $runs = $this->getOldLiveRuns();
55 | /** @var Run $run */
56 | $delete = [];
57 |
58 | foreach ($runs as $run) {
59 | $lastHeartbeat = $run->getLastHeartbeatAt();
60 | $time = time() - 3600;
61 | $processTimeout = $run->getProcessTimeout();
62 | $time -= $processTimeout;
63 | $oldDate = \DateTime::createFromFormat('U', strval($time));
64 | if (false === $oldDate) {
65 | throw new \Exception("Could not create DateTime object from $time");
66 | }
67 | $oldDate->setTimezone(new \DateTimeZone(date_default_timezone_get()));
68 | if (null === $lastHeartbeat || $oldDate > $lastHeartbeat) {
69 | $delete[] = $run;
70 | }
71 | }
72 |
73 | return $this->deleteOldRuns($delete);
74 | }
75 |
76 | /**
77 | * @return int
78 | */
79 | protected function deleteOldRuns(array $delete)
80 | {
81 | $count = count($delete);
82 | $objectManager = $this->getObjectManager();
83 | for ($i = 0; $i < $count; $i += 100) {
84 | $deleteList = array_slice($delete, $i, 100);
85 | foreach ($deleteList as $object) {
86 | $objectManager->remove($object);
87 | }
88 | $this->flush();
89 | }
90 |
91 | return $count;
92 | }
93 |
94 | protected function flush()
95 | {
96 | $this->getObjectManager()->flush();
97 | }
98 |
99 | /**
100 | * @param string $action
101 | */
102 | protected function persistRun(Run $run, $action = 'persist')
103 | {
104 | parent::persistRun($run, $action);
105 | $this->persist($run, $action);
106 | }
107 |
108 | /**
109 | * @return array
110 | */
111 | abstract protected function getOldLiveRuns();
112 |
113 | abstract protected function persist($object, $action = 'persist');
114 |
115 | abstract protected function removeOlderThan($objectName, $field, \DateTime $olderThan);
116 |
117 | public function pruneArchivedRuns(\DateTime $olderThan)
118 | {
119 | return $this->removeOlderThan($this->getRunArchiveClass(), 'endedAt', $olderThan);
120 | }
121 | }
122 |
--------------------------------------------------------------------------------
/Doctrine/ProgressCallbackTrait.php:
--------------------------------------------------------------------------------
1 | id;
20 | }
21 |
22 | /**
23 | * @param mixed $id
24 | */
25 | public function setId($id)
26 | {
27 | $this->id = $id;
28 | }
29 |
30 | #[ODM\Field(type: 'date')]
31 | #[ODM\Index(unique: false, order: 'asc')]
32 | protected $finishedAt;
33 |
34 | #[ODM\Field(type: 'integer')]
35 | #[ODM\Index(unique: false, order: 'asc')]
36 | protected $status;
37 | }
38 |
--------------------------------------------------------------------------------
/Document/Run.php:
--------------------------------------------------------------------------------
1 | addCompilerPass(new WorkerCompilerPass());
19 | $container->addCompilerPass(new RabbitMQCompilerPass());
20 | $container->addCompilerPass(new BeanstalkdCompilerPass());
21 | $container->addCompilerPass(new RedisCompilerPass());
22 | $container->addCompilerPass(new GridCompilerPass());
23 | }
24 | }
25 |
--------------------------------------------------------------------------------
/Entity/BaseJob.php:
--------------------------------------------------------------------------------
1 | runId;
110 | }
111 |
112 | /**
113 | * @param mixed $runId
114 | */
115 | public function setRunId($runId)
116 | {
117 | $this->runId = $runId;
118 | }
119 |
120 | public function setArgs($args)
121 | {
122 | $args = serialize($args);
123 | parent::setArgs($args);
124 | }
125 |
126 | public function getArgs()
127 | {
128 | $args = parent::getArgs();
129 |
130 | return unserialize($args);
131 | }
132 | }
133 |
--------------------------------------------------------------------------------
/Entity/BaseRun.php:
--------------------------------------------------------------------------------
1 | id;
30 | }
31 |
32 | /**
33 | * @param mixed $id
34 | */
35 | public function setId($id)
36 | {
37 | $this->id = $id;
38 | }
39 | }
40 |
--------------------------------------------------------------------------------
/Entity/Run.php:
--------------------------------------------------------------------------------
1 | job = $job;
17 | }
18 |
19 | /**
20 | * @param mixed $job
21 | */
22 | public function setJob($job)
23 | {
24 | $this->job = $job;
25 | }
26 |
27 | /**
28 | * @return mixed
29 | */
30 | public function getJob()
31 | {
32 | return $this->job;
33 | }
34 | }
35 |
--------------------------------------------------------------------------------
/EventDispatcher/EventDispatcher.php:
--------------------------------------------------------------------------------
1 | getSubscribedEvents() as $key => $value) {
12 | $this->listeners[$key][] = [$subscriber, $value];
13 | }
14 | }
15 |
16 | public function hasListeners($eventName)
17 | {
18 | if (!isset($this->listeners[$eventName])) {
19 | return false;
20 | }
21 |
22 | return $this->listeners[$eventName] ? true : false;
23 | }
24 |
25 | public function dispatch($eventName, Event $event)
26 | {
27 | if (!isset($this->listeners[$eventName])) {
28 | return;
29 | }
30 |
31 | foreach ($this->listeners[$eventName] as $callback) {
32 | call_user_func($callback, $event);
33 | }
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/EventDispatcher/EventSubscriberInterface.php:
--------------------------------------------------------------------------------
1 | 'eventHandler'),
13 | * array(...),
14 | * )
15 | *
16 | * @return array
17 | */
18 | public static function getSubscribedEvents();
19 | }
20 |
--------------------------------------------------------------------------------
/Exception/ArgumentsNotSetException.php:
--------------------------------------------------------------------------------
1 | runManager = $runManager;
18 | $this->jobTiminigManager = $jobTimingManager;
19 | $this->jobClass = $jobClass;
20 | }
21 |
22 | /**
23 | * @return array
24 | */
25 | public static function getAllStatuses()
26 | {
27 | return [
28 | BaseJob::STATUS_NEW => 0,
29 | BaseJob::STATUS_RUNNING => 0,
30 | BaseJob::STATUS_SUCCESS => 0,
31 | BaseJob::STATUS_FAILURE => 0,
32 | BaseJob::STATUS_EXCEPTION => 0,
33 | \Dtc\QueueBundle\Model\Job::STATUS_EXPIRED => 0, ];
34 | }
35 |
36 | /**
37 | * @throws UnsupportedException
38 | */
39 | public function getStatus(): array
40 | {
41 | $count = $this->getWaitingJobCount();
42 | $allStatuses = static::getAllStatuses();
43 | foreach (array_keys($allStatuses) as $status) {
44 | $allStatuses[$status] = 'N/A';
45 | }
46 | $allStatuses[BaseJob::STATUS_NEW] = $count;
47 |
48 | return ['all' => $allStatuses];
49 | }
50 |
51 | /**
52 | * @return string
53 | */
54 | public function getJobClass()
55 | {
56 | return $this->jobClass;
57 | }
58 |
59 | /**
60 | * @return JobTimingManager
61 | */
62 | public function getJobTimingManager()
63 | {
64 | return $this->jobTiminigManager;
65 | }
66 |
67 | /**
68 | * @return RunManager
69 | */
70 | public function getRunManager()
71 | {
72 | return $this->runManager;
73 | }
74 |
75 | abstract public function getJob($workerName = null, $methodName = null, $prioritize = true, $runId = null);
76 |
77 | abstract public function save(Job $job);
78 |
79 | abstract public function saveHistory(Job $job);
80 |
81 | public function resetExceptionJobs($workerName = null, $methodName = null)
82 | {
83 | throw new UnsupportedException('Unsupported');
84 | }
85 |
86 | public function pruneExceptionJobs($workerName = null, $methodName = null)
87 | {
88 | throw new UnsupportedException('Unsupported');
89 | }
90 |
91 | public function getWaitingJobCount($workerName = null, $methodName = null)
92 | {
93 | throw new UnsupportedException('Unsupported');
94 | }
95 |
96 | public function deleteJob(Job $job)
97 | {
98 | throw new UnsupportedException('Unsupported');
99 | }
100 |
101 | public function pruneExpiredJobs($workerName = null, $methodName = null)
102 | {
103 | throw new UnsupportedException('Unsupported');
104 | }
105 |
106 | public function pruneArchivedJobs(\DateTime $olderThan)
107 | {
108 | throw new UnsupportedException('Unsupported');
109 | }
110 | }
111 |
--------------------------------------------------------------------------------
/Manager/ArchivableJobManager.php:
--------------------------------------------------------------------------------
1 | jobArchiveClass = $jobArchiveClass;
25 | parent::__construct($runManager, $jobTimingManager, $jobClass);
26 | }
27 |
28 | /**
29 | * @return string
30 | */
31 | public function getJobArchiveClass()
32 | {
33 | return $this->jobArchiveClass;
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/Manager/JobIdTrait.php:
--------------------------------------------------------------------------------
1 | pid) ? $this->pid : null;
13 | $hostname = isset($this->hostname) ? $this->hostname : null;
14 | if (!$job->getId()) {
15 | $job->setId(uniqid($hostname.'-'.$pid, true));
16 | }
17 | }
18 | }
19 |
--------------------------------------------------------------------------------
/Manager/JobManagerInterface.php:
--------------------------------------------------------------------------------
1 | jobTimingClass = $jobTimingClass;
18 | $this->recordTimings = $recordTimings;
19 | }
20 |
21 | /**
22 | * @throws UnsupportedException
23 | *
24 | * @return int Number of archived runs pruned
25 | */
26 | public function pruneJobTimings(\DateTime $olderThan)
27 | {
28 | throw new UnsupportedException('not supported');
29 | }
30 |
31 | /**
32 | * Subclasses should overrride this function instead of recordTiming.
33 | *
34 | * @param $status
35 | */
36 | protected function performRecording($status, \DateTime $dateTime = null)
37 | {
38 | }
39 |
40 | /**
41 | * @param $status
42 | */
43 | public function recordTiming($status, \DateTime $dateTime = null)
44 | {
45 | if (!$this->recordTimings) {
46 | return;
47 | }
48 | $this->performRecording($status, $dateTime);
49 | }
50 |
51 | /**
52 | * @return string
53 | */
54 | public function getJobTimingClass()
55 | {
56 | return $this->jobTimingClass;
57 | }
58 | }
59 |
--------------------------------------------------------------------------------
/Manager/PriorityJobManager.php:
--------------------------------------------------------------------------------
1 | maxPriority;
23 | }
24 |
25 | /**
26 | * @param mixed $maxPriority
27 | */
28 | public function setMaxPriority($maxPriority)
29 | {
30 | $this->maxPriority = $maxPriority;
31 | }
32 |
33 | /**
34 | * @return mixed
35 | */
36 | public function getPriorityDirection()
37 | {
38 | return $this->priorityDirection;
39 | }
40 |
41 | /**
42 | * @param mixed $priorityDirection
43 | */
44 | public function setPriorityDirection($priorityDirection)
45 | {
46 | $this->priorityDirection = $priorityDirection;
47 | }
48 |
49 | protected function validatePriority($priority)
50 | {
51 | if (null === $priority) {
52 | return;
53 | }
54 |
55 | if (!ctype_digit(strval($priority))) {
56 | throw new PriorityException("Priority ($priority) needs to be a positive integer");
57 | }
58 | if (strval(intval($priority)) !== strval($priority)) {
59 | throw new PriorityException("Priority ($priority) needs to be less than ".PHP_INT_MAX);
60 | }
61 | $maxPriority = $this->getMaxPriority();
62 | if (null !== $maxPriority && intval($priority) > $maxPriority) {
63 | throw new PriorityException("Priority ($priority) must be less than ".$maxPriority);
64 | }
65 | }
66 |
67 | /**
68 | * Returns the prioirty in ASCENDING order regardless of the User's choice of direction
69 | * (for storing RabbitMQ, Mysql, others).
70 | *
71 | * @param $priority
72 | *
73 | * @return mixed
74 | */
75 | protected function calculatePriority($priority)
76 | {
77 | if (null === $priority) {
78 | return $priority;
79 | }
80 |
81 | if (null === $this->maxPriority) {
82 | return null;
83 | }
84 |
85 | if (self::PRIORITY_DESC === $this->priorityDirection) {
86 | $priority = $this->maxPriority - $priority;
87 | }
88 |
89 | return $priority;
90 | }
91 |
92 | abstract protected function prioritySave(Job $job);
93 |
94 | /**
95 | * @param Job $job
96 | *
97 | * @throws PriorityException
98 | */
99 | protected function retryableSave(RetryableJob $job)
100 | {
101 | $this->validatePriority($job->getPriority());
102 | if (!$job->getId()) { // An unsaved job needs it's priority potentially adjusted
103 | $job->setPriority($this->calculatePriority($job->getPriority()));
104 | }
105 |
106 | $result = $this->prioritySave($job);
107 |
108 | return $result;
109 | }
110 | }
111 |
--------------------------------------------------------------------------------
/Manager/RunManager.php:
--------------------------------------------------------------------------------
1 | runClass = $runClass;
18 | }
19 |
20 | /**
21 | * @return string
22 | */
23 | public function getRunClass()
24 | {
25 | return $this->runClass;
26 | }
27 |
28 | /**
29 | * @param string $runClass
30 | */
31 | public function setRunClass($runClass)
32 | {
33 | $this->runClass = $runClass;
34 | }
35 |
36 | /**
37 | * @return int Number of archived runs pruned
38 | *
39 | * @throws UnsupportedException
40 | */
41 | public function pruneArchivedRuns(\DateTime $olderThan)
42 | {
43 | throw new UnsupportedException('not supported - '.$olderThan->getTimestamp());
44 | }
45 |
46 | /**
47 | * Prunes stalled runs.
48 | *
49 | * @return int Number of stalled runs pruned
50 | *
51 | * @throws UnsupportedException
52 | */
53 | public function pruneStalledRuns()
54 | {
55 | throw new UnsupportedException('not supported');
56 | }
57 |
58 | /**
59 | * @param float $start
60 | */
61 | public function recordHeartbeat(Run $run, $start, Job $job = null)
62 | {
63 | $jobId = null;
64 | if (null !== $job) {
65 | $jobId = $job->getId();
66 | }
67 |
68 | $heartbeat = microtime(true);
69 | $run->setLastHeartbeatAt(Util::getMicrotimeFloatDateTime($heartbeat));
70 | $run->setCurrentJobId($jobId);
71 | $run->setElapsed($heartbeat - $start);
72 | $this->persistRun($run);
73 | }
74 |
75 | /**
76 | * @param string $action
77 | */
78 | protected function persistRun(Run $run, $action = 'persist')
79 | {
80 | // To be overridden
81 | }
82 |
83 | /**
84 | * @param int $count
85 | */
86 | public function updateProcessed(Run $run, $count)
87 | {
88 | $run->setProcessed($count);
89 | $this->persistRun($run);
90 | }
91 |
92 | /**
93 | * Sets up the runManager (document / entity persister) if appropriate.
94 | *
95 | * @param float $start
96 | * @param int|null $maxCount
97 | * @param int|null $duration
98 | * @param int $processTimeout
99 | *
100 | * @return Run
101 | */
102 | public function runStart($start, $maxCount = null, $duration = null, $processTimeout = null)
103 | {
104 | $runClass = $this->getRunClass();
105 | /** @var Run $run */
106 | $run = new $runClass();
107 | $startDate = Util::getMicrotimeFloatDateTime($start);
108 | $run->setLastHeartbeatAt($startDate);
109 | $run->setStartedAt($startDate);
110 | if (null !== $maxCount) {
111 | $run->setMaxCount($maxCount);
112 | }
113 | if (null !== $duration) {
114 | $run->setDuration($duration);
115 | }
116 | $run->setHostname(gethostname());
117 | $run->setPid(getmypid());
118 | $run->setProcessed(0);
119 | $run->setProcessTimeout($processTimeout);
120 | $this->persistRun($run);
121 |
122 | return $run;
123 | }
124 |
125 | /**
126 | * @param int|null $start
127 | */
128 | public function runStop(Run $run, $start)
129 | {
130 | $end = microtime(true);
131 | $run->setEndedAt(Util::getMicrotimeFloatDateTime($end));
132 | $run->setElapsed($end - $start);
133 | $this->persistRun($run, 'remove');
134 | }
135 | }
136 |
--------------------------------------------------------------------------------
/Manager/SaveableTrait.php:
--------------------------------------------------------------------------------
1 | getPriority() && !isset($this->maxPriority)) {
18 | throw new PriorityException('This queue does not support priorities');
19 | }
20 |
21 | if (!$job instanceof RetryableJob) {
22 | throw new ClassNotSubclassException('Job needs to be instance of '.RetryableJob::class);
23 | }
24 | }
25 | }
26 |
--------------------------------------------------------------------------------
/Manager/StallableJobManager.php:
--------------------------------------------------------------------------------
1 | getId()) {
35 | if ($job instanceof StallableJob) {
36 | $this->setStallableJobDefaults($job);
37 | }
38 | }
39 |
40 | return $this->stallableSave($job);
41 | }
42 |
43 | /**
44 | * @param $retry true if retry
45 | *
46 | * @return bool true if retry
47 | */
48 | abstract protected function stallableSaveHistory(StallableJob $job, $retry);
49 |
50 | /**
51 | * @return bool true if retry
52 | *
53 | * @param Job $job
54 | */
55 | protected function retryableSaveHistory(RetryableJob $job, $retry)
56 | {
57 | if (!$job instanceof StallableJob) {
58 | throw new \InvalidArgumentException('job not instance of '.StallableJob::class);
59 | }
60 |
61 | if ($retry) {
62 | return $this->stallableSaveHistory($job, $retry);
63 | }
64 |
65 | if (StallableJob::STATUS_STALLED === $job->getStatus()) {
66 | return $this->stallableSaveHistory($job, $this->updateJobStalled($job));
67 | }
68 |
69 | return $this->stallableSaveHistory($job, false);
70 | }
71 |
72 | /**
73 | * @return bool false if
74 | */
75 | private function updateJobStalled(StallableJob $job)
76 | {
77 | return $this->updateJobMax($job, 'Stalls', StallableJob::STATUS_MAX_STALLS, true);
78 | }
79 |
80 | protected function setStallableJobDefaults(StallableJob $job)
81 | {
82 | if (null === $job->getMaxStalls()) {
83 | $job->setMaxStalls($this->defaultMaxStalls);
84 | }
85 | }
86 |
87 | /**
88 | * @return int|null
89 | */
90 | public function getDefaultMaxStalls()
91 | {
92 | return $this->defaultMaxStalls;
93 | }
94 |
95 | /**
96 | * @param int|null $defaultMaxStalls
97 | */
98 | public function setDefaultMaxStalls($defaultMaxStalls)
99 | {
100 | $this->defaultMaxStalls = $defaultMaxStalls;
101 | }
102 | }
103 |
--------------------------------------------------------------------------------
/Manager/VerifyTrait.php:
--------------------------------------------------------------------------------
1 | maxPriority) && true !== $prioritize)) {
19 | throw new UnsupportedException('Unsupported');
20 | }
21 | }
22 | }
23 |
--------------------------------------------------------------------------------
/Model/BaseStatistics.php:
--------------------------------------------------------------------------------
1 | ['label' => 'Finished: SUCCESS', 'color' => 'green'],
26 | self::STATUS_FINISHED_FAILURE => ['label' => 'Finished: FAILURE', 'color' => 'red'],
27 | self::STATUS_FINISHED_EXCEPTION => ['label' => 'Finished: EXCEPTION', 'color' => 'orange'],
28 | self::STATUS_FINISHED_EXPIRED => ['label' => 'Finished: EXPIRED', 'color' => 'maroon'],
29 | self::STATUS_FINISHED_STALLED => ['label' => 'Finished: STALLED', 'color' => 'gold'],
30 | self::STATUS_INSERT => ['label' => 'INSERT', 'color' => 'navy'],
31 | self::STATUS_INSERT_DELAYED => ['label' => 'INSERT (Delayed)', 'color' => 'purple'], ];
32 | }
33 |
34 | /**
35 | * @return mixed
36 | */
37 | public function getFinishedAt()
38 | {
39 | return $this->finishedAt;
40 | }
41 |
42 | /**
43 | * @param mixed $finishedAt
44 | */
45 | public function setFinishedAt($finishedAt)
46 | {
47 | $this->finishedAt = $finishedAt;
48 | }
49 |
50 | /**
51 | * @return int
52 | */
53 | public function getStatus()
54 | {
55 | return $this->status;
56 | }
57 |
58 | /**
59 | * @param int $status
60 | */
61 | public function setStatus($status)
62 | {
63 | $this->status = $status;
64 | }
65 | }
66 |
--------------------------------------------------------------------------------
/Model/MessageableJob.php:
--------------------------------------------------------------------------------
1 | getWhenAt()) {
13 | $this->setWhenAt(Util::getMicrotimeDateTime());
14 | }
15 | }
16 |
17 | protected function toMessageArray()
18 | {
19 | return [
20 | 'worker' => $this->getWorkerName(),
21 | 'args' => $this->getArgs(),
22 | 'method' => $this->getMethod(),
23 | 'priority' => $this->getPriority(),
24 | 'whenAt' => $this->getWhenAt()->format('U.u'),
25 | 'createdAt' => $this->getCreatedAt()->format('U.u'),
26 | 'updatedAt' => $this->getUpdatedAt()->format('U.u'),
27 | 'expiresAt' => ($expiresAt = $this->getExpiresAt()) ? $expiresAt->format('U.u') : null,
28 | 'retries' => $this->getRetries(),
29 | 'maxRetries' => $this->getMaxRetries(),
30 | 'failures' => $this->getFailures(),
31 | 'maxFailures' => $this->getMaxFailures(),
32 | 'exceptions' => $this->getExceptions(),
33 | 'maxExceptions' => $this->getMaxExceptions(),
34 | ];
35 | }
36 |
37 | /**
38 | * @return string A json_encoded version of a queueable version of the object
39 | */
40 | public function toMessage()
41 | {
42 | return json_encode($this->toMessageArray());
43 | }
44 |
45 | /**
46 | * @param string $message a json_encoded version of the object
47 | */
48 | public function fromMessage($message)
49 | {
50 | $arr = json_decode($message, true);
51 | if (is_array($arr)) {
52 | $this->fromMessageArray($arr);
53 | }
54 | }
55 |
56 | protected function fromMessageArray(array $arr)
57 | {
58 | $this->setByList(['args', 'priority', 'method'], $arr);
59 | if (isset($arr['worker'])) {
60 | $this->setWorkerName($arr['worker']);
61 | }
62 |
63 | $this->fromMessageArrayRetries($arr);
64 | foreach (['expiresAt', 'createdAt', 'updatedAt', 'whenAt'] as $dateField) {
65 | if (isset($arr[$dateField])) {
66 | $this->setDateTimeField($dateField, $arr[$dateField]);
67 | }
68 | }
69 | }
70 |
71 | /**
72 | * @param $dateField string
73 | * @param $timeStr string|null
74 | */
75 | private function setDateTimeField($dateField, $timeStr)
76 | {
77 | if ($timeStr) {
78 | $dateTime = \DateTime::createFromFormat('U.u', $timeStr);
79 | if (false !== $dateTime) {
80 | $dateTime->setTimezone(new \DateTimeZone(date_default_timezone_get()));
81 | $method = 'set'.ucfirst($dateField);
82 | $this->$method($dateTime);
83 | }
84 | }
85 | }
86 |
87 | private function setByList(array $list, array $arr)
88 | {
89 | foreach ($list as $key) {
90 | if (isset($arr[$key])) {
91 | $method = 'set'.ucfirst($key);
92 | $this->$method($arr[$key]);
93 | }
94 | }
95 | }
96 |
97 | protected function fromMessagearrayRetries(array $arr)
98 | {
99 | $this->setByList(['retries', 'maxRetries', 'failures', 'maxFailures', 'exceptions', 'maxExceptions'], $arr);
100 | }
101 | }
102 |
--------------------------------------------------------------------------------
/Model/MessageableJobWithId.php:
--------------------------------------------------------------------------------
1 | toMessageArray();
13 | $arr['id'] = $this->getId();
14 |
15 | return json_encode($arr);
16 | }
17 |
18 | /**
19 | * @param string $message a json_encoded version of the object
20 | */
21 | public function fromMessage($message)
22 | {
23 | $arr = json_decode($message, true);
24 | if (is_array($arr)) {
25 | $this->fromMessageArray($arr);
26 | if (isset($arr['id'])) {
27 | $this->setId($arr['id']);
28 | }
29 | }
30 | }
31 | }
32 |
--------------------------------------------------------------------------------
/Model/MicrotimeTrait.php:
--------------------------------------------------------------------------------
1 | setWhenUs(Util::getMicrotimeIntegerFormat($whenAt));
14 | }
15 |
16 | /**
17 | * @return \DateTime|null
18 | */
19 | public function getWhenAt()
20 | {
21 | $whenUs = isset($this->whenUs) ? $this->whenUs : null;
22 | if ($whenUs) {
23 | return Util::getDateTimeFromDecimalFormat($whenUs);
24 | }
25 |
26 | return null;
27 | }
28 |
29 | /**
30 | * @param string
31 | */
32 | public function setWhenUs($whenUs)
33 | {
34 | $this->whenUs = $whenUs;
35 |
36 | return $this;
37 | }
38 |
39 | /**
40 | * @param string
41 | */
42 | public function getWhenUs()
43 | {
44 | return isset($this->whenUs) ? $this->whenUs : null;
45 | }
46 | }
47 |
--------------------------------------------------------------------------------
/Model/RetryableJob.php:
--------------------------------------------------------------------------------
1 | setUpdatedAt($this->getCreatedAt());
27 | }
28 |
29 | /**
30 | * @return int|null
31 | */
32 | public function getMaxFailures()
33 | {
34 | return $this->maxFailures;
35 | }
36 |
37 | /**
38 | * @param int|null $maxFailures
39 | */
40 | public function setMaxFailures($maxFailures)
41 | {
42 | $this->maxFailures = $maxFailures;
43 |
44 | return $this;
45 | }
46 |
47 | /**
48 | * @return int|null
49 | */
50 | public function getFailures()
51 | {
52 | return $this->failures;
53 | }
54 |
55 | /**
56 | * @param int|null $failures
57 | */
58 | public function setFailures($failures)
59 | {
60 | $this->failures = $failures;
61 |
62 | return $this;
63 | }
64 |
65 | /**
66 | * @return int|null
67 | */
68 | public function getRetries()
69 | {
70 | return $this->retries;
71 | }
72 |
73 | /**
74 | * @param int|null $retries
75 | *
76 | * @return RetryableJob
77 | */
78 | public function setRetries($retries)
79 | {
80 | $this->retries = $retries;
81 |
82 | return $this;
83 | }
84 |
85 | /**
86 | * @return int|null
87 | */
88 | public function getMaxRetries()
89 | {
90 | return $this->maxRetries;
91 | }
92 |
93 | /**
94 | * @param int|null $maxRetries
95 | *
96 | * @return RetryableJob
97 | */
98 | public function setMaxRetries($maxRetries)
99 | {
100 | $this->maxRetries = $maxRetries;
101 |
102 | return $this;
103 | }
104 |
105 | /**
106 | * @return \DateTime
107 | */
108 | public function getUpdatedAt()
109 | {
110 | return $this->updatedAt;
111 | }
112 |
113 | public function setUpdatedAt(\DateTime $updatedAt)
114 | {
115 | $this->updatedAt = $updatedAt;
116 |
117 | return $this;
118 | }
119 |
120 | /**
121 | * @return int|null
122 | */
123 | public function getMaxExceptions()
124 | {
125 | return $this->maxExceptions;
126 | }
127 |
128 | /**
129 | * @param int|null $maxExceptions
130 | */
131 | public function setMaxExceptions($maxExceptions)
132 | {
133 | $this->maxExceptions = $maxExceptions;
134 |
135 | return $this;
136 | }
137 |
138 | /**
139 | * @return int|null
140 | */
141 | public function getExceptions()
142 | {
143 | return $this->exceptions;
144 | }
145 |
146 | /**
147 | * @param int|null $exceptions
148 | */
149 | public function setExceptions($exceptions)
150 | {
151 | $this->exceptions = $exceptions;
152 |
153 | return $this;
154 | }
155 | }
156 |
--------------------------------------------------------------------------------
/Model/StallableJob.php:
--------------------------------------------------------------------------------
1 | maxStalls;
19 | }
20 |
21 | /**
22 | * @param int|null $maxStalls
23 | *
24 | * @return StallableJob
25 | */
26 | public function setMaxStalls($maxStalls)
27 | {
28 | $this->maxStalls = $maxStalls;
29 |
30 | return $this;
31 | }
32 |
33 | /**
34 | * @return int
35 | */
36 | public function getStalls()
37 | {
38 | return $this->stalls;
39 | }
40 |
41 | /**
42 | * @param int $stalls
43 | *
44 | * @return StallableJob
45 | */
46 | public function setStalls($stalls)
47 | {
48 | $this->stalls = $stalls;
49 |
50 | return $this;
51 | }
52 | }
53 |
--------------------------------------------------------------------------------
/ODM/CommonTrait.php:
--------------------------------------------------------------------------------
1 | getObjectManager();
23 | $qb = $objectManager->createQueryBuilder($objectName);
24 | $qb
25 | ->remove()
26 | ->field($field)->lt($olderThan);
27 |
28 | $query = $qb->getQuery();
29 | $result = $query->execute();
30 | if ($result instanceof DeleteResult) {
31 | return $result->getDeletedCount();
32 | } elseif (isset($result['n'])) {
33 | return $result['n'];
34 | }
35 |
36 | return 0;
37 | }
38 |
39 | /**
40 | * @param Run|Job|JobTiming $object
41 | * @param string $action
42 | */
43 | protected function persist($object, $action = 'persist')
44 | {
45 | $objectManager = $this->getObjectManager();
46 | $objectManager->$action($object);
47 | $objectManager->flush();
48 | }
49 |
50 | /**
51 | * @return ObjectManager
52 | */
53 | abstract public function getObjectManager();
54 |
55 | /**
56 | * @param string $objectName
57 | */
58 | public function stopIdGenerator($objectName)
59 | {
60 | // Not needed for ODM
61 | }
62 |
63 | public function restoreIdGenerator($objectName)
64 | {
65 | // Not needed for ODM
66 | }
67 | }
68 |
--------------------------------------------------------------------------------
/ODM/JobTimingManager.php:
--------------------------------------------------------------------------------
1 | isRunning() ? 'running' : 'waiting').'.odm';
21 | }
22 |
23 | public function setRunning($flag)
24 | {
25 | $this->running = $flag;
26 | }
27 |
28 | public function isRunning()
29 | {
30 | return $this->running;
31 | }
32 |
33 | public function setColumnSource(\Dtc\GridBundle\Grid\Source\ColumnSource $columnSource)
34 | {
35 | $this->columnSource = $columnSource;
36 | }
37 |
38 | protected function getRunningQueryBuilder()
39 | {
40 | /** @var DocumentManager $documentManager */
41 | $documentManager = $this->jobManager->getObjectManager();
42 | $builder = $documentManager->createQueryBuilder($this->jobManager->getJobClass());
43 | $builder->field('status')->equals(BaseJob::STATUS_RUNNING);
44 | $builder->sort('startedAt', 'desc');
45 | $builder->limit($this->limit);
46 | $builder->skip($this->offset);
47 |
48 | return $builder;
49 | }
50 |
51 | public function __construct(JobManager $jobManager)
52 | {
53 | $this->jobManager = $jobManager;
54 |
55 | /** @var DocumentManager $documentManager */
56 | $documentManager = $jobManager->getObjectManager();
57 | parent::__construct($documentManager, $jobManager->getJobClass());
58 | }
59 |
60 | public function setColumns($columns)
61 | {
62 | if ($columns) {
63 | foreach ($columns as $column) {
64 | if ($column instanceof GridColumn) {
65 | $column->setOption('sortable', false);
66 | }
67 | }
68 | }
69 |
70 | parent::setColumns($columns);
71 | }
72 |
73 | public function getColumns()
74 | {
75 | if ($columns = parent::getColumns()) {
76 | return $columns;
77 | }
78 |
79 | if (!$this->columnSource) {
80 | $this->autoDiscoverColumns();
81 |
82 | return parent::getColumns();
83 | }
84 |
85 | $columnSourceInfo = $this->columnSource->getColumnSourceInfo($this->objectManager, 'Dtc\QueueBundle\Document\Job', false);
86 | $this->setColumns($columnSourceInfo->columns);
87 | $this->setDefaultSort($columnSourceInfo->sort);
88 | $this->setIdColumn($columnSourceInfo->idColumn);
89 |
90 | return parent::getColumns();
91 | }
92 |
93 | public function getDefaultSort()
94 | {
95 | return null;
96 | }
97 |
98 | protected function getQueryBuilder()
99 | {
100 | if ($this->isRunning()) {
101 | return $this->getRunningQueryBuilder();
102 | }
103 | $builder = $this->jobManager->getJobQueryBuilder();
104 | $builder->limit($this->limit);
105 | $builder->skip($this->offset);
106 |
107 | return $builder;
108 | }
109 | }
110 |
--------------------------------------------------------------------------------
/ODM/RunManager.php:
--------------------------------------------------------------------------------
1 | getObjectManager();
22 | /** @var Builder $queryBuilder */
23 | $queryBuilder = $objectManager->createQueryBuilder($this->getRunClass());
24 | $queryBuilder->find();
25 | $time = time() - 86400;
26 | $date = \DateTime::createFromFormat('U', strval($time));
27 | if (false === $date) {
28 | throw new \Exception("Could not create DateTime object from $time");
29 | }
30 | $date->setTimezone(new \DateTimeZone(date_default_timezone_get()));
31 | $queryBuilder->field('lastHeartbeatAt')->lt($date);
32 |
33 | return $queryBuilder->getQuery()->toArray();
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/ODM/StubLiveJobsGridSource.php:
--------------------------------------------------------------------------------
1 | isRunning() ? 'running' : 'waiting').'.odm';
13 | }
14 |
15 | public function setRunning($flag)
16 | {
17 | $this->running = $flag;
18 | }
19 |
20 | public function isRunning()
21 | {
22 | return $this->running;
23 | }
24 |
25 | public function __construct(JobManager $jobManager)
26 | {
27 | $this->jobManager = $jobManager;
28 | }
29 | }
30 |
--------------------------------------------------------------------------------
/ORM/JobTimingManager.php:
--------------------------------------------------------------------------------
1 | getObjectManagerReset();
14 | }
15 | }
16 |
--------------------------------------------------------------------------------
/ORM/LiveJobsGridSource.php:
--------------------------------------------------------------------------------
1 | isRunning() ? 'running' : 'waiting').'.orm';
21 | }
22 |
23 | public function __construct(JobManager $jobManager)
24 | {
25 | $this->jobManager = $jobManager;
26 | /** @var EntityManager $entityManager */
27 | $entityManager = $jobManager->getObjectManager();
28 | parent::__construct($entityManager, $jobManager->getJobClass());
29 | }
30 |
31 | /**
32 | * @param bool $flag
33 | */
34 | public function setRunning($flag)
35 | {
36 | $this->running = $flag;
37 | }
38 |
39 | public function isRunning()
40 | {
41 | return $this->running;
42 | }
43 |
44 | public function setColumnSource(\Dtc\GridBundle\Grid\Source\ColumnSource $columnSource)
45 | {
46 | $this->columnSource = $columnSource;
47 | }
48 |
49 | protected function getRunningQueryBuilder()
50 | {
51 | /** @var EntityManager $entityManager */
52 | $entityManager = $this->jobManager->getObjectManager();
53 | $queryBuilder = $entityManager->createQueryBuilder()->add('select', 'j');
54 | $queryBuilder->from($this->jobManager->getJobClass(), 'j');
55 | $queryBuilder->where('j.status = :status')->setParameter(':status', BaseJob::STATUS_RUNNING);
56 | $queryBuilder->orderBy('j.startedAt', 'DESC');
57 | $queryBuilder->setFirstResult($this->offset)
58 | ->setMaxResults($this->limit);
59 |
60 | return $queryBuilder;
61 | }
62 |
63 | protected function getQueryBuilder()
64 | {
65 | if ($this->isRunning()) {
66 | return $this->getRunningQueryBuilder();
67 | }
68 |
69 | $queryBuilder = $this->jobManager->getJobQueryBuilder();
70 | $queryBuilder->add('select', 'j');
71 | $queryBuilder->setFirstResult($this->offset)
72 | ->setMaxResults($this->limit);
73 |
74 | return $queryBuilder;
75 | }
76 |
77 | public function setColumns($columns)
78 | {
79 | if ($columns) {
80 | foreach ($columns as $column) {
81 | if ($column instanceof GridColumn) {
82 | $column->setOption('sortable', false);
83 | }
84 | }
85 | }
86 |
87 | parent::setColumns($columns);
88 | }
89 |
90 | public function getColumns()
91 | {
92 | if ($columns = parent::getColumns()) {
93 | return $columns;
94 | }
95 |
96 | if (!$this->columnSource) {
97 | $this->autoDiscoverColumns();
98 |
99 | return parent::getColumns();
100 | }
101 |
102 | $columnSourceInfo = $this->columnSource->getColumnSourceInfo($this->objectManager, 'Dtc\QueueBundle\Entity\Job', false);
103 | $this->setColumns($columnSourceInfo->columns);
104 | $this->setDefaultSort($columnSourceInfo->sort);
105 | $this->setIdColumn($columnSourceInfo->idColumn);
106 |
107 | return parent::getColumns();
108 | }
109 |
110 | public function getDefaultSort()
111 | {
112 | return null;
113 | }
114 |
115 | public function getCount()
116 | {
117 | $qb = $this->getQueryBuilder();
118 | $qb->resetDQLPart('orderBy');
119 | $qb->add('select', 'count(j)')
120 | ->setFirstResult(null)
121 | ->setMaxResults(null);
122 |
123 | return $qb->getQuery()
124 | ->getSingleScalarResult();
125 | }
126 | }
127 |
--------------------------------------------------------------------------------
/ORM/RunManager.php:
--------------------------------------------------------------------------------
1 | getObjectManagerReset();
16 | }
17 |
18 | /**
19 | * @return array|mixed
20 | *
21 | * @throws \Exception
22 | */
23 | protected function getOldLiveRuns()
24 | {
25 | /** @var EntityManager $objectManager */
26 | $objectManager = $this->getObjectManager();
27 | /** @var QueryBuilder $queryBuilder */
28 | $queryBuilder = $objectManager->createQueryBuilder();
29 | $queryBuilder->select(['r'])
30 | ->from($this->getRunClass(), 'r');
31 | $time = time() - 86400;
32 | $date = \DateTime::createFromFormat('U', strval($time));
33 | if (false === $date) {
34 | throw new \Exception("Could not create DateTime object from $time");
35 | }
36 | $date->setTimezone(new \DateTimeZone(date_default_timezone_get()));
37 | $queryBuilder->where('r.lastHeartbeatAt < :date');
38 | $queryBuilder->setParameter(':date', $date);
39 |
40 | return $queryBuilder->getQuery()->getResult();
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/ORM/StubLiveJobsGridSource.php:
--------------------------------------------------------------------------------
1 | isRunning() ? 'running' : 'waiting').'.orm';
14 | }
15 |
16 | public function __construct(JobManager $jobManager)
17 | {
18 | $this->jobManager = $jobManager;
19 | }
20 |
21 | /**
22 | * @param bool $flag
23 | */
24 | public function setRunning($flag)
25 | {
26 | $this->running = $flag;
27 | }
28 |
29 | public function isRunning()
30 | {
31 | return $this->running;
32 | }
33 | }
34 |
--------------------------------------------------------------------------------
/RabbitMQ/Job.php:
--------------------------------------------------------------------------------
1 | deliveryTag;
15 | }
16 |
17 | /**
18 | * @param mixed $deliveryTag
19 | */
20 | public function setDeliveryTag($deliveryTag)
21 | {
22 | $this->deliveryTag = $deliveryTag;
23 | }
24 | }
25 |
--------------------------------------------------------------------------------
/Redis/BaseJobManager.php:
--------------------------------------------------------------------------------
1 | cacheKeyPrefix = $cacheKeyPrefix;
37 | $this->hostname = gethostname() ?: '';
38 | $this->pid = getmypid();
39 |
40 | parent::__construct($runManager, $jobTimingManager, $jobClass);
41 | }
42 |
43 | public function setRedis(RedisInterface $redis)
44 | {
45 | $this->redis = $redis;
46 | }
47 |
48 | protected function getJobCacheKey($jobId)
49 | {
50 | return $this->cacheKeyPrefix.'_job_'.$jobId;
51 | }
52 |
53 | /**
54 | * @param string $jobCrc
55 | */
56 | protected function getJobCrcHashKey($jobCrc)
57 | {
58 | return $this->cacheKeyPrefix.'_job_crc_'.$jobCrc;
59 | }
60 |
61 | protected function getPriorityQueueCacheKey()
62 | {
63 | return $this->cacheKeyPrefix.'_priority';
64 | }
65 |
66 | protected function getWhenQueueCacheKey()
67 | {
68 | return $this->cacheKeyPrefix.'_when';
69 | }
70 |
71 | protected function getStatusCacheKey()
72 | {
73 | return $this->cacheKeyPrefix.'_status';
74 | }
75 |
76 | abstract protected function saveJob(Job $job);
77 |
78 | public function resetJob(RetryableJob $job)
79 | {
80 | if (!$job instanceof Job) {
81 | throw new \InvalidArgumentException('$job must be instance of '.Job::class);
82 | }
83 | $job->setStatus(BaseJob::STATUS_NEW);
84 | $job->setMessage(null);
85 | $job->setStartedAt(null);
86 | $job->setRetries($job->getRetries() + 1);
87 | $job->setUpdatedAt(Util::getMicrotimeDateTime());
88 | $this->saveJob($job);
89 |
90 | return true;
91 | }
92 |
93 | public function deleteJob(\Dtc\QueueBundle\Model\Job $job)
94 | {
95 | $jobId = $job->getId();
96 | $priorityQueue = $this->getPriorityQueueCacheKey();
97 | $whenQueue = $this->getWhenQueueCacheKey();
98 |
99 | $this->redis->zRem($priorityQueue, $jobId);
100 | $this->redis->zRem($whenQueue, $jobId);
101 | $this->redis->del([$this->getJobCacheKey($jobId)]);
102 | $this->redis->lRem($this->getJobCrcHashKey($job->getCrcHash()), 1, $jobId);
103 | }
104 | }
105 |
--------------------------------------------------------------------------------
/Redis/Job.php:
--------------------------------------------------------------------------------
1 | redis = $redis;
13 | $this->maxRetries = $maxRetries;
14 | }
15 |
16 | public function mGet(array $keys)
17 | {
18 | return $this->redis->mGet($keys);
19 | }
20 |
21 | public function hScan($key, &$cursor, $pattern = '', $count = 0)
22 | {
23 | return $this->redis->hScan($key, $cursor, $pattern, $count);
24 | }
25 |
26 | public function zScan($key, &$cursor, $pattern = '', $count = 0)
27 | {
28 | return $this->redis->zScan($key, $cursor, $pattern, $count);
29 | }
30 |
31 | public function zCount($key, $min, $max)
32 | {
33 | return $this->redis->zCount($key, $min, $max);
34 | }
35 |
36 | public function zAdd($zkey, $score, $value)
37 | {
38 | return $this->redis->zadd($zkey, $score, $value);
39 | }
40 |
41 | public function set($key, $value)
42 | {
43 | return $this->redis->set($key, $value);
44 | }
45 |
46 | public function get($key)
47 | {
48 | return $this->redis->get($key);
49 | }
50 |
51 | public function hIncrBy($key, $hashKey, $value)
52 | {
53 | return $this->redis->hIncrBy($key, $hashKey, $value);
54 | }
55 |
56 | public function hGetAll($key)
57 | {
58 | return $this->redis->hGetAll($key);
59 | }
60 |
61 | public function setEx($key, $seconds, $value)
62 | {
63 | return $this->redis->setex($key, $seconds, $value);
64 | }
65 |
66 | public function lRem($lKey, $count, $value)
67 | {
68 | return $this->redis->lrem($lKey, $value, $count);
69 | }
70 |
71 | public function lPush($lKey, array $values)
72 | {
73 | $args = $values;
74 | array_unshift($args, $lKey);
75 |
76 | return call_user_func_array([$this->redis, 'lPush'], $args);
77 | }
78 |
79 | public function lRange($lKey, $start, $stop)
80 | {
81 | return $this->redis->lrange($lKey, $start, $stop);
82 | }
83 |
84 | public function del(array $keys)
85 | {
86 | return $this->redis->del($keys);
87 | }
88 |
89 | public function zRem($zkey, $value)
90 | {
91 | return $this->redis->zrem($zkey, $value);
92 | }
93 |
94 | public function zPop($key)
95 | {
96 | $retries = 0;
97 | do {
98 | $this->redis->watch($key);
99 | $elements = $this->redis->zrange($key, 0, 0);
100 | if (empty($elements)) {
101 | $this->redis->unwatch();
102 |
103 | return null;
104 | }
105 | $result = $this->redis->multi()
106 | ->zrem($key, $elements[0])
107 | ->exec();
108 | if (false !== $result) {
109 | return $elements[0];
110 | }
111 | ++$retries;
112 | } while ($retries < $this->maxRetries);
113 |
114 | return null;
115 | }
116 |
117 | public function zPopByMaxScore($key, $max)
118 | {
119 | $retries = 0;
120 | do {
121 | $this->redis->watch($key);
122 | $elements = $this->redis->zrangebyscore($key, 0, $max, ['limit' => [0, 1]]);
123 | if (empty($elements)) {
124 | $this->redis->unwatch();
125 |
126 | return null;
127 | }
128 | $result = $this->redis->multi()
129 | ->zrem($key, $elements[0])
130 | ->exec();
131 | if (false !== $result) {
132 | return $elements[0];
133 | }
134 | ++$retries;
135 | } while ($retries < $this->maxRetries);
136 |
137 | return null;
138 | }
139 | }
140 |
--------------------------------------------------------------------------------
/Redis/RedisInterface.php:
--------------------------------------------------------------------------------
1 | redis->zScan($cacheKey, $cursor, '', 100)) {
13 | $jobs = $this->redis->mget(array_map(function ($item) {
14 | return $this->getJobCacheKey($item);
15 | }, array_keys($jobs)));
16 | $this->extractStatusResults($jobs, $results);
17 | if (0 === $cursor) {
18 | break;
19 | }
20 | }
21 |
22 | return $results;
23 | }
24 |
25 | protected function extractStatusResults(array $jobs, array &$results)
26 | {
27 | foreach ($jobs as $jobMessage) {
28 | if (is_string($jobMessage)) {
29 | $job = new Job();
30 | $job->fromMessage($jobMessage);
31 | $resultHashKey = $job->getWorkerName().'->'.$job->getMethod().'()';
32 | if (!isset($results[$resultHashKey][BaseJob::STATUS_NEW])) {
33 | $results[$resultHashKey] = static::getAllStatuses();
34 | }
35 | if (!isset($results[$resultHashKey][BaseJob::STATUS_NEW])) {
36 | $results[$resultHashKey][BaseJob::STATUS_NEW] = 0;
37 | }
38 | ++$results[$resultHashKey][BaseJob::STATUS_NEW];
39 | }
40 | }
41 | }
42 |
43 | protected function extractStatusHashResults(array $hResults, array &$results)
44 | {
45 | foreach ($hResults as $key => $value) {
46 | list($workerName, $method, $status) = explode(',', $key);
47 | $resultHashKey = $workerName.'->'.$method.'()';
48 | if (!isset($results[$resultHashKey])) {
49 | $results[$resultHashKey] = static::getAllStatuses();
50 | }
51 | if (!isset($results[$resultHashKey][$status])) {
52 | $results[$resultHashKey][$status] = 0;
53 | }
54 | $results[$resultHashKey][$status] += $value;
55 | }
56 | }
57 | }
58 |
--------------------------------------------------------------------------------
/Resources/config/dtc_queue.yaml:
--------------------------------------------------------------------------------
1 | dtc_queue:
2 | # For full configuration options see:
3 | # https://github.com/mmucklo/DtcQueueBundle/blob/master/Resources/doc/full-configuration.md
4 | manager:
5 | # This parameter is required and should typically be set to one of:
6 | # [odm|orm|beanstalkd|rabbit_mq|redis]
7 | job: orm
8 | timings:
9 | # Set this to true to record job timing history in a separate table
10 | # (note: for beanstalkd|rabbit_mq|redis, job_timing or run needs to
11 | # be set to one of orm|odm under the manager section)
12 | record: false
13 | priority:
14 | max: 255
15 | direction: desc
16 | admin:
17 | # chartjs is used to render the job timings graph in the admin section
18 | chartjs: 'https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.7.2/Chart.bundle.min.js'
19 |
20 |
--------------------------------------------------------------------------------
/Resources/config/routing.yml:
--------------------------------------------------------------------------------
1 | dtc_queue:
2 | path: /dtc_queue/
3 | defaults: { _controller: dtc_queue.controller.queue:status }
4 |
5 | dtc_queue_queue_status:
6 | path: /dtc_queue/status
7 | defaults: { _controller: dtc_queue.controller.queue:status }
8 |
9 | dtc_queue_jobs_all:
10 | path: /dtc_queue/jobs_all
11 | defaults: { _controller: dtc_queue.controller.queue:jobsAll }
12 |
13 | dtc_queue_jobs:
14 | path: /dtc_queue/jobs
15 | defaults: { _controller: dtc_queue.controller.queue:jobs }
16 |
17 | dtc_queue_runs:
18 | path: /dtc_queue/runs
19 | defaults: { _controller: dtc_queue.controller.queue:runs }
20 |
21 | dtc_queue_workers:
22 | path: /dtc_queue/workers
23 | defaults: { _controller: dtc_queue.controller.queue:workers }
24 |
25 | dtc_queue_jobs_running:
26 | path: /dtc_queue/jobs-running
27 | defaults: { _controller: dtc_queue.controller.queue:jobsRunning }
28 |
29 | dtc_queue_archive:
30 | path: /dtc_queue/archive
31 | defaults: { _controller: dtc_queue.controller.queue:archive }
32 | methods: POST
33 |
34 | dtc_queue_reset_stalled:
35 | path: /dtc_queue/reset-stalled
36 | defaults: { _controller: dtc_queue.controller.queue:resetStalled }
37 |
38 | dtc_queue_prune_stalled:
39 | path: /dtc_queue/prune-stalled
40 | defaults: { _controller: dtc_queue.controller.queue:pruneStalled }
41 |
42 | dtc_queue_trends:
43 | path: /dtc_queue/trends
44 | defaults: { _controller: dtc_queue.controller.trends:trends }
45 |
46 | dtc_queue_timings:
47 | path: /dtc_queue/timings
48 | defaults: { _controller: dtc_queue.controller.trends:timings }
49 |
50 |
--------------------------------------------------------------------------------
/Resources/doc/images/trends-example.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mmucklo/DtcQueueBundle/c560509477f965171de77c7d63f08acd49d92827/Resources/doc/images/trends-example.png
--------------------------------------------------------------------------------
/Resources/doc/symfony2-3.md:
--------------------------------------------------------------------------------
1 | # Symfony 2 / 3 Installation Instructions
2 |
3 | Then add the bundle to __AppKernel.php__:
4 |
5 | ```php
6 |
58 | ```
59 |
60 | #### Error
61 |
62 | When visiting /dtc_queue/ in the browser, the following error is encountered:
63 |
64 | ```
65 | The parameter "dtc_grid.theme.css" must be defined
66 | ```
67 |
68 | Solution:
69 |
70 | ```
71 | composer require mmucklo/grid-bundle
72 | ```
73 |
--------------------------------------------------------------------------------
/Resources/docker/php/ubuntu/Dockerfile:
--------------------------------------------------------------------------------
1 | # For building PHP from source for debugging purposes (specifically used in troubleshooting Issue #98)
2 | FROM ubuntu:19.10
3 | ENV DEBIAN_FRONTEND=noninteractive
4 | RUN apt update
5 | RUN apt install -y apt-utils
6 | RUN apt install -y perl-modules
7 | RUN apt upgrade -y
8 | RUN apt install -y libfreetype6-dev
9 | RUN apt install -y git
10 | RUN apt install -y vim
11 | RUN apt install -y libzip-dev libldap2-dev libxml2-dev libpng-dev libicu-dev libbz2-dev libtidy-dev
12 | RUN apt install -y libmemcached-dev
13 | RUN apt install -y libssl-dev
14 | RUN apt install -y build-essential
15 | RUN apt install -y autoconf automake re2c libtool bison
16 | RUN apt install -y curl wget
17 | RUN apt install -y netcat
18 | RUN git clone https://git.php.net/repository/php-src.git
19 | RUN apt install -y sendmail gawk
20 | RUN apt install -y libcurl4-openssl-dev zlibc libgd-dev libfreetype6 libjpeg9 libgdbm-dev libsodium-dev mysql-common mysql-client postgresql-11 libreadline5 libreadline-dev
21 | RUN apt install -y libsqlite3-dev
22 |
--------------------------------------------------------------------------------
/Resources/views/Queue/grid.html.twig:
--------------------------------------------------------------------------------
1 | {% extends "@DtcGrid/Page/datatables.html.twig" %}
2 |
3 | {% block grid %}
4 | {% include '@DtcQueue/Queue/nav.html.twig' %}
5 |
6 | {{ dtc_queue_grid_label1 }}
7 | {{ dtc_grid.render | raw }}
8 | {{ dtc_queue_grid_label2 }}
9 | {{ archive_grid.render | raw }}
10 | {% endblock %}
11 |
12 |
--------------------------------------------------------------------------------
/Resources/views/Queue/jobs.html.twig:
--------------------------------------------------------------------------------
1 | {% extends "@DtcGrid/Page/datatables.html.twig" %}
2 | {% import "@DtcQueue/Queue/macros.html.twig" as queue_macros %}
3 | {% block title %}DtcQueue - Admin - Waiting Jobs{% endblock %}
4 | {% block grid %}
5 | {% include '@DtcQueue/Queue/nav.html.twig' %}
6 |
7 | Waiting Jobs
8 | {{ dtc_grid.render | raw }}
9 | {{ queue_macros.worker_method_button('Archive', 'dtc_queue_archive', worker_methods, prompt_message, 'archive') }}
10 |
11 | {% endblock %}
12 |
--------------------------------------------------------------------------------
/Resources/views/Queue/jobs_running.html.twig:
--------------------------------------------------------------------------------
1 | {% extends "@DtcGrid/Page/datatables.html.twig" %}
2 | {% import "@DtcQueue/Queue/macros.html.twig" as queue_macros %}
3 | {% block title %}DtcQueue - Admin - Running Jobs{% endblock %}
4 | {% block grid %}
5 | {% include '@DtcQueue/Queue/nav.html.twig' %}
6 |
7 | Running Jobs
8 | (Including Stalled Jobs)
9 | {{ dtc_grid.render | raw }}
10 | {{ queue_macros.worker_method_button('Prune Stalled', 'dtc_queue_prune_stalled', worker_methods, prompt_message, 'prunestalled') }}
11 | {{ queue_macros.worker_method_button('Reset Stalled', 'dtc_queue_reset_stalled', worker_methods, prompt_message, 'resetstalled') }}
12 |
13 | {% endblock %}
14 |
--------------------------------------------------------------------------------
/Resources/views/Queue/macros.html.twig:
--------------------------------------------------------------------------------
1 | {% macro worker_method_button(button_text, route_name, worker_methods, prompt_message, tag) %}
2 |
3 |
19 |
24 | {% endmacro %}
25 |
--------------------------------------------------------------------------------
/Resources/views/Queue/nav.html.twig:
--------------------------------------------------------------------------------
1 |
20 |
--------------------------------------------------------------------------------
/Resources/views/Queue/status.html.twig:
--------------------------------------------------------------------------------
1 | {% extends "@DtcQueue/layout.html.twig" %}
2 | {% block title %}DtcQueue - Admin - Status{% endblock %}
3 | {% block content %}
4 | {% for worker, info in status %}
5 | {{ worker }}
6 |
7 | {% for job_status, count in info %}
8 | - {{ job_status }}: {{ count|e }}
9 | {% endfor %}
10 |
11 | {% endfor %}
12 | {% endblock %}
13 |
--------------------------------------------------------------------------------
/Resources/views/Queue/trends.html.twig:
--------------------------------------------------------------------------------
1 | {% extends "@DtcQueue/layout.html.twig" %}
2 | {% block title %}DtcQueue - Admin - Trends{% endblock %}
3 | {% block headcontent %}
4 |
26 | {% endblock %}
27 |
28 | {% block content %}
29 | {% if not record_timings %}
30 | Timings Not Recorded
31 |
32 | Job timings are not being recorded.
33 |
34 |
35 | The following configuration entry needs to be enabled in config/packages/dtc_queue.yaml:
36 |
37 |
38 | dtc_queue:
39 | # ...
40 | record_timings: true
41 |
42 | {% else %}
43 | {% if not found_year_function %}
44 | {{ found_year_reason }}
45 | {% else %}
46 |
47 |
48 |
49 |
56 | Loading...
57 |
58 |
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
71 |
72 | {% endif %}
73 | {% endif %}
74 | {% endblock %}
75 |
--------------------------------------------------------------------------------
/Resources/views/Queue/workers.html.twig:
--------------------------------------------------------------------------------
1 | {% extends "@DtcQueue/layout.html.twig" %}
2 | {% block title %}DtcQueue - Admin - Workers{% endblock %}
3 | {% block content %}
4 |
5 | {% for name, className in workers %}
6 | - {{ name }}: {{ className }}
7 | {% endfor %}
8 |
9 | {% endblock %}
10 |
--------------------------------------------------------------------------------
/Resources/views/layout.html.twig:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | {% block title %}DtcQueue - Admin{% endblock %}
6 | {% for stylesheet in css %}
7 | {% if stylesheet.url is defined %}
8 |
12 | {% else %}
13 |
14 | {% endif %}
15 | {% endfor %}
16 | {% for javascript in js %}
17 | {% if javascript.url is defined %}
18 |
22 | {% else %}
23 |
24 | {% endif %}
25 | {% endfor %}
26 | {% block headcontent %}
27 | {% endblock %}
28 |
29 |
30 | {% block main %}
31 | {% include '@DtcQueue/Queue/nav.html.twig' %}
32 |
33 |
34 |
35 | {% block content %}
36 | {% endblock %}
37 |
38 |
39 |