├── .gitignore ├── composer.json ├── src └── ProcessExecutive │ ├── ExecutiveDaemonControl.php │ ├── ExecutiveControl.php │ ├── ExecutiveDaemon.php │ └── Executive.php ├── LICENSE └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | /.buildpath 2 | /.project 3 | /.settings/ 4 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "jayesbe/php-process-executive", 3 | "type" : "project", 4 | "description" : "Control execution of script in child processes", 5 | "version": "0.9.5", 6 | "homepage" : "https://github.com/jayesbe/php-process-executive", 7 | "license" : "MIT", 8 | "support" : { 9 | "email" : "jayesbe@users.noreply.github.com", 10 | "wiki" : "https://github.com/jayesbe/php-process-executive/wiki", 11 | "source" : "https://github.com/jayesbe/php-process-executive" 12 | }, 13 | "authors" : [{ 14 | "name" : "Jayes Be", 15 | "email" : "jayesbe@users.noreply.github.com", 16 | "role" : "Architect" 17 | } 18 | ], 19 | "require" : { 20 | "php" : ">=5.3.3" 21 | }, 22 | "minimum-stability" : "stable", 23 | "autoload" : { 24 | "psr-4" : { 25 | "ProcessExecutive\\" : "src/ProcessExecutive" 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/ProcessExecutive/ExecutiveDaemonControl.php: -------------------------------------------------------------------------------- 1 | 6 | * 7 | * For the full copyright and license information, please view the LICENSE 8 | * file that was distributed with this source code. 9 | */ 10 | namespace ProcessExecutive; 11 | 12 | /** 13 | * Interface for the process executive to control its operation 14 | * 15 | * @author Jayes 16 | */ 17 | interface ExecutiveDaemonControl extends ExecutiveControl 18 | { 19 | 20 | /** 21 | * return a queue of items 22 | * 23 | * @return array 24 | */ 25 | public function getQueue(); 26 | 27 | /** 28 | * Should the daemon stop ? 29 | * 30 | * @return boolean true to stop | false to continue 31 | */ 32 | public function stop(); 33 | } 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Jayes 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /src/ProcessExecutive/ExecutiveControl.php: -------------------------------------------------------------------------------- 1 | 6 | * 7 | * For the full copyright and license information, please view the LICENSE 8 | * file that was distributed with this source code. 9 | */ 10 | namespace ProcessExecutive; 11 | 12 | /** 13 | * Interface for the process executive to control its operation 14 | * 15 | * @author Jayes 16 | */ 17 | interface ExecutiveControl 18 | { 19 | 20 | /** 21 | * Close parent resources 22 | */ 23 | public function closeResources(); 24 | 25 | /** 26 | * Reload parent resources 27 | */ 28 | public function reloadResources(); 29 | 30 | /** 31 | * Returns the max number of processes the executive will allow to be executed at once 32 | * 33 | * @return int 34 | */ 35 | public function getMaxProcesses(); 36 | 37 | /** 38 | * Return's an item or items to process from the queue 39 | * 40 | * @return mixed 41 | */ 42 | public function getProcessItem(&$queue); 43 | 44 | /** 45 | * Executes in the child process 46 | */ 47 | public function executeChildProcess($item); 48 | } 49 | -------------------------------------------------------------------------------- /src/ProcessExecutive/ExecutiveDaemon.php: -------------------------------------------------------------------------------- 1 | 6 | * 7 | * For the full copyright and license information, please view the LICENSE 8 | * file that was distributed with this source code. 9 | */ 10 | namespace ProcessExecutive; 11 | 12 | /** 13 | * The ExecutiveDaemon will use the provided control interface to launch child processes indefinitely. 14 | * 15 | * @author Jayes 16 | */ 17 | class ExecutiveDaemon extends Executive 18 | { 19 | 20 | public function __construct(ExecutiveDaemonControl $control) 21 | { 22 | parent::__construct($control); 23 | } 24 | 25 | public function run($sleepTime = 60) 26 | { 27 | while (1) { 28 | $queue = $this->getControl()->getQueue(); 29 | 30 | $this->execute($queue); 31 | 32 | if ($this->getControl()->stop()) { 33 | break; 34 | } 35 | 36 | if ($sleepTime == 0) { 37 | break; 38 | } 39 | $this->real_sleep($sleepTime); 40 | } 41 | 42 | // daemon is being exited.. 43 | if ($this->areResourcesClosed()) { 44 | $this->getControl()->reloadResources(); 45 | } 46 | 47 | while (!empty($this->procs)) { 48 | $pid = array_shift($this->procs); 49 | $waitpid = pcntl_waitpid($pid, $status, WNOHANG | WUNTRACED); 50 | if ($waitpid == 0) { 51 | array_push($this->procs, $pid); 52 | } 53 | } 54 | } 55 | 56 | /** 57 | * (non-PHPdoc) 58 | * 59 | * @see \ProcessExecutive\Executive::getControl() 60 | * 61 | * @return ExecutiveDaemonControl 62 | */ 63 | protected function getControl() 64 | { 65 | return parent::getControl(); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/ProcessExecutive/Executive.php: -------------------------------------------------------------------------------- 1 | 6 | * 7 | * For the full copyright and license information, please view the LICENSE 8 | * file that was distributed with this source code. 9 | */ 10 | namespace ProcessExecutive; 11 | 12 | /** 13 | * The Executive will use the provided control interface to launch child processes. 14 | * 15 | * @author Jayes 16 | */ 17 | class Executive 18 | { 19 | private $control; 20 | 21 | private $maxProcs; 22 | 23 | private $resourcesClosed; 24 | 25 | protected $procs; 26 | 27 | public function __construct(ExecutiveControl $control) 28 | { 29 | $this->control = $control; 30 | $this->procs = array(); 31 | $this->maxProcs = $control->getMaxProcesses(); 32 | $this->resourcesClosed = false; 33 | } 34 | 35 | /** 36 | * 37 | * @return ExecutiveControl 38 | */ 39 | protected function getControl() 40 | { 41 | return $this->control; 42 | } 43 | 44 | /** 45 | * 46 | * @return boolean 47 | */ 48 | protected function areResourcesClosed() 49 | { 50 | return $this->resourcesClosed; 51 | } 52 | 53 | /** 54 | * 55 | * @param array $queue 56 | */ 57 | public function execute(&$queue) 58 | { 59 | while (!empty($queue)) { 60 | 61 | // how many concurrent processes ? 62 | if (count($this->procs) < $this->maxProcs) { 63 | 64 | // we generally begin with our connections closed so our forks are connection clean 65 | if (! $this->areResourcesClosed()) { 66 | $this->getControl()->closeResources(); 67 | $this->resourcesClosed = true; 68 | } 69 | 70 | $item = $this->getControl()->getProcessItem($queue); 71 | 72 | // fork 73 | $pid = pcntl_fork(); 74 | if ($pid == - 1) { 75 | // this should probably do something better 76 | die('Could not fork '); 77 | } 78 | // parent.. immediately check the child's status 79 | else if ($pid) { 80 | if (pcntl_waitpid($pid, $status, WNOHANG) == 0) { 81 | $this->procs[$pid] = $pid; 82 | continue; // until max_procs is reached 83 | } 84 | } 85 | // child.. 86 | else { 87 | $this->getControl()->executeChildProcess($item); 88 | exit(0); 89 | } 90 | } 91 | 92 | // reopen our connection if closed 93 | if ($this->areResourcesClosed()) { 94 | $this->getControl()->reloadResources(); 95 | $this->resourcesClosed = false; 96 | } 97 | 98 | // loop our currently processing procs and clear out the completed ones 99 | $this->wait(); 100 | } 101 | 102 | // we have to check here again if the connection is closed. 103 | if ($this->areResourcesClosed()) { 104 | $this->getControl()->reloadResources(); 105 | $this->resourcesClosed = false; 106 | } 107 | 108 | // now clean up any remaining process 109 | $this->waitpids(); 110 | } 111 | 112 | /** 113 | * 114 | */ 115 | protected function waitpids() 116 | { 117 | foreach ($this->procs as $pid) { 118 | $waitpid = pcntl_waitpid($pid, $status, WNOHANG | WUNTRACED); 119 | if ($waitpid !== 0) { 120 | $this->check($pid, $status); 121 | } 122 | } 123 | 124 | // if proces count isnt 0, lets just wait (suspend execution) 125 | if (count($this->procs) !== 0) { 126 | $pid = pcntl_wait($status, WUNTRACED); 127 | $this->check($pid, $status); 128 | $this->waitpids(); 129 | } 130 | } 131 | 132 | /** 133 | * 134 | */ 135 | protected function wait() 136 | { 137 | while (($pid = pcntl_wait($status, WNOHANG | WUNTRACED)) > 0) { 138 | $this->check($pid, $status); 139 | } 140 | 141 | // if the procs count is at the max lets just wait (suspend execution) 142 | if (count($this->procs) == $this->maxProcs) { 143 | $pid = pcntl_wait($status, WUNTRACED); 144 | $this->check($pid, $status); 145 | $this->wait(); 146 | } 147 | } 148 | 149 | /** 150 | * 151 | * @param int $pid 152 | * @param int $status 153 | */ 154 | protected function check($pid, $status) 155 | { 156 | if (pcntl_wifstopped($status)) { 157 | posix_kill($pid, SIGKILL); 158 | } 159 | unset($this->procs[$pid]); 160 | } 161 | 162 | /** 163 | * @deprecated 164 | * @param int $seconds 165 | */ 166 | protected function real_sleep($seconds) 167 | { 168 | $start = microtime(true); 169 | for ($i = 1; $i <= $seconds; $i ++) { 170 | @time_sleep_until($start + $i); 171 | } 172 | } 173 | } 174 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | php-process-executive 2 | ===================== 3 | 4 | Execute forked processing easily. 5 | 6 | **Features** 7 | 8 | * Execute many concurrent child processes keeping the parent process clean. 9 | * Execute as a daemon to keep the parent process alive. Combine with CRON and supervisor for an effective and lightweight PHP Daemon. 10 | 11 | **Requirements** 12 | 13 | * Process Control (PCNTL) 14 | 15 | Installation 16 | ------------ 17 | 18 | In order to install php-process-executive you need the PHP Process Control Extension. 19 | 20 | ### On Ubuntu 21 | 22 | Open a command console and execute the 23 | following command to install PCNTL: 24 | 25 | ```bash 26 | $ pecl install pcntl 27 | ``` 28 | 29 | Usage 30 | ----- 31 | 32 | Here is a sample implementation of a Symfony2 Command using ProcessExecutive 33 | 34 | ```php 35 | namespace FooBar\AppBundle\Command; 36 | 37 | // use Symfony\Component\Console\Command\Command; 38 | use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand; 39 | use Symfony\Component\Console\Input\InputArgument; 40 | use Symfony\Component\Console\Input\InputInterface; 41 | use Symfony\Component\Console\Output\OutputInterface; 42 | use Symfony\Component\DependencyInjection\ContainerAware; 43 | use FooBar\UserBundle\Entity\User; 44 | 45 | use ProcessExecutive\Executive; 46 | use ProcessExecutive\ExecutiveControl; 47 | 48 | /** 49 | * GenerateUsers command for testing purposes. 50 | * 51 | * Will generate random users 52 | * 53 | * You could also extend from Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand 54 | * to get access to the container via $this->getContainer(). 55 | * 56 | * @author Jayesbe 57 | */ 58 | class GenerateUsersCommand extends ContainerAwareCommand implements ExecutiveControl 59 | { 60 | const MAX_USERS = 5000000; 61 | 62 | private 63 | 64 | $userSize, 65 | 66 | $totalGenerated, 67 | 68 | $output; 69 | 70 | /** 71 | * {@inheritdoc} 72 | */ 73 | protected function configure() 74 | { 75 | $this->userSize = 0; 76 | $this->totalGenerated = 0; 77 | 78 | $this 79 | ->setName('foobar:generate:users') 80 | ->setDescription('Populate database with random users.') 81 | ->addArgument('size', InputArgument::OPTIONAL, 'Number of Users to generate', self::MAX_USERS) 82 | ->setHelp(<<%command.name% will populate the database with randomly generated users. 84 | 85 | The optional argument specifies the size of the population to generate (up to a maximum of 5 million): 86 | 87 | php %command.full_name% 5000000 88 | EOF 89 | ); 90 | } 91 | 92 | /** 93 | * {@inheritdoc} 94 | */ 95 | protected function execute(InputInterface $input, OutputInterface $output) 96 | { 97 | $this->output = $output; 98 | 99 | $this->userSize = intval($input->getArgument('size')); 100 | if ($this->userSize > self::MAX_USERS) { 101 | $output->writeln("Attempted to populate with ".$this->userSize.' users. Max allowed '.self::MAX_USERS); 102 | return; 103 | } 104 | $output->writeln("Attempting to populate with ".$this->userSize.' users...'); 105 | 106 | // we call these to make sure they are created in the parent 107 | $doctrine = $this->getContainer()->get('doctrine'); 108 | $em = $doctrine->getManager(); 109 | 110 | // we need to create our Executive here 111 | $processor = new Executive($this); 112 | 113 | // your queue can be anything. 114 | $queue = array(0 => $this->userSize); 115 | 116 | // execute our queue 117 | $processor->execute($queue); 118 | 119 | $output->writeln(sprintf('Population Created: %s users!', $this->totalGenerated)); 120 | } 121 | 122 | public function closeResources() 123 | { 124 | // close all db connections and memcache connections. 125 | // anything that requires the child to have its own resource since 126 | // the parent cannot share its resources with its children. 127 | $this->getContainer()->get('doctrine')->getManager()->getConnection()->close(); 128 | } 129 | 130 | public function reloadResources() 131 | { 132 | $this->getContainer()->get('doctrine')->getManager()->getConnection()->connect(); 133 | } 134 | 135 | public function getMaxProcesses() 136 | { 137 | // will create and maintain 8 concurrent processes 138 | return 8; 139 | } 140 | 141 | public function getProcessItem(&$queue) 142 | { 143 | // handle your queue item. 144 | // since we are generating users and our queue only contains a count 145 | // we will generate a user id and return that as our item 146 | // we will then decrease the size of users we need to generate 147 | // the system will halt processing when the queue is empty 148 | 149 | if ($queue[0] == 0) { 150 | throw new \Exception('Empty Queue.'); 151 | } 152 | 153 | $uid = uniqid("u",true); 154 | --$queue[0]; 155 | $this->totalGenerated++; 156 | 157 | if ($queue[0] == 0) { 158 | $queue = null; 159 | $queue = array(); 160 | } 161 | 162 | return $uid; 163 | } 164 | 165 | public function executeChildProcess($uid) 166 | { 167 | // now the main bits of processing we want done in each child. 168 | 169 | // get doctrine and entity manager and reload connection 170 | // this part must not use the parent process as we need to create all new object references for each child 171 | $doctrine = $this->getContainer()->get('doctrine'); 172 | $em = $doctrine->getManager(); 173 | 174 | // and a completely separate db connection per child 175 | $em->getConnection()->connect(); 176 | 177 | $user = new User(); 178 | $user->setEmail($uid.'@example.org'); 179 | $user->setUsername($uid); 180 | $user->setPlainPassword($uid); 181 | $user->setEnabled(true); 182 | 183 | // if you use random number generate mt_rand() you need to seed for each child 184 | // otherwise the childs will all use the same seed from the parent. 185 | // seed mt_rand 186 | // mt_srand(); 187 | 188 | $em->persist($user); 189 | $em->flush(); 190 | 191 | $this->output->writeln("User Generated: ".$user->getId()); 192 | } 193 | } 194 | ``` 195 | 196 | Notes 197 | ----- 198 | 199 | Doctrine is known to continually increase memory in a CLI environment. Its recommended to use PDO. However if you really want to use Doctrine, the only efficient way is to make sure that memory is cleared out when youre done with it. We have been utilizing this same code for the past two years in production. We use it for a variety of applications in both single run and daemonized modes. We use it for both Symfony 1 / Doctrine 1 and Symfony 2 / Doctrine 2 with the same results. A parent process that doesn't move when it comes to memory consumption and cpu usage. We also combine our execution with 'nice' which allows us to control the process priority of the parent and child processes executed. 200 | 201 | for example, the above Symfony2 Command can be run as 202 | 203 | ```bash 204 | nice -n 10 -- app/console "foobar:generate:users" -size=100 205 | ``` 206 | 207 | The performance will be based on how much work your child processes do and potentially how often they hit the disk. Vary the getMaxProcess() return value between 1, 2, 4, 8 or more to see how much you can eak out of it. 208 | 209 | The above Symfony2 Command is perfect to compare timing. 210 | 211 | for example, on a simple i5 with 4GB of available memor, creating 100 users with: 212 | 213 | ```bash 214 | time app/console foobar:generate:users 100 215 | ``` 216 | 217 | 2 processes 218 | 219 | ```bash 220 | real 0m12.589s 221 | user 0m12.875s 222 | sys 0m2.936s 223 | ``` 224 | 225 | 4 processes 226 | 227 | ```bash 228 | real 0m10.658s 229 | user 0m8.501s 230 | sys 0m2.628s 231 | ``` 232 | 233 | 8 processes 234 | 235 | ```bash 236 | real 0m15.807s 237 | user 0m9.249s 238 | sys 0m2.757s 239 | ``` 240 | 241 | Most importantly is memory consumption. The above Symfony2 Command results in a parent process that consumes 40 MB of memory and does not move. The process can run indefinitely without chewing up all the memory in the system. 242 | 243 | LEGAL DISCLAIMER 244 | ---------------- 245 | 246 | This software is published under the MIT License, which states that: 247 | 248 | > THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 249 | > IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 250 | > FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 251 | > AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 252 | > LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 253 | > OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 254 | > SOFTWARE. 255 | 256 | ----- 257 | --------------------------------------------------------------------------------