├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── bin └── resque-pool ├── composer.json ├── composer.lock ├── config └── resque-pool.yml.example ├── lib └── Resque │ └── Pool │ ├── Cli.php │ ├── Configuration.php │ ├── Logger.php │ ├── Platform.php │ └── Pool.php ├── phpstan.neon ├── phpunit.xml └── test ├── Resque └── Pool │ └── Tests │ ├── BaseTestCase.php │ ├── ConfigurationTest.php │ ├── Mock │ └── Worker.php │ └── PoolTest.php ├── bootstrap.php └── misc ├── resque-pool-custom.yml.php └── resque-pool.yml /.gitignore: -------------------------------------------------------------------------------- 1 | .*.swp 2 | vendor/* 3 | resque-pool.yml 4 | 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: php 2 | php: 3 | - "5.5" 4 | - "5.4" 5 | install: 6 | - composer install 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (C) 2010 by Nicholas Evans , 2012 by Erik Bernhardson , et al. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | PHP Resque Pool 2 | =============== 3 | 4 | Php resque pool is a port of [resque-pool](http://github.com/nevans/resque-pool) 5 | for managing [php-resque](http://github.com/chrisboulton/php-resque) workers. 6 | Given a config file, it manages your workers for you, starting up the appropriate 7 | number of workers for each worker type. 8 | 9 | Benefits 10 | --------- 11 | 12 | * Less config - With a simple YAML file, you can start up a pool daemon. 13 | * Monitoring - If a worker dies for some reason, php-resque-pool will start 14 | another. 15 | * Easily change worker distribution - To change your worker counts just update 16 | the YAML file and send the manager a HUP signal. 17 | 18 | How to use 19 | ---------- 20 | 21 | ### YAML file config 22 | 23 | Create a `config/resque-pool.yml` (or `resque-pool.yml`) with your worker 24 | counts. The YAML file supports both using root level defaults as well as 25 | environment specific overrides (`RESQUE_ENV` environment variables can be 26 | used to determine environment). For example in `config/resque-pool.yml`: 27 | 28 | foo: 1 29 | bar: 2 30 | "foo,bar,baz": 1 31 | 32 | production: 33 | "foo,bar,baz": 4 34 | ### Start the pool manager 35 | 36 | Then you can start the queues via: 37 | 38 | bin/resque-pool --daemon --environment production 39 | 40 | This will start up seven worker processes, one exclusively for the foo queue, 41 | two exclusively for the bar queue, and four workers looking at all queues in 42 | priority. With the config above, this is similar to if you ran the following: 43 | 44 | QUEUES=foo php resque.php 45 | QUEUES=bar php resque.php 46 | QUEUES=bar php resque.php 47 | QUEUES=foo,bar,baz php resque.php 48 | QUEUES=foo,bar,baz php resque.php 49 | QUEUES=foo,bar,baz php resque.php 50 | QUEUES=foo,bar,baz php resque.php 51 | 52 | The pool manager will stay around monitoring the resque worker parents, giving 53 | three levels: a single pool manager, many worker parents, and one worker child 54 | per worker (when the actual job is being processed). For example, `ps -ef f | 55 | grep [r]esque` (in Linux) might return something like the following: 56 | 57 | resque 13858 1 0 13:44 ? S 0:02 resque-pool-manager: managing [13867, 13875, 13871, 13872, 13868, 13870, 13876] 58 | resque 13867 13858 0 13:44 ? S 0:00 \_ resque-1.0: Waiting for foo 59 | resque 13868 13858 0 13:44 ? S 0:00 \_ resque-1.0: Waiting for bar 60 | resque 13870 13858 0 13:44 ? S 0:00 \_ resque-1.0: Waiting for bar 61 | resque 13871 13858 0 13:44 ? S 0:00 \_ resque-1.0: Waiting for foo,bar,baz 62 | resque 13872 13858 0 13:44 ? S 0:00 \_ resque-1.0: Forked 7481 at 1280343254 63 | resque 7481 13872 0 14:54 ? S 0:00 \_ resque-1.0: Processing foo since 1280343254 64 | resque 13875 13858 0 13:44 ? S 0:00 \_ resque-1.0: Waiting for foo,bar,baz 65 | resque 13876 13858 0 13:44 ? S 0:00 \_ resque-1.0: Forked 7485 at 1280343255 66 | resque 7485 13876 0 14:54 ? S 0:00 \_ resque-1.0: Processing bar since 1280343254 67 | 68 | Running as a daemon will currently output to stdout, although this will be configurable 69 | in the future. 70 | 71 | SIGNALS 72 | ------- 73 | 74 | The pool manager responds to the following signals: 75 | 76 | * `HUP` - reload the config file, restart all workers. 77 | * `QUIT` - gracefully shut down workers (via `QUIT`) and shutdown the manager 78 | after all workers are done. 79 | * `INT` - gracefully shut down workers (via `QUIT`) and immediately shutdown manager 80 | * `TERM` - immediately shut down workers (via `INT`) and immediately shutdown manager 81 | _(configurable via command line options)_ 82 | * `WINCH` - _(only when running as a daemon)_ send `QUIT` to each worker, but 83 | keep manager running (send `HUP` to reload config and restart workers) 84 | * `USR1`/`USR2`/`CONT` - pass the signal on to all worker parents (see Resque docs). 85 | 86 | Use `HUP` to change the number of workers per worker type. Signals can be sent via the 87 | `kill` command, e.g. `kill -HUP $master_pid` 88 | 89 | Other Features 90 | -------------- 91 | 92 | You can specify an alternate config file by setting the `RESQUE_POOL_CONFIG` or 93 | with the `--config` command line option. 94 | 95 | Owner 96 | ------------ 97 | 98 | @ebernhardson 99 | 100 | 101 | Contributors 102 | ------------ 103 | 104 | @michael34435 105 | -------------------------------------------------------------------------------- /bin/resque-pool: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env php 2 | run(); 13 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ebernhardson/php-resque-pool", 3 | "type": "library", 4 | "description": "php-resque worker pool manager", 5 | "keywords": [], 6 | "homepage": "http://github.com/ebernhardson/php-resque-pool", 7 | "license": "", 8 | "authors": [ 9 | { 10 | "name": "Erik Bernhardson", 11 | "email": "bernhardsonerik@gmail.com" 12 | }, 13 | { 14 | "name": "Nicholas Evans", 15 | "email": "nick@ekenosen.net" 16 | } 17 | ], 18 | "require": { 19 | "php": ">=5.4.0", 20 | "symfony/yaml": "~2.1|~3.0|~4.0|~5.0|~6.0", 21 | "resque/php-resque": "~1.3" 22 | }, 23 | "require-dev": { 24 | "phpstan/phpstan": "~1.8" 25 | }, 26 | "autoload": { 27 | "psr-0": { "Resque\\Pool": "lib/" } 28 | }, 29 | "bin": [ 30 | "bin/resque-pool" 31 | ], 32 | "target-dir": "", 33 | "scripts": { 34 | "phpstan": "phpstan analyse --memory-limit=2G" 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /composer.lock: -------------------------------------------------------------------------------- 1 | { 2 | "_readme": [ 3 | "This file locks the dependencies of your project to a known state", 4 | "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", 5 | "This file is @generated automatically" 6 | ], 7 | "content-hash": "4b825bc6929d891a1ece448031a0760d", 8 | "packages": [ 9 | { 10 | "name": "colinmollenhour/credis", 11 | "version": "v1.13.1", 12 | "source": { 13 | "type": "git", 14 | "url": "https://github.com/colinmollenhour/credis.git", 15 | "reference": "85df015088e00daf8ce395189de22c8eb45c8d49" 16 | }, 17 | "dist": { 18 | "type": "zip", 19 | "url": "https://api.github.com/repos/colinmollenhour/credis/zipball/85df015088e00daf8ce395189de22c8eb45c8d49", 20 | "reference": "85df015088e00daf8ce395189de22c8eb45c8d49", 21 | "shasum": "" 22 | }, 23 | "require": { 24 | "php": ">=5.6.0" 25 | }, 26 | "suggest": { 27 | "ext-redis": "Improved performance for communicating with redis" 28 | }, 29 | "type": "library", 30 | "autoload": { 31 | "classmap": [ 32 | "Client.php", 33 | "Cluster.php", 34 | "Sentinel.php", 35 | "Module.php" 36 | ] 37 | }, 38 | "notification-url": "https://packagist.org/downloads/", 39 | "license": [ 40 | "MIT" 41 | ], 42 | "authors": [ 43 | { 44 | "name": "Colin Mollenhour", 45 | "email": "colin@mollenhour.com" 46 | } 47 | ], 48 | "description": "Credis is a lightweight interface to the Redis key-value store which wraps the phpredis library when available for better performance.", 49 | "homepage": "https://github.com/colinmollenhour/credis", 50 | "support": { 51 | "issues": "https://github.com/colinmollenhour/credis/issues", 52 | "source": "https://github.com/colinmollenhour/credis/tree/v1.13.1" 53 | }, 54 | "time": "2022-06-20T22:56:59+00:00" 55 | }, 56 | { 57 | "name": "psr/log", 58 | "version": "1.1.4", 59 | "source": { 60 | "type": "git", 61 | "url": "https://github.com/php-fig/log.git", 62 | "reference": "d49695b909c3b7628b6289db5479a1c204601f11" 63 | }, 64 | "dist": { 65 | "type": "zip", 66 | "url": "https://api.github.com/repos/php-fig/log/zipball/d49695b909c3b7628b6289db5479a1c204601f11", 67 | "reference": "d49695b909c3b7628b6289db5479a1c204601f11", 68 | "shasum": "" 69 | }, 70 | "require": { 71 | "php": ">=5.3.0" 72 | }, 73 | "type": "library", 74 | "extra": { 75 | "branch-alias": { 76 | "dev-master": "1.1.x-dev" 77 | } 78 | }, 79 | "autoload": { 80 | "psr-4": { 81 | "Psr\\Log\\": "Psr/Log/" 82 | } 83 | }, 84 | "notification-url": "https://packagist.org/downloads/", 85 | "license": [ 86 | "MIT" 87 | ], 88 | "authors": [ 89 | { 90 | "name": "PHP-FIG", 91 | "homepage": "https://www.php-fig.org/" 92 | } 93 | ], 94 | "description": "Common interface for logging libraries", 95 | "homepage": "https://github.com/php-fig/log", 96 | "keywords": [ 97 | "log", 98 | "psr", 99 | "psr-3" 100 | ], 101 | "support": { 102 | "source": "https://github.com/php-fig/log/tree/1.1.4" 103 | }, 104 | "time": "2021-05-03T11:20:27+00:00" 105 | }, 106 | { 107 | "name": "resque/php-resque", 108 | "version": "v1.3.6", 109 | "source": { 110 | "type": "git", 111 | "url": "https://github.com/resque/php-resque.git", 112 | "reference": "fe41c04763699b1318d97ed14cc78583e9380161" 113 | }, 114 | "dist": { 115 | "type": "zip", 116 | "url": "https://api.github.com/repos/resque/php-resque/zipball/fe41c04763699b1318d97ed14cc78583e9380161", 117 | "reference": "fe41c04763699b1318d97ed14cc78583e9380161", 118 | "shasum": "" 119 | }, 120 | "require": { 121 | "colinmollenhour/credis": "~1.7", 122 | "php": ">=5.6.0", 123 | "psr/log": "~1.0" 124 | }, 125 | "require-dev": { 126 | "phpunit/phpunit": "^5.7" 127 | }, 128 | "suggest": { 129 | "ext-pcntl": "REQUIRED for forking processes on platforms that support it (so anything but Windows).", 130 | "ext-proctitle": "Allows php-resque to rename the title of UNIX processes to show the status of a worker.", 131 | "ext-redis": "Native PHP extension for Redis connectivity. Credis will automatically utilize when available." 132 | }, 133 | "bin": [ 134 | "bin/resque", 135 | "bin/resque-scheduler" 136 | ], 137 | "type": "library", 138 | "extra": { 139 | "branch-alias": { 140 | "dev-master": "1.0-dev" 141 | } 142 | }, 143 | "autoload": { 144 | "psr-0": { 145 | "Resque": "lib", 146 | "ResqueScheduler": "lib" 147 | } 148 | }, 149 | "notification-url": "https://packagist.org/downloads/", 150 | "license": [ 151 | "MIT" 152 | ], 153 | "authors": [ 154 | { 155 | "name": "Dan Hunsaker", 156 | "email": "danhunsaker+resque@gmail.com", 157 | "role": "Maintainer" 158 | }, 159 | { 160 | "name": "Rajib Ahmed", 161 | "homepage": "https://github.com/rajibahmed", 162 | "role": "Maintainer" 163 | }, 164 | { 165 | "name": "Steve Klabnik", 166 | "email": "steve@steveklabnik.com", 167 | "role": "Maintainer" 168 | }, 169 | { 170 | "name": "Chris Boulton", 171 | "email": "chris@bigcommerce.com", 172 | "role": "Creator" 173 | } 174 | ], 175 | "description": "Redis backed library for creating background jobs and processing them later. Based on resque for Ruby.", 176 | "homepage": "http://www.github.com/resque/php-resque/", 177 | "keywords": [ 178 | "background", 179 | "job", 180 | "redis", 181 | "resque" 182 | ], 183 | "support": { 184 | "issues": "https://github.com/resque/php-resque/issues", 185 | "source": "https://github.com/resque/php-resque/tree/v1.3.6" 186 | }, 187 | "time": "2020-04-16T16:39:50+00:00" 188 | }, 189 | { 190 | "name": "symfony/polyfill-ctype", 191 | "version": "v1.26.0", 192 | "source": { 193 | "type": "git", 194 | "url": "https://github.com/symfony/polyfill-ctype.git", 195 | "reference": "6fd1b9a79f6e3cf65f9e679b23af304cd9e010d4" 196 | }, 197 | "dist": { 198 | "type": "zip", 199 | "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/6fd1b9a79f6e3cf65f9e679b23af304cd9e010d4", 200 | "reference": "6fd1b9a79f6e3cf65f9e679b23af304cd9e010d4", 201 | "shasum": "" 202 | }, 203 | "require": { 204 | "php": ">=7.1" 205 | }, 206 | "provide": { 207 | "ext-ctype": "*" 208 | }, 209 | "suggest": { 210 | "ext-ctype": "For best performance" 211 | }, 212 | "type": "library", 213 | "extra": { 214 | "branch-alias": { 215 | "dev-main": "1.26-dev" 216 | }, 217 | "thanks": { 218 | "name": "symfony/polyfill", 219 | "url": "https://github.com/symfony/polyfill" 220 | } 221 | }, 222 | "autoload": { 223 | "files": [ 224 | "bootstrap.php" 225 | ], 226 | "psr-4": { 227 | "Symfony\\Polyfill\\Ctype\\": "" 228 | } 229 | }, 230 | "notification-url": "https://packagist.org/downloads/", 231 | "license": [ 232 | "MIT" 233 | ], 234 | "authors": [ 235 | { 236 | "name": "Gert de Pagter", 237 | "email": "BackEndTea@gmail.com" 238 | }, 239 | { 240 | "name": "Symfony Community", 241 | "homepage": "https://symfony.com/contributors" 242 | } 243 | ], 244 | "description": "Symfony polyfill for ctype functions", 245 | "homepage": "https://symfony.com", 246 | "keywords": [ 247 | "compatibility", 248 | "ctype", 249 | "polyfill", 250 | "portable" 251 | ], 252 | "support": { 253 | "source": "https://github.com/symfony/polyfill-ctype/tree/v1.26.0" 254 | }, 255 | "funding": [ 256 | { 257 | "url": "https://symfony.com/sponsor", 258 | "type": "custom" 259 | }, 260 | { 261 | "url": "https://github.com/fabpot", 262 | "type": "github" 263 | }, 264 | { 265 | "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", 266 | "type": "tidelift" 267 | } 268 | ], 269 | "time": "2022-05-24T11:49:31+00:00" 270 | }, 271 | { 272 | "name": "symfony/yaml", 273 | "version": "v6.1.3", 274 | "source": { 275 | "type": "git", 276 | "url": "https://github.com/symfony/yaml.git", 277 | "reference": "cc48dd42ae1201abced04ae38284e23ce2d2d8f3" 278 | }, 279 | "dist": { 280 | "type": "zip", 281 | "url": "https://api.github.com/repos/symfony/yaml/zipball/cc48dd42ae1201abced04ae38284e23ce2d2d8f3", 282 | "reference": "cc48dd42ae1201abced04ae38284e23ce2d2d8f3", 283 | "shasum": "" 284 | }, 285 | "require": { 286 | "php": ">=8.1", 287 | "symfony/polyfill-ctype": "^1.8" 288 | }, 289 | "conflict": { 290 | "symfony/console": "<5.4" 291 | }, 292 | "require-dev": { 293 | "symfony/console": "^5.4|^6.0" 294 | }, 295 | "suggest": { 296 | "symfony/console": "For validating YAML files using the lint command" 297 | }, 298 | "bin": [ 299 | "Resources/bin/yaml-lint" 300 | ], 301 | "type": "library", 302 | "autoload": { 303 | "psr-4": { 304 | "Symfony\\Component\\Yaml\\": "" 305 | }, 306 | "exclude-from-classmap": [ 307 | "/Tests/" 308 | ] 309 | }, 310 | "notification-url": "https://packagist.org/downloads/", 311 | "license": [ 312 | "MIT" 313 | ], 314 | "authors": [ 315 | { 316 | "name": "Fabien Potencier", 317 | "email": "fabien@symfony.com" 318 | }, 319 | { 320 | "name": "Symfony Community", 321 | "homepage": "https://symfony.com/contributors" 322 | } 323 | ], 324 | "description": "Loads and dumps YAML files", 325 | "homepage": "https://symfony.com", 326 | "support": { 327 | "source": "https://github.com/symfony/yaml/tree/v6.1.3" 328 | }, 329 | "funding": [ 330 | { 331 | "url": "https://symfony.com/sponsor", 332 | "type": "custom" 333 | }, 334 | { 335 | "url": "https://github.com/fabpot", 336 | "type": "github" 337 | }, 338 | { 339 | "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", 340 | "type": "tidelift" 341 | } 342 | ], 343 | "time": "2022-07-20T14:45:06+00:00" 344 | } 345 | ], 346 | "packages-dev": [ 347 | { 348 | "name": "phpstan/phpstan", 349 | "version": "1.8.2", 350 | "source": { 351 | "type": "git", 352 | "url": "https://github.com/phpstan/phpstan.git", 353 | "reference": "c53312ecc575caf07b0e90dee43883fdf90ca67c" 354 | }, 355 | "dist": { 356 | "type": "zip", 357 | "url": "https://api.github.com/repos/phpstan/phpstan/zipball/c53312ecc575caf07b0e90dee43883fdf90ca67c", 358 | "reference": "c53312ecc575caf07b0e90dee43883fdf90ca67c", 359 | "shasum": "" 360 | }, 361 | "require": { 362 | "php": "^7.2|^8.0" 363 | }, 364 | "conflict": { 365 | "phpstan/phpstan-shim": "*" 366 | }, 367 | "bin": [ 368 | "phpstan", 369 | "phpstan.phar" 370 | ], 371 | "type": "library", 372 | "autoload": { 373 | "files": [ 374 | "bootstrap.php" 375 | ] 376 | }, 377 | "notification-url": "https://packagist.org/downloads/", 378 | "license": [ 379 | "MIT" 380 | ], 381 | "description": "PHPStan - PHP Static Analysis Tool", 382 | "support": { 383 | "issues": "https://github.com/phpstan/phpstan/issues", 384 | "source": "https://github.com/phpstan/phpstan/tree/1.8.2" 385 | }, 386 | "funding": [ 387 | { 388 | "url": "https://github.com/ondrejmirtes", 389 | "type": "github" 390 | }, 391 | { 392 | "url": "https://github.com/phpstan", 393 | "type": "github" 394 | }, 395 | { 396 | "url": "https://www.patreon.com/phpstan", 397 | "type": "patreon" 398 | }, 399 | { 400 | "url": "https://tidelift.com/funding/github/packagist/phpstan/phpstan", 401 | "type": "tidelift" 402 | } 403 | ], 404 | "time": "2022-07-20T09:57:31+00:00" 405 | } 406 | ], 407 | "aliases": [], 408 | "minimum-stability": "stable", 409 | "stability-flags": [], 410 | "prefer-stable": false, 411 | "prefer-lowest": false, 412 | "platform": { 413 | "php": ">=5.4.0" 414 | }, 415 | "platform-dev": [], 416 | "plugin-api-version": "2.3.0" 417 | } 418 | -------------------------------------------------------------------------------- /config/resque-pool.yml.example: -------------------------------------------------------------------------------- 1 | foo: 2 2 | -------------------------------------------------------------------------------- /lib/Resque/Pool/Cli.php: -------------------------------------------------------------------------------- 1 | 10 | * @author Michael Kuan 11 | * @copyright (c) 2012 Erik Bernhardson 12 | * @license http://www.opensource.org/licenses/mit-license.php 13 | */ 14 | class Cli 15 | { 16 | 17 | /** 18 | * @var array> 19 | */ 20 | private static $optionDefs = array( 21 | 'help' => array('Show usage information', 'default' => false, 'short' => 'h'), 22 | 'config' => array('Alternate path to config file', 'short' => 'c'), 23 | 'appName' => array('Alternate appname', 'short' => 'a'), 24 | 'daemon' => array('Run as a background daemon', 'default' => false, 'short' => 'd'), 25 | 'pidfile' => array('PID file location', 'short' => 'p'), 26 | 'environment' => array('Set RESQUE_ENV', 'short' => 'E'), 27 | 'term-graceful-wait' => array('On TERM signal, wait for workers to shut down gracefully'), 28 | 'term-graceful' => array('On TERM signal, shut down workers gracefully'), 29 | 'term_immediate' => array('On TERM signal, shut down workers immediately (default)'), 30 | ); 31 | 32 | /** 33 | * @return void 34 | */ 35 | public function run() 36 | { 37 | $opts = $this->parseOptions(); 38 | 39 | if ($opts['daemon']) { 40 | $this->daemonize(); 41 | } 42 | $this->managePidfile($opts['pidfile']); 43 | $config = $this->buildConfiguration($opts); 44 | $this->startPool($config); 45 | } 46 | 47 | /** 48 | * @return array 49 | * @phpstan-return array{ 50 | * help: bool, config: string, appName: string, daemon: bool, pidfile: string, 51 | * environment: string, term-graceful-wait: string, term-graceful: string, term_immediate: string 52 | * } 53 | */ 54 | public function parseOptions() 55 | { 56 | $shortopts = ''; 57 | $longopts = array(); 58 | /** @var array $defaults */ 59 | $defaults = array(); 60 | $shortmap = array(); 61 | 62 | foreach (self::$optionDefs as $name => $def) { 63 | $def += array('default' => '', 'short' => false); 64 | 65 | $defaults[$name] = $def['default']; 66 | $postfix = is_bool($defaults[$name]) ? '' : ':'; 67 | 68 | $longopts[] = $name.$postfix; 69 | if ($def['short']) { 70 | $shortmap[$def['short']] = $name; 71 | $shortopts .= $def['short'].$postfix; 72 | } 73 | } 74 | 75 | /** @var array $received */ 76 | $received = getopt($shortopts, $longopts); 77 | 78 | foreach (array_keys($received) as $key) { 79 | if (strlen($key) === 1) { 80 | $received[$shortmap[$key]] = $received[$key]; 81 | unset($received[$key]); 82 | } 83 | } 84 | 85 | // getopt is odd ... it returns false for received args with no options allowed 86 | foreach (array_keys($received) as $key) { 87 | if (false === $received[$key]) { 88 | $received[$key] = true; 89 | } 90 | } 91 | 92 | $received += $defaults; 93 | 94 | if ($received['help']) { 95 | $this->usage(); 96 | exit(0); 97 | } 98 | 99 | return $received; // @phpstan-ignore-line 100 | } 101 | 102 | /** 103 | * @return void 104 | */ 105 | public function usage() 106 | { 107 | $cmdname = isset($GLOBALS['argv'][0]) ? $GLOBALS['argv'][0] : 'resque-pool'; 108 | echo "\n" 109 | ."Usage:" 110 | ."\t$cmdname [OPTION]...\n" 111 | ."\n"; 112 | foreach (self::$optionDefs as $name => $def) { 113 | $def += array('default' => '', 'short' => false); 114 | printf(" %2s %-20s %s\n", 115 | $def['short'] ? ('-'.$def['short']) : '', 116 | "--{$name}", 117 | $def[0] 118 | ); 119 | } 120 | echo "\n\n"; 121 | } 122 | 123 | /** 124 | * @return void 125 | */ 126 | public function daemonize() 127 | { 128 | $pid = pcntl_fork(); 129 | if ($pid === -1) { 130 | throw new \RuntimeException("Failed pcntl_fork"); 131 | } 132 | if ($pid) { 133 | // parent 134 | echo "Started background process: {$pid}\n\n"; 135 | exit(0); 136 | } 137 | } 138 | 139 | /** 140 | * @param string $pidfile 141 | * @return void 142 | */ 143 | public function managePidfile($pidfile) 144 | { 145 | if (!$pidfile) { 146 | return; 147 | } 148 | 149 | if (file_exists($pidfile)) { 150 | if ($this->processStillRunning($pidfile)) { 151 | throw new \Exception("Pidfile already exists at '{$pidfile}' and process is still running."); 152 | } else { 153 | unlink($pidfile); 154 | } 155 | } elseif (!is_dir($piddir = dirname($pidfile))) { 156 | mkdir($piddir, 0777, true); 157 | } 158 | 159 | file_put_contents($pidfile, getmypid(), LOCK_EX); 160 | register_shutdown_function(function() use ($pidfile) { 161 | if (file_exists($pidfile)) { 162 | @unlink($pidfile); 163 | } 164 | }); 165 | } 166 | 167 | /** 168 | * @param string $pidfile 169 | * @return bool 170 | */ 171 | public function processStillRunning($pidfile) 172 | { 173 | $contents = file_get_contents($pidfile); 174 | if (false === $contents) { 175 | return true; 176 | } 177 | 178 | $oldPid = (int)trim($contents); 179 | 180 | return posix_kill($oldPid, 0); 181 | } 182 | 183 | /** 184 | * @param array $options 185 | * @phpstan-param array{ 186 | * appName?: string, environment?: string, config?: string, daemon?: bool, 187 | * term-graceful-wait?: string, term-graceful?: string 188 | * } $options 189 | * @return Configuration 190 | */ 191 | public function buildConfiguration(array $options) 192 | { 193 | $config = new Configuration; 194 | if (isset($options['appName'])) { 195 | $config->appName = $options['appName']; 196 | } 197 | if (isset($options['environment'])) { 198 | $config->environment = $options['environment']; 199 | } 200 | if (isset($options['config'])) { 201 | $config->queueConfigFile = $options['config']; 202 | } 203 | if (isset($options['daemon'])) { 204 | $config->handleWinch = true; 205 | } 206 | if (isset($options['term-graceful-wait'])) { 207 | $config->termBehavior = 'graceful_worker_shutdown_and_wait'; 208 | } elseif (isset($options['term-graceful'])) { 209 | $config->termBehavior = 'graceful_worker_shutdown'; 210 | } 211 | 212 | return $config; 213 | } 214 | 215 | /** 216 | * @return void 217 | */ 218 | public function startPool(Configuration $config) 219 | { 220 | $pool = new Pool($config); 221 | $pool->start(); 222 | $pool->join(); 223 | } 224 | } 225 | -------------------------------------------------------------------------------- /lib/Resque/Pool/Configuration.php: -------------------------------------------------------------------------------- 1 | 13 | * @author Michael Kuan 14 | * @copyright (c) 2012 Erik Bernhardson 15 | * @license http://www.opensource.org/licenses/mit-license.php 16 | */ 17 | class Configuration 18 | { 19 | const LOG_NONE = 0; 20 | const LOG_NORMAL = 1; 21 | const LOG_VERBOSE = 2; 22 | 23 | const DEFAULT_WORKER_INTERVAL = 5; 24 | 25 | /** 26 | * @var null|callable 27 | */ 28 | public $afterPreFork; 29 | /** 30 | * Tag used in log output 31 | * 32 | * @var null|string 33 | */ 34 | public $appName; 35 | /** 36 | * Possible configuration file locations 37 | * 38 | * @var string[] 39 | */ 40 | public $configFiles = array('resque-pool.yml', 'config/resque-pool.yml'); 41 | /** 42 | * Environment to use from configuration 43 | * @var string 44 | */ 45 | public $environment = 'dev'; 46 | /** 47 | * Reset worker counts to 0 when SIGWINCH is received 48 | * 49 | * @var bool 50 | */ 51 | public $handleWinch = false; 52 | /** 53 | * @var Logger 54 | */ 55 | public $logger; 56 | /** 57 | * @var integer self::LOG_* 58 | */ 59 | public $logLevel = self::LOG_NONE; 60 | /** 61 | * @var Platform 62 | */ 63 | public $platform; 64 | /** 65 | * Active configuration file location. When null self::$configFiles will be tried. 66 | * 67 | * @var string|null 68 | */ 69 | public $queueConfigFile; 70 | /** 71 | * @var integer 72 | */ 73 | public $sleepTime = 60; 74 | /** 75 | * What to do when receiving SIGTERM 76 | * 77 | * @var string 78 | */ 79 | public $termBehavior = ''; 80 | /** 81 | * @var string 82 | */ 83 | public $workerClass = '\\Resque_Worker'; 84 | /** 85 | * @var integer 86 | */ 87 | public $workerInterval = self::DEFAULT_WORKER_INTERVAL; 88 | 89 | /** 90 | * @var array 91 | */ 92 | protected $queueConfig; 93 | 94 | /** 95 | * @param array|string|null $config Either a configuration array, path to yml 96 | * file containing config, or null 97 | * @param Logger|null $logger If not provided one will be instantiated 98 | * @param Platform|null $platform If not provided one will be instantiated 99 | */ 100 | public function __construct($config = null, Logger $logger = null, Platform $platform = null) 101 | { 102 | $this->loadEnvironment(); 103 | $this->logger = $logger ?: new Logger($this->appName); 104 | $this->platform = $platform ?: new Platform; 105 | 106 | if (is_array($config)) { 107 | $this->queueConfig = $config; 108 | $this->queueConfigFile = null; 109 | } elseif (is_string($config)) { 110 | $this->queueConfigFile = $config; 111 | } elseif ($config !== null) { // @phpstan-ignore-line 112 | throw new \InvalidArgumentException('Unknown $config argument passed'); 113 | } 114 | } 115 | 116 | /** @return void */ 117 | public function initialize() 118 | { 119 | if (!$this->queueConfig) { 120 | $this->chooseConfigFile(); 121 | $this->loadQueueConfig(); 122 | } 123 | if ($this->environment && isset($this->queueConfig[$this->environment])) { 124 | $this->queueConfig = $this->queueConfig[$this->environment] + $this->queueConfig; // @phpstan-ignore-line 125 | } 126 | // filter out the environments 127 | $this->queueConfig = array_filter($this->queueConfig, 'is_integer'); 128 | $this->logger->log("Configured queues: " . implode(" ", $this->knownQueues())); 129 | } 130 | 131 | /** 132 | * @param string $queues 133 | * 134 | * @return integer Desired number of workers for specified queue combination 135 | */ 136 | public function workerCount($queues) 137 | { 138 | return isset($this->queueConfig[$queues]) ? $this->queueConfig[$queues] : 0; 139 | } 140 | 141 | /** 142 | * @return string[] All configured queue combinations 143 | */ 144 | public function knownQueues() 145 | { 146 | return $this->queueConfig ? array_keys($this->queueConfig) : array(); 147 | } 148 | 149 | /** 150 | * @return array Map of queue combination to desired worker count 151 | */ 152 | public function queueConfig() 153 | { 154 | return $this->queueConfig; 155 | } 156 | 157 | /** 158 | * Resets the current queue configuration 159 | * @return void 160 | */ 161 | public function resetQueues() 162 | { 163 | $this->queueConfig = array(); 164 | } 165 | 166 | /** 167 | * @return void 168 | */ 169 | protected function loadEnvironment() 170 | { 171 | $this->appName = basename(getcwd() ?: '.'); 172 | $this->environment = (string)getenv('RESQUE_ENV'); 173 | $this->workerInterval = (int)getenv('INTERVAL') ?: $this->workerInterval; 174 | $this->queueConfigFile = (string)getenv('RESQUE_POOL_CONFIG'); 175 | 176 | if (getenv('VVERBOSE')) { 177 | $this->logLevel = self::LOG_VERBOSE; 178 | } elseif (getenv('LOGGING') || getenv('VERBOSE')) { 179 | $this->logLevel = self::LOG_NORMAL; 180 | } 181 | } 182 | 183 | /** 184 | * @return void 185 | */ 186 | protected function chooseConfigFile() 187 | { 188 | if ($this->queueConfigFile) { 189 | if (file_exists($this->queueConfigFile)) { 190 | return; 191 | } 192 | $this->logger->log("Chosen config file '{$this->queueConfigFile}' not found. Looking for others."); 193 | } 194 | $this->queueConfigFile = null; 195 | foreach ($this->configFiles as $file) { 196 | if (file_exists($file)) { 197 | $this->queueConfigFile = $file; 198 | break; 199 | } 200 | } 201 | } 202 | 203 | /** 204 | * @return void 205 | */ 206 | protected function loadQueueConfig() 207 | { 208 | if ($this->queueConfigFile && file_exists($this->queueConfigFile)) { 209 | $this->logger->log("Loading config file: {$this->queueConfigFile}"); 210 | try { 211 | if (preg_match("/\.php/", $this->queueConfigFile)) { 212 | ob_start(); 213 | include($this->queueConfigFile); 214 | $queueConfig = (string)ob_get_clean(); 215 | } else { 216 | $queueConfig = (string)file_get_contents($this->queueConfigFile); 217 | } 218 | 219 | $this->queueConfig = Yaml::parse($queueConfig); // @phpstan-ignore-line 220 | } catch (ParseException $e) { 221 | $msg = "Invalid config file: ".$e->getMessage(); 222 | $this->logger->log($msg); 223 | 224 | throw new \RuntimeException($msg, 0, $e); 225 | } 226 | } 227 | if (!$this->queueConfig) { 228 | $this->logger->log('No configuration loaded.'); 229 | $this->queueConfig = array(); 230 | } 231 | } 232 | } 233 | -------------------------------------------------------------------------------- /lib/Resque/Pool/Logger.php: -------------------------------------------------------------------------------- 1 | 10 | * @author Michael Kuan 11 | * @copyright (c) 2012 Erik Bernhardson 12 | * @license http://www.opensource.org/licenses/mit-license.php 13 | */ 14 | class Logger 15 | { 16 | /** @var string */ 17 | private $appName; 18 | 19 | /** 20 | * @param null|string $appName 21 | */ 22 | public function __construct($appName = null) 23 | { 24 | $this->appName = $appName ? "[{$appName}]" : ""; 25 | } 26 | 27 | /** 28 | * @param string $string 29 | * @return void 30 | */ 31 | public function procline($string) 32 | { 33 | if (function_exists('setproctitle')) { 34 | setproctitle("resque-pool-manager{$this->appName}: {$string}"); 35 | } elseif (function_exists('cli_set_process_title') && PHP_OS !== 'Darwin') { 36 | cli_set_process_title("resque-pool-manager{$this->appName}: {$string}"); 37 | } 38 | } 39 | 40 | /** 41 | * @param string $message 42 | * @return void 43 | */ 44 | public function log($message) 45 | { 46 | $pid = getmypid(); 47 | echo "resque-pool-manager{$this->appName}[{$pid}]: {$message}\n"; 48 | } 49 | 50 | /** 51 | * @param string $message 52 | * @return void 53 | */ 54 | public function logWorker($message) 55 | { 56 | $pid = getmypid(); 57 | echo "resque-pool-worker{$this->appName}[{$pid}]: {$message}\n"; 58 | } 59 | 60 | /** 61 | * This function closes and re-opens the output log 62 | * @return void 63 | */ 64 | public function rotate() 65 | { 66 | // not possible in php? 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /lib/Resque/Pool/Platform.php: -------------------------------------------------------------------------------- 1 | 12 | * @author Michael Kuan 13 | * @copyright (c) 2012 Erik Bernhardson 14 | * @license http://www.opensource.org/licenses/mit-license.php 15 | */ 16 | class Platform 17 | { 18 | /** @var int */ 19 | private static $SIG_QUEUE_MAX_SIZE = 5; 20 | 21 | /** @var null|Logger */ 22 | protected $logger; 23 | /** @var bool */ 24 | private $quitOnExitSignal; 25 | 26 | /** @var list */ 27 | private $sigQueue = array(); 28 | /** @var array */ 29 | private $trappedSignals = array(); 30 | 31 | /** 32 | * @param Logger|null $logger 33 | * @return void 34 | */ 35 | public function setLogger(Logger $logger = null) 36 | { 37 | $this->logger = $logger; 38 | } 39 | 40 | /** 41 | * @param bool $bool 42 | * @return void 43 | */ 44 | public function setQuitOnExitSignal($bool) 45 | { 46 | $this->quitOnExitSignal = $bool; 47 | } 48 | 49 | /** 50 | * exit is reserved word 51 | * @param int $status 52 | * @return never 53 | */ 54 | public function _exit($status = 0) 55 | { 56 | exit($status); 57 | } 58 | 59 | /** 60 | * @return int 61 | */ 62 | public function pcntl_fork() 63 | { 64 | return pcntl_fork(); 65 | } 66 | 67 | /** 68 | * @param int $seconds 69 | * @return false|int 70 | */ 71 | public function sleep($seconds) 72 | { 73 | return sleep($seconds); 74 | } 75 | 76 | /** 77 | * @param list|int $pids 78 | * @param int $sig 79 | * @return void 80 | */ 81 | public function signalPids($pids, $sig) 82 | { 83 | if (!is_array($pids)) { 84 | $pids = array($pids); 85 | } 86 | 87 | foreach ($pids as $pid) { 88 | posix_kill($pid, $sig); 89 | } 90 | } 91 | 92 | /** 93 | * @param list $signals 94 | * @return void 95 | */ 96 | public function trapSignals(array $signals) 97 | { 98 | foreach ($signals as $sig) { 99 | $this->trappedSignals[$sig] = true; 100 | pcntl_signal($sig, array($this, 'trapDeferred')); 101 | } 102 | } 103 | 104 | /** 105 | * @return void 106 | */ 107 | public function releaseSignals() 108 | { 109 | $noop = function() {}; 110 | foreach (array_keys($this->trappedSignals) as $sig) { 111 | pcntl_signal($sig, $noop); 112 | } 113 | 114 | $this->trappedSignals = array(); 115 | } 116 | 117 | 118 | /** 119 | * called by php signal handling 120 | * @internal 121 | * @param int $signal 122 | * @return void 123 | */ 124 | public function trapDeferred($signal) 125 | { 126 | if (count($this->sigQueue) < self::$SIG_QUEUE_MAX_SIZE) { 127 | if ($this->quitOnExitSignal && in_array($signal, array(SIGINT, SIGTERM))) { 128 | $this->log("Received {$signal}: short circuiting QUIT waitpid"); 129 | $this->_exit(1); // TODO: should this return a failed exit code? 130 | } 131 | 132 | $this->sigQueue[] = $signal; 133 | } else { 134 | $this->log("Ignoring SIG{$signal}, queue=" . json_encode($this->sigQueue, true)); // @phpstan-ignore-line 135 | } 136 | } 137 | 138 | /** 139 | * @return int 140 | */ 141 | public function numSignalsPending() 142 | { 143 | pcntl_signal_dispatch(); 144 | 145 | return count($this->sigQueue); 146 | } 147 | 148 | /** 149 | * @return int|null 150 | */ 151 | public function nextSignal() 152 | { 153 | // this will queue up signals into $this->sigQueue 154 | pcntl_signal_dispatch(); 155 | 156 | return array_shift($this->sigQueue); 157 | } 158 | 159 | /** 160 | * @param bool $wait When non-false and there are no dead children, wait for the next one 161 | * @return array|null Returns either the pid and exit code of a dead child process, or null 162 | */ 163 | public function nextDeadChild($wait = false) 164 | { 165 | $wpid = pcntl_waitpid(-1, $status, $wait === false ? WNOHANG : 0); 166 | // 0 is WNOHANG and no dead children, -1 is no children exist 167 | if ($wpid === 0 || $wpid === -1) { 168 | return null; 169 | } 170 | 171 | /** @var int $exit */ 172 | $exit = pcntl_wexitstatus($status); 173 | 174 | return array($wpid, $exit); 175 | } 176 | 177 | /** 178 | * @param string $msg 179 | * @return void 180 | */ 181 | protected function log($msg) 182 | { 183 | if ($this->logger) { 184 | $this->logger->log($msg); 185 | } 186 | } 187 | } 188 | -------------------------------------------------------------------------------- /lib/Resque/Pool/Pool.php: -------------------------------------------------------------------------------- 1 | 10 | * @author Michael Kuan 11 | * @copyright (c) 2012 Erik Bernhardson 12 | * @license http://www.opensource.org/licenses/mit-license.php 13 | */ 14 | class Pool 15 | { 16 | /** 17 | * @var list 18 | */ 19 | private static $QUEUE_SIGS = array(SIGQUIT, SIGINT, SIGTERM, SIGUSR1, SIGUSR2, SIGCONT, SIGHUP, SIGWINCH, SIGCHLD); 20 | 21 | /** 22 | * @var Configuration 23 | */ 24 | private $config; 25 | 26 | /** 27 | * @var Logger 28 | */ 29 | private $logger; 30 | /** 31 | * @var Platform 32 | */ 33 | private $platform; 34 | 35 | /** 36 | * @var array> 37 | */ 38 | private $workers = array(); 39 | 40 | public function __construct(Configuration $config) 41 | { 42 | $this->config = $config; 43 | $this->logger = $config->logger; 44 | $this->platform = $config->platform; 45 | } 46 | 47 | /** 48 | * @return void 49 | */ 50 | public function start() 51 | { 52 | $this->config->initialize(); 53 | $this->logger->procline('(starting)'); 54 | $this->platform->trapSignals(self::$QUEUE_SIGS); 55 | $this->maintainWorkerCount(); 56 | 57 | $this->logger->procline('(started)'); 58 | $this->logger->log("started manager"); 59 | $this->reportWorkerPoolPids(); 60 | } 61 | 62 | /** 63 | * @return void 64 | */ 65 | public function join() 66 | { 67 | while (true) { 68 | $this->reapAllWorkers(); 69 | if ($this->handleSignalQueue()) { 70 | break; 71 | } 72 | 73 | if (0 === $this->platform->numSignalsPending()) { 74 | $this->maintainWorkerCount(); 75 | $this->platform->sleep($this->config->sleepTime); 76 | } 77 | 78 | $this->logger->procline(sprintf("managing [%s]", implode(' ', $this->allPids()))); 79 | } 80 | $this->logger->procline("(shutting down)"); 81 | $this->logger->log("manager finished"); 82 | } 83 | 84 | /** 85 | * @return bool When true the pool manager must shut down 86 | */ 87 | protected function handleSignalQueue() 88 | { 89 | switch ($signal = $this->platform->nextSignal()) { 90 | case null: 91 | break; 92 | case SIGUSR1: 93 | case SIGUSR2: 94 | case SIGCONT: 95 | $this->logger->log("{$signal}: sending to all workers"); 96 | $this->signalAllWorkers($signal); 97 | break; 98 | case SIGHUP: 99 | $this->logger->log("HUP: reload config file"); 100 | $this->config->resetQueues(); 101 | $this->config->initialize(); 102 | $this->logger->log('HUP: gracefully shutdown old children (which have old logfiles open)'); 103 | $this->signalAllWorkers(SIGQUIT); 104 | $this->logger->log('HUP: new children will inherit new logfiles'); 105 | $this->maintainWorkerCount(); 106 | break; 107 | case SIGWINCH: 108 | if ($this->config->handleWinch) { 109 | $this->logger->log('WINCH: gracefully stopping all workers'); 110 | $this->config->resetQueues(); 111 | $this->maintainWorkerCount(); 112 | } 113 | break; 114 | case SIGQUIT: 115 | $this->platform->setQuitOnExitSignal(true); 116 | $this->gracefulWorkerShutdownAndWait($signal); 117 | 118 | return true; 119 | case SIGINT: 120 | $this->gracefulWorkerShutdown($signal); 121 | 122 | return true; 123 | case SIGTERM: 124 | switch ($this->config->termBehavior) { 125 | case "graceful_worker_shutdown_and_wait": 126 | $this->gracefulWorkerShutdownAndWait($signal); 127 | break; 128 | case "graceful_worker_shutdown": 129 | $this->gracefulWorkerShutdown($signal); 130 | break; 131 | default: 132 | $this->shutdownEverythingNow($signal); 133 | break; 134 | } 135 | 136 | return true; 137 | } 138 | 139 | return false; 140 | } 141 | 142 | /** 143 | * @return void 144 | */ 145 | public function reportWorkerPoolPids() 146 | { 147 | if (count($this->workers) === 0) { 148 | $this->logger->log('Pool is empty'); 149 | } else { 150 | $pids = $this->allPids(); 151 | $this->logger->log("Pool contains worker PIDs: ".implode(', ', $pids)); 152 | } 153 | } 154 | 155 | /** 156 | * Creates or shuts down workers to match the configured worker counts. 157 | * @return void 158 | */ 159 | public function maintainWorkerCount() 160 | { 161 | foreach ($this->allKnownQueues() as $queues) { 162 | $delta = $this->workerDeltaFor($queues); 163 | if ($delta > 0) { 164 | while ($delta-- > 0) { 165 | $this->spawnWorker($queues); 166 | } 167 | } elseif ($delta < 0) { 168 | $pids = array_slice($this->pidsFor($queues), 0, -$delta); 169 | $this->platform->signalPids($pids, SIGQUIT); 170 | } 171 | } 172 | } 173 | 174 | /** 175 | * Finds and unsets dead workers. 176 | * 177 | * @param boolean $wait When true waits for all children to shutdown. 178 | * @return void 179 | */ 180 | public function reapAllWorkers($wait = false) 181 | { 182 | while ($exited = $this->platform->nextDeadChild($wait)) { 183 | list($wpid, $exit) = $exited; 184 | $this->logger->log("Reaped resque worker {$wpid} (status: {$exit}) queues: ". $this->workerQueues($wpid)); 185 | $this->deleteWorker($wpid); 186 | } 187 | } 188 | 189 | /** 190 | * @param int $pid 191 | * @return string|null The queues $pid was created to work on 192 | */ 193 | public function workerQueues($pid) 194 | { 195 | foreach ($this->workers as $queues => $workers) { 196 | if (isset($workers[$pid])) { 197 | return $queues; 198 | } 199 | } 200 | 201 | return null; 202 | } 203 | 204 | /** 205 | * @return list The pids of all living worker daemons 206 | */ 207 | public function allPids() 208 | { 209 | if (!$this->workers) { 210 | return array(); 211 | } 212 | 213 | $result = array(); 214 | foreach ($this->workers as $queues) { 215 | $result[] = array_keys($queues); 216 | } 217 | 218 | return call_user_func_array('array_merge', $result); // @phpstan-ignore-line 219 | } 220 | 221 | /** 222 | * @return list 223 | */ 224 | public function allKnownQueues() 225 | { 226 | return array_unique(array_merge($this->config->knownQueues(), array_keys($this->workers))); 227 | } 228 | 229 | /** 230 | * @param int $signal 231 | * @return void 232 | */ 233 | public function signalAllWorkers($signal) 234 | { 235 | $this->platform->signalPids($this->allPids(), $signal); 236 | } 237 | 238 | /** 239 | * @param int $signal 240 | * @return void 241 | */ 242 | public function gracefulWorkerShutdownAndWait($signal) 243 | { 244 | $this->logger->log("{$signal}: graceful shutdown, waiting for children"); 245 | $this->signalAllWorkers(SIGQUIT); 246 | $this->reapAllWorkers(true); // will hang until all workers are shutdown 247 | } 248 | 249 | /** 250 | * @param int $signal 251 | * @return void 252 | */ 253 | public function gracefulWorkerShutdown($signal) 254 | { 255 | $this->logger->log("{$signal}: immediate shutdown (graceful worker shutdown)"); 256 | $this->signalAllWorkers(SIGQUIT); 257 | } 258 | 259 | /** 260 | * @param int $signal 261 | * @return void 262 | */ 263 | public function shutdownEverythingNow($signal) 264 | { 265 | $this->logger->log("{$signal}: immediate shutdown (and immediate worker shutdown)"); 266 | $this->signalAllWorkers(SIGTERM); 267 | } 268 | 269 | /** 270 | * @param string $queues 271 | * @return int 272 | */ 273 | protected function workerDeltaFor($queues) 274 | { 275 | $max = $this->config->workerCount($queues); 276 | $active = isset($this->workers[$queues]) ? count($this->workers[$queues]) : 0; 277 | 278 | return $max - $active; 279 | } 280 | 281 | /** 282 | * @param string $queues 283 | * @return int[] 284 | */ 285 | protected function pidsFor($queues) 286 | { 287 | return isset($this->workers[$queues]) ? array_keys($this->workers[$queues]) : array(); 288 | } 289 | 290 | /** 291 | * @param int $pid 292 | * @return void 293 | */ 294 | protected function deleteWorker($pid) 295 | { 296 | foreach (array_keys($this->workers) as $queues) { 297 | if (isset($this->workers[$queues][$pid])) { 298 | unset($this->workers[$queues][$pid]); 299 | 300 | return ; 301 | } 302 | } 303 | } 304 | 305 | /** 306 | * NOTE: the only time resque code is ever loaded is *after* this fork. 307 | * this way resque(and application) code is loaded per fork and 308 | * will pick up changed files. 309 | * TODO: the other possibility here is to load all the resque(and possibly application) 310 | * code pre-fork so that the copy-on-write functionality of the linux memory model 311 | * can share the compiled code between workers. Some investigation into the facts 312 | * would be usefull 313 | * @param string $queues 314 | * @return void 315 | */ 316 | protected function spawnWorker($queues) 317 | { 318 | $pid = $this->platform->pcntl_fork(); 319 | if ($pid === -1) { 320 | $this->logger->log('pcntl_fork failed'); 321 | $this->platform->_exit(1); 322 | } elseif ($pid === 0) { 323 | $this->platform->releaseSignals(); 324 | $worker = $this->createWorker($queues); 325 | $this->logger->logWorker("Starting worker {$worker}"); 326 | $this->logger->procline("Starting worker {$worker}"); 327 | $this->callAfterPrefork($worker); 328 | $worker->work($this->config->workerInterval); // @phpstan-ignore-line 329 | $this->platform->_exit(0); 330 | } else { 331 | $this->workers[$queues][$pid] = true; 332 | } 333 | } 334 | 335 | /** 336 | * @param object $worker 337 | * @return void 338 | */ 339 | protected function callAfterPrefork($worker) 340 | { 341 | if ($callable = $this->config->afterPreFork) { 342 | call_user_func($callable, $this, $worker); 343 | } 344 | } 345 | 346 | /** 347 | * @param string $queues 348 | * @return object 349 | */ 350 | protected function createWorker($queues) 351 | { 352 | $queues = explode(',', $queues); 353 | $class = $this->config->workerClass; 354 | $worker = new $class($queues); 355 | if ($this->config->logLevel === Configuration::LOG_VERBOSE) { 356 | $worker->logLevel = \Resque_Worker::LOG_VERBOSE; // @phpstan-ignore-line 357 | } elseif ($this->config->logLevel === Configuration::LOG_NORMAL) { 358 | $worker->logLevel = \Resque_Worker::LOG_NORMAL; // @phpstan-ignore-line 359 | } 360 | 361 | return $worker; 362 | } 363 | } 364 | -------------------------------------------------------------------------------- /phpstan.neon: -------------------------------------------------------------------------------- 1 | parameters: 2 | level: max 3 | paths: 4 | - lib 5 | -------------------------------------------------------------------------------- /phpunit.xml: -------------------------------------------------------------------------------- 1 | 2 | 13 | 14 | 15 | ./test/Resque/Pool/ 16 | 17 | 18 | 19 | 20 | 21 | ./lib/Resque/Pool/ 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /test/Resque/Pool/Tests/BaseTestCase.php: -------------------------------------------------------------------------------- 1 | assertInternalType('array', $expect, $message); 10 | $this->assertInternalType('array', $subject, $message); 11 | $this->assertEquals(count($expect), count($subject), $message); 12 | foreach ($expect as $item) { 13 | $this->assertContains($item, $subject, $message); 14 | } 15 | } 16 | 17 | public function mockLogger() 18 | { 19 | return $this->getMock('Resque\\Pool\\Logger'); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /test/Resque/Pool/Tests/ConfigurationTest.php: -------------------------------------------------------------------------------- 1 | mockLogger()); 13 | $this->assertEquals(Configuration::DEFAULT_WORKER_INTERVAL, $config->workerInterval); 14 | putenv('INTERVAL=20'); 15 | $config = new Configuration(null, $this->mockLogger()); 16 | $this->assertEquals(20, $config->workerInterval); 17 | } 18 | 19 | public function testThrowsExceptionOnInvalidInstantiation() 20 | { 21 | $this->setExpectedException('InvalidArgumentException'); 22 | $config = new Configuration(new \StdClass); 23 | } 24 | 25 | public function loadingThePoolConfigurationProvider() 26 | { 27 | $simpleConfig = array('foo' => 1, 'bar' => 2, 'foo,bar' => 3, "bar,foo" => 4); 28 | $config = array( 29 | 'foo' => 8, 30 | 'test' => array('bar' => 10, 'foo,bar' => 12), 31 | 'development' => array('baz' => 14, 'foo,bar' => 16), 32 | ); 33 | $configFile = 'test/misc/resque-pool.yml'; 34 | $customConfigFile = 'test/misc/resque-pool-custom.yml.php'; 35 | 36 | $testEnv = function() { putenv('RESQUE_ENV=test'); }; 37 | $devEnv = function() { putenv('RESQUE_ENV=development'); }; 38 | $noEnv = function() { putenv('RESQUE_ENV='); }; 39 | 40 | return array( 41 | array($simpleConfig, $noEnv, function($test, $subject) { 42 | $msg = 'passing a simple configuration array should load the values from the array'; 43 | $test->assertEquals(1, $subject->workerCount('foo'), $msg); 44 | $test->assertEquals(2, $subject->workerCount('bar'), $msg); 45 | $test->assertEquals(3, $subject->workerCount('foo,bar'), $msg); 46 | $test->assertEquals(4, $subject->workerCount('bar,foo'), $msg); 47 | $test->assertArrayEquals(array('foo', 'bar', 'foo,bar', 'bar,foo'), $subject->knownQueues(), $msg); 48 | }), 49 | array($config, $testEnv, function($test, $subject) { 50 | $test->assertEquals(8, $subject->workerCount('foo'), 'should load the default values from the Hash'); 51 | }), 52 | array($config, $testEnv, function($test, $subject) { 53 | $msg = 'Only queues should be returned from knownQueues'; 54 | $test->assertArrayEquals(array('foo', 'bar', 'foo,bar'), $subject->knownQueues(), $msg); 55 | }), 56 | array($config, $testEnv, function($test, $subject) { 57 | $msg = 'When RESQUE_ENV is set it should merge the values for the correct RESQUE_ENV'; 58 | $test->assertEquals(10, $subject->workerCount('bar'), $msg); 59 | $test->assertEquals(12, $subject->workerCount('foo,bar'), $msg); 60 | }), 61 | array($config, $testEnv, function($test, $subject) { 62 | $msg = 'When RESQUE_ENV is set it should not load the values for the other environments'; 63 | $test->assertEquals(12, $subject->workerCount('foo,bar'), $msg); 64 | $test->assertEquals(0, $subject->workerCount('baz'), $msg); 65 | }), 66 | array($config, $devEnv, function($test, $subject) { 67 | $msg = "When RESQUE_ENV is set it should load the config for that environment"; 68 | $test->assertEquals(8, $subject->workerCount('foo'), $msg); 69 | $test->assertEquals(16, $subject->workerCount('foo,bar'), $msg); 70 | $test->assertEquals(14, $subject->workerCount('baz'), $msg); 71 | $test->assertEquals(0, $subject->workerCount('bar'), $msg); 72 | }), 73 | array($config, $noEnv, function($test, $subject) { 74 | $msg = 'when there is no environment it should load the default values only'; 75 | $test->assertEquals(8, $subject->workerCount('foo'), $msg); 76 | $test->assertEquals(0, $subject->workerCount('bar'), $msg); 77 | $test->assertEquals(0, $subject->workerCount('foo,bar'), $msg); 78 | $test->assertEquals(0, $subject->workerCount('baz'), $msg); 79 | }), 80 | array(array(), $noEnv, function($test, $subject) { 81 | $test->assertEquals(array(), $subject->queueConfig(), 'given no configuration it should have no worker types'); 82 | }), 83 | array($configFile, $testEnv, function($test, $subject) { 84 | $test->assertEquals(1, $subject->workerCount('foo'), "when RESQUE_ENV is set it should load the default YAML"); 85 | }), 86 | array($configFile, $testEnv, function($test, $subject) { 87 | $msg = 'when RAILS_ENV is set it should merge the YAML for the correct RESQUE_ENV'; 88 | $test->assertEquals(5, $subject->workerCount('bar'), $msg); 89 | $test->assertEquals(3, $subject->workerCount('foo,bar'), $msg); 90 | }), 91 | array($configFile, $testEnv, function($test, $subject) { 92 | $msg = 'when RAILS_ENV is set it should not load the YAML for the other environments'; 93 | $test->assertEquals(1, $subject->workerCount('foo'), $msg); 94 | $test->assertEquals(5, $subject->workerCount('bar'), $msg); 95 | $test->assertEquals(3, $subject->workerCount('foo,bar'), $msg); 96 | $test->assertEquals(0, $subject->workerCount('baz'), $msg); 97 | }), 98 | array($configFile, $devEnv, function($test, $subject) { 99 | $msg = "When RESQUE_ENV is set it should load the config for that environment"; 100 | $test->assertEquals(1, $subject->workerCount('foo'), $msg); 101 | $test->assertEquals(4, $subject->workerCount('foo,bar'), $msg); 102 | $test->assertEquals(23, $subject->workerCount('baz'), $msg); 103 | $test->assertEquals(0, $subject->workerCount('bar'), $msg); 104 | }), 105 | array($configFile, $noEnv, function($test, $subject) { 106 | $msg = 'when there is no environment it should load the default values only'; 107 | $test->assertEquals(1, $subject->workerCount('foo'), $msg); 108 | $test->assertEquals(0, $subject->workerCount('bar'), $msg); 109 | $test->assertEquals(0, $subject->workerCount('foo,bar'), $msg); 110 | $test->assertEquals(0, $subject->workerCount('baz'), $msg); 111 | }), 112 | array($customConfigFile, $noEnv, function($test, $subject) { 113 | $test->assertEquals(2, $subject->workerCount('foo'), 'when there is php in the yaml it should be parsed'); 114 | }), 115 | ); 116 | 117 | } 118 | 119 | /** 120 | * @dataProvider loadingThePoolConfigurationProvider 121 | */ 122 | public function testLoadingThePoolConfiguration($config, $before, $test) 123 | { 124 | $before && $before(); 125 | 126 | $config = new Configuration($config, $this->mockLogger()); 127 | $config->initialize(); 128 | 129 | $test($this, $config); 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /test/Resque/Pool/Tests/Mock/Worker.php: -------------------------------------------------------------------------------- 1 | queues = $queues; 16 | } 17 | 18 | public function work() 19 | { 20 | ++$this->calledWork; 21 | } 22 | 23 | public function __toString() 24 | { 25 | return __CLASS__; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /test/Resque/Pool/Tests/PoolTest.php: -------------------------------------------------------------------------------- 1 | 1)), 16 | array(array('foo'=>1,'bar'=>1)), 17 | array(array('foo'=>7,'bar'=>5,'baz'=>3)), 18 | ); 19 | } 20 | 21 | /** 22 | * @dataProvider maintainWorkerCountUpwardsProvider 23 | */ 24 | public function testMaintainWorkerCountUpwards(array $queueConfig) 25 | { 26 | list($pool, $pids) = $this->poolForSpawn($queueConfig); 27 | $pool->maintainWorkerCount(); 28 | $this->assertCount(array_sum($queueConfig), $pool->allPids()); 29 | } 30 | 31 | public function testAllPidsEmpty() 32 | { 33 | $pool = new Pool(new Configuration, $this->mockLogger()); 34 | $this->assertEquals(array(), $pool->allPids()); 35 | } 36 | 37 | public function testAllKnownQueues() 38 | { 39 | list($pool, $config, $pids) = $this->poolForSpawn(array('foo'=>1,'bar,baz'=>3), false); 40 | $this->assertArrayEquals(array('foo','bar,baz'), $pool->allKnownQueues()); 41 | $config->resetQueues(); 42 | $this->assertEquals(array(), $pool->allKnownQueues()); 43 | 44 | list($pool, $config, $pids) = $this->poolForSpawn(array('foo'=>1,'bar,baz'=>3)); 45 | $pool->maintainWorkerCount(); 46 | $this->assertArrayEquals(array('foo','bar,baz'), $pool->allKnownQueues()); 47 | $config->resetQueues(); 48 | // These queues are still known because they haven't been reaped yet 49 | $this->assertArrayEquals(array('foo','bar,baz'), $pool->allKnownQueues()); 50 | } 51 | 52 | public function testWorkerQueues() 53 | { 54 | list($pool, $config, $pids) = $this->poolForSpawn(array('baz'=>1)); 55 | $this->assertNull($pool->workerQueues(null)); 56 | $this->assertNull($pool->workerQueues(array())); 57 | $this->assertNull($pool->workerQueues('foo')); 58 | 59 | $this->assertNull($pool->workerQueues(reset($pids))); 60 | $pool->maintainWorkerCount(); 61 | $this->assertEquals('baz', $pool->workerQueues(reset($pids))); 62 | } 63 | 64 | public function testCallAfterPreFork() 65 | { 66 | $config = new Configuration(array('bang,boom'=>1)); 67 | $config->logger = $this->mockLogger(); 68 | $config->workerClass = __NAMESPACE__.'\\Mock\\Worker'; 69 | $config->platform = $this->mockPlatform(); 70 | $config->platform->expects($this->once()) 71 | ->method('pcntl_fork') 72 | ->will($this->returnValue(0)); // 0 means pretend its the child process 73 | 74 | $pool = new Pool($config); 75 | $called = 0; 76 | $test = $this; 77 | $config->afterPreFork = function($pool, $worker) use (&$called, $test) { 78 | $test->assertInstanceOf('Resque\\Pool\\Pool', $pool); 79 | $test->assertInstanceOf(__NAMESPACE__.'\\Mock\\Worker', $worker); 80 | $called++; 81 | }; 82 | 83 | $pool->maintainWorkerCount(); 84 | $this->assertEquals(1, $called); 85 | $this->assertEquals(1, Mock\Worker::$instances[0]->calledWork); 86 | $this->assertArrayEquals(array('bang','boom'), Mock\Worker::$instances[0]->queues); 87 | } 88 | 89 | protected function poolForSpawn($queueConfig = null, $mockFork=true) 90 | { 91 | $config = new Configuration($queueConfig); 92 | $config->logger = $this->mockLogger(); 93 | $config->platform = $this->mockPlatform(); 94 | $pool = new Pool($config); 95 | 96 | $workers = array_sum($queueConfig); 97 | $pids = range(1,$workers); 98 | 99 | if ($mockFork) { 100 | $config->platform->expects($this->exactly($workers)) 101 | ->method('pcntl_fork') 102 | ->will(new \PHPUnit_Framework_MockObject_Stub_ConsecutiveCalls($pids)); 103 | } 104 | 105 | return array($pool, $config, $pids); // NOTE: the returned $pids is a copy, not a reference like in the closure 106 | } 107 | 108 | protected function mockPlatform() 109 | { 110 | return $this->getMock('Resque\\Pool\\Platform'); 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /test/bootstrap.php: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /test/misc/resque-pool.yml: -------------------------------------------------------------------------------- 1 | foo: 1 2 | 3 | production: 4 | "foo,bar": 10 5 | 6 | development: 7 | "foo,bar": 4 8 | "baz": 23 9 | 10 | test: 11 | "bar": 5 12 | "foo,bar": 3 13 | 14 | --------------------------------------------------------------------------------