├── .github ├── actions │ └── composite │ │ └── setup-composer-cache │ │ └── action.yml └── workflows │ └── bedrock-php.yml ├── .gitignore ├── .phan └── config.php ├── .php-cs-fixer.php ├── README.md ├── bin └── BedrockWorkerManager.php ├── ci ├── phan.php ├── php-style-fixer ├── src │ ├── CommandLine.php │ ├── PHPStyler.php │ ├── PhanAnalyzer.php │ └── Travis.php └── style.php ├── composer.json ├── composer.lock ├── package.sh ├── sample ├── SampleWorker.php └── demo.sh └── src ├── Cache.php ├── Client.php ├── DB.php ├── DB └── Response.php ├── Exceptions ├── BedrockError.php ├── Cache │ └── NotFound.php ├── ConnectionFailure.php └── Jobs │ ├── DoesNotExist.php │ ├── GenericError.php │ ├── IllegalAction.php │ ├── MalformedAttribute.php │ ├── RetryableException.php │ └── SqlFailed.php ├── Jobs.php ├── LocalDB.php ├── Plugin.php ├── Stats ├── NullStats.php └── StatsInterface.php └── Status.php /.github/actions/composite/setup-composer-cache/action.yml: -------------------------------------------------------------------------------- 1 | name: Set up Composer Cache 2 | description: Set up Composer Cache 3 | 4 | runs: 5 | using: composite 6 | steps: 7 | - name: Get composer cache directory 8 | id: composer-cache 9 | run: echo "dir=$(composer config cache-files-dir)" >> $GITHUB_OUTPUT 10 | shell: bash 11 | 12 | - name: Cache Composer Files 13 | # v4.2.0 14 | uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 15 | with: 16 | path: ${{ steps.composer-cache.outputs.dir }} 17 | key: ${{ runner.os }}-composer-${{ hashFiles('**/composer.lock') }} 18 | restore-keys: ${{ runner.os }}-composer- 19 | 20 | - name: Composer install 21 | run: composer install --prefer-dist --no-interaction --dev 22 | shell: bash 23 | -------------------------------------------------------------------------------- /.github/workflows/bedrock-php.yml: -------------------------------------------------------------------------------- 1 | name: Test Suite 2 | on: 3 | push: 4 | branches: 5 | - "**/*" 6 | - "!main" 7 | concurrency: 8 | group: "${{ github.ref }}" 9 | cancel-in-progress: true 10 | env: 11 | TRAVIS_COMMIT: ${{ github.sha }} 12 | TRAVIS_BRANCH: ${{ github.ref }} 13 | jobs: 14 | PHP_Phan: 15 | name: Phan 16 | runs-on: ubuntu-24.04 17 | timeout-minutes: 15 18 | steps: 19 | - name: checkout 20 | # v4.1.0 21 | uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 22 | with: 23 | # Set fetch-depth to 100 so that we can compare HEAD with a good chunk of git log history 24 | fetch-depth: 100 25 | 26 | # Run this after the packages are installed to ensure our apt-mirror is set up. 27 | - name: Install PHP and Libraries 28 | uses: shivammathur/setup-php@9e72090525849c5e82e596468b86eb55e9cc5401 # v2.32.0 29 | with: 30 | tools: composer:v2.7.1 31 | php-version: 8.3.20 32 | coverage: none 33 | env: 34 | GITHUB_TOKEN: ${{ secrets.CODE_EXPENSIFY_TOKEN }} 35 | fail-fast: true 36 | 37 | - name: Setup Composer Cache 38 | uses: ./.github/actions/composite/setup-composer-cache 39 | 40 | - name: Run Phan tests 41 | run: "php ./ci/phan.php" 42 | PHP_Style: 43 | name: PHP Style 44 | runs-on: ubuntu-24.04 45 | timeout-minutes: 15 46 | steps: 47 | - name: checkout 48 | # v4.1.0 49 | uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 50 | with: 51 | # Set fetch-depth to 100 so that we can compare HEAD with a good chunk of git log history 52 | fetch-depth: 100 53 | 54 | - name: Install PHP and Libraries 55 | uses: shivammathur/setup-php@9e72090525849c5e82e596468b86eb55e9cc5401 # v2.32.0 56 | with: 57 | tools: composer:v2.7.1 58 | php-version: 8.3.20 59 | coverage: none 60 | env: 61 | GITHUB_TOKEN: ${{ secrets.CODE_EXPENSIFY_TOKEN }} 62 | fail-fast: true 63 | 64 | - name: Setup Composer Cache 65 | uses: ./.github/actions/composite/setup-composer-cache 66 | 67 | - name: Test for Style 68 | run: "php ./ci/style.php" 69 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | vendor 2 | .idea/ 3 | .php_cs.cache 4 | composer.json-e 5 | .DS_Store 6 | -------------------------------------------------------------------------------- /.phan/config.php: -------------------------------------------------------------------------------- 1 | [ 15 | 'src/', 16 | 'bin/', 17 | 'vendor/', 18 | ], 19 | 'file_list' => [ 20 | ], 21 | // A directory list that defines files that will be excluded 22 | // from static analysis, but whose class and method 23 | // information should be included. 24 | // Generally, you'll want to include the directories for 25 | // third-party code (such as "vendor/") in this list. 26 | // n.b.: If you'd like to parse but not analyze 3rd 27 | // party code, directories containing that code 28 | // should be added to the `directory_list` as 29 | // to `excluce_analysis_directory_list`. 30 | "exclude_analysis_directory_list" => [ 31 | 'vendor/', 32 | ], 33 | ]; 34 | -------------------------------------------------------------------------------- /.php-cs-fixer.php: -------------------------------------------------------------------------------- 1 | exclude('vendor') 6 | ->exclude('externalLib') 7 | ->in(__DIR__) 8 | ; 9 | $config = new PhpCsFixer\Config(); 10 | $config 11 | ->setRules([ 12 | '@PSR12' => true, 13 | '@Symfony' => true, 14 | 'phpdoc_annotation_without_dot' => false, 15 | 'phpdoc_summary' => false, 16 | 'single_quote' => true, 17 | 'ordered_imports' => true, 18 | 'no_break_comment' => false, 19 | 'blank_line_before_statement' => false, 20 | 'increment_style' => false, 21 | 'yoda_style' => false, 22 | 'phpdoc_to_comment' => false, // Need to disable this to use single-line suppressions with Psalm 23 | 'array_syntax' => ['syntax' => 'short'], 24 | 'no_useless_else' => true, 25 | 'single_line_throw' => false, 26 | 'heredoc_to_nowdoc' => true, 27 | 'global_namespace_import' => true, 28 | 'fully_qualified_strict_types' => true, 29 | ]) 30 | ->setUsingCache(true) 31 | ->setFinder($finder); 32 | 33 | return $config; 34 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Bedrock-PHP 2 | This is a library to interact with [Bedrock](https://github.com/Expensify/Bedrock) 3 | 4 | # Publishing Your Changes 5 | When you want to publish a new version: 6 | 7 | 1. Create a new branch 8 | 1. Commit your changes 9 | 1. Update `/composer.json` to have the new version number and commit those changes as well 10 | 1. Tag your branch with the new version number, for example `git tag 1.0.4` 11 | 1. Push you branch to the remote `git push origin HEAD --tags` 12 | 1. Create a PR and assign it to someone for review 13 | -------------------------------------------------------------------------------- /bin/BedrockWorkerManager.php: -------------------------------------------------------------------------------- 1 | --workerPath= --maxLoad= [--maxIterations= --versionWatchFile= --writeConsistency= --enableLoadHandler --minSafeJobs= --maxSafeTime= --localJobsDBPath= --debugThrottle]` 26 | */ 27 | 28 | // Verify it's being started correctly 29 | if (php_sapi_name() !== 'cli') { 30 | // Throw an exception rather than just output because we assume this is 31 | // being executed on a webserver, so no STDOUT. Hopefully they've 32 | // configured a general uncaught exception handler, and this will trigger 33 | // that. 34 | throw new Exception('This script is cli only'); 35 | } 36 | 37 | // Parse the command line and verify the required settings are provided 38 | $options = getopt('', ['maxLoad::', 'maxIterations::', 'jobName::', 'logger::', 'stats::', 'workerPath::', 39 | 'versionWatchFile::', 'writeConsistency::', 'enableLoadHandler', 'minSafeJobs::', 'maxJobsInSingleRun::', 40 | 'maxSafeTime::', 'localJobsDBPath::', 'debugThrottle', 'backoffThreshold::', 41 | 'intervalDurationSeconds::', 'doubleBackoffPreventionIntervalFraction::', 'multiplicativeDecreaseFraction::', 42 | 'jobsToAddPerSecond::', 'profileChangeThreshold::', ]); 43 | 44 | $workerPath = $options['workerPath'] ?? null; 45 | if (!$workerPath) { 46 | echo "Usage: sudo -u user php ./bin/BedrockWorkerManager.php --workerPath= [--jobName= --maxLoad= --maxIterations= --writeConsistency= --enableLoadHandler --minSafeJobs= --maxJobsInSingleRun= --maxSafeTime= --localJobsDBPath= --debugThrottle]\r\n"; 47 | exit(1); 48 | } 49 | 50 | // Add defaults 51 | $jobName = $options['jobName'] ?? '*'; // Process all jobs by default 52 | $maxLoad = floatval($options['maxLoad'] ?? 1.0); // Max load of 1.0 by default 53 | $maxIterations = intval($options['maxIterations'] ?? -1); // Unlimited iterations by default 54 | $pathToDB = $options['localJobsDBPath'] ?? '/tmp/localJobsDB.sql'; 55 | $minSafeJobs = intval($options['minSafeJobs'] ?? 10); // The floor of the number of jobs we'll target running simultaneously. 56 | $maxJobsForSingleRun = intval($options['maxJobsInSingleRun'] ?? 10); 57 | $maxSafeTime = intval($options['maxSafeTime'] ?? 0); // The maximum job time before we start paying attention 58 | $debugThrottle = isset($options['debugThrottle']); // Set to true to maintain a debug history 59 | $enableLoadHandler = isset($options['enableLoadHandler']); // Enables the AIMD load handler 60 | 61 | // The fraction of run time the current batch of jobs needs to be in relation to the previous batch to cause us to 62 | // back off our target number of jobs. 63 | $backoffThreshold = floatval($options['backoffThreshold'] ?? 1.1); 64 | 65 | // The length of time in seconds that we use to determine the average speed of jobs currently running. 66 | $intervalDurationSeconds = floatval($options['intervalDurationSeconds'] ?? 10.0); 67 | 68 | // The fraction of $intervalDurationSeconds that we block a double backoff from happening in. This keeps us from 69 | // backing off to the base value on each successive run when we really only want to backoff by one step. 70 | $doubleBackoffPreventionIntervalFraction = floatval($options['doubleBackoffPreventionIntervalFraction'] ?? 1.0); 71 | 72 | // When we back off, we'll multiply the target by this value. Should be between 0 and 1. 73 | $multiplicativeDecreaseFraction = floatval($options['multiplicativeDecreaseFraction'] ?? 0.8); 74 | 75 | // Try to increase the target by this many jobs every second. 76 | $jobsToAddPerSecond = floatval($options['jobsToAddPerSecond'] ?? 1.0); 77 | 78 | $profileChangeThreshold = floatval($options['profileChangeThreshold'] ?? 0.25); 79 | 80 | // Internal state variables for determining the number of jobs to run at one time. 81 | // $target is the number of jobs that we think we can safely run at one time. It defaults to the number of jobs we've 82 | // decided is always safe, and is continually adjusted by `getNumberOfJobsToQueue`. 83 | $target = $minSafeJobs; 84 | $lastRun = microtime(true); 85 | $lastBackoff = 0; 86 | 87 | // This is the name of a particular job if it made up over 50% of the jobs previously returned. Its purpose is to 88 | // detect when we switch job types so that we can react appropriately. 89 | $lastJobProfile = 'none'; 90 | 91 | $bedrock = Client::getInstance(); 92 | 93 | // Prepare to use the host logger and stats client, if configured 94 | $logger = $bedrock->getLogger(); 95 | $logger->info('Starting BedrockWorkerManager', ['maxIterations' => $maxIterations]); 96 | $stats = $bedrock->getStats(); 97 | 98 | // Set up the database for the AIMD load handler. 99 | $localDB = new LocalDB($pathToDB, $logger, $stats); 100 | if ($enableLoadHandler) { 101 | $localDB->open(); 102 | $query = 'CREATE TABLE IF NOT EXISTS localJobs ( 103 | localJobID integer PRIMARY KEY AUTOINCREMENT NOT NULL, 104 | jobID integer NOT NULL, 105 | jobName text NOT NULL, 106 | started text NOT NULL, 107 | ended text, 108 | workerPID integer NOT NULL, 109 | retryAfter text 110 | ); 111 | CREATE INDEX IF NOT EXISTS localJobsLocalJobID ON localJobs (localJobID); 112 | PRAGMA journal_mode = WAL;'; 113 | $localDB->write($query); 114 | } 115 | 116 | // If --versionWatch is enabled, begin watching a version file for changes. If the file doesn't exist, create it. 117 | $versionWatchFile = @$options['versionWatchFile']; 118 | if ($versionWatchFile && !file_exists($versionWatchFile)) { 119 | touch($versionWatchFile); 120 | } 121 | $versionWatchFileTimestamp = $versionWatchFile && file_exists($versionWatchFile) ? filemtime($versionWatchFile) : false; 122 | 123 | // Wrap everything in a general exception handler so we can handle error 124 | // conditions as gracefully as possible. 125 | try { 126 | // Validate details now that we have exception handling 127 | if (!is_dir($workerPath)) { 128 | throw new Exception("Invalid --workerPath path '$workerPath'"); 129 | } 130 | if ($maxLoad <= 0) { 131 | throw new Exception('--maxLoad must be greater than zero'); 132 | } 133 | $jobs = new Jobs($bedrock); 134 | 135 | // If --maxIterations is set, loop a finite number of times and then self 136 | // destruct. This is to guard against memory leaks, as we assume there is 137 | // some other script that will restart this when it dies. 138 | $iteration = 0; 139 | $loopStartTime = 0; 140 | while (true) { 141 | // Is it time to self destruct? 142 | if ($maxIterations > 0 && $iteration >= $maxIterations) { 143 | $logger->info('We did all our loops iteration, shutting down'); 144 | break; 145 | } 146 | $iteration++; 147 | $logger->info('Loop iteration', ['iteration' => $iteration]); 148 | 149 | // Step One wait for resources to free up 150 | $isFirstTry = true; 151 | $jobsToQueue = 0; 152 | while (true) { 153 | $childProcesses = []; 154 | // Get the latest load 155 | if (!file_exists('/proc/loadavg')) { 156 | throw new Exception('are you in a chroot? If so, please make sure /proc is mounted correctly'); 157 | } 158 | 159 | if ($versionWatchFile && checkVersionFile($versionWatchFile, $versionWatchFileTimestamp, $stats)) { 160 | $logger->info('Version watch file changed, stop processing new jobs'); 161 | 162 | // We break out of this loop and the outer one too. We don't want to process anything more, 163 | // just wait for child processes to finish. 164 | break 2; 165 | } 166 | 167 | // Check if we can fork based on the load of our webservers 168 | $load = sys_getloadavg()[0]; 169 | $jobsToQueue = getNumberOfJobsToQueue(); 170 | if ($load > $maxLoad) { 171 | $logger->info('[AIMD] Not safe to start a new job, load is too high, waiting 1s and trying again.', ['load' => $load, 'MAX_LOAD' => $maxLoad]); 172 | sleep(1); 173 | } elseif ($jobsToQueue >= 3) { 174 | // We ensure we ask minimum 3 jobs per GetJobs call, to avoid running tons of GetJobs calls back to back 175 | // to return just 1 job 176 | $logger->info('[AIMD] Safe to start a new job, checking for more work', ['jobsToQueue' => $jobsToQueue, 'target' => $target, 'load' => $load, 'MAX_LOAD' => $maxLoad]); 177 | $stats->counter('bedrockWorkerManager.currentJobsToQueue', $jobsToQueue); 178 | $stats->counter('bedrockWorkerManager.targetJobsToQueue', (int) $target); 179 | break; 180 | } else { 181 | $logger->info('[AIMD] Not enough jobs to queue, waiting 0.5s and trying again.', ['jobsToQueue' => $jobsToQueue, 'target' => $target, 'load' => $load, 'MAX_LOAD' => $maxLoad]); 182 | $localDB->write('DELETE FROM localJobs WHERE started < '.(microtime(true) - 60 * 60).' AND ended IS NULL;'); 183 | $isFirstTry = false; 184 | usleep(500000); 185 | } 186 | } 187 | 188 | // Check to see if BWM was able to get jobs on the first attempt. If not, it would add a full second each time it failed, skewing the timer numbers. 189 | if ($isFirstTry) { 190 | $stats->timer('bedrockWorkerManager.fullLoop', microtime(true) - $loopStartTime); 191 | } 192 | 193 | // Poll the server until we successfully get a job 194 | $response = null; 195 | while (!$response) { 196 | if ($versionWatchFile && checkVersionFile($versionWatchFile, $versionWatchFileTimestamp, $stats)) { 197 | $logger->info('Version watch file changed, stop processing new jobs'); 198 | 199 | // We break out of this loop and the outer one too. We don't want to process anything more, 200 | // just wait for child processes to finish. 201 | break 2; 202 | } 203 | 204 | if (adminDownStatusEnabled($stats)) { 205 | $logger->info('ADMIN_DOWN status detected. Not spawning more child processes. Trying again after 1s.'); 206 | sleep(1); 207 | 208 | // Don't query for new jobs until the status has been lifted 209 | continue; 210 | } 211 | 212 | // Ready to get a new job 213 | try { 214 | // Query the server for a job 215 | $response = $jobs->getJobs($jobName, $jobsToQueue, ['getMockedJobs' => true]); 216 | } catch (Exception $e) { 217 | // Try again in a random amount of time between 1 and 10 seconds 218 | $wait = rand(1, 10); 219 | $logger->info('Problem getting job, retrying in a few seconds', ['message' => $e->getMessage(), 'wait' => $wait]); 220 | sleep($wait); 221 | } 222 | } 223 | 224 | // Found a job 225 | $loopStartTime = microtime(true); 226 | $response = $response ?? []; 227 | if ($response['code'] == 200) { 228 | // BWM jobs are '/' separated names, the last component of which 229 | // indicates the name of the worker to instantiate to execute this 230 | // job: 231 | // 232 | // arbitrary/optional/path/to/workerName 233 | // 234 | // We look for a file: 235 | // 236 | // /.php 237 | // 238 | // If it's there, we include it, and then create an object and run 239 | // it like: 240 | // 241 | // $worker = new $workerName( $job ); 242 | // $worker->run( ); 243 | // 244 | // The optional path info allows for jobs to be scheduled 245 | // selectively. For example, you may have separate jobs scheduled 246 | // as production/jobName and staging/jobName, with a WorkerManager 247 | // in each environment looking for each path. 248 | $jobsToRun = $response['body']['jobs']; 249 | 250 | // Check what's running now. 251 | $runningCounts = []; 252 | $runningTotal = 0; 253 | $running = $localDB->read('SELECT jobName FROM localJobs WHERE ended IS NULL;'); 254 | if ($running) { 255 | foreach ($running as $job) { 256 | $jobParts = explode('?', $job ?? ''); 257 | $runningName = explode('/', $jobParts[0])[1]; 258 | if (isset($runningCounts[$runningName])) { 259 | $runningCounts[$runningName]++; 260 | } else { 261 | $runningCounts[$runningName] = 1; 262 | } 263 | $runningTotal++; 264 | } 265 | $logger->info('[AIMD] current jobs (running):', $runningCounts); 266 | } 267 | 268 | // Now make a modified version of what's running that includes the jobs we just selected. 269 | $targetCounts = $runningCounts; 270 | $targetTotal = $runningTotal; 271 | foreach ($jobsToRun as $job) { 272 | $jobParts = explode('?', $job['name'] ?? ''); 273 | $job['name'] = $jobParts[0]; 274 | $workerName = explode('/', $job['name'])[1]; 275 | if (isset($targetCounts[$workerName])) { 276 | $targetCounts[$workerName]++; 277 | } else { 278 | $targetCounts[$workerName] = 1; 279 | } 280 | $targetTotal++; 281 | } 282 | $logger->info('[AIMD] current jobs (target):', $targetCounts); 283 | 284 | // Now we want to detect if the new target profile is "significantly different" to the existing profile. 285 | // How? 286 | // What if we compute the percentages of the total for each job. This results in a sort of "stacked line 287 | // graph" model that totals to 100%, with each job type being some fraction of this total. 288 | // We can compute this for each currently running job, and then again for each target job, and we can 289 | // detect if any job's percentage has changed drastically between the two. 290 | // For example, imagine jobs A, B, C and D, each with 25% of the currently running set of jobs. 291 | // When we re-compute the averages for the new targets, suppose that we end up with: 292 | // Job A: 10% 293 | // Job B: 10% 294 | // Job C: 25% 295 | // Job D: 55% 296 | // This shows an increase in D of 30%, which may go over some threshold (it's unclear what to set this 297 | // threshold at) and indicate a "change of job profile". 298 | // Note that the change is detected not in the number of any particular jobs, but in the percentage of jobs 299 | // as a whole. This prevents a single job type going from 1% to 3% of running jobs as counting as an 300 | // increase of 200%, which would likely be significant, when it makes up only a small fraction of all the 301 | // work currently being done. 302 | // 303 | // This deliberately fails to detect a gradual change in job profile. If a job goes from 5 to 7 to 10 to 12 304 | // to 15 to 20% of total jobs over several iterations, it may at no point hit a (for example) 10% increase 305 | // threshold, but a gradual increase like this should be handled by existing mechanisms. We are only trying 306 | // to detect sudden changes in job profiles with this code. 307 | $jobIncreases = []; 308 | foreach ($targetCounts as $name => $count) { 309 | $targetPercent = $count / max($targetTotal, 1); 310 | $runningPercent = ($runningCounts[$name] ?? 0) / max($runningTotal, 1); 311 | if ($runningPercent + $profileChangeThreshold < $targetPercent) { 312 | $jobIncreases[$name] = ['from' => $runningPercent, 'to' => $targetPercent]; 313 | } 314 | } 315 | if (count($jobIncreases)) { 316 | $logger->info('[AIMD] Job profile changed, increases: ', $jobIncreases); 317 | } 318 | 319 | foreach ($jobsToRun as $job) { 320 | $jobParts = explode('?', $job['name']); 321 | $extraParams = count($jobParts) > 1 ? $jobParts[1] : null; 322 | $job['name'] = $jobParts[0]; 323 | $workerName = explode('/', $job['name'])[1]; 324 | $workerFilename = $workerPath."/$workerName.php"; 325 | $stats->timer('bedrockJob.lateBy.'.$job['name'], (time() - strtotime($job['nextRun'])) * 1000); 326 | if (file_exists($workerFilename)) { 327 | // The file seems to exist -- fork it so we can run it. 328 | // 329 | // Note: By explicitly ignoring SIGCHLD we tell the kernel to 330 | // "reap" finished child processes automatically, rather 331 | // than creating "zombie" processes. (We don't care 332 | // about the child's exit code, so we have no use for the 333 | // zombie process.) 334 | $logger->info('Forking and running a worker.', [ 335 | 'workerFileName' => $workerFilename, 336 | ]); 337 | 338 | // Do the fork 339 | pcntl_signal(SIGCHLD, SIG_IGN); 340 | $pid = $stats->benchmark('bedrockWorkerManager.fork', function () { return pcntl_fork(); }); 341 | if ($pid == -1) { 342 | // Something went wrong, couldn't fork 343 | $errorMessage = pcntl_strerror(pcntl_get_last_error()); 344 | throw new Exception("Unable to fork because '$errorMessage', aborting."); 345 | } elseif ($pid == 0) { 346 | // Get a new localDB handle 347 | $localDB = new LocalDB($pathToDB, $logger, $stats); 348 | $localDB->open(); 349 | 350 | // Track each job that we've successfully forked 351 | $myPid = getmypid(); 352 | $localJobID = 0; 353 | if ($enableLoadHandler) { 354 | $safeJobName = SQLite3::escapeString($job['name']); 355 | $safeRetryAfter = SQLite3::escapeString($job['retryAfter'] ?? ''); 356 | $jobStartTime = microtime(true); 357 | $localDB->write("INSERT INTO localJobs (jobID, jobName, started, workerPID, retryAfter) VALUES ({$job['jobID']}, '$safeJobName', '$jobStartTime', $myPid, '$safeRetryAfter');"); 358 | $localJobID = $localDB->getLastInsertedRowID(); 359 | } 360 | 361 | // We forked, so we need to make sure the bedrock client opens new sockets inside this for, 362 | // instead of reusing the ones created by the parent process. But we also want to make sure we 363 | // keep the same commitCount because we need the finishJob call below to run in a server that has 364 | // the commit of the GetJobs call above or the job we are trying to finish might be in QUEUED state. 365 | $commitCount = Client::getInstance()->commitCount; 366 | /* @phan-suppress-next-line PhanTypeMismatchDimFetch */ 367 | Client::clearInstancesAfterFork($job['data']['_commitCounts'] ?? []); 368 | $bedrock = Client::getInstance(); 369 | $bedrock->commitCount = $commitCount; 370 | $jobs = new Jobs($bedrock); 371 | 372 | // For each child PID, we need a fresh random seed to prevent duplicate "random" numbers generated. 373 | // Otherwise similar jobs run in the same BWM batch that call rand() will return the same result. 374 | mt_srand(); 375 | 376 | // If we are using a global REQUEST_ID, reset it to indicate this is a new process. 377 | if (isset($GLOBALS['REQUEST_ID'])) { 378 | // Reset the REQUEST_ID and re-log the line so we see 379 | // it when searching for either the parent and child 380 | // REQUEST_IDs. 381 | $GLOBALS['REQUEST_ID'] = substr(str_replace(['+', '/'], 'x', base64_encode(openssl_random_pseudo_bytes(6))), 0, 6); // random 6 character ID 382 | } 383 | $logger->info('Fork succeeded, child process, running job', [ 384 | 'name' => $job['name'], 385 | 'id' => $job['jobID'], 386 | 'extraParams' => $extraParams, 387 | 'pid' => $myPid, 388 | ]); 389 | $stats->counter('bedrockJob.create.'.$job['name']); 390 | 391 | // Include the worker now (not in the parent thread) such 392 | // that we automatically pick up new versions over the 393 | // worker without needing to restart the parent. 394 | include_once $workerFilename; 395 | $stats->benchmark('bedrockJob.finish.'.$job['name'], function () use ($workerName, $bedrock, $jobs, $job, $extraParams, $logger, $localDB, $enableLoadHandler, $localJobID, $stats, $jobStartTime) { 396 | $worker = new $workerName($bedrock, $job); 397 | 398 | // Open the DB connection after the fork in the child process. 399 | try { 400 | if (!$worker->getParam('mockRequest')) { 401 | // Run the worker. If it completes successfully, finish the job. 402 | $worker->run(); 403 | 404 | // Success 405 | $logger->info('Job completed successfully, exiting.', [ 406 | 'name' => $job['name'], 407 | 'id' => $job['jobID'], 408 | 'extraParams' => $extraParams, 409 | ]); 410 | } else { 411 | $logger->info('Mock job, not running and marking as finished.', [ 412 | 'name' => $job['name'], 413 | 'id' => $job['jobID'], 414 | 'extraParams' => $extraParams, 415 | ]); 416 | } 417 | 418 | try { 419 | $jobs->finishJob((int) $job['jobID'], $worker->getData()); 420 | } catch (DoesNotExist $e) { 421 | // Job does not exist, but we know it had to exist because we were running it, so 422 | // we assume this is happening because we retried the command in a different server 423 | // after the first server actually ran the command (but we lost the response). 424 | $logger->info('Failed to FinishJob we probably retried the command so it is safe to ignore', ['job' => $job, 'exception' => $e]); 425 | } catch (IllegalAction $e) { 426 | // IllegalAction is returned when we try to finish a job not in RUNNING state (child 427 | // jobs are put in FINISHED state when they are finished), which can happen if we 428 | // retried the command in a different server after the first server actually ran the 429 | // command (but we lost the response). 430 | $logger->info('Failed to FinishJob we probably retried the command on a child job so it is safe to ignore', ['job' => $job, 'exception' => $e]); 431 | } catch (BedrockError $e) { 432 | if (!$job['retryAfter']) { 433 | throw $e; 434 | } 435 | $logger->info('Could not call finishJob successfully, but job has retryAfter, so not failing the job and letting it be processed again'); 436 | } 437 | } catch (RetryableException $e) { 438 | // Worker had a recoverable failure; retry again later. 439 | $logger->info('Job could not complete, retrying.', [ 440 | 'name' => $job['name'], 441 | 'id' => $job['jobID'], 442 | 'extraParams' => $extraParams, 443 | 'exception' => $e, 444 | ]); 445 | try { 446 | $jobs->retryJob((int) $job['jobID'], $e->getDelay(), $worker->getData(), $e->getName(), $e->getNextRun(), null, $e->getIgnoreRepeat()); 447 | } catch (IllegalAction|DoesNotExist $e) { 448 | // IllegalAction is returned when we try to finish a job that's not RUNNING, this 449 | // can happen if we retried the command in a different server 450 | // after the first server actually ran the command (but we lost the response). 451 | $logger->info('Failed to RetryJob we probably retried the command so it is safe to ignore', ['job' => $job, 'exception' => $e]); 452 | } 453 | } catch (Throwable $e) { 454 | $logger->alert('Job failed with errors, exiting.', [ 455 | 'name' => $job['name'], 456 | 'id' => $job['jobID'], 457 | 'extraParams' => $extraParams, 458 | 'exception' => $e, 459 | ]); 460 | // Worker had a fatal error -- mark as failed. 461 | try { 462 | $jobs->failJob((int) $job['jobID']); 463 | } catch (IllegalAction|DoesNotExist $e) { 464 | // IllegalAction is returned when we try to finish a job that's not RUNNING, this 465 | // can happen if we retried the command in a different server 466 | // after the first server actually ran the command (but we lost the response). 467 | $logger->info('Failed to FailJob we probably retried a repeat command so it is safe to ignore', ['job' => $job, 'exception' => $e]); 468 | } 469 | } finally { 470 | if ($enableLoadHandler) { 471 | $time = microtime(true); 472 | $jobDuration = $time - $jobStartTime; 473 | if ($jobDuration > 60) { 474 | $logger->notice('Job took longer than 1 minute', ['name' => $job['name'], 'jobID' => $job['jobID'], 'duration' => $jobDuration]); 475 | } 476 | $stats->benchmark('bedrockWorkerManager.db.write.update', function () use ($localDB, $localJobID, $time) { 477 | $localDB->write("UPDATE localJobs SET ended=$time WHERE localJobID=$localJobID;"); 478 | }); 479 | $localDB->close(); 480 | } 481 | } 482 | }); 483 | 484 | // The forked worker process is all done. 485 | $stats->counter('bedrockJob.finish.'.$job['name']); 486 | exit(1); 487 | } 488 | // Otherwise we are the parent thread -- continue execution 489 | $logger->info('Successfully ran job', [ 490 | 'name' => $job['name'], 491 | 'id' => $job['jobID'], 492 | 'pid' => $pid, 493 | ]); 494 | } else { 495 | // No worker for this job 496 | $logger->warning('No worker found, ignoring', ['jobName' => $job['name']]); 497 | $jobs->failJob((int) $job['jobID']); 498 | } 499 | } 500 | } elseif ($response['code'] == 303) { 501 | $logger->info('No job found, retrying.'); 502 | } else { 503 | $logger->warning('Failed to get job'); 504 | } 505 | } 506 | } catch (Throwable $e) { 507 | $message = $e->getMessage(); 508 | $logger->alert('BedrockWorkerManager.php exited abnormally', ['exception' => $e]); 509 | echo "Error: $message\r\n"; 510 | } 511 | 512 | $logger->info('Stopped BedrockWorkerManager, will not wait for children'); 513 | 514 | /** 515 | * Determines whether or not we call GetJob and try to start a new job 516 | * 517 | * @return int How many jobs it is safe to queue. 518 | */ 519 | function getNumberOfJobsToQueue(): int 520 | { 521 | global $backoffThreshold, 522 | $doubleBackoffPreventionIntervalFraction, 523 | $enableLoadHandler, 524 | $intervalDurationSeconds, 525 | $jobsToAddPerSecond, 526 | $lastBackoff, 527 | $lastRun, 528 | $localDB, 529 | $logger, 530 | $maxJobsForSingleRun, 531 | $minSafeJobs, 532 | $multiplicativeDecreaseFraction, 533 | $target; 534 | 535 | // Allow for disabling of the load handler. 536 | if (!$enableLoadHandler) { 537 | $logger->info('[AIMD] Load handler not enabled, scheduling max jobs.', ['maxJobsForSingleRun' => $maxJobsForSingleRun]); 538 | 539 | return $maxJobsForSingleRun; 540 | } 541 | $now = microtime(true); 542 | 543 | // Following line is only for testing. 544 | // $secondElapsed = (intval($now) === intval($lastRun)) ? 0 : intval($now); 545 | 546 | // Get timing info for the last two intervals. 547 | $timeSinceLastRun = $now - $lastRun; 548 | $lastRun = $now; 549 | $oneIntervalAgo = $now - $intervalDurationSeconds; 550 | $twoIntervalsAgo = $oneIntervalAgo - $intervalDurationSeconds; 551 | 552 | // Look up how many jobs are currently in progress. 553 | $numActive = intval($localDB->read('SELECT COUNT(*) FROM localJobs WHERE ended IS NULL;')[0]); 554 | 555 | // Look up how many jobs we've finished recently. 556 | $lastIntervalData = $localDB->read('SELECT COUNT(*), AVG(ended - started) FROM localJobs WHERE ended IS NOT NULL AND ended > '.$oneIntervalAgo.';'); 557 | $lastIntervalCount = $lastIntervalData[0]; 558 | $lastIntervalAverage = floatval($lastIntervalData[1]); 559 | 560 | $previousIntervalData = $localDB->read('SELECT COUNT(*), AVG(ended - started) FROM localJobs WHERE ended IS NOT NULL AND ended > '.$twoIntervalsAgo.' AND ended < '.$oneIntervalAgo.';'); 561 | $previousIntervalCount = $previousIntervalData[0]; 562 | $previousIntervalAverage = floatval($previousIntervalData[1]); 563 | 564 | // Following block is only for testing. 565 | // if ($secondElapsed) { 566 | // echo "$secondElapsed, $numActive, $target, $lastIntervalAverage\n"; 567 | // } 568 | 569 | // Delete old stuff. 570 | $localDB->write('DELETE FROM localJobs WHERE ended IS NOT NULL AND ended < '.$twoIntervalsAgo.';'); 571 | 572 | // If we don't have enough data, we'll return a value based on the current target and active job count. 573 | if ($lastIntervalCount === 0) { 574 | $logger->info('[AIMD] No jobs finished this interval, returning default value.', ['minSafeJobs' => $minSafeJobs, 'returnValue' => max($target - $numActive, 0)]); 575 | 576 | return intval(max($target - $numActive, 0)); 577 | } elseif ($previousIntervalCount === 0) { 578 | $logger->info('[AIMD] No jobs finished previous interval, returning default value.', ['minSafeJobs' => $minSafeJobs, 'returnValue' => max($target - $numActive, 0)]); 579 | 580 | return intval(max($target - $numActive, 0)); 581 | } 582 | 583 | // Update our target. If the last interval average run time exceeds the previous one by too much, back off. 584 | // Options: 585 | // 1. Make intervalDurationSeconds longer for more data to average. 586 | // 2. Make backoffThreshold higher (this seems riskier) 587 | // 3. Back off by less (increase multiplicativeDecreaseFraction closer to 1). 588 | // 589 | // Possibly helpful ideas: 590 | // Log the count and type of jobs used to calculate lastIntervalData and previousIntervalData. 591 | // Also log the times for each type of job. 592 | // 593 | // Just knowing the count of completed jobs in the previous intervals is interesting. If it's a very small number 594 | // of jobs, a high degree of variability is expected. 595 | if ($lastIntervalAverage > ($previousIntervalAverage * $backoffThreshold)) { 596 | // Skip backoff if we've done so too recently in the past. (within 10 seconds by default) 597 | if ($lastBackoff < $now - ($intervalDurationSeconds * $doubleBackoffPreventionIntervalFraction)) { 598 | $target = max($target * $multiplicativeDecreaseFraction, $minSafeJobs); 599 | $lastBackoff = $now; 600 | $logger->info('[AIMD] Backing off jobs target.', [ 601 | 'target' => $target, 602 | 'lastIntervalAverage' => $lastIntervalAverage, 603 | 'previousIntervalAverage' => $previousIntervalAverage, 604 | 'backoffThreshold' => $backoffThreshold, 605 | ]); 606 | } 607 | } else { 608 | // Otherwise, slowly ramp up. Increase by $jobsToAddPerSecond every second, except don't increase past 2x the 609 | // number of currently running jobs. 610 | if (($target + $timeSinceLastRun * $jobsToAddPerSecond) < $numActive * 2) { 611 | // Ok, we're running at least half this many jobs, we can increment. 612 | $target += $timeSinceLastRun * $jobsToAddPerSecond; 613 | } 614 | $logger->info('[AIMD] Congestion Avoidance, incrementing target', ['target' => $target]); 615 | } 616 | 617 | // Now we know how many jobs we want to be running, and how many are running, so we can return the difference. 618 | $numJobsToRun = intval(max($target - $numActive, 0)); 619 | $logger->info('[AIMD] Found number of jobs to run.', [ 620 | 'numJobsToRun' => $numJobsToRun, 621 | 'target' => $target, 622 | 'numActive' => $numActive, 623 | 'lastIntervalAverage' => $lastIntervalAverage, 624 | 'previousIntervalAverage' => $previousIntervalAverage, 625 | 'lastIntervalCount' => $lastIntervalCount, 626 | 'previousIntervalCount' => $previousIntervalCount, 627 | 'timeSinceLastRun' => $timeSinceLastRun, 628 | ]); 629 | 630 | return $numJobsToRun; 631 | } 632 | 633 | /** 634 | * Watch a version file that will cause us to automatically shut 635 | * down if it changes. This enables triggering a restart if new 636 | * PHP is deployed. 637 | * 638 | * Note: php's filemtime results are cached, so we need to clear 639 | * that cache or we'll be getting a stale modified time. 640 | * 641 | * @param Expensify\Bedrock\Stats\StatsInterface $stats 642 | */ 643 | function checkVersionFile(string $versionWatchFile, int $versionWatchFileTimestamp, $stats): bool 644 | { 645 | return $stats->benchmark('bedrockWorkerManager.checkVersionFile', function () use ($versionWatchFile, $versionWatchFileTimestamp) { 646 | clearstatcache(true, $versionWatchFile); 647 | $newVersionWatchFileTimestamp = ($versionWatchFile && file_exists($versionWatchFile)) ? filemtime($versionWatchFile) : false; 648 | $versionChanged = $versionWatchFile && $newVersionWatchFileTimestamp !== $versionWatchFileTimestamp; 649 | 650 | return $versionChanged; 651 | }); 652 | } 653 | 654 | /** 655 | * Watch for an ADMIN_DOWN file that can be touched. If in place, we 656 | * will not spawn new child processes until it has been removed. 657 | * 658 | * @param Expensify\Bedrock\Stats\StatsInterface $stats 659 | */ 660 | function adminDownStatusEnabled($stats): bool 661 | { 662 | return $stats->benchmark('bedrockWorkerManager.adminDownStatusEnabled', function () { 663 | clearstatcache(true, Jobs::ADMIN_DOWN_FILE_LOCATION); 664 | return file_exists(Jobs::ADMIN_DOWN_FILE_LOCATION); 665 | }); 666 | } 667 | -------------------------------------------------------------------------------- /ci/phan.php: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env php 2 | analyze()) { 9 | exit(0); 10 | } else { 11 | exit(1); 12 | } 13 | -------------------------------------------------------------------------------- /ci/php-style-fixer: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # php-cs-fixer expects it's first parameter to be the command name 3 | COMMAND=$1; shift 4 | DIR=`git rev-parse --show-toplevel` 5 | REPO_NAME=`basename $DIR` 6 | php $DIR/vendor/bin/php-cs-fixer $COMMAND --config $DIR/.php-cs-fixer.php "$@" 7 | -------------------------------------------------------------------------------- /ci/src/CommandLine.php: -------------------------------------------------------------------------------- 1 | call("git checkout $branch 2>&1"); 19 | $this->call('git fetch origin main:main 2>&1'); 20 | Travis::fold("end", "git.checkout.".get_called_class()); 21 | } 22 | 23 | /** 24 | * Get the new/modified PHP files in a branch, with respect to main. 25 | * 26 | * @param string $branch 27 | * 28 | * @return array 29 | */ 30 | protected function getModifiedFiles($branch) 31 | { 32 | return $this->eexec("git diff main...$branch --name-status | grep -v 'vendor/' | egrep \"^[A|M].*\\.php$\" | cut -f 2"); 33 | } 34 | 35 | /** 36 | * @param string $cmd 37 | * @param bool $exitOnFail 38 | * 39 | * @return bool 40 | */ 41 | protected function call($cmd, $exitOnFail = true) 42 | { 43 | echo $cmd.PHP_EOL; 44 | system($cmd, $ret); 45 | if ($ret !== 0) { 46 | if ($exitOnFail) { 47 | echo 'Error: '.$ret.PHP_EOL; 48 | exit($ret); 49 | } 50 | 51 | return false; 52 | } 53 | 54 | return true; 55 | } 56 | 57 | /** 58 | * @param string $cmd 59 | * @param bool $exitOnFail 60 | * 61 | * @return array 62 | */ 63 | protected function eexec($cmd, $exitOnFail = true) 64 | { 65 | echo $cmd.PHP_EOL; 66 | exec($cmd, $output, $ret); 67 | if ($ret !== 0 && $exitOnFail) { 68 | echo 'Error: '.$ret.PHP_EOL; 69 | exit($ret); 70 | } 71 | 72 | return $output; 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /ci/src/PHPStyler.php: -------------------------------------------------------------------------------- 1 | branch = $branchToCheck; 29 | $this->commit = $commit; 30 | } 31 | 32 | /** 33 | * Checks the style. 34 | * 35 | * @return bool true if all is good, false if errors were found. 36 | */ 37 | public function check() 38 | { 39 | $PHPLintCommand = "find . -name '*.php' -not \\( -path './externalLib/*' -or -path './vendor/*' -or -path './build/*' \\) -print0 | xargs -0 -L 1 -n 1 -P 8 php -l 1>/dev/null"; 40 | 41 | if ($this->branch === 'main') { 42 | Travis::foldCall("lintmaster.php", $PHPLintCommand); 43 | 44 | echo 'Skipping style check for merge commits'; 45 | 46 | return true; 47 | } 48 | 49 | $this->checkoutBranch($this->branch); 50 | 51 | Travis::foldCall("lint.php", $PHPLintCommand); 52 | 53 | Travis::fold("start", "style.php"); 54 | Travis::timeStart(); 55 | echo 'Enforce PHP style'.PHP_EOL; 56 | $output = $this->getModifiedFiles($this->branch); 57 | $lintedFiles = []; 58 | $lintOK = true; 59 | $dir = $this->eexec('git rev-parse --show-toplevel')[0]; 60 | $gitRepoName = $this->eexec("basename $dir")[0]; 61 | foreach ($output as $file) { 62 | echo "Linting $file... ".PHP_EOL; 63 | 64 | $fixerCmd = "$dir/ci/php-style-fixer fix --diff $file"; 65 | $fileResult = $this->eexec($fixerCmd, true); 66 | $fileOK = true; 67 | foreach ($fileResult as $index => $line) { 68 | // When a file is fixed, it outputs ` 1) File.php` and this is the only way we have to detect if 69 | // something was fixed or not as the linter only exits with an error exit code when the fixer actually 70 | // fails (not when it fixes something). 71 | if (preg_match('/^\W*1\)/', $line)) { 72 | $fileOK = false; 73 | } 74 | } 75 | $lintedFiles[] = [$file, $fileOK, join(PHP_EOL, $fileResult)]; 76 | $lintOK = $lintOK && $fileOK; 77 | } 78 | 79 | echo "\n\nLinted files:\n\n"; 80 | 81 | foreach ($lintedFiles as $lint) { 82 | echo $lint[0].'... '.($lint[1] ? 'OK!' : 'FAIL!'.PHP_EOL.$lint[2]).PHP_EOL; 83 | } 84 | 85 | Travis::timeFinish(); 86 | Travis::fold("end", "style.php"); 87 | 88 | if (!$lintOK) { 89 | return false; 90 | } 91 | 92 | Travis::foldCall("git.checkout2", "git checkout {$this->commit} 2>&1"); 93 | 94 | return true; 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /ci/src/PhanAnalyzer.php: -------------------------------------------------------------------------------- 1 | branch = $branchToCheck; 21 | } 22 | 23 | /** 24 | * Analyzes the code. 25 | * 26 | * @return bool true if all is good, false if errors were found. 27 | */ 28 | public function analyze() 29 | { 30 | if ($this->branch === 'main') { 31 | echo 'Skipping style check for merge commits'; 32 | 33 | return true; 34 | } 35 | 36 | $this->checkoutBranch($this->branch); 37 | 38 | Travis::fold("start", "phan.analyze"); 39 | Travis::timeStart(); 40 | echo 'Analyze PHP using Phan'.PHP_EOL; 41 | $changedFiles = $this->eexec("git diff main...{$this->branch} --name-status | egrep \"^[A|M].*\\.php$\" | cut -f 2"); 42 | echo "Analyzing files:".PHP_EOL; 43 | $lintErrors = $this->eexec("./vendor/bin/phan -p -z --processes 5", false); 44 | $lintOK = true; 45 | foreach ($lintErrors as $lintError) { 46 | foreach ($changedFiles as $file) { 47 | if (strpos($lintError, $file) === 0) { 48 | echo 'FAIL! '.$file.': '.$lintError.PHP_EOL; 49 | $lintOK = false; 50 | break; 51 | } 52 | } 53 | } 54 | Travis::timeFinish(); 55 | Travis::fold("end", "phan.analyze"); 56 | 57 | return $lintOK; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /ci/src/Travis.php: -------------------------------------------------------------------------------- 1 | check(); 10 | exit((int) !$valid); 11 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "expensify/bedrock-php", 3 | "description": "Bedrock PHP Library", 4 | "type": "library", 5 | "version": "2.1.0", 6 | "authors": [ 7 | { 8 | "name": "Expensify", 9 | "email": "jobs@expensify.com" 10 | } 11 | ], 12 | "minimum-stability": "stable", 13 | "repositories": [ 14 | { 15 | "type": "git", 16 | "url": "https://github.com/Expensify/Bedrock-PHP.git" 17 | } 18 | ], 19 | "require": { 20 | "psr/log": "1.0.1" 21 | }, 22 | "require-dev": { 23 | "friendsofphp/php-cs-fixer": "^3.64.0", 24 | "phan/phan": "^5.4.5" 25 | }, 26 | "autoload": { 27 | "psr-4": { 28 | "Expensify\\Bedrock\\": "src" 29 | } 30 | }, 31 | "autoload-dev": { 32 | "psr-4": { 33 | "Expensify\\Bedrock\\CI\\": "ci/src" 34 | } 35 | }, 36 | "bin": [ 37 | "bin/BedrockWorkerManager.php" 38 | ] 39 | } 40 | -------------------------------------------------------------------------------- /package.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo Current version: 4 | sed -n -e '/"version"/p' composer.json 5 | 6 | echo What is the new version? 7 | read targetVersion 8 | 9 | sed -i -e 's/"version": "[0-9]*\.[0-9]*\.[0-9]*",/"version": "'$targetVersion'",/g' composer.json 10 | 11 | echo Committing build 12 | git add . 13 | git commit -m "build version $targetVersion" 14 | 15 | echo Tagging release... 16 | git tag -a "$targetVersion" -m "$targetVersion" 17 | 18 | echo Pushing tag 19 | git push origin --tags -------------------------------------------------------------------------------- /sample/SampleWorker.php: -------------------------------------------------------------------------------- 1 | bedrock->getLogger()->info("Running SampleWorker for '{$this->job['name']}'"); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /sample/demo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # This is a trival script to demonstrate Bedrock::Jobs 3 | 4 | # ----------------------- 5 | echo "Confirming bedrock is running" 6 | BEDROCK_PID=`pgrep bedrock` 7 | if [ -z "$BEDROCK_PID" ] 8 | then 9 | echo "Please start bedrock, eg: sudo ./bedrock -clean -fork" 10 | exit 11 | fi 12 | 13 | # ----------------------- 14 | echo "Clean up after the last demo" 15 | RESULT=`echo 'Query 16 | query: DELETE FROM jobs; 17 | connection: close 18 | 19 | ' | nc localhost 8888 | head -n 1` 20 | if [[ "$RESULT" != 200* ]] 21 | then 22 | echo "ERROR: Cleanup failed ($RESULT)" 23 | exit 24 | fi 25 | 26 | 27 | # ----------------------- 28 | echo 'Creating a SampleWorker job...' 29 | echo "CreateJob 30 | name: SampleWorker 31 | connection: close 32 | 33 | " | nc localhost 8888 > /dev/null 34 | sleep 1 35 | 36 | # ----------------------- 37 | echo "Confirming job is QUEUED" 38 | COUNT=`echo "Query: SELECT COUNT(*) FROM jobs; 39 | connection: close 40 | 41 | " | nc localhost 8888 | tail -n 1` 42 | if [ "$COUNT" != 1 ] 43 | then 44 | echo "ERROR: Failed to queue job (count=$COUNT)" 45 | exit 46 | fi 47 | 48 | # ----------------------- 49 | echo "Starting BWM..." 50 | php ../bin/BedrockWorkerManager.php --workerPath=. & 51 | PID=$! 52 | 53 | # ----------------------- 54 | while [ "$COUNT" != 0 ] 55 | do 56 | echo "Waiting for job to finish" 57 | COUNT=`echo "Query: SELECT COUNT(*) FROM jobs; 58 | connection: close 59 | 60 | " | nc localhost 8888 | tail -n 1` 61 | sleep 1 62 | done 63 | 64 | # ----------------------- 65 | echo "Done" 66 | kill $PID 67 | -------------------------------------------------------------------------------- /src/Cache.php: -------------------------------------------------------------------------------- 1 | client->getLogger()->info("BedrockCache read", [ 30 | 'key' => $name, 31 | 'version' => $version, 32 | ]); 33 | $response = $this->call("ReadCache", ["name" => $fullName]); 34 | if ($response['code'] === 404) { 35 | throw new NotFound('The cache entry could not be found', 666); 36 | } 37 | return $response['body']; 38 | } 39 | 40 | /** 41 | * Reads from the cache, but if it does not find the entry, it returns the passed default. 42 | * 43 | * @param string $name 44 | * @param mixed $default 45 | * @param string $version 46 | * 47 | * @return mixed 48 | */ 49 | public function readWithDefault($name, $default, $version = null) 50 | { 51 | try { 52 | return $this->read($name, $version); 53 | } catch (NotFound $e) { 54 | return $default; 55 | } 56 | } 57 | 58 | /** 59 | * Gets data from a cache, if it is not present, it computes it by calling $computeFunction and saves the result in the cache. 60 | * 61 | * @param string $name 62 | * @param null|string $version 63 | * @param callable $computeFunction 64 | * 65 | * @return array 66 | */ 67 | public function get(string $name, ?string $version, callable $computeFunction) 68 | { 69 | try { 70 | return $this->read($name, $version); 71 | } catch (NotFound $e) { 72 | $value = $computeFunction(); 73 | $this->write($name, $value, $version); 74 | return $value; 75 | } 76 | } 77 | 78 | /** 79 | * Writes a named value to the cache, overriding any value of the same 80 | * name. If a version is provided, also invalidates all other versions of 81 | * the value. This write is asynchronous (eg, it returns when it has been 82 | * successfully queued with the server, but before the write itself has 83 | * completed). 84 | * 85 | * @param string $name Arbitrary string used to uniquely name this value. 86 | * @param mixed $value Raw binary data to associate with this name 87 | * @param string $version (optional) Version identifier (eg, a timestamp, counter, name, etc) 88 | */ 89 | public function write($name, $value, $version = null, array $headers = []) 90 | { 91 | // By default, unless specified otherwise, we want writes to be async 92 | $headers = array_merge([ 93 | 'Connection' => 'forget', 94 | ], $headers); 95 | 96 | // If we have a version, invalidate previous versions 97 | if ($version) { 98 | // Invalidate all other versions of this name before setting 99 | $headers["invalidateName"] = "$name/*"; 100 | $headers["name"] = "$name/$version"; 101 | } else { 102 | // Just set this name 103 | $headers["name"] = "$name/"; 104 | } 105 | 106 | $this->call("WriteCache", $headers, json_encode($value)); 107 | } 108 | 109 | /** 110 | * Call the bedrock cache methods, and handle connection error. 111 | * 112 | * @param string $body 113 | * 114 | * @return mixed|null 115 | */ 116 | private function call(string $method, array $headers, $body = '') 117 | { 118 | // Both writing to and reading from the cache are always idempotent operations 119 | $headers['idempotent'] = true; 120 | 121 | try { 122 | return $this->client->getStats()->benchmark("bedrock.cache.$method", function () use ($method, $headers, $body) { 123 | return $this->client->call($method, $headers, $body); 124 | }); 125 | } catch (ConnectionFailure $e) { 126 | $this->client->getLogger()->alert('Bedrock Cache failure', ['exception' => $e]); 127 | } 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /src/Client.php: -------------------------------------------------------------------------------- 1 | clusterName = $config['clusterName']; 167 | $this->mainHostConfigs = $config['mainHostConfigs']; 168 | $this->failoverHostConfigs = $config['failoverHostConfigs']; 169 | $this->connectionTimeout = $config['connectionTimeout']; 170 | $this->readTimeout = $config['readTimeout']; 171 | $this->bedrockTimeout = $config['bedrockTimeout']; 172 | $this->logger = $config['logger']; 173 | $this->stats = $config['stats']; 174 | $this->writeConsistency = $config['writeConsistency']; 175 | $this->maxBlackListTimeout = $config['maxBlackListTimeout']; 176 | $this->commandPriority = $config['commandPriority']; 177 | $this->logParam = $config['logParam']; 178 | 179 | // If the caller explicitly set `mockRequests`, use that value. 180 | if (isset($config['mockRequests'])) { 181 | $this->mockRequests = $config['mockRequests']; 182 | } elseif (isset($_SERVER['HTTP_X_MOCK_REQUEST'])) { 183 | // otherwise check the http headers and set it 184 | $this->mockRequests = isset($_SERVER['HTTP_X_MOCK_REQUEST']); 185 | } 186 | 187 | // Make sure we have at least one host configured 188 | $this->logger->debug('Bedrock\Client - Constructed', ['clusterName' => $this->clusterName, 'mainHostConfigs' => $this->mainHostConfigs, 'failoverHostConfigs' => $this->failoverHostConfigs]); 189 | if (empty($this->mainHostConfigs)) { 190 | throw new BedrockError('Main hosts are not set, cannot instantiate bedrock client'); 191 | } 192 | } 193 | 194 | /** 195 | * Returns an instance of this class for the specified configuration. It will return the same instance if the same 196 | * configuration was passed (not counting the logger and stats params), unless clearInstancesAfterFork is called. 197 | */ 198 | public static function getInstance(array $config = []): Client 199 | { 200 | $config = array_merge(self::$defaultConfig, $config); 201 | ksort($config); 202 | $configForHash = $config; 203 | unset($configForHash['logger']); 204 | unset($configForHash['stats']); 205 | unset($configForHash['logParam']); 206 | $hash = sha1(print_r($configForHash, true)); 207 | if (isset(self::$instances[$hash])) { 208 | return self::$instances[$hash]; 209 | } 210 | $instance = new self($config); 211 | 212 | // If we had a preloaded commitCount, set it so that the first request uses it. 213 | if (isset(self::$preloadedCommitCounts[$instance->getClusterName()])) { 214 | $instance->commitCount = self::$preloadedCommitCounts[$instance->getClusterName()]; 215 | } 216 | self::$instances[$hash] = $instance; 217 | 218 | return $instance; 219 | } 220 | 221 | /** 222 | * After forking, you need to call this method to make sure the forks don't share the same socket and instead open 223 | * a new connection. 224 | * 225 | * @param int[] $preloadedCommitCounts 226 | */ 227 | public static function clearInstancesAfterFork(array $preloadedCommitCounts) 228 | { 229 | self::$instances = []; 230 | self::$preloadedCommitCounts = $preloadedCommitCounts; 231 | } 232 | 233 | /** 234 | * Sets the default config to use, these are used as defaults each time you create a new instance. 235 | */ 236 | public static function configure(array $config) 237 | { 238 | // Store the configuration 239 | self::$defaultConfig = array_merge([ 240 | 'clusterName' => 'bedrock', 241 | 'mainHostConfigs' => ['localhost' => ['blacklistedUntil' => 0, 'port' => 8888]], 242 | 'failoverHostConfigs' => ['localhost' => ['blacklistedUntil' => 0, 'port' => 8888]], 243 | 'connectionTimeout' => 1, 244 | 'readTimeout' => 120, 245 | 'bedrockTimeout' => 110, 246 | 'logger' => new NullLogger(), 247 | 'stats' => new NullStats(), 248 | 'writeConsistency' => 'ASYNC', 249 | 'maxBlackListTimeout' => 1, 250 | 'commandPriority' => null, 251 | 'logParam' => null, 252 | ], self::$defaultConfig, $config); 253 | } 254 | 255 | /** 256 | * @return LoggerInterface 257 | */ 258 | public function getLogger() 259 | { 260 | return $this->logger; 261 | } 262 | 263 | /** 264 | * Sets a logger instance on the object. 265 | */ 266 | public function setLogger(LoggerInterface $logger) 267 | { 268 | $this->logger = $logger; 269 | 270 | return null; 271 | } 272 | 273 | /** 274 | * @return StatsInterface 275 | */ 276 | public function getStats() 277 | { 278 | if (is_string($this->stats)) { 279 | $this->stats = new $this->stats(); 280 | } 281 | 282 | return $this->stats; 283 | } 284 | 285 | /** 286 | * Returns the last host successfully used. 287 | */ 288 | public function getLastHost(): string 289 | { 290 | return $this->lastHost; 291 | } 292 | 293 | /** 294 | * Returns the cluster name. 295 | */ 296 | public function getClusterName(): string 297 | { 298 | return $this->clusterName; 299 | } 300 | 301 | /** 302 | * Makes a direct call to Bedrock. 303 | * 304 | * @param string $method Request method 305 | * @param array $headers Request headers (optional) 306 | * @param string $body Request body (optional) 307 | * 308 | * @return array JSON response, or null on error 309 | * 310 | * @throws BedrockError 311 | * @throws ConnectionFailure 312 | */ 313 | public function call($method, $headers = [], $body = '') 314 | { 315 | // Start timing the entire end-to-end 316 | $timeStart = microtime(true); 317 | 318 | // Include the last CommitCount, if we have one 319 | if ($this->commitCount) { 320 | $headers['commitCount'] = $this->commitCount; 321 | } 322 | 323 | // Include the requestID for logging purposes 324 | if (isset($GLOBALS['REQUEST_ID'])) { 325 | $headers['requestID'] = $GLOBALS['REQUEST_ID']; 326 | } 327 | $headers['lastIP'] = $_SERVER['HTTP_X_FORWARDED_FOR'] ?? $_SERVER['REMOTE_ADDR'] ?? null; 328 | 329 | // Set the write consistency 330 | if ($this->writeConsistency) { 331 | $headers['writeConsistency'] = $this->writeConsistency; 332 | } 333 | 334 | // Add mock request header if set. 335 | if ($this->mockRequests) { 336 | $headers['mockRequest'] = true; 337 | } 338 | 339 | if ($this->commandPriority && !isset($headers['priority'])) { 340 | $headers['priority'] = $this->commandPriority; 341 | } 342 | 343 | if (!array_key_exists('timeout', $headers) && $this->bedrockTimeout) { 344 | $headers['timeout'] = $this->bedrockTimeout * 1000; 345 | } 346 | 347 | if (!array_key_exists('logParam', $headers) && $this->logParam) { 348 | $headers['logParam'] = $this->logParam; 349 | } 350 | 351 | $this->logger->info('Bedrock\Client - Starting a request', [ 352 | 'command' => $method, 353 | 'clusterName' => $this->clusterName, 354 | 'headers' => $headers, 355 | ]); 356 | 357 | // Construct the request 358 | $rawRequest = "$method\r\n"; 359 | foreach ($headers as $name => $value) { 360 | if (is_array($value) || is_object($value)) { 361 | // Passing flag JSON_PRESERVE_ZERO_FRACTION because otherwise PHP will serialize floating numbers like 2.0 to 2 instead of 2.0. 362 | // This can cause bugs in Auth since we can't know what type a float number will be and accessing an int with the accessor for 363 | // floats (or vice versa) would throw. 364 | $rawRequest .= "$name: ".addcslashes(json_encode($value, JSON_PRESERVE_ZERO_FRACTION), '\\')."\r\n"; 365 | } elseif (is_bool($value)) { 366 | $rawRequest .= "$name: ".($value ? 'true' : 'false')."\r\n"; 367 | } elseif ($value === null || $value === '') { 368 | // skip empty values 369 | } else { 370 | $rawRequest .= "$name: ".self::toUTF8(addcslashes($value, "\r\n\t\\"))."\r\n"; 371 | } 372 | } 373 | $rawRequest .= 'Content-Length: '.strlen($body)."\r\n"; 374 | $rawRequest .= "\r\n"; 375 | $rawRequest .= $body; 376 | 377 | $response = null; 378 | $preferredHost = null; 379 | if (isset($headers['host'])) { 380 | $preferredHost = $headers['host']; 381 | unset($headers['host']); 382 | } 383 | $hostConfigs = $this->getPossibleHosts($preferredHost); 384 | 385 | // If we passed a preferred host and we already had a connected socket, but to a different host and the preferred 386 | // host is not blacklisted (the preferred host is returned first in the possible hosts array only when it's not blacklisted) 387 | // then we close the socket in order to connect to the preferred one. 388 | $closeSocketAfterRequest = array_key_exists('Connection', $headers) ? $headers['Connection'] === 'close' : false; 389 | 390 | if ($preferredHost && $this->socket && key($hostConfigs) !== $this->lastHost) { 391 | @socket_close($this->socket); 392 | $this->socket = null; 393 | $closeSocketAfterRequest = true; 394 | } 395 | 396 | $hostName = null; 397 | $retriedAllHosts = false; 398 | while (!$response && count($hostConfigs)) { 399 | $exception = null; 400 | reset($hostConfigs); 401 | $numRetriesLeft = count($hostConfigs) - 1; 402 | 403 | if ($this->lastHost && $this->socket && !isset($hostConfigs[$this->lastHost])) { 404 | // If we have a socket connection, but the current host is no longer in the list of available host 405 | // configs, close the socket so it can be reset. 406 | @socket_close($this->socket); 407 | $this->socket = null; 408 | $this->logger->info('Bedrock\Client - Not reusing socket because the host it was connected to is no longer available', ['host' => $this->lastHost]); 409 | } 410 | // If we already have a socket for this instance, then we first try to reuse it 411 | if ($this->socket) { 412 | $hostName = $this->lastHost; 413 | } else { 414 | // Try the first possible host. 415 | $hostName = (string) key($hostConfigs); 416 | $this->lastHost = $hostName; 417 | } 418 | try { 419 | // We get the port from either the main or failover host configs, due to socket reuse, the host we are 420 | // trying to use might not be in the picked host configs, because getPossibleHosts randomizes them. 421 | $port = $this->mainHostConfigs[$hostName]['port'] ?? $this->failoverHostConfigs[$hostName]['port']; 422 | $this->sendRawRequest($hostName, $port, $rawRequest); 423 | $response = $this->receiveResponse(); 424 | } catch (ConnectionFailure $e) { 425 | // The error happened during connection (or before we sent any data, or in a case where we know the 426 | // command was never processed) so we can retry it safely. 427 | $this->markHostAsFailed($hostName); 428 | if ($numRetriesLeft) { 429 | $this->logger->info('Bedrock\Client - Failed to connect or send the request; retrying', ['host' => $hostName, 'message' => $e->getMessage(), 'retriesLeft' => $numRetriesLeft, 'exception' => $e]); 430 | } else { 431 | if ($retriedAllHosts) { 432 | $this->logger->error('Bedrock\Client - Failed to connect or send the request; not retrying because we are out of retries', ['host' => $hostName, 'message' => $e->getMessage(), 'exception' => $e]); 433 | } else { 434 | $this->logger->info('Bedrock\Client - Failed to connect or send the request; retrying in all hosts', ['host' => $hostName, 'message' => $e->getMessage(), 'exception' => $e]); 435 | } 436 | $exception = $e; 437 | } 438 | } catch (BedrockError $e) { 439 | // This error happen after sending some data to the server, so we only can retry it if it is an idempotent command 440 | $this->markHostAsFailed($hostName); 441 | /* @phan-suppress-next-line PhanTypeInvalidDimOffset for some reason phan says idempotent does not exist, but I have the ?? so it should not matter */ 442 | if (!($headers['idempotent'] ?? false)) { 443 | $this->logger->error('Bedrock\Client - Failed to send the whole request or to receive it; not retrying because command is not idempotent', ['host' => $hostName, 'message' => $e->getMessage(), 'exception' => $e]); 444 | throw $e; 445 | } 446 | if ($numRetriesLeft) { 447 | $this->logger->info('Bedrock\Client - Failed to send the whole request or to receive it; retrying because command is idempotent', ['host' => $hostName, 'message' => $e->getMessage(), 'retriesLeft' => $numRetriesLeft, 'exception' => $e]); 448 | } else { 449 | if ($retriedAllHosts) { 450 | $this->logger->error('Bedrock\Client - Failed to send the whole request or to receive it; not retrying because we are out of retries', ['host' => $hostName, 'message' => $e->getMessage(), 'exception' => $e]); 451 | } else { 452 | $this->logger->info('Bedrock\Client - Failed to send the whole request or to receive it; retrying in all hosts', ['host' => $hostName, 'message' => $e->getMessage(), 'exception' => $e]); 453 | } 454 | $exception = $e; 455 | } 456 | } finally { 457 | // We remove the host we just used from the possible hosts to use, in case we are retrying 458 | $hostConfigs = array_filter($hostConfigs, function ($possibleHost) use ($hostName) { 459 | return $possibleHost !== $hostName; 460 | }, ARRAY_FILTER_USE_KEY); 461 | } 462 | 463 | // All non blacklisted hosts failed, this could be because we are in the middle of a cluster version flip. 464 | // ie: We have version 1 and version 2, we've installed version 2 in less than half the cluster, a previous 465 | // request already marked all servers in version 2 as failed (since 1 is the current version) in this request 466 | // we have installed version 2 in one more server, making it the current version. So now all the servers that 467 | // were marked as failed in the previous request are the ones serving requests and all the ones that were good 468 | // before are now in the old version and not serving requests. So to cover this, we retry in all servers 469 | // once hoping it will find a server that works. 470 | if ($exception) { 471 | if ($retriedAllHosts) { 472 | throw $exception; 473 | } 474 | $retriedAllHosts = true; 475 | $this->logger->info('All non blacklisted hosts failed, as a last resort try again in all hosts'); 476 | $hostConfigs = $this->getPossibleHosts($preferredHost, true); 477 | } 478 | } 479 | 480 | if (is_null($response)) { 481 | throw new ConnectionFailure('Could not connect to Bedrock hosts or failovers'); 482 | } 483 | 484 | // If we had to close the socket after using it (because we connected to a preferred host or because the command 485 | // had Connection:forget), disconnect from it so we don't send all future requests to it. 486 | if ($closeSocketAfterRequest || ($response['headers']['Connection'] ?? '') === 'close') { 487 | $this->logger->info('Closing socket after use'); 488 | @socket_close($this->socket); 489 | $this->socket = null; 490 | } 491 | 492 | // Log how long this particular call took 493 | $processingTime = (isset($response['headers']['processTime']) ? $response['headers']['processTime'] : 0) / 1000; 494 | $serverTime = (isset($response['headers']['totalTime']) ? $response['headers']['totalTime'] : 0) / 1000; 495 | $clientTime = round(microtime(true) - $timeStart, 3) * 1000; 496 | $networkTime = $clientTime - $serverTime; 497 | $waitTime = $serverTime - $processingTime; 498 | $this->logger->info('Bedrock\Client - Request finished', [ 499 | 'host' => $hostName, 500 | 'command' => $method, 501 | 'jsonCode' => isset($response['codeLine']) ? $response['codeLine'] : null, 502 | 'duration' => $clientTime, 503 | 'net' => $networkTime, 504 | 'wait' => $waitTime, 505 | 'proc' => $processingTime, 506 | 'commitCount' => $this->commitCount, 507 | ]); 508 | 509 | // Done! 510 | return $response; 511 | } 512 | 513 | /** 514 | * Sends the request on a new socket, if a previous one existed, it closes the connection first. 515 | * 516 | * @throws ConnectionFailure When the failure is before sending any data to the server 517 | * @throws BedrockError When we already sent some data 518 | */ 519 | private function sendRawRequest(string $host, int $port, string $rawRequest) 520 | { 521 | // Try to connect to the requested host 522 | $pid = getmypid(); 523 | if (!$this->socket) { 524 | $this->logger->info('Bedrock\Client - Opening new socket', ['host' => $host, 'cluster' => $this->clusterName, 'pid' => $pid]); 525 | $this->socket = @socket_create(AF_INET, SOCK_STREAM, getprotobyname('tcp')); 526 | 527 | // Make sure we succeed to create a socket 528 | if ($this->socket === false) { 529 | $socketError = socket_strerror(socket_last_error()); 530 | throw new ConnectionFailure("Could not connect to create socket: $socketError"); 531 | } 532 | 533 | // Configure this socket and try to connect to it 534 | socket_set_option($this->socket, SOL_SOCKET, SO_SNDTIMEO, ['sec' => $this->connectionTimeout, 'usec' => 0]); 535 | socket_set_option($this->socket, SOL_SOCKET, SO_RCVTIMEO, ['sec' => $this->readTimeout, 'usec' => 0]); 536 | @socket_connect($this->socket, $host, $port); 537 | $socketErrorCode = socket_last_error($this->socket); 538 | if ($socketErrorCode === 115) { 539 | $this->logger->info('Bedrock\Client - socket_connect returned error 115, continuing.'); 540 | } elseif ($socketErrorCode) { 541 | $socketError = socket_strerror($socketErrorCode); 542 | throw new ConnectionFailure("Could not connect to Bedrock host $host:$port. Error: $socketErrorCode $socketError"); 543 | } 544 | } else { 545 | $this->logger->info('Bedrock\Client - Reusing socket', ['host' => $host, 'cluster' => $this->clusterName, 'pid' => $pid]); 546 | } 547 | socket_clear_error($this->socket); 548 | 549 | // Send the information to the socket 550 | $bytesSent = @socket_send($this->socket, $rawRequest, strlen($rawRequest), MSG_EOF); 551 | 552 | // Failed to send anything 553 | if ($bytesSent === false) { 554 | $socketErrorCode = socket_last_error(); 555 | $socketError = socket_strerror($socketErrorCode); 556 | throw new ConnectionFailure("Failed to send request to bedrock host $host:$port. Error: $socketErrorCode $socketError"); 557 | } 558 | 559 | // We sent something; can't retry or else we might double-send the same request. Let's make sure we sent the 560 | // whole thing, else there's a problem. 561 | if ($bytesSent < strlen($rawRequest)) { 562 | $this->logger->info('Bedrock\Client - Could not send the whole request', ['bytesSent' => $bytesSent, 'expected' => strlen($rawRequest)]); 563 | throw new ConnectionFailure("Sent partial request to bedrock host $host:$port"); 564 | } elseif ($bytesSent > strlen($rawRequest)) { 565 | $this->logger->info('Bedrock\Client - sent more data than needed', ['bytesSent' => $bytesSent, 'expected' => strlen($rawRequest)]); 566 | throw new BedrockError("Sent more content than expected to host $host:$port"); 567 | } 568 | } 569 | 570 | /** 571 | * @param ?string $preferredHost If passed, it will prefer this host over any of the configured ones. This does not 572 | * ensure it will use that host, but it will try to use it if its not blacklisted. 573 | * 574 | * @suppress PhanUndeclaredConstant - suppresses TRAVIS_RUNNING 575 | */ 576 | private function getPossibleHosts(?string $preferredHost, bool $resetHosts = false) 577 | { 578 | // We get the host configs from the APC cache. Then, we check the configuration there with the passed 579 | // configuration and if it's outdated (ie: it has different hosts from the one in the config), we reset it. This 580 | // is so that we don't keep the old cache after changing the hosts or failover configuration. 581 | if (!defined('TRAVIS_RUNNING') || !TRAVIS_RUNNING) { 582 | $apcuKey = self::APCU_CACHE_PREFIX.$this->clusterName; 583 | if ($resetHosts) { 584 | $this->logger->info('Bedrock\Client - Resetting host configs'); 585 | $cachedHostConfigs = []; 586 | } else { 587 | $cachedHostConfigs = apcu_fetch($apcuKey) ?: []; 588 | } 589 | $this->logger->debug('Bedrock\Client - APC fetch host configs', array_keys($cachedHostConfigs)); 590 | 591 | // If the hosts and ports in the cache don't match the ones in the config, reset the cache. 592 | $cachedHostsAndPorts = []; 593 | foreach ($cachedHostConfigs as $hostName => $config) { 594 | $cachedHostsAndPorts[$hostName] = $config['port']; 595 | } 596 | asort($cachedHostsAndPorts); 597 | $uncachedHostsAndPort = []; 598 | foreach (array_merge($this->mainHostConfigs, $this->failoverHostConfigs) as $hostName => $config) { 599 | $uncachedHostsAndPort[$hostName] = $config['port']; 600 | } 601 | asort($uncachedHostsAndPort); 602 | if ($cachedHostsAndPorts !== $uncachedHostsAndPort) { 603 | $cachedHostConfigs = array_merge($this->mainHostConfigs, $this->failoverHostConfigs); 604 | $this->logger->info('Bedrock\Client - APC store host configs', array_keys($cachedHostConfigs)); 605 | apcu_store($apcuKey, $cachedHostConfigs); 606 | } 607 | } else { 608 | $cachedHostConfigs = array_merge($this->mainHostConfigs, $this->failoverHostConfigs); 609 | } 610 | 611 | // Get one main host and all the failovers, then remove any of them that we know already failed. 612 | // Assemble the list of servers we'll try, in order. First, pick one of the main hosts. We pick randomly 613 | // because we want to equally balance each server across all of its local databases. This allows us to have an 614 | // unequal number of servers and databases in a given datacenter. Also, we only pick one (versus trying both) 615 | // because if our first attempt fails we want to equally balance across *all* databases -- including the remote 616 | // ones. Otherwise if a database node goes down, the other databases in the same datacenter would get more load 617 | // (whereas this approach ensures the load is spread evenly across all). 618 | $failoverHostNames = array_keys($this->failoverHostConfigs); 619 | shuffle($failoverHostNames); 620 | $mainHostName = array_rand($this->mainHostConfigs); 621 | $preferredHost = array_key_exists((string) $preferredHost, $cachedHostConfigs) ? [$preferredHost] : []; 622 | $hostNames = array_filter(array_unique(array_merge($preferredHost, [$mainHostName], $failoverHostNames))); 623 | 624 | $nonBlackListedHosts = []; 625 | foreach ($hostNames as $hostName) { 626 | $blackListedUntil = $cachedHostConfigs[$hostName]['blacklistedUntil'] ?? null; 627 | if (!$blackListedUntil || $blackListedUntil < time()) { 628 | $nonBlackListedHosts[$hostName] = $cachedHostConfigs[$hostName]; 629 | } 630 | } 631 | 632 | if (empty($nonBlackListedHosts)) { 633 | $this->getLogger()->info('Bedrock\Client - All possible hosts have been blacklisted, using full list instead'); 634 | $nonBlackListedHosts = $cachedHostConfigs; 635 | } 636 | $this->getLogger()->info('Bedrock\Client - Possible hosts', ['nonBlacklistedHosts' => array_keys($nonBlackListedHosts)]); 637 | 638 | return $nonBlackListedHosts; 639 | } 640 | 641 | /** 642 | * Receives and parses the response. 643 | * 644 | * @return array Response object including 'code', 'codeLine', 'headers', `size` and 'body' 645 | * 646 | * @throws BedrockError 647 | */ 648 | private function receiveResponse() 649 | { 650 | // Make sure bedrock is returning something https://github.com/Expensify/Expensify/issues/11010 651 | if (@socket_recv($this->socket, $buf, self::PACKET_LENGTH, MSG_PEEK) === false) { 652 | throw new BedrockError('Socket failed to read data'); 653 | } 654 | 655 | $totalDataReceived = 0; 656 | $responseHeaders = []; 657 | $responseLength = null; 658 | $response = ''; 659 | $dataOnSocket = ''; 660 | $codeLine = null; 661 | 662 | // Read the data on the socket block by block until we got them all 663 | do { 664 | $sizeDataOnSocket = @socket_recv($this->socket, $dataOnSocket, self::PACKET_LENGTH, 0); 665 | if ($sizeDataOnSocket === false) { 666 | $errorCode = socket_last_error($this->socket); 667 | $errorMsg = socket_strerror($errorCode); 668 | throw new BedrockError("Error receiving data: $errorCode - $errorMsg"); 669 | } 670 | if ($sizeDataOnSocket === 0 || strlen($dataOnSocket) === 0) { 671 | throw new BedrockError('Bedrock response was empty'); 672 | } 673 | $totalDataReceived += $sizeDataOnSocket; 674 | $response .= $dataOnSocket; 675 | 676 | // The first time are reading data from the socket, we need to extract the headers 677 | // to be able to get the size of the response 678 | // It is use to know when to stop to read data from the socket 679 | if ($responseLength === null && strpos($response, self::HEADER_DELIMITER) !== false) { 680 | $dataOffset = strpos($response, self::HEADER_DELIMITER); 681 | $responseHeadersStr = substr($response, 0, $dataOffset + strlen(self::HEADER_DELIMITER)); 682 | $responseHeaderLines = explode(self::HEADER_FIELD_SEPARATOR, $responseHeadersStr); 683 | $codeLine = array_shift($responseHeaderLines); 684 | $responseHeaders = $this->extractResponseHeaders($responseHeaderLines); 685 | $responseLength = (int) $responseHeaders['Content-Length']; 686 | $response = substr($response, $dataOffset + strlen(self::HEADER_DELIMITER)); 687 | } 688 | } while (is_null($responseLength) || strlen($response) < $responseLength); 689 | 690 | // We save the commitCount for future requests. This is useful if for some reason we change the bedrock node we 691 | // are talking to. 692 | // We only set it if process time was returned, which means we did a write. We don't care about saving the commit 693 | // count for reads, since we did not change anything in the DB. 694 | if (isset($responseHeaders['commitCount']) && (($responseHeaders['processTime'] ?? 0) > 0 || ($responseHeaders['upstreamProcessTime'] ?? 0) > 0)) { 695 | $this->commitCount = (int) $responseHeaders['commitCount']; 696 | } 697 | 698 | // We treat this '555 Timeout', which is a command timeout (not a query timeout), as a ConnectionFailure so that it gets retried regardless of if it is idempotent or not. 699 | // For other exceptions that have different error codes/messages, we do not throw here, so it gets handled like any regular exception. 700 | if ($codeLine === '555 Timeout') { 701 | throw new ConnectionFailure('Internal Bedrock command timeout (555 Timeout)'); 702 | } 703 | 704 | if ($codeLine === '500 Internal Server Error') { 705 | throw new ConnectionFailure('Bedrock responded with 500 Internal Server Error'); 706 | } 707 | 708 | // We'll parse the body *only* if this is `application/json` or blank. 709 | $isJSON = !isset($responseHeaders['Content-Type']) || !strcasecmp($responseHeaders['Content-Type'], 'application/json'); 710 | 711 | $result = [ 712 | 'headers' => $responseHeaders, 713 | 'body' => $isJSON ? $this->parseRawBody($responseHeaders, $response) : $response, 714 | 'size' => $totalDataReceived, 715 | 'codeLine' => $codeLine, 716 | 'code' => intval($codeLine), 717 | ]; 718 | if ($isJSON) { 719 | $result['rawBody'] = $response; 720 | } 721 | return $result; 722 | } 723 | 724 | /** 725 | * Parse a raw response from bedrock. 726 | * 727 | * @return array|null the decoded json, or null on error 728 | * 729 | * @throws BedrockError 730 | */ 731 | private function parseRawBody(array $headers, string $body) 732 | { 733 | // Detect if we are using Gzip (TODO: can we remove this?) 734 | if (isset($headers['Content-Encoding']) && $headers['Content-Encoding'] === 'gzip') { 735 | $body = gzdecode($body); 736 | if ($body === false) { 737 | throw new BedrockError('Could not gzip decode bedrock response'); 738 | } 739 | } else { 740 | // Who knows why we need to trim in this case? 741 | $body = trim($body); 742 | } 743 | 744 | if ($body === '') { 745 | return []; 746 | } 747 | 748 | $json = json_decode($body, true); 749 | if (json_last_error() !== JSON_ERROR_NONE) { 750 | // This will remove unwanted characters. 751 | // Check http://stackoverflow.com/a/20845642 and http://www.php.net/chr for details 752 | for ($i = 0; $i <= 31; $i++) { 753 | $body = str_replace(chr($i), '', $body); 754 | } 755 | $jsonStr = str_replace(chr(127), '', $body); 756 | 757 | // We've seen occurrences of this happen when the string is not UTF-8. Forcing it fixes it. 758 | // See https://github.com/Expensify/Expensify/issues/21805 for example. 759 | $json = json_decode(mb_convert_encoding($jsonStr, 'UTF-8', 'UTF-8'), true); 760 | if (json_last_error() !== JSON_ERROR_NONE) { 761 | throw new BedrockError('Could not parse JSON from bedrock'); 762 | } 763 | } 764 | 765 | return $json; 766 | } 767 | 768 | private function extractResponseHeaders(array $responseHeaderLines) 769 | { 770 | $responseHeaders = []; 771 | foreach ($responseHeaderLines as $responseHeaderLine) { 772 | // Try to split this line 773 | $nameValue = explode(':', $responseHeaderLine, 2); 774 | if (count($nameValue) === 2) { 775 | $responseHeaders[trim($nameValue[0])] = trim($nameValue[1]); 776 | } elseif (strlen($responseHeaderLine)) { 777 | $this->logger->warning('Bedrock\Client - Malformed response header, ignoring.', ['responseHeaderLine' => $responseHeaderLine]); 778 | } 779 | } 780 | 781 | return $responseHeaders; 782 | } 783 | 784 | /** 785 | * Converts a string to UTF8. 786 | * 787 | * @param string $str 788 | * 789 | * @return string 790 | */ 791 | private static function toUTF8($str) 792 | { 793 | // Get the current encoding, default to UTF-8 if we can't tell. Then convert 794 | // the string to UTF-8 and ignore any characters that can't be converted. 795 | $encoding = mb_detect_encoding($str) ?: 'UTF-8'; 796 | 797 | return iconv($encoding, 'UTF-8//IGNORE', $str); 798 | } 799 | 800 | /** 801 | * When a host fails, we blacklist that server for a certain amount of time, so we don't send requests to it when we 802 | * know it's down. The blacklist time is a random amount of time between 1 second and the maxBlackListTimeout 803 | * configuration. 804 | * We also close and clear the socket from the cache, so we don't reuse it. 805 | * 806 | * @suppress PhanUndeclaredConstant - suppresses TRAVIS_RUNNING 807 | */ 808 | private function markHostAsFailed(string $host) 809 | { 810 | $blacklistedUntil = time() + rand(1, $this->maxBlackListTimeout); 811 | if (!defined('TRAVIS_RUNNING') || !TRAVIS_RUNNING) { 812 | $apcuKey = self::APCU_CACHE_PREFIX.$this->clusterName; 813 | $hostConfigs = apcu_fetch($apcuKey); 814 | $hostConfigs[$host]['blacklistedUntil'] = $blacklistedUntil; 815 | apcu_store($apcuKey, $hostConfigs); 816 | } 817 | $this->logger->info('Bedrock\Client - Marking server as failed', ['host' => $host, 'time' => date('Y-m-d H:i:s', $blacklistedUntil)]); 818 | 819 | // Since there was a problem with the host and we want to talk to a different one, we close and clear the socket. 820 | if ($this->socket) { 821 | @socket_close($this->socket); 822 | $this->socket = null; 823 | } 824 | } 825 | 826 | /** 827 | * Returns the highest commitCount of each cluster name instantiated in this request. 828 | * 829 | * @return int[] 830 | */ 831 | public static function getCommitCounts(): array 832 | { 833 | $commitCounts = []; 834 | foreach (self::$instances as $instance) { 835 | $commitCounts[$instance->getClusterName()][] = $instance->commitCount; 836 | } 837 | foreach ($commitCounts as $name => $values) { 838 | $commitCounts[$name] = max($values) ?? 0; 839 | } 840 | 841 | return array_filter($commitCounts); 842 | } 843 | } 844 | -------------------------------------------------------------------------------- /src/DB.php: -------------------------------------------------------------------------------- 1 | run($sql, true); 40 | } 41 | return $this->run($sql, false); 42 | } 43 | 44 | /** 45 | * Executes an SQL query. 46 | * 47 | * @param string $sql The query to run 48 | * @param bool $idempotent Is this command idempotent? If the command is run twice is the final result the same? 49 | * @param int $timeout Time in microseconds, defaults to 60 seconds 50 | * 51 | * @throws BedrockError 52 | */ 53 | public function run(string $sql, bool $idempotent, int $timeout = 60000): Response 54 | { 55 | $sql = substr($sql, -1) === ";" ? $sql : $sql.";"; 56 | $matches = []; 57 | preg_match('/\s*(select|insert|delete|update).*/i', $sql, $matches); 58 | $operation = isset($matches[1]) && in_array(strtolower($matches[1]), ['insert', 'update', 'delete', 'select']) ? strtolower($matches[1]) : 'unknown'; 59 | $response = $this->client->getStats()->benchmark("bedrock.db.query.$operation", function () use ($sql, $idempotent, $timeout) { 60 | return new Response($this->client->call( 61 | 'Query', 62 | [ 63 | 'query' => $sql, 64 | 'format' => "json", 65 | 'idempotent' => $idempotent, 66 | 'timeout' => $timeout, 67 | ] 68 | )); 69 | }); 70 | 71 | if ($response->getCode() === self::CODE_QUERY_FAILED) { 72 | throw new BedrockError($response->getCodeLine()." - ".$response->getError(), $response->getCode()); 73 | } 74 | 75 | if ($response->getCode() !== self::CODE_OK) { 76 | throw new BedrockError($response->getCodeLine(), $response->getCode()); 77 | } 78 | 79 | return $response; 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/DB/Response.php: -------------------------------------------------------------------------------- 1 | container = $leValue; 18 | } 19 | 20 | /** 21 | * @return string 22 | */ 23 | public function __toString() 24 | { 25 | return json_encode($this->toArray()); 26 | } 27 | 28 | /** 29 | * @return array 30 | */ 31 | public function toArray() 32 | { 33 | return $this->container; 34 | } 35 | 36 | public function count(): int 37 | { 38 | return count($this->container); 39 | } 40 | 41 | public function jsonSerialize(): array 42 | { 43 | return $this->toArray(); 44 | } 45 | 46 | /** 47 | * Shortcut method to access the value inside the container. 48 | * 49 | * @param string|array $index 50 | * @param mixed $default 51 | * 52 | * @return mixed 53 | */ 54 | protected function getFromContainer($index, $default = null) 55 | { 56 | if (is_array($index)) { 57 | $array = $this->container; 58 | foreach ($index as $step) { 59 | if (!isset($array[$step])) { 60 | return $default; 61 | } 62 | $array = $array[$step]; 63 | } 64 | 65 | return $array; 66 | } 67 | 68 | if (!is_string($index) && !is_int($index)) { 69 | return $default; 70 | } 71 | 72 | if (!isset($this->container[$index])) { 73 | return $default; 74 | } 75 | 76 | return $this->container[$index]; 77 | } 78 | 79 | /** 80 | * Shortcut function to set data inside the container. 81 | * 82 | * @param string|int|array $index 83 | * @param mixed $value 84 | * 85 | * @return mixed 86 | */ 87 | protected function setContainerValue($index, $value) 88 | { 89 | return $this->container[$index] = $value; 90 | } 91 | 92 | /** 93 | * Shortcut to make sure the value of an index is strictly true. 94 | * 95 | * @param string|array $index 96 | * 97 | * @return bool 98 | */ 99 | protected function isTrue($index) 100 | { 101 | return $this->getFromContainer($index) === true; 102 | } 103 | 104 | /** 105 | * Get the resposne code. 106 | * 107 | * @return int 108 | */ 109 | public function getCode() 110 | { 111 | return (int) $this->getFromContainer('code'); 112 | } 113 | 114 | public function getCodeLine(): string 115 | { 116 | return $this->getFromContainer('codeLine'); 117 | } 118 | 119 | /** 120 | * Get the headers (column names). 121 | * 122 | * @return string[] 123 | */ 124 | public function getHeaders() 125 | { 126 | return $this->getFromContainer(['body', 'headers'], []); 127 | } 128 | 129 | /** 130 | * Get the rows returned. 131 | * 132 | * @param bool $assoc Do we want to return each row keyed by the column name? If not, they will be keyed by index. 133 | * 134 | * @return array[] 135 | */ 136 | public function getRows(bool $assoc = false): array 137 | { 138 | $rows = $this->getFromContainer(['body', 'rows'], []); 139 | if (!$assoc) { 140 | return $rows; 141 | } 142 | $results = []; 143 | $headers = $this->getHeaders(); 144 | foreach ($rows as $row) { 145 | $result = []; 146 | foreach ($headers as $index => $header) { 147 | $result[$header] = $row[$index]; 148 | } 149 | $results[] = $result; 150 | } 151 | return $results; 152 | } 153 | 154 | /** 155 | * Get the error message. 156 | * 157 | * @return string 158 | */ 159 | public function getError() 160 | { 161 | return $this->getFromContainer(['headers', 'error'], ''); 162 | } 163 | 164 | /** 165 | * Returns true if no rows were returned. 166 | * 167 | * @return bool 168 | */ 169 | public function isEmpty() 170 | { 171 | return count($this->getRows()) === 0; 172 | } 173 | 174 | /** 175 | * Returns the last insert row ID. 176 | * 177 | * @return int 178 | */ 179 | public function getLastInsertRowID() 180 | { 181 | return $this->getFromContainer(['headers', 'lastInsertRowID']); 182 | } 183 | } 184 | -------------------------------------------------------------------------------- /src/Exceptions/BedrockError.php: -------------------------------------------------------------------------------- 1 | delay = $delay; 50 | $this->nextRun = $nextRun; 51 | $this->name = $name; 52 | $this->ignoreRepeat = $ignoreRepeat; 53 | parent::__construct($message, $code, $previous); 54 | } 55 | 56 | /** 57 | * Returns the time to delay the retry (in seconds) 58 | */ 59 | public function getDelay(): int 60 | { 61 | return $this->delay; 62 | } 63 | 64 | /** 65 | * Returns the nextRun time 66 | */ 67 | public function getNextRun(): string 68 | { 69 | return $this->nextRun; 70 | } 71 | 72 | /** 73 | * Returns the new name 74 | */ 75 | public function getName(): string 76 | { 77 | return $this->name; 78 | } 79 | 80 | /** 81 | * Returns if we should ignore the jobs repeat parameter when calculating when to retry 82 | */ 83 | public function getIgnoreRepeat(): bool 84 | { 85 | return $this->ignoreRepeat; 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /src/Exceptions/Jobs/SqlFailed.php: -------------------------------------------------------------------------------- 1 | client->getStats()->counter('bedrockJob.call.'.$method); 142 | $response = $this->client->getStats()->benchmark('bedrock.jobs.'.$method, function () use ($method, $headers, $body) { 143 | return $this->client->call($method, $headers, $body); 144 | }); 145 | 146 | $job = $headers['name'] ?? $headers['jobID'] ?? null; 147 | $responseCode = $response['code'] ?? null; 148 | $codeLine = $response['codeLine'] ?? null; 149 | 150 | $this->client->getStats()->counter('bedrockJob.call.response.'.$method.$responseCode); 151 | 152 | if ($responseCode === 402) { 153 | throw new MalformedAttribute("Malformed attribute. Job :$job, message: $codeLine"); 154 | } 155 | 156 | if ($responseCode === 404) { 157 | throw new DoesNotExist("Job $job does not exist"); 158 | } 159 | 160 | if ($responseCode === 405) { 161 | throw new IllegalAction("Cannot perform `$method` on job $job"); 162 | } 163 | 164 | if ($responseCode === 502) { 165 | throw new SqlFailed("SQL failed for job $job: {$codeLine}"); 166 | } 167 | 168 | // 202 code is a successful job creation using the "Connection: forget" header 169 | if (!in_array($responseCode, [200, 202])) { 170 | throw new GenericError("Generic error for job $job"); 171 | } 172 | 173 | return $response; 174 | } 175 | 176 | /** 177 | * Schedules a new job, optionally in the future, optionally to repeat. 178 | * 179 | * @param string $name 180 | * @param array|null $data (optional) 181 | * @param string|null $firstRun (optional) 182 | * @param string|null $repeat (optional) see https://github.com/Expensify/Bedrock/blob/master/plugins/Jobs.md#repeat-syntax 183 | * @param bool|null $unique (optional) Do we want only one job with this name to exist? 184 | * @param int|null $priority (optional) Specify a job priority. Jobs with higher priorities will be run first. 185 | * @param int|null $parentJobID (optional) Specify this job's parent job. 186 | * @param string|null $connection (optional) Specify 'Connection' header using constants defined in this class. 187 | * @param string|null $retryAfter (optional) Specify after what time in RUNNING this job should be retried (same syntax as repeat) 188 | * @param bool $overwrite (optional) Only applicable when unique is is true. When set to true it will overwrite the existing job with the new jobs data 189 | * 190 | * @return array Containing "jobID" 191 | */ 192 | public function createJob($name, $data = null, $firstRun = null, $repeat = null, $unique = false, $priority = self::PRIORITY_MEDIUM, $parentJobID = null, $connection = self::CONNECTION_WAIT, $retryAfter = null, $overwrite = true) 193 | { 194 | $this->client->getLogger()->info('Create job', ['name' => $name]); 195 | $commitCounts = Client::getCommitCounts(); 196 | 197 | $response = $this->call( 198 | 'CreateJob', 199 | [ 200 | 'name' => $name, 201 | 'data' => array_merge($data ?? [], count($commitCounts) ? ['_commitCounts' => $commitCounts] : []), 202 | 'firstRun' => $firstRun, 203 | 'repeat' => $repeat, 204 | 'unique' => $unique, 205 | 'jobPriority' => $priority, 206 | 'parentJobID' => $parentJobID, 207 | 'Connection' => $connection, 208 | // If the name of the job has to be unique, Bedrock will return any existing job that exists with the 209 | // given name instead of making a new one, which essentially makes the command idempotent. 210 | 'idempotent' => $unique, 211 | 'retryAfter' => $retryAfter, 212 | 'overwrite' => $overwrite, 213 | ] 214 | ); 215 | 216 | $this->client->getLogger()->info('Job created', ['name' => $name, 'id' => $response['body']['jobID'] ?? null]); 217 | 218 | return $response; 219 | } 220 | 221 | /** 222 | * Schedules a list of jobs. 223 | * 224 | * @param array $jobs JSON array containing each job. Each job should include the same parameters as jobs define in CreateJob 225 | * @param string $connection (optional) Specify 'Connection' header using constants defined in this class. 226 | * 227 | * @return array - contain the jobIDs with the unique identifier of the created jobs 228 | */ 229 | public function createJobs(array $jobs, string $connection = self::CONNECTION_WAIT): array 230 | { 231 | $this->client->getLogger()->info('Create jobs', ['jobs' => $jobs]); 232 | 233 | // We renamed the `priority` param to `jobPriority` because `priority` is a generic param of any bedrock command. 234 | foreach ($jobs as $i => $job) { 235 | if (isset($jobs[$i]['priority'])) { 236 | $jobs[$i]['jobPriority'] = $jobs[$i]['priority']; 237 | unset($jobs[$i]['priority']); 238 | } 239 | } 240 | 241 | // If the name of the job has to be unique, Bedrock will return any existing job that exists with the 242 | // given name instead of making a new one, which essentially makes the command idempotent. 243 | $areAllJobsUnique = true; 244 | $commitCounts = Client::getCommitCounts(); 245 | foreach ($jobs as $i => $job) { 246 | $jobs[$i]['data'] = array_merge($jobs[$i]['data'] ?? [], count($commitCounts) ? ['_commitCounts' => $commitCounts] : []); 247 | $areAllJobsUnique = ($jobs[$i]['unique'] ?? false) && $areAllJobsUnique; 248 | } 249 | 250 | $response = $this->call( 251 | 'CreateJobs', 252 | [ 253 | 'jobs' => $jobs, 254 | 'idempotent' => $areAllJobsUnique, 255 | 'Connection' => $connection, 256 | ], 257 | ); 258 | 259 | $this->client->getLogger()->info('Jobs created', ['jobIDs' => $response['body']['jobIDs'] ?? null]); 260 | 261 | return $response; 262 | } 263 | 264 | /** 265 | * Waits for a match (if requested) and atomically dequeues exactly one job. 266 | * 267 | * @param string $name 268 | * 269 | * @return array Containing all job details 270 | */ 271 | public function getJob($name) 272 | { 273 | $headers = ['name' => $name]; 274 | 275 | return $this->call('GetJob', $headers); 276 | } 277 | 278 | /** 279 | * Waits for a match (if requested) and atomically dequeues $numResults jobs. 280 | * 281 | * @return array Containing all job details 282 | */ 283 | public function getJobs(string $name, int $numResults, array $params = []): array 284 | { 285 | $headers = [ 286 | 'name' => $name, 287 | 'numResults' => $numResults, 288 | ]; 289 | 290 | $headers = array_merge($headers, $params); 291 | 292 | return $this->call('GetJobs', $headers); 293 | } 294 | 295 | /** 296 | * Updates the data associated with a job. 297 | * 298 | * @param int $jobID 299 | * @param array $data 300 | * @param string $repeat (optional) see https://github.com/Expensify/Bedrock/blob/master/plugins/Jobs.md#repeat-syntax 301 | * @param int|null $priority (optional) The new priority of the job 302 | * 303 | * @return array 304 | */ 305 | public function updateJob($jobID, $data, $repeat = null, ?int $priority = null, ?string $nextRun = null) 306 | { 307 | $commitCounts = Client::getCommitCounts(); 308 | return $this->call( 309 | 'UpdateJob', 310 | [ 311 | 'jobID' => $jobID, 312 | 'data' => array_merge($data ?? [], count($commitCounts) ? ['_commitCounts' => $commitCounts] : []), 313 | 'repeat' => $repeat, 314 | 'jobPriority' => $priority, 315 | 'idempotent' => true, 316 | 'nextRun' => $nextRun, 317 | ] 318 | ); 319 | } 320 | 321 | /** 322 | * Marks a job as finished, which causes it to repeat if requested. 323 | * 324 | * @param int $jobID 325 | * @param array $data (optional) 326 | * 327 | * @return array 328 | */ 329 | public function finishJob($jobID, $data = null) 330 | { 331 | return $this->call( 332 | 'FinishJob', 333 | [ 334 | 'jobID' => $jobID, 335 | 'data' => $data, 336 | 'idempotent' => true, 337 | ] 338 | ); 339 | } 340 | 341 | /** 342 | * Cancel a QUEUED, RUNQUEUED, FAILED job from a sibling. 343 | */ 344 | public function cancelJob(int $jobID): array 345 | { 346 | return $this->call( 347 | 'CancelJob', 348 | [ 349 | 'jobID' => $jobID, 350 | ] 351 | ); 352 | } 353 | 354 | /** 355 | * Removes all trace of a job. 356 | * 357 | * @param int $jobID 358 | * 359 | * @return array 360 | */ 361 | public function deleteJob($jobID) 362 | { 363 | return $this->call( 364 | 'DeleteJob', 365 | [ 366 | 'jobID' => $jobID, 367 | 'idempotent' => true, 368 | ] 369 | ); 370 | } 371 | 372 | /** 373 | * Mark a job as failed. 374 | * 375 | * @param int $jobID 376 | * 377 | * @return array 378 | */ 379 | public function failJob($jobID) 380 | { 381 | return $this->call( 382 | 'FailJob', 383 | [ 384 | 'jobID' => $jobID, 385 | 'idempotent' => true, 386 | ] 387 | ); 388 | } 389 | 390 | /** 391 | * Retry a job. Job must be in a RUNNING state to be able to be retried. 392 | */ 393 | public function retryJob(int $jobID, int $delay = 0, array $data = null, string $name = '', string $nextRun = '', ?int $priority = null, bool $ignoreRepeat = false): array 394 | { 395 | return $this->call( 396 | 'RetryJob', 397 | [ 398 | 'jobID' => $jobID, 399 | 'delay' => $delay, 400 | 'data' => $data, 401 | 'name' => $name, 402 | 'nextRun' => $nextRun, 403 | 'idempotent' => true, 404 | 'jobPriority' => $priority, 405 | 'ignoreRepeat' => $ignoreRepeat, 406 | ] 407 | ); 408 | } 409 | 410 | /** 411 | * Requeues a job back to QUEUED state. Updates the job name if provided as well. 412 | */ 413 | public function requeueJobs(array $jobIDs, string $name = ''): array 414 | { 415 | return $this->call( 416 | 'RequeueJobs', 417 | [ 418 | 'jobIDs' => implode(',', $jobIDs), 419 | 'name' => $name, 420 | 'idempotent' => true, 421 | ] 422 | ); 423 | } 424 | 425 | /** 426 | * Query a job's info. 427 | * Bedrock will return: 428 | * - 200 - OK 429 | * . created - creation time of this job 430 | * . jobID - unique ID of the job 431 | * . state - One of QUEUED, RUNNING, FINISHED 432 | * . name - name of the actual job matched 433 | * . nextRun - timestamp of next scheduled run 434 | * . lastRun - timestamp it was last run 435 | * . repeat - recurring description 436 | * . data - JSON data associated with this job. 437 | * 438 | * @param int $jobID 439 | * 440 | * @return array|null 441 | */ 442 | public function queryJob($jobID) 443 | { 444 | $bedrockResponse = $this->call( 445 | 'QueryJob', 446 | [ 447 | 'jobID' => $jobID, 448 | 'idempotent' => true, 449 | ] 450 | ); 451 | 452 | return $bedrockResponse['body'] ?? null; 453 | } 454 | 455 | /** 456 | * Schedules a new job, optionally in the future, optionally to repeat. 457 | * Silently fails in case of an exception and logs the error. 458 | * 459 | * @param string $name 460 | * @param array|null $data (optional) 461 | * @param string|null $firstRun (optional) 462 | * @param string|null $repeat (optional) see https://github.com/Expensify/Bedrock/blob/master/plugins/Jobs.md#repeat-syntax 463 | * @param bool $unique Do we want only one job with this name to exist? 464 | * @param int $priority (optional) Specify a job priority. Jobs with higher priorities will be run first. 465 | * @param int|null $parentJobID (optional) Specify this job's parent job. 466 | * @param string $connection (optional) Specify 'Connection' header using constants defined in this class. 467 | * @param bool $overwrite (optional) Only applicable when unique is is true. When set to true it will overwrite the existing job with the new jobs data 468 | * 469 | * @return array Containing "jobID" 470 | */ 471 | public static function queueJob($name, $data = null, $firstRun = null, $repeat = null, $unique = false, $priority = self::PRIORITY_MEDIUM, $parentJobID = null, $connection = self::CONNECTION_WAIT, string $retryAfter = '', bool $overwrite = true) 472 | { 473 | $bedrock = Client::getInstance(); 474 | try { 475 | $jobs = new self($bedrock); 476 | 477 | return $jobs->createJob($name, $data, $firstRun, $repeat, $unique, $priority, $parentJobID, $connection, $retryAfter, $overwrite); 478 | } catch (Exception $e) { 479 | $bedrock->getLogger()->alert('Could not create Bedrock job', ['exception' => $e]); 480 | 481 | return []; 482 | } 483 | } 484 | } 485 | -------------------------------------------------------------------------------- /src/LocalDB.php: -------------------------------------------------------------------------------- 1 | location = $location; 36 | $this->logger = $logger; 37 | $this->stats = $stats; 38 | } 39 | 40 | /** 41 | * Opens a DB connection. 42 | */ 43 | public function open() 44 | { 45 | if ($this->handle === null) { 46 | $this->handle = new SQLite3($this->location); 47 | $this->handle->busyTimeout(15000); 48 | $this->handle->enableExceptions(true); 49 | } 50 | } 51 | 52 | /** 53 | * Close the DB connection and unset the object. 54 | */ 55 | public function close() 56 | { 57 | if ($this->handle !== null) { 58 | $this->handle->close(); 59 | $this->handle = null; 60 | } 61 | } 62 | 63 | /** 64 | * Runs a read query on a local database. 65 | * 66 | * @return array 67 | */ 68 | public function read(string $query) 69 | { 70 | $result = null; 71 | $returnValue = []; 72 | while (true) { 73 | try { 74 | $result = $this->handle->query($query); 75 | break; 76 | } catch (Exception $e) { 77 | if ($e->getMessage() === 'database is locked') { 78 | $this->logger->info('Query failed, retrying', ['query' => $query, 'error' => $e->getMessage()]); 79 | } else { 80 | $this->logger->info('Query failed, not retrying', ['query' => $query, 'error' => $e->getMessage()]); 81 | throw $e; 82 | } 83 | } 84 | } 85 | 86 | if ($result) { 87 | $returnValue = $result->fetchArray(SQLITE3_NUM); 88 | } 89 | 90 | return $returnValue ?? []; 91 | } 92 | 93 | /** 94 | * Runs a write query on a local database. 95 | */ 96 | public function write(string $query) 97 | { 98 | while (true) { 99 | try { 100 | $this->handle->query($query); 101 | break; 102 | } catch (Exception $e) { 103 | if ($e->getMessage() === 'database is locked') { 104 | $this->logger->info('Query failed, retrying', ['query' => $query, 'error' => $e->getMessage()]); 105 | } else { 106 | $this->logger->info('Query failed, not retrying', ['query' => $query, 'error' => $e->getMessage()]); 107 | throw $e; 108 | } 109 | } 110 | } 111 | } 112 | 113 | /** 114 | * Gets last inserted row. 115 | * 116 | * @return int|null 117 | */ 118 | public function getLastInsertedRowID() 119 | { 120 | if ($this->handle === null) { 121 | return null; 122 | } 123 | 124 | return $this->handle->lastInsertRowID(); 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /src/Plugin.php: -------------------------------------------------------------------------------- 1 | client = $client; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/Stats/NullStats.php: -------------------------------------------------------------------------------- 1 | client->call("Ping"); 18 | } 19 | } 20 | --------------------------------------------------------------------------------