├── .github └── workflows │ └── tests.yml ├── .gitignore ├── FUNDING.yml ├── LICENSE ├── README.md ├── SECURITY.md ├── UPGRADE.md ├── bin └── file-watcher.js ├── composer.json ├── config └── tmi-cluster.php ├── migrations ├── 2020_09_11_191542_create_supervisors_table.php ├── 2020_10_08_161019_create_supervisor_processes_table.php ├── 2021_03_21_172842_create_tmi_cluster_channels_table.php └── 2021_04_06_073453_add_tmi_cluster_table_prefixes.php ├── package-lock.json ├── package.json ├── phpunit.xml ├── public ├── favicon-degraded.ico ├── favicon.ico ├── mix-manifest.json ├── tmi-cluster.css ├── tmi-cluster.js └── tmi-cluster.js.LICENSE.txt ├── resources ├── cli │ └── motd.txt ├── js │ ├── components │ │ ├── Charts.vue │ │ └── TmiDashboard.vue │ └── tmi-cluster.js ├── scss │ ├── tmi-cluster.scss │ └── variables.scss └── views │ ├── dashboard │ └── index.blade.php │ ├── invite │ └── index.blade.php │ └── layouts │ ├── app.blade.php │ └── astro.blade.php ├── src ├── AutoCleanup.php ├── AutoReconnect.php ├── AutoScale.php ├── BackgroundProcess.php ├── Commands │ ├── TmiClusterAuthorizeCommand.php │ ├── TmiClusterCommand.php │ ├── TmiClusterFeaturesCommand.php │ ├── TmiClusterHealthCommand.php │ ├── TmiClusterInstallCommand.php │ ├── TmiClusterJoinCommand.php │ ├── TmiClusterListCommand.php │ ├── TmiClusterProcessCommand.php │ ├── TmiClusterPublishCommand.php │ ├── TmiClusterPurgeCommand.php │ └── TmiClusterTerminateCommand.php ├── Contracts │ ├── ChannelDistributor.php │ ├── ChannelManager.php │ ├── ClusterClient.php │ ├── ClusterClientOptions.php │ ├── CommandQueue.php │ ├── HasPrometheusMetrics.php │ ├── Invitable.php │ ├── Pausable.php │ ├── Restartable.php │ ├── SupervisorJoinHandler.php │ ├── SupervisorRepository.php │ └── Terminable.php ├── EventMap.php ├── Events │ ├── ClusterClientBootstrap.php │ ├── ClusterClientRegistered.php │ ├── ClusterClientTerminated.php │ ├── PeriodicTimerCalled.php │ ├── ProcessScaled.php │ ├── SupervisorBootstrap.php │ ├── SupervisorLooped.php │ ├── UnableToLaunchProcess.php │ └── WorkerProcessRestarting.php ├── Exceptions │ └── NotFoundSupervisorException.php ├── Facades │ └── TmiCluster.php ├── Http │ └── Controllers │ │ ├── Controller.php │ │ ├── DashboardController.php │ │ ├── InviteController.php │ │ └── MetricsController.php ├── Limiters │ ├── DurationLimiter.php │ └── DurationLimiterBuilder.php ├── Listeners │ ├── MonitorClusterClientMemory.php │ ├── MonitorSupervisorMemory.php │ └── SendNotification.php ├── ListensForSignals.php ├── Lock.php ├── Models │ ├── Channel.php │ ├── Supervisor.php │ └── SupervisorProcess.php ├── Notifications │ ├── AutoScaleScaled.php │ └── Notification.php ├── PhpBinary.php ├── Process │ ├── BackgroundProcess.php │ ├── CommandString.php │ ├── Process.php │ ├── ProcessOptions.php │ └── ProcessPool.php ├── Providers │ └── TmiClusterServiceProvider.php ├── Repositories │ ├── DatabaseChannelManager.php │ ├── DummyChannelManager.php │ ├── RedisChannelDistributor.php │ ├── RedisCommandQueue.php │ └── SupervisorRepository.php ├── RouteRegistrar.php ├── ServiceBindings.php ├── Supervisor.php ├── Support │ ├── Arr.php │ ├── Composer.php │ ├── Health.php │ ├── Os.php │ ├── Str.php │ └── Url.php ├── TmiCluster.php ├── TmiClusterClient.php ├── Traits │ ├── Lockable.php │ └── TmiClusterHelpers.php ├── TwitchLogin.php └── Watchable.php ├── tests ├── TestCase.php ├── TestCases │ ├── ArrayTest.php │ ├── AutoScaleTest.php │ ├── ChannelDistributorTest.php │ ├── CommandQueueTest.php │ ├── ComposerTest.php │ ├── DatabaseChannelManagerTest.php │ ├── MetricsTest.php │ ├── MonitorClusterClientMemoryTest.php │ ├── MonitorSupervisorMemoryTest.php │ ├── NotificationTest.php │ ├── SendMessageTest.php │ ├── StaleTest.php │ └── TmiClusterTest.php ├── Traits │ └── CreatesSupervisors.php └── laravel │ └── artisan └── webpack.mix.js /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | tests: 7 | 8 | services: 9 | redis: 10 | image: redis 11 | ports: 12 | - 6379:6379 13 | options: --entrypoint redis-server 14 | 15 | strategy: 16 | fail-fast: false 17 | matrix: 18 | php: [8.0, 8.1, 8.2] 19 | composer-dependency: [prefer-stable, prefer-lowest] 20 | 21 | name: "PHP ${{ matrix.php }} - ${{ matrix.composer-dependency }}" 22 | 23 | runs-on: ubuntu-latest 24 | 25 | steps: 26 | - uses: actions/checkout@v2 27 | 28 | - name: Setup PHP 29 | uses: shivammathur/setup-php@v2 30 | with: 31 | php-version: ${{ matrix.php }} 32 | extensions: dom, curl, libxml, mbstring, redis, zip 33 | coverage: none 34 | 35 | - name: Install dependencies 36 | run: composer update --no-interaction --no-scripts --no-suggest --no-progress --prefer-dist --${{ matrix.composer-dependency }} 37 | 38 | - name: PHPUnit Tests 39 | run: vendor/bin/phpunit 40 | env: 41 | CLIENT_ID: ${{ secrets.TMI_IDENTITY_USERNAME }} 42 | CLIENT_SECRET: ${{ secrets.TMI_IDENTITY_PASSWORD }} -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /vendor 2 | /node_modules 3 | /.idea 4 | composer.lock 5 | .phpunit.result.cache 6 | .env* 7 | -------------------------------------------------------------------------------- /FUNDING.yml: -------------------------------------------------------------------------------- 1 | ko_fi: ene 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 René Preuß 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # TMI Cluster for Twitch Chatbots 2 | 3 |

4 | 5 |

6 | 7 |

8 | Total Downloads 9 | Latest Stable Version 10 | License 11 | Discord 12 |

13 | 14 | ## Introduction 15 | 16 | TMI Cluster enables a highly scalable IRC client cluster for Twitch. TMI Cluster consists of multiple supervisors that can be deployed on multiple hosts. The core is inspired by [Horizon](https://github.com/laravel/horizon), which handles the complex IRC process management. It is designed to work within the Laravel ecosystem. 17 | 18 | The cluster stores its data in the database and has a Redis Command Queue to send IRC commands and receive messages. 19 | 20 | ## Features 21 | 22 | - Supervisors can be deployed on multiple servers 23 | - Up-to-date Twitch IRC Client written in PHP 8 24 | - Scalable message input/output queue 25 | - Advanced cluster status dashboard 26 | - Channel management and invite screen 27 | 28 | ## PHP Twitch Messaging Interface 29 | 30 | The TMI Cluster is powered by the [PHP Twitch Messaging Interface](https://github.com/ghostzero/tmi) client to communicate with Twitch. It's a full featured, high performance Twitch IRC client written in PHP 8. 31 | 32 | ## Official Documentation 33 | 34 | You can view our official documentation [here](https://tmiphp.com/docs/tmi-cluster.html). 35 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Supported Versions 4 | 5 | Due to the fact that I do the project alone, I only support the latest major version of the project. 6 | 7 | | Version | Supported | 8 | | ------- | ------------------ | 9 | | 3.x | :white_check_mark: | 10 | | 2.x | :x: | 11 | 12 | ## Reporting a Vulnerability 13 | 14 | I ask you to report vulnerabilities as a direct message to `GhostZero#0001` in Discord. 15 | Alternatively, an email can be sent to rene@preuss.io, but I very rarely check my emails. 16 | -------------------------------------------------------------------------------- /UPGRADE.md: -------------------------------------------------------------------------------- 1 | # Upgrade Guide 2 | 3 | When upgrading to any new TMI Cluster version, you should re-publish TMI Cluster's assets: 4 | 5 | ```bash 6 | php artisan tmi-cluster:publish 7 | ``` 8 | 9 | To ensure TMI Cluster's assets are updated when a new version is downloaded, you may add a Composer hook inside your project's composer.json file to automatically publish TMI Cluster's latest assets: 10 | 11 | ```bash 12 | "scripts": { 13 | "post-update-cmd": [ 14 | "@php artisan tmi-cluster:publish --ansi" 15 | ] 16 | } 17 | ``` 18 | 19 | ## Upgrading To 3.0 From 2.x 20 | 21 | ### TMI Cluster Auto Cleanup 22 | 23 | PR: https://github.com/ghostzero/tmi-cluster/pull/14 24 | 25 | The ChannelManager now manages the channels and decides whether to remove channels from the TMI cluster. This was previously decided by the Twitch API. Because the API sometimes returns wrong values, we have completely removed the Twitch API. 26 | 27 | ### TMI Cluster Channel Manager 28 | 29 | PR: https://github.com/ghostzero/tmi-cluster/pull/14 30 | 31 | Channels must have given an authorization before the TMI cluster joins the channels. This prevents the TMI cluster from joining channels that have revoked authorization, for example. 32 | 33 | ### TMI Cluster Database 34 | 35 | PR: https://github.com/ghostzero/tmi-cluster/pull/14 36 | 37 | We added a prefix for all our tables with `tmi_cluster_` e.g. `tmi_cluster_channels`. Don't forget to update your database. We recommend that you turn off the TMI Custer for this migration. 38 | 39 | ```bash 40 | php artisan migrate 41 | ``` -------------------------------------------------------------------------------- /bin/file-watcher.js: -------------------------------------------------------------------------------- 1 | const chokidar = require('chokidar'); 2 | 3 | const paths = JSON.parse(process.argv[2]); 4 | const poll = !!process.argv[3]; 5 | 6 | const watcher = chokidar.watch(paths, { 7 | ignoreInitial: true, 8 | usePolling: poll, 9 | }); 10 | 11 | watcher 12 | .on('add', () => console.log('File added...')) 13 | .on('change', () => console.log('File changed...')) 14 | .on('unlink', () => console.log('File deleted...')) 15 | .on('unlinkDir', () => console.log('Directory deleted...')); -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ghostzero/tmi-cluster", 3 | "description": "Laravel package to create a tmi cluster.", 4 | "type": "library", 5 | "license": "MIT", 6 | "authors": [ 7 | { 8 | "name": "René Preuß", 9 | "email": "rene@preuss.io" 10 | } 11 | ], 12 | "require": { 13 | "php": "^8.0", 14 | "ext-json": "*", 15 | "ext-pcntl": "*", 16 | "ext-posix": "*", 17 | "ghostzero/tmi": "^2.1", 18 | "illuminate/console": "^v8.83|^9.0|^10.0", 19 | "illuminate/support": "^v8.83|^9.0|^10.0", 20 | "predis/predis": "^1.1", 21 | "nesbot/carbon": "^2.62.1" 22 | }, 23 | "require-dev": { 24 | "mockery/mockery": "^1.3.2", 25 | "orchestra/testbench": "^6.0", 26 | "phpunit/phpunit": "^9.3", 27 | "romanzipp/laravel-twitch": "^3.0" 28 | }, 29 | "autoload": { 30 | "psr-4": { 31 | "GhostZero\\TmiCluster\\": "src/" 32 | } 33 | }, 34 | "autoload-dev": { 35 | "psr-4": { 36 | "GhostZero\\TmiCluster\\Tests\\": "tests/" 37 | } 38 | }, 39 | "extra": { 40 | "laravel": { 41 | "providers": [ 42 | "GhostZero\\TmiCluster\\Providers\\TmiClusterServiceProvider" 43 | ], 44 | "aliases": { 45 | "TmiCluster": "GhostZero\\TmiCluster\\Facades\\TmiCluster" 46 | } 47 | } 48 | }, 49 | "scripts": { 50 | "lowest-test": [ 51 | "Composer\\Config::disableProcessTimeout", 52 | "composer update --prefer-lowest --ignore-platform-req=ext-pcntl --ignore-platform-req=ext-posix" 53 | ] 54 | }, 55 | "suggest": { 56 | "romanzipp/laravel-twitch": "Rquired to use the cluster auto cleanup feature (^3.0)." 57 | }, 58 | "config": { 59 | "sort-packages": true 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /migrations/2020_09_11_191542_create_supervisors_table.php: -------------------------------------------------------------------------------- 1 | string('id')->primary(); 18 | $table->json('options'); 19 | $table->json('metrics')->nullable(); 20 | $table->timestamp('last_ping_at'); 21 | $table->timestamps(); 22 | $table->softDeletes(); 23 | }); 24 | } 25 | 26 | /** 27 | * Reverse the migrations. 28 | * 29 | * @return void 30 | */ 31 | public function down() 32 | { 33 | Schema::dropIfExists('supervisors'); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /migrations/2020_10_08_161019_create_supervisor_processes_table.php: -------------------------------------------------------------------------------- 1 | uuid('id')->primary(); 18 | $table->string('supervisor_id')->nullable(); 19 | $table->string('state'); 20 | $table->json('channels'); 21 | $table->json('metrics')->nullable(); 22 | $table->timestamp('last_ping_at'); 23 | $table->timestamps(); 24 | }); 25 | } 26 | 27 | /** 28 | * Reverse the migrations. 29 | * 30 | * @return void 31 | */ 32 | public function down() 33 | { 34 | Schema::dropIfExists('supervisor_processes'); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /migrations/2021_03_21_172842_create_tmi_cluster_channels_table.php: -------------------------------------------------------------------------------- 1 | string('id', 25)->primary(); 18 | $table->boolean('reconnect')->default(false); 19 | $table->boolean('revoked')->default(false); 20 | $table->timestamp('acknowledged_at')->nullable(); 21 | $table->uuid('supervisor_process_id')->index()->nullable(); 22 | $table->timestamps(); 23 | }); 24 | } 25 | 26 | /** 27 | * Reverse the migrations. 28 | * 29 | * @return void 30 | */ 31 | public function down() 32 | { 33 | Schema::dropIfExists('tmi_cluster_channels'); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /migrations/2021_04_06_073453_add_tmi_cluster_table_prefixes.php: -------------------------------------------------------------------------------- 1 | 2 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | ./tests/ 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /public/favicon-degraded.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghostzero/tmi-cluster/6f0884974fb4aa29bd786171e1faf2d41a3917ed/public/favicon-degraded.ico -------------------------------------------------------------------------------- /public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghostzero/tmi-cluster/6f0884974fb4aa29bd786171e1faf2d41a3917ed/public/favicon.ico -------------------------------------------------------------------------------- /public/mix-manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "/tmi-cluster.js": "/tmi-cluster.js?id=2fcff7006166774aa6cfa35c1af9443c", 3 | "/tmi-cluster.css": "/tmi-cluster.css?id=4b89ddcd445831ec010f57ae492e65ae" 4 | } 5 | -------------------------------------------------------------------------------- /public/tmi-cluster.js.LICENSE.txt: -------------------------------------------------------------------------------- 1 | /*! 2 | * Vue.js v2.7.13 3 | * (c) 2014-2022 Evan You 4 | * Released under the MIT License. 5 | */ 6 | 7 | /*! @preserve 8 | * numeral.js 9 | * version : 2.0.6 10 | * author : Adam Draper 11 | * license : MIT 12 | * http://adamwdraper.github.com/Numeral-js/ 13 | */ 14 | -------------------------------------------------------------------------------- /resources/cli/motd.txt: -------------------------------------------------------------------------------- 1 | _ _ _ _ 2 | | | (_) | | | | 3 | | |_ _ __ ___ _ ______ ___| |_ _ ___| |_ ___ _ __ 4 | | __| '_ ` _ \| |______/ __| | | | / __| __/ _ \ '__| 5 | | |_| | | | | | | | (__| | |_| \__ \ || __/ | 6 | \__|_| |_| |_|_| \___|_|\__,_|___/\__\___|_| 7 | 8 | Thanks for using the tmi-cluster! If you have any questions, suggestions, or 9 | complaints, please contact me at: https://ghostzero.dev/discord 10 | 11 | Official Documentation: https://tmiphp.com/docs/tmi-cluster.html 12 | -------------------------------------------------------------------------------- /resources/js/components/Charts.vue: -------------------------------------------------------------------------------- 1 | 6 | 7 | 93 | 94 | -------------------------------------------------------------------------------- /resources/js/tmi-cluster.js: -------------------------------------------------------------------------------- 1 | const numeral = require('numeral'); 2 | 3 | import Vue from 'vue'; 4 | 5 | Vue.prototype.$http = require('axios'); 6 | 7 | window.chartColors = { 8 | color_1: '#6d00ff', 9 | color_2: '#994fff', 10 | color_3: '#bb7fff', 11 | color_4: '#d6adff', 12 | }; 13 | 14 | window.chartBackgroundColors = { 15 | color_1: 'rgba(109, 0, 255, .5)', 16 | color_2: 'rgba(153, 79, 255, .5)', 17 | color_3: 'rgba(187, 127, 255, .5)', 18 | color_4: 'rgba(214, 173, 255, .5)', 19 | }; 20 | 21 | Vue.filter('formatNumber', function (value) { 22 | return numeral(value).format("0,0"); 23 | }); 24 | 25 | 26 | Vue.component('tmi-dashboard', require('./components/TmiDashboard').default); 27 | 28 | const app = new Vue({ 29 | el: '#app', 30 | }); 31 | -------------------------------------------------------------------------------- /resources/scss/tmi-cluster.scss: -------------------------------------------------------------------------------- 1 | @use "sass:math"; 2 | 3 | // Variables 4 | @import 'variables'; 5 | 6 | // Bootstrap 7 | @import '~bootstrap/scss/bootstrap'; 8 | 9 | html, 10 | body { 11 | height: 100%; 12 | } 13 | html, 14 | body { 15 | height: 100%; 16 | } 17 | 18 | .app-center { 19 | display: flex; 20 | align-items: center; 21 | padding-top: 40px; 22 | padding-bottom: 40px; 23 | background-color: #0a0117; 24 | height: 100%; 25 | } 26 | 27 | .form-signin { 28 | width: 100%; 29 | max-width: 400px; 30 | padding: 16px; 31 | margin: auto; 32 | } 33 | .form-signin .checkbox { 34 | font-weight: 400; 35 | } 36 | .form-signin .form-control { 37 | position: relative; 38 | box-sizing: border-box; 39 | height: auto; 40 | padding: 10px; 41 | font-size: 16px; 42 | } 43 | .form-signin .form-control:focus { 44 | z-index: 2; 45 | } 46 | .form-signin input[type="email"] { 47 | margin-bottom: -1px; 48 | border-bottom-right-radius: 0; 49 | border-bottom-left-radius: 0; 50 | } 51 | .form-signin input[type="password"] { 52 | margin-bottom: 10px; 53 | border-top-left-radius: 0; 54 | border-top-right-radius: 0; 55 | } 56 | 57 | @font-face { 58 | font-family: 'JetBrains Mono'; 59 | font-weight: 500; 60 | font-style: normal; 61 | font-display: swap; 62 | /* Read next point */ 63 | unicode-range: U+000-5FF; /* Download only latin glyphs */ 64 | src: local('JetBrains Mono'), 65 | url('https://cdn.bitinflow.com/fonts/JetBrainsMono-Regular.woff2') format('woff2'), 66 | url('https://cdn.bitinflow.com/fonts/JetBrainsMono-Regular.woff') format('woff'); 67 | } 68 | 69 | @font-face { 70 | font-family: 'JetBrains Mono Extra Bold'; 71 | font-weight: 400; 72 | font-style: normal; 73 | font-display: swap; 74 | /* Read next point */ 75 | unicode-range: U+000-5FF; /* Download only latin glyphs */ 76 | src: local('JetBrains Mono Extra Bold'), 77 | url('https://cdn.bitinflow.com/fonts/JetBrainsMono-ExtraBold.woff2') format('woff2'), 78 | url('https://cdn.bitinflow.com/fonts/JetBrainsMono-ExtraBold.woff') format('woff'); 79 | } 80 | 81 | body { 82 | font-family: "JetBrains Mono", sans-serif; 83 | } 84 | 85 | .bg-primary-light-1 { 86 | background-color: #994fff; 87 | } 88 | 89 | .bg-primary-light-2 { 90 | background-color: #bb7fff; 91 | } 92 | 93 | .bg-primary-light-3 { 94 | background-color: #d6adff; 95 | } 96 | 97 | .bg-gradient-tmi { 98 | background: linear-gradient(90deg, rgba(83,5,165,1) 0%, rgba(69,5,141,1) 60%, rgba(36,4,87,1) 100%); 99 | } 100 | 101 | // Utilities 102 | 103 | .opacity-0{ 104 | opacity: 0; 105 | } 106 | 107 | @for $i from 1 through 9 { 108 | $opacity: math.div($i, 10); 109 | .opacity-#{($i*10)}{ 110 | opacity: $opacity; 111 | } 112 | } 113 | 114 | // Bubbles 115 | 116 | .bg-bubbles { 117 | position: absolute; 118 | padding: 0 0 0 0; 119 | opacity: .5; 120 | // Fill the bubble background with the whole screen. 121 | top: 0; 122 | left: 0; 123 | width: 100%; 124 | height: 100%; 125 | // If the element content exceeds the given width and height, the overflow attribute can determine whether the scrollbar behavior is displayed or not. 126 | overflow: hidden; 127 | li { 128 | border-radius: 4px; 129 | position: absolute; 130 | // Bottom is set to create the effect of bubbles emerging from the bottom of the page. 131 | bottom: -160px; 132 | // Default bubble size; 133 | width: 40px; 134 | height: 40px; 135 | background-color: rgba(255, 255, 255, 0.15); 136 | list-style: none; 137 | // Use custom animation to make bubbles appear, rise and roll. 138 | animation: square 15s infinite; 139 | transition-timing-function: linear; 140 | // Different positions, sizes, transparency and velocities of each bubble are set to make it appear hierarchical. 141 | &:nth-child(1) { 142 | left: 10%; 143 | } 144 | &:nth-child(2) { 145 | left: 20%; 146 | width: 90px; 147 | height: 90px; 148 | animation-delay: 2s; 149 | animation-duration: 7s; 150 | } 151 | &:nth-child(3) { 152 | left: 25%; 153 | animation-delay: 4s; 154 | } 155 | &:nth-child(4) { 156 | left: 40%; 157 | width: 60px; 158 | height: 60px; 159 | animation-duration: 8s; 160 | background-color: rgba(255, 255, 255, 0.3); 161 | } 162 | &:nth-child(5) { 163 | left: 70%; 164 | } 165 | &:nth-child(6) { 166 | left: 80%; 167 | width: 120px; 168 | height: 120px; 169 | animation-delay: 3s; 170 | background-color: rgba(255, 255, 255, 0.2); 171 | } 172 | &:nth-child(7) { 173 | left: 32%; 174 | width: 160px; 175 | height: 160px; 176 | animation-delay: 2s; 177 | } 178 | &:nth-child(8) { 179 | left: 55%; 180 | width: 20px; 181 | height: 20px; 182 | animation-delay: 4s; 183 | animation-duration: 15s; 184 | } 185 | &:nth-child(9) { 186 | left: 25%; 187 | width: 20px; 188 | height: 20px; 189 | animation-delay: 2s; 190 | animation-duration: 12s; 191 | background-color: rgba(255, 255, 255, 0.3); 192 | } 193 | &:nth-child(10) { 194 | left: 85%; 195 | width: 160px; 196 | height: 160px; 197 | animation-delay: 5s; 198 | } 199 | } 200 | // Custom square animation; 201 | @keyframes square { 202 | 0% { 203 | opacity: 0.5; 204 | transform: translateY(0px) rotate(45deg); 205 | } 206 | 25% { 207 | opacity: 0.75; 208 | transform: translateY(-400px) rotate(90deg) 209 | } 210 | 50% { 211 | opacity: 1; 212 | transform: translateY(-600px) rotate(135deg); 213 | } 214 | 100% { 215 | opacity: 0; 216 | transform: translateY(-1000px) rotate(180deg); 217 | } 218 | } 219 | } -------------------------------------------------------------------------------- /resources/scss/variables.scss: -------------------------------------------------------------------------------- 1 | $white: #fff !default; 2 | $gray-100: #f8f9fa !default; 3 | $gray-200: #ebebeb !default; 4 | $gray-300: #dee2e6 !default; 5 | $gray-400: #ced4da !default; 6 | $gray-500: #adb5bd !default; 7 | $gray-600: #888 !default; 8 | $gray-700: #181e23 !default; 9 | $gray-800: #141a1e !default; 10 | $gray-900: #0d1114 !default; 11 | $black: #000 !default; 12 | 13 | // Colors 14 | $primary: #6d00ff; 15 | $danger: #c00; 16 | $dark: #150130; 17 | 18 | // Body 19 | $body-bg: $white; 20 | $body-color: #150130; 21 | 22 | // Table 23 | .thead-primary { 24 | background-color: $primary; 25 | color: $white; 26 | } 27 | 28 | .thead-secondary { 29 | background-color: #250459; 30 | color: $white; 31 | } 32 | 33 | .small-notice { 34 | font-size: .8rem; 35 | } 36 | 37 | // Grid containers 38 | 39 | $container-max-widths: ( 40 | sm: 540px, 41 | md: 720px, 42 | lg: 960px, 43 | xl: 1140px 44 | ) !default; -------------------------------------------------------------------------------- /resources/views/dashboard/index.blade.php: -------------------------------------------------------------------------------- 1 | @extends('tmi-cluster::layouts.app') 2 | 3 | @section('content') 4 | 7 | 8 | @endsection 9 | -------------------------------------------------------------------------------- /resources/views/invite/index.blade.php: -------------------------------------------------------------------------------- 1 | @extends('tmi-cluster::layouts.astro') 2 | 3 | @section('content') 4 |
5 |
6 |
8 | @csrf 9 |
10 |
11 | 13 | 14 | 16 |
17 |
18 |

19 | {{ config('tmi-cluster.tmi.identity.username') }} 20 | 21 |

22 | @if($invited) 23 |

24 | Is now in your Twitch chat 25 |

26 | @else 27 |

28 | Likes to join your Twitch chat 29 |

30 | @endif 31 |

32 | Signed in as {{ $user_username }} 33 |

34 |
35 |
36 |
37 | Used in {{ $connections }} channels

38 | This bot cannot read your private messages or send messages as you. 39 |
40 |
41 | 53 |
54 |
55 |
56 | @endsection 57 | -------------------------------------------------------------------------------- /resources/views/layouts/app.blade.php: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | TMI Cluster 9 | @stack('tmi-cluster::meta') 10 | 11 | 12 | 14 | 15 | 16 | 18 | 20 | 23 | 26 | 27 | 29 | 30 | 31 | 32 | 33 | 34 |
35 | @yield('content') 36 | 37 | 42 |
43 | 44 | 47 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /resources/views/layouts/astro.blade.php: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | TMI Cluster 9 | @stack('tmi-cluster::meta') 10 | 11 | 12 | 13 | 14 | 15 | 17 | 19 | 22 | 25 | 26 | 27 | 28 | 29 | 30 | 31 |
32 | 35 | @yield('content') 36 |
37 | 38 | 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /src/AutoCleanup.php: -------------------------------------------------------------------------------- 1 | channelManager = app(ChannelManager::class); 15 | } 16 | 17 | public static function shouldCleanup(): bool 18 | { 19 | return config('tmi-cluster.channel_manager.auto_cleanup.enabled'); 20 | } 21 | 22 | public function register(TmiClusterClient $client) 23 | { 24 | $cleanupInterval = $this->getCleanupInterval(); 25 | $client->log(sprintf('Register cleanup loop for every %s sec.', $cleanupInterval)); 26 | $client->getTmiClient()->getLoop()->addPeriodicTimer($cleanupInterval, fn() => $this->handle($client)); 27 | } 28 | 29 | private function handle(TmiClusterClient $client) 30 | { 31 | try { 32 | if (!self::shouldCleanup()) { 33 | return; 34 | } 35 | 36 | $channels = $this->channelManager->stale($client->getTmiClient()->getChannels()); 37 | 38 | foreach ($channels as $channel) { 39 | $client->getTmiClient()->part($channel); 40 | } 41 | 42 | $client->log(sprintf('Cleanup complete! Parted: %s', count($channels))); 43 | } catch (Exception $e) { 44 | $client->log($e->getMessage()); 45 | $client->log($e->getTraceAsString()); 46 | } 47 | } 48 | 49 | private function getCleanupInterval(): int 50 | { 51 | return config('tmi-cluster.channel_manager.auto_cleanup.interval', 300) 52 | + random_int(0, max(0, config('tmi-cluster.channel_manager.auto_cleanup.max_delay', 600))); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/AutoReconnect.php: -------------------------------------------------------------------------------- 1 | counter = random_int(0, 30); 29 | } catch (Exception $e) { 30 | $this->counter = 0; 31 | } 32 | } 33 | 34 | public function handle(Supervisor $supervisor): void 35 | { 36 | try { 37 | $this->counter++; 38 | 39 | $channelManager = app(ChannelManager::class); 40 | 41 | if ($channelManager instanceof DatabaseChannelManager) { 42 | $this->doReconnect($supervisor, $channelManager); 43 | } 44 | } catch (Throwable $exception) { 45 | $supervisor->output(null, $exception->getMessage()); 46 | $supervisor->output(null, $exception->getTraceAsString()); 47 | } 48 | } 49 | 50 | private function doReconnect(Supervisor $supervisor, DatabaseChannelManager $channelManager) 51 | { 52 | if ($this->counter % 30 !== 0) return; 53 | $lock = $this->lock('reconnect-disconnected-channels', 10); 54 | 55 | try { 56 | $lock->block(5); 57 | // Lock acquired after waiting maximum of 5 seconds... 58 | $connectedChannels = SupervisorProcess::all() 59 | ->filter(fn(SupervisorProcess $process) => !$process->is_stale) 60 | ->map(fn(SupervisorProcess $process) => $process->channels) 61 | ->collapse() 62 | ->toArray(); 63 | 64 | $disconnectedChannels = $channelManager->disconnected($connectedChannels); 65 | 66 | $count = count($disconnectedChannels); 67 | 68 | if ($count > 0) { 69 | $supervisor->output(null, sprintf('Reconnecting %d disconnected channels...', $count)); 70 | app(ChannelDistributor::class)->join($disconnectedChannels); 71 | } 72 | } catch (LockTimeoutException $e) { 73 | $supervisor->output(null, 'Unable to acquire reconnect lock...'); 74 | } finally { 75 | optional($lock)->release(); 76 | } 77 | } 78 | 79 | 80 | } -------------------------------------------------------------------------------- /src/AutoScale.php: -------------------------------------------------------------------------------- 1 | counter = random_int(0, 10); 40 | } catch (Exception $e) { 41 | $this->counter = 0; 42 | } 43 | } 44 | 45 | public function scale(Supervisor $supervisor): void 46 | { 47 | try { 48 | $this->counter++; 49 | 50 | $channels = $this->getCurrentChannels($supervisor); 51 | $averageUsage = $this->getCurrentAverageChannelUsage($channels); 52 | $nextUsage = $this->getNextAverageChannelUsage($channels); 53 | 54 | if ($this->shouldScaleOut($averageUsage)) { 55 | $this->scaleOut($supervisor); 56 | } elseif ($this->shouldScaleIn($averageUsage)) { 57 | if (!$this->shouldScaleOut($nextUsage)) { 58 | $this->scaleIn($supervisor); 59 | } 60 | } 61 | 62 | $this->setMinimumScale($supervisor->processes()->count()); 63 | 64 | $this->releaseStaleSupervisors($supervisor); 65 | } catch (Throwable $exception) { 66 | $supervisor->output(null, $exception->getMessage()); 67 | $supervisor->output(null, $exception->getTraceAsString()); 68 | } 69 | } 70 | 71 | private function releaseStaleSupervisors(Supervisor $supervisor): void 72 | { 73 | if ($this->counter % 10 !== 0) return; 74 | $lock = $this->lock('release-stale-supervisors', 10); 75 | 76 | try { 77 | $lock->block(5); 78 | // Lock acquired after waiting maximum of 5 seconds... 79 | app(SupervisorRepository::class)->flushStale(); 80 | } catch (LockTimeoutException $e) { 81 | $supervisor->output(null, 'Unable to acquire flush stale lock...'); 82 | } finally { 83 | optional($lock)->release(); 84 | } 85 | } 86 | 87 | public function shouldScaleOut(float $usage): bool 88 | { 89 | return $usage > config('tmi-cluster.auto_scale.thresholds.scale_out', 70); 90 | } 91 | 92 | public function shouldRestoreScale(): bool 93 | { 94 | return config('tmi-cluster.auto_scale.restore', true); 95 | } 96 | 97 | public function shouldScaleIn(float $usage): bool 98 | { 99 | return $usage < config('tmi-cluster.auto_scale.thresholds.scale_in', 50); 100 | } 101 | 102 | private function getCurrentAverageChannelUsage(Collection $c) 103 | { 104 | $channelLimit = config('tmi-cluster.auto_scale.thresholds.channels', 50); 105 | $channelCount = $c->sum(); 106 | $serverCount = $c->count() + 1; 107 | 108 | return (($channelCount / $serverCount) / $channelLimit) * 100; 109 | } 110 | 111 | private function getNextAverageChannelUsage(Collection $c) 112 | { 113 | $channelLimit = config('tmi-cluster.auto_scale.thresholds.channels', 50); 114 | $channelCount = $c->sum(); 115 | $serverCount = $c->count(); 116 | 117 | if ($serverCount <= 0) { 118 | return 0; 119 | } 120 | 121 | return (($channelCount / $serverCount) / $channelLimit) * 100; 122 | } 123 | 124 | public function scaleOut(Supervisor $supervisor): void 125 | { 126 | $count = $supervisor->processes()->count(); 127 | $supervisor->output(null, 'Scale out: ' . ($count + 1)); 128 | 129 | if ($count >= config('tmi-cluster.auto_scale.processes.max', 25)) { 130 | return; // skip scale out, keep a maximum of instance 131 | } 132 | 133 | $supervisor->scale($count + 1); 134 | } 135 | 136 | public function scaleIn(Supervisor $supervisor): void 137 | { 138 | $count = $supervisor->processes()->count(); 139 | 140 | if ($count <= $this->getMinimumScale()) { 141 | return; // skip scale in, keep a minimum of instance 142 | } 143 | 144 | $supervisor->output(null, 'Scale in: ' . ($count - 1)); 145 | $supervisor->scale($count - 1); 146 | } 147 | 148 | private function getCurrentChannels(Supervisor $supervisor): Collection 149 | { 150 | $c = collect(); 151 | $supervisor->model->processes() 152 | ->each(function (SupervisorProcess $process) use ($c) { 153 | $c->push(count($process->channels)); 154 | }); 155 | 156 | return $c; 157 | } 158 | 159 | public function getMinimumScale(): int 160 | { 161 | $default = config('tmi-cluster.auto_scale.processes.min', 2); 162 | if (!$this->shouldRestoreScale()) { 163 | return $default; 164 | } 165 | 166 | return max($this->connection()->get('auto-scale:minimum-scale') ?? $default, $default); 167 | } 168 | 169 | public function setMinimumScale(int $scale) 170 | { 171 | return $this->connection()->set('auto-scale:minimum-scale', $scale, 'EX', 60); 172 | } 173 | } 174 | -------------------------------------------------------------------------------- /src/BackgroundProcess.php: -------------------------------------------------------------------------------- 1 | argument('channel'); 32 | 33 | if ($this->option('revoke')) { 34 | $this->revokeAuthorization($channels); 35 | 36 | $this->info('Channels authorization revoked successfully.'); 37 | } else { 38 | $this->authorize($channels); 39 | 40 | $this->info('Channels authorized successfully.'); 41 | } 42 | 43 | return 0; 44 | } 45 | 46 | private function authorize($channels): void 47 | { 48 | foreach ($channels as $channel) { 49 | $this->getChannelManager()->authorize(new TwitchLogin($channel)); 50 | } 51 | } 52 | 53 | private function revokeAuthorization(array $channels) 54 | { 55 | foreach ($channels as $channel) { 56 | $this->getChannelManager()->revokeAuthorization(new TwitchLogin($channel)); 57 | } 58 | } 59 | 60 | private function getChannelManager(): ChannelManager 61 | { 62 | return app(ChannelManager::class); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/Commands/TmiClusterCommand.php: -------------------------------------------------------------------------------- 1 | printMotd(); 43 | 44 | event(new SupervisorBootstrap()); 45 | 46 | try { 47 | /** @var Supervisor $supervisor */ 48 | $supervisor = app(SupervisorRepository::class)->create(); 49 | } catch (DomainException) { 50 | $this->error('A supervisor with this name is already running.'); 51 | 52 | return 13; 53 | } 54 | 55 | $this->info("Supervisor {$supervisor->model->getKey()} started successfully!"); 56 | $this->info('Press Ctrl+C to stop the supervisor safely.'); 57 | 58 | $this->getOutput()->newLine(); 59 | 60 | return $this->start($supervisor); 61 | } 62 | 63 | private function start(Supervisor $supervisor): int 64 | { 65 | if ($supervisor->model->options['nice']) { 66 | proc_nice($supervisor->model->options['nice']); 67 | } 68 | 69 | $supervisor->handleOutputUsing(function ($type, $line) { 70 | $method = $this->matchOutputType($type); 71 | $time = date('Y-m-d H:i:s'); 72 | $this->{$method}('[' . $time . '] ' . trim($line)); 73 | }); 74 | 75 | $supervisor->scale(config('tmi-cluster.auto_scale.processes.min')); 76 | 77 | if ($this->option('watch')) { 78 | $watcher = $this->startServerWatcher(); 79 | Event::listen(function (SupervisorLooped $event) use ($watcher) { 80 | if ($watcher->isRunning() && 81 | $watcher->getIncrementalOutput()) { 82 | $this->info('Application change detected. Restarting supervisor…'); 83 | $event->supervisor->restart(); 84 | } elseif ($watcher->isTerminated()) { 85 | $this->error( 86 | 'Watcher process has terminated. Please ensure Node and chokidar are installed.' . PHP_EOL . 87 | $watcher->getErrorOutput() 88 | ); 89 | } 90 | }); 91 | } 92 | 93 | $supervisor->monitor(); 94 | 95 | return 0; 96 | } 97 | 98 | private function printMotd(): void 99 | { 100 | $lines = file_get_contents(__DIR__ . '/../../resources/cli/motd.txt'); 101 | $lines = explode("\n", $lines); 102 | foreach ($lines as $line) { 103 | $this->comment($line); 104 | } 105 | 106 | $detectedTmiClusterVersion = Composer::detectTmiClusterVersion(); 107 | $this->comment(sprintf('You are running version %s of tmi-cluster.', $detectedTmiClusterVersion)); 108 | 109 | // print an unsupported message if the detected version is not within the supported range of dev-master or semantic version of ^2.3 or ^3.0 110 | if (!Composer::isSupportedVersion($detectedTmiClusterVersion)) { 111 | $this->warn('This version of tmi-cluster is not supported. Please upgrade to the latest version.'); 112 | } 113 | 114 | $this->getOutput()->newLine(); 115 | } 116 | 117 | private function matchOutputType(?string $type): string 118 | { 119 | return match ($type) { 120 | null, 'info', 'out' => 'info', 121 | 'error', 'err' => 'error', 122 | 'comment' => 'comment', 123 | 'question' => 'question', 124 | default => throw new DomainException(sprintf('Unknown output type "%s"', $type)), 125 | }; 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /src/Commands/TmiClusterFeaturesCommand.php: -------------------------------------------------------------------------------- 1 | comment('TMI Cluster:'); 33 | $this->info(sprintf('Framework: %s', config('tmi-cluster.framework'))); 34 | $this->info(sprintf('Fast Termination: %s', (config('tmi-cluster.fast_termination')) ? 'Enabled' : 'Disabled')); 35 | 36 | $this->comment('Connections:'); 37 | $this->info(sprintf('Database: %s', config('database.default'))); 38 | $this->info(sprintf('Redis: %s', config('tmi-cluster.use'))); 39 | $this->info(sprintf('Redis Prefix: %s', config('tmi-cluster.prefix'))); 40 | 41 | $this->comment('TMI:'); 42 | $this->info(sprintf('TMI Username: %s', config('tmi-cluster.tmi.identity.username') ?? 'Not set')); 43 | $this->info(sprintf('TMI Reconnect: %s', (config('tmi-cluster.tmi.connection.reconnect') ? 'Enabled' : 'Disabled'))); 44 | $this->info(sprintf('TMI Rejoin: %s', (config('tmi-cluster.tmi.connection.rejoin') ? 'Enabled' : 'Disabled'))); 45 | 46 | $this->comment('You will use the following implementations:'); 47 | 48 | $channelManagerName = get_class(app(ChannelManager::class)); 49 | $channelDistributorName = get_class(app(ChannelDistributor::class)); 50 | 51 | $this->info(sprintf("Your channel manager is %s", $channelManagerName)); 52 | $this->info(sprintf("Your channel distributor is %s", $channelDistributorName)); 53 | 54 | $this->comment('Auto Reconnect:'); 55 | 56 | if ($channelManagerName === DatabaseChannelManager::class) { 57 | $this->info('Your channel manager supports auto reconnect.'); 58 | } else { 59 | $this->comment('Your channel manager does not support auto reconnect.'); 60 | } 61 | 62 | $this->comment('Auto Cleanup:'); 63 | 64 | if (AutoCleanup::shouldCleanup()) { 65 | $this->info('The auto cleanup feature is enabled.'); 66 | 67 | $settings = config('tmi-cluster.channel_manager.auto_cleanup'); 68 | 69 | $this->info(sprintf('The auto cleanup interval is %d seconds.', $settings['interval'])); 70 | $this->info(sprintf('The auto cleanup will have a max start delay of %d seconds.', $settings['max_delay'])); 71 | } else { 72 | $this->comment('The auto cleanup feature is disabled.'); 73 | } 74 | 75 | return 0; 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/Commands/TmiClusterHealthCommand.php: -------------------------------------------------------------------------------- 1 | option('host-only') 39 | ? $this->getHostSupervisors($repository) 40 | : $repository->all(); 41 | 42 | if (Health::isOperational($supervisors, $this->option('at-least-one'))) { 43 | $this->info('TMI cluster is healthy.'); 44 | return 0; 45 | } 46 | 47 | $this->error('TMI cluster is unhealthy.'); 48 | return 1; 49 | } 50 | 51 | private function getHostSupervisors(SupervisorRepository $repository): Collection 52 | { 53 | return $repository->all()->filter(function (Supervisor $supervisor) { 54 | return str_starts_with(sprintf('%s-', $supervisor->getKey()), $this->getHostname()); 55 | }); 56 | } 57 | 58 | private function getHostname(): string 59 | { 60 | return $this->option('hostname') ?? gethostname(); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/Commands/TmiClusterInstallCommand.php: -------------------------------------------------------------------------------- 1 | comment('Publishing TMI Cluster assets...'); 29 | $this->callSilent('vendor:publish', ['--tag' => 'tmi-cluster-assets', '--force' => true]); 30 | 31 | $this->comment('Publishing TMI Cluster configuration...'); 32 | $this->callSilent('vendor:publish', ['--tag' => 'tmi-cluster-config']); 33 | 34 | $this->info('TMI Cluster scaffolding installed successfully.'); 35 | 36 | return 0; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/Commands/TmiClusterJoinCommand.php: -------------------------------------------------------------------------------- 1 | argument('channel'); 33 | 34 | $this->printChannelNames(); 35 | 36 | if ($this->option('authorize')) { 37 | $result = Artisan::call('tmi-cluster:authorize', [ 38 | 'channel' => $channels, 39 | ]); 40 | 41 | if ($result === 0) { 42 | $this->info(Artisan::output()); 43 | } else { 44 | $this->error(Artisan::output()); 45 | } 46 | } 47 | 48 | $results = $this->getChannelDistributor()->joinNow($channels); 49 | 50 | foreach ($results as $type => $channels) { 51 | if (empty($channels)) { 52 | continue; 53 | } 54 | 55 | $this->info(sprintf('%s:', ucfirst($type))); 56 | foreach ($channels as $channel) { 57 | $this->info(sprintf("\t%s", $channel)); 58 | } 59 | } 60 | 61 | return 0; 62 | } 63 | 64 | private function getChannelDistributor(): ChannelDistributor 65 | { 66 | return app(ChannelDistributor::class); 67 | } 68 | 69 | private function getChannelManager(): ChannelManager 70 | { 71 | return app(ChannelManager::class); 72 | } 73 | 74 | private function printChannelNames(): void 75 | { 76 | $channelManagerName = get_class($this->getChannelManager()); 77 | $channelDistributorName = get_class($this->getChannelDistributor()); 78 | 79 | $this->info(sprintf("Your channel manager is %s", $channelManagerName)); 80 | $this->info(sprintf("Your channel distributor is %s", $channelDistributorName)); 81 | 82 | $this->getOutput()->newLine(); 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /src/Commands/TmiClusterListCommand.php: -------------------------------------------------------------------------------- 1 | all() 36 | ->map(function (Supervisor $supervisor) { 37 | return [ 38 | $supervisor->getKey(), 39 | $supervisor->getKey(), 40 | $supervisor->last_ping_at->diffInSeconds(), 41 | $supervisor->processes()->count(), 42 | ]; 43 | }); 44 | 45 | $this->table([ 46 | 'ID', 'Name', 'Last Ping', 'Processes' 47 | ], $supervisors); 48 | 49 | 50 | $processes = SupervisorProcess::all() 51 | ->map(function (SupervisorProcess $process) { 52 | return [ 53 | $this->option('uuid') ? $process->getKey() : explode('-', $process->getKey())[0], 54 | $process->supervisor ? $process->supervisor->getKey() : 'N/A', 55 | $process->state, 56 | $process->last_ping_at ? $process->last_ping_at->diffInSeconds() : 'N/A', 57 | count($process->channels), 58 | $process->memory_usage, 59 | ]; 60 | }); 61 | 62 | $this->table([ 63 | 'ID', 'Supervisor', 'State', 'Last Ping', 'Channels', 'Memory Usage' 64 | ], $processes); 65 | 66 | return 0; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/Commands/TmiClusterProcessCommand.php: -------------------------------------------------------------------------------- 1 | clusterClientOptions(); 37 | 38 | event(new ClusterClientBootstrap($options)); 39 | 40 | TmiClusterClient::make($options, function ($type, $line) { 41 | $this->info($line); 42 | })->connect(); 43 | 44 | return 0; 45 | } 46 | 47 | /** 48 | * Returns the given cluster client configuration. 49 | * 50 | * @return ClusterClientOptions 51 | */ 52 | private function clusterClientOptions(): ClusterClientOptions 53 | { 54 | return new ClusterClientOptions( 55 | $this->argument('uuid'), 56 | $this->option('supervisor'), 57 | $this->option('stop-when-empty'), 58 | $this->option('memory'), 59 | $this->option('force') 60 | ); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/Commands/TmiClusterPublishCommand.php: -------------------------------------------------------------------------------- 1 | comment('Publishing TMI Cluster assets...'); 29 | $this->callSilent('vendor:publish', ['--tag' => 'tmi-cluster-assets', '--force' => true]); 30 | 31 | $this->info('TMI Cluster assets published successfully.'); 32 | 33 | return 0; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/Commands/TmiClusterPurgeCommand.php: -------------------------------------------------------------------------------- 1 | flushStale(); 32 | } catch (Exception $e) { 33 | $this->error($e->getMessage()); 34 | 35 | return 1; 36 | } 37 | 38 | $this->info('Stale supervisors have been purged.'); 39 | 40 | return 0; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/Commands/TmiClusterTerminateCommand.php: -------------------------------------------------------------------------------- 1 | argument('supervisor')) { 31 | $supervisors = $repository->all()->whereIn('id', [$id]); 32 | } else { 33 | $supervisors = $repository->all(); 34 | } 35 | 36 | foreach ($supervisors as $supervisor) { 37 | $this->info(sprintf('Terminating %s...', $supervisor->getKey())); 38 | $commandQueue->push($supervisor->getKey(), CommandQueue::COMMAND_SUPERVISOR_TERMINATE); 39 | } 40 | 41 | return 0; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/Contracts/ChannelDistributor.php: -------------------------------------------------------------------------------- 1 | options = $options; 17 | $this->output = $output; 18 | } 19 | 20 | /** 21 | * Starts a new TMI Client process. 22 | * 23 | * @param ClusterClientOptions $options 24 | * @param Closure $output 25 | * @return self 26 | */ 27 | abstract public static function make(ClusterClientOptions $options, Closure $output): self; 28 | 29 | public static function getQueueName(string $id, string $name): string 30 | { 31 | return $id . '-' . $name; 32 | } 33 | 34 | abstract public function connect(): void; 35 | 36 | abstract public function pause(): void; 37 | 38 | abstract public function continue(): void; 39 | 40 | abstract public function restart(): void; 41 | 42 | abstract public function terminate(int $status = 0): void; 43 | 44 | abstract public function getTmiClient(): Client; 45 | 46 | abstract public function log(string $message): void; 47 | 48 | abstract public function getUuid(); 49 | 50 | public function memoryUsage(): float 51 | { 52 | return memory_get_usage() / 1024 / 1024; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/Contracts/ClusterClientOptions.php: -------------------------------------------------------------------------------- 1 | uuid = $uuid; 22 | $this->supervisor = $supervisor; 23 | $this->stopWhenEmpty = $stopWhenEmpty; 24 | $this->memory = $memory; 25 | $this->force = $force; 26 | } 27 | 28 | public function getUuid(): string 29 | { 30 | return $this->uuid; 31 | } 32 | 33 | public function getSupervisor(): string 34 | { 35 | return $this->supervisor; 36 | } 37 | 38 | public function isStopWhenEmpty(): bool 39 | { 40 | return $this->stopWhenEmpty; 41 | } 42 | 43 | public function getMemory(): int 44 | { 45 | return $this->memory; 46 | } 47 | 48 | public function isForce(): bool 49 | { 50 | return $this->force; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/Contracts/CommandQueue.php: -------------------------------------------------------------------------------- 1 | #example] 11 | */ 12 | public const COMMAND_TMI_WRITE = 'tmi:write'; 13 | 14 | /** 15 | * Joins a irc channel. 16 | * 17 | * Payload: [channel => #example] 18 | */ 19 | public const COMMAND_TMI_JOIN = 'tmi:join'; 20 | 21 | /** 22 | * Leaves a irc channel. 23 | * 24 | * Payload: [channel => #example] 25 | */ 26 | public const COMMAND_TMI_PART = 'tmi:part'; 27 | 28 | /** 29 | * Executes a exit(6) within the tmi cluster process. 30 | */ 31 | public const COMMAND_CLIENT_EXIT = 'client:exit'; 32 | 33 | /** 34 | * Scales out the supervisor by one process. 35 | */ 36 | public const COMMAND_SUPERVISOR_SCALE_OUT = 'supervisor:scale-out'; 37 | 38 | /** 39 | * Scales in the supervisor by one process. 40 | */ 41 | public const COMMAND_SUPERVISOR_SCALE_IN = 'supervisor:scale-in'; 42 | 43 | /** 44 | * Executes a exit(0) within the tmi cluster supervisor. 45 | */ 46 | public const COMMAND_SUPERVISOR_TERMINATE = 'supervisor:terminate'; 47 | 48 | /** 49 | * Queue to handle lost and found cases. Eg. a channel cannot be joined. 50 | */ 51 | public const NAME_JOIN_HANDLER = 'join-handler'; 52 | 53 | /** 54 | * Supervisor queue: First-come, First-served. 55 | */ 56 | public const NAME_ANY_SUPERVISOR = '*'; 57 | 58 | /** 59 | * Push a command onto a queue. 60 | * 61 | * @param string $name 62 | * @param string $command 63 | * @param array $options 64 | */ 65 | public function push(string $name, string $command, array $options = []); 66 | 67 | /** 68 | * Get the pending commands for a given queue name. 69 | * 70 | * @param string $name 71 | * @return array 72 | */ 73 | public function pending(string $name): array; 74 | 75 | /** 76 | * Flush the command queue for a given queue name. 77 | * 78 | * @param string $name 79 | * @return void 80 | */ 81 | public function flush(string $name): void; 82 | } 83 | -------------------------------------------------------------------------------- /src/Contracts/HasPrometheusMetrics.php: -------------------------------------------------------------------------------- 1 | 13 | */ 14 | public function getPrometheusMetrics(): Collection; 15 | } -------------------------------------------------------------------------------- /src/Contracts/Invitable.php: -------------------------------------------------------------------------------- 1 | [ 14 | Listeners\SendNotification::class, 15 | ], 16 | 17 | Events\SupervisorLooped::class => [ 18 | Listeners\MonitorSupervisorMemory::class, 19 | ], 20 | 21 | Events\PeriodicTimerCalled::class => [ 22 | Listeners\MonitorClusterClientMemory::class, 23 | ], 24 | ]; 25 | } 26 | -------------------------------------------------------------------------------- /src/Events/ClusterClientBootstrap.php: -------------------------------------------------------------------------------- 1 | options = $options; 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/Events/ClusterClientRegistered.php: -------------------------------------------------------------------------------- 1 | clusterClient = $clusterClient; 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/Events/ClusterClientTerminated.php: -------------------------------------------------------------------------------- 1 | clusterClient = $clusterClient; 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/Events/PeriodicTimerCalled.php: -------------------------------------------------------------------------------- 1 | clusterClient = $clusterClient; 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/Events/ProcessScaled.php: -------------------------------------------------------------------------------- 1 | message = $message; 19 | $this->description = $description; 20 | $this->event = $event; 21 | $this->cause = $cause; 22 | } 23 | 24 | public function toNotification(): ?AutoScaleScaledNotification 25 | { 26 | if (config('tmi-cluster.framework') === 'laravel-zero') { 27 | return null; // notifications are not supported 28 | } 29 | 30 | return new AutoScaleScaledNotification( 31 | $this->message, $this->description, $this->event, $this->cause 32 | ); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/Events/SupervisorBootstrap.php: -------------------------------------------------------------------------------- 1 | supervisor = $supervisor; 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/Events/UnableToLaunchProcess.php: -------------------------------------------------------------------------------- 1 | process = $process; 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/Events/WorkerProcessRestarting.php: -------------------------------------------------------------------------------- 1 | process = $process; 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/Exceptions/NotFoundSupervisorException.php: -------------------------------------------------------------------------------- 1 | get())) { 22 | return response('', 204); 23 | } 24 | 25 | return response('', 503); 26 | } 27 | 28 | public function statistics(Request $request): array 29 | { 30 | /** @var AutoScale $autoScale */ 31 | $autoScale = app(AutoScale::class); 32 | 33 | $withChannels = !$request->has('time'); 34 | 35 | $supervisors = Supervisor::query() 36 | ->with('processes:' . implode(',', $this->getProcessColumns($withChannels))) 37 | ->get(); 38 | 39 | $ircMessages = $supervisors 40 | ->sum(function (Supervisor $supervisor) { 41 | return $supervisor->processes->sum(function (SupervisorProcess $process) { 42 | return $process->metrics['irc_messages'] ?? 0; 43 | }); 44 | }); 45 | 46 | $ircCommands = $supervisors 47 | ->sum(function (Supervisor $supervisor) { 48 | return $supervisor->processes->sum(function (SupervisorProcess $process) { 49 | return $process->metrics['irc_commands'] ?? 0; 50 | }); 51 | }); 52 | 53 | $supervisors->map(function (Supervisor $supervisor) { 54 | $tokens = explode('-', $supervisor->getKey()); 55 | $supervisor->id_short = $tokens[count($tokens) - 1]; 56 | $supervisor->processes->transform(function (SupervisorProcess $process) { 57 | $process->id_short = explode('-', $process->getKey())[0]; 58 | $process->last_ping_at_in_seconds = $process->last_ping_at->diffInSeconds(); 59 | return $process; 60 | }); 61 | }); 62 | 63 | $time = round(microtime(true) * 1000); 64 | 65 | return [ 66 | 'time' => $time, 67 | 'operational' => Health::isOperational($supervisors), 68 | 'supervisors' => $supervisors, 69 | 'irc_messages' => $ircMessages, 70 | 'irc_commands' => $ircCommands, 71 | 'irc_messages_per_second' => $this->getDataPerSecond($request, 'irc_messages', $ircMessages, $time), 72 | 'irc_commands_per_second' => $this->getDataPerSecond($request, 'irc_commands', $ircCommands, $time), 73 | 'channels' => $supervisors 74 | ->sum(function (Supervisor $supervisor) { 75 | return $supervisor->processes->sum(function (SupervisorProcess $process) { 76 | return $process->metrics['channels'] ?? 0; 77 | }); 78 | }), 79 | 'processes' => $supervisors->sum(fn(Supervisor $supervisor) => count($supervisor->processes)), 80 | 'auto_scale' => array_merge(config('tmi-cluster.auto_scale'), [ 81 | 'minimum_scale' => $autoScale->getMinimumScale(), 82 | ]), 83 | ]; 84 | } 85 | 86 | public function getDataPerSecond(Request $request, string $field, int $current, float $time): float 87 | { 88 | if ($request->has(['time', $field])) { 89 | $lastIrcMessages = $request->get($field, $current); 90 | $lastTime = (int)$request->get('time', $time); 91 | $timeDiff = $time - $lastTime; 92 | $diff = $current - $lastIrcMessages; 93 | 94 | return round($diff / ($timeDiff / 1000)); 95 | } 96 | 97 | return 0; 98 | } 99 | 100 | private function getProcessColumns(bool $withChannels): array 101 | { 102 | $columns = [ 103 | 'id', 104 | 'state', 105 | 'metrics', 106 | 'created_at', 107 | 'last_ping_at', 108 | 'supervisor_id', 109 | ]; 110 | 111 | if ($withChannels) { 112 | $columns[] = 'channels'; 113 | } 114 | 115 | return $columns; 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /src/Http/Controllers/InviteController.php: -------------------------------------------------------------------------------- 1 | getInvitableUser($request); 21 | 22 | return view('tmi-cluster::invite.index', [ 23 | 'connections' => Channel::query()->count(), 24 | 'bot_username' => config('tmi-cluster.tmi.identity.username'), 25 | 'invited' => $channelManager->authorized([$user->getTwitchLogin()]), 26 | 'user_username' => $user->getTwitchLogin(), 27 | ]); 28 | } 29 | 30 | public function store(Request $request, ChannelManager $channelManager) 31 | { 32 | $user = $this->getInvitableUser($request); 33 | 34 | if ($channelManager->authorized([$user->getTwitchLogin()])) { 35 | $channelManager->revokeAuthorization($user); 36 | } else { 37 | $channelManager->authorize($user, [ 38 | 'reconnect' => true, 39 | ]); 40 | } 41 | 42 | return back(); 43 | } 44 | 45 | private function getInvitableUser(Request $request): Invitable 46 | { 47 | $user = $request->user(); 48 | 49 | if (!($user instanceof Invitable) || !$user->getTwitchLogin()) { 50 | throw new HttpException(409, 'You cannot invite this bot into your channel.'); 51 | } 52 | 53 | return $user; 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/Http/Controllers/MetricsController.php: -------------------------------------------------------------------------------- 1 | 'Number of processes of the supervisor', 15 | 'tmi_cluster_process_channels' => 'Memory usage of the process', 16 | 'tmi_cluster_process_irc_commands' => 'IRC commands processed by the process', 17 | 'tmi_cluster_process_irc_messages' => 'IRC messages processed by the process', 18 | 'tmi_cluster_process_memory_usage' => 'Memory usage of the process', 19 | 'tmi_cluster_process_command_queue_commands' => 'Commands in the command queue', 20 | 'tmi_cluster_avg_usage' => 'Average usage of all processes', 21 | 'tmi_cluster_avg_response_time' => 'Average response time of all processes', 22 | ]; 23 | 24 | private const METRIC_TYPE_HINTS = [ 25 | 'tmi_cluster_supervisor_processes' => 'gauge', 26 | 'tmi_cluster_process_channels' => 'gauge', 27 | 'tmi_cluster_process_irc_commands' => 'counter', 28 | 'tmi_cluster_process_irc_messages' => 'counter', 29 | 'tmi_cluster_process_memory_usage' => 'gauge', 30 | 'tmi_cluster_process_command_queue_commands' => 'gauge', 31 | 'tmi_cluster_avg_usage' => 'gauge', 32 | 'tmi_cluster_avg_response_time' => 'gauge', 33 | ]; 34 | 35 | public function handle() 36 | { 37 | $metrics = collect(); 38 | 39 | /** @var SupervisorRepository $repository */ 40 | $repository = app(SupervisorRepository::class); 41 | 42 | $repository->all()->each(function (Supervisor $supervisor) use (&$metrics) { 43 | foreach ($supervisor->processes as $process) { 44 | $metrics->push($this->getProcessMetrics($supervisor, $process)); 45 | } 46 | 47 | $metrics->push($this->getSupervisorMetrics($supervisor)); 48 | }); 49 | 50 | $metrics->push($this->getGlobalMetrics()); 51 | 52 | return response($metrics->collapse()->join(PHP_EOL) . PHP_EOL) 53 | ->header('Content-Type', 'text/plain'); 54 | } 55 | 56 | protected function getProcessMetrics(Supervisor $supervisor, SupervisorProcess $process): array 57 | { 58 | return $process->getPrometheusMetrics() 59 | ->map(function ($value, $key) use ($supervisor, $process) { 60 | return $this->ensureIsDocumented(sprintf('tmi_cluster_process_%s', $key), $value, [ 61 | 'supervisor' => $supervisor->getKey(), 62 | 'process' => $process->getKey(), 63 | ]); 64 | })->values()->toArray(); 65 | } 66 | 67 | protected function getSupervisorMetrics(Supervisor $supervisor): array 68 | { 69 | return $supervisor->getPrometheusMetrics() 70 | ->map(function ($value, $key) use ($supervisor) { 71 | return $this->ensureIsDocumented(sprintf('tmi_cluster_supervisor_%s', $key), $value, [ 72 | 'supervisor' => $supervisor->getKey(), 73 | ]); 74 | })->values()->toArray(); 75 | } 76 | 77 | private function getGlobalMetrics(): array 78 | { 79 | return [ 80 | $this->ensureIsDocumented('tmi_cluster_avg_usage', 10), 81 | $this->ensureIsDocumented('tmi_cluster_avg_response_time', 700) 82 | ]; 83 | } 84 | 85 | private function ensureIsDocumented(string $key, float $value, array $tags = []): string 86 | { 87 | $line = sprintf('%s%s %s', $key, $this->formatTags($tags), $value); 88 | 89 | if (isset($this->hintedMetrics[$key])) { 90 | return $line; 91 | } 92 | 93 | $this->hintedMetrics[$key] = true; 94 | 95 | return implode(PHP_EOL, [ 96 | sprintf('# HELP %s %s', $key, self::METRIC_HELP_TEXT[$key] ?? 'N/A'), 97 | sprintf('# TYPE %s %s', $key, self::METRIC_TYPE_HINTS[$key] ?? 'gauge'), 98 | $line, 99 | ]); 100 | } 101 | 102 | /** 103 | * Format tags for prometheus. 104 | */ 105 | private function formatTags(array $tags): string 106 | { 107 | $formatted = collect($tags)->map(function ($value, $key) { 108 | return sprintf('%s="%s"', $key, $value); 109 | })->join(','); 110 | 111 | return $formatted ? sprintf('{%s}', $formatted) : ''; 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /src/Limiters/DurationLimiter.php: -------------------------------------------------------------------------------- 1 | name = $name; 70 | $this->decay = $decay; 71 | $this->redis = $redis; 72 | $this->maxLocks = $maxLocks; 73 | $this->locks = $locks; 74 | } 75 | 76 | /** 77 | * Attempt to acquire the lock for the given number of seconds. 78 | * 79 | * @param int $timeout 80 | * @param callable|null $callback 81 | * @return mixed 82 | * 83 | * @throws \Illuminate\Contracts\Redis\LimiterTimeoutException 84 | */ 85 | public function block($timeout, $callback = null) 86 | { 87 | $starting = time(); 88 | 89 | while (!$this->acquire()) { 90 | if (time() - $timeout >= $starting) { 91 | throw new LimiterTimeoutException; 92 | } 93 | 94 | usleep(750 * 1000); 95 | } 96 | 97 | if (is_callable($callback)) { 98 | return $callback(); 99 | } 100 | 101 | return true; 102 | } 103 | 104 | /** 105 | * Attempt to acquire the lock. 106 | * 107 | * @return bool 108 | */ 109 | public function acquire() 110 | { 111 | $results = $this->redis->eval( 112 | $this->luaScript(), 1, $this->name, microtime(true), time(), $this->decay, $this->maxLocks, $this->locks 113 | ); 114 | 115 | $this->decaysAt = $results[1]; 116 | 117 | $this->remaining = max(0, $results[2]); 118 | 119 | return (bool)$results[0]; 120 | } 121 | 122 | /** 123 | * Get the Lua script for acquiring a lock. 124 | * 125 | * KEYS[1] - The limiter name 126 | * ARGV[1] - Current time in microseconds 127 | * ARGV[2] - Current time in seconds 128 | * ARGV[3] - Duration of the bucket 129 | * ARGV[4] - Allowed number of tasks 130 | * ARGV[5] - Tasks per execution 131 | * 132 | * @return string 133 | */ 134 | protected function luaScript() 135 | { 136 | return <<<'LUA' 137 | local function reset() 138 | redis.call('HMSET', KEYS[1], 'start', ARGV[2], 'end', ARGV[2] + ARGV[3], 'count', ARGV[5]) 139 | return redis.call('EXPIRE', KEYS[1], ARGV[3] * 2) 140 | end 141 | 142 | if redis.call('EXISTS', KEYS[1]) == 0 then 143 | return {reset(), ARGV[2] + ARGV[3], ARGV[4] - ARGV[5]} 144 | end 145 | 146 | if ARGV[1] >= redis.call('HGET', KEYS[1], 'start') and ARGV[1] <= redis.call('HGET', KEYS[1], 'end') then 147 | return { 148 | tonumber(redis.call('HINCRBY', KEYS[1], 'count', ARGV[5])) <= tonumber(ARGV[4]), 149 | redis.call('HGET', KEYS[1], 'end'), 150 | ARGV[4] - redis.call('HGET', KEYS[1], 'count') 151 | } 152 | end 153 | 154 | return {reset(), ARGV[2] + ARGV[3], ARGV[4] - ARGV[5]} 155 | LUA; 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /src/Limiters/DurationLimiterBuilder.php: -------------------------------------------------------------------------------- 1 | name = $name; 64 | $this->connection = $connection; 65 | } 66 | 67 | /** 68 | * Set the maximum number of locks that can obtained per time window. 69 | * 70 | * @param int $maxLocks 71 | * @return $this 72 | */ 73 | public function allow($maxLocks) 74 | { 75 | $this->maxLocks = $maxLocks; 76 | 77 | return $this; 78 | } 79 | 80 | /** 81 | * Set the amount of time the lock window is maintained. 82 | * 83 | * @param \DateTimeInterface|\DateInterval|int $decay 84 | * @return $this 85 | */ 86 | public function every($decay) 87 | { 88 | $this->decay = $this->secondsUntil($decay); 89 | 90 | return $this; 91 | } 92 | 93 | /** 94 | * Set the amount of time to block until a lock is available. 95 | * 96 | * @param int $timeout 97 | * @return $this 98 | */ 99 | public function block($timeout) 100 | { 101 | $this->timeout = $timeout; 102 | 103 | return $this; 104 | } 105 | 106 | /** 107 | * Set the amount of locks, that should be taken per execution. 108 | * 109 | * @param int $locks 110 | * @return $this 111 | */ 112 | public function take($locks) 113 | { 114 | $this->locks = $locks; 115 | 116 | return $this; 117 | } 118 | 119 | /** 120 | * Execute the given callback if a lock is obtained, otherwise call the failure callback. 121 | * 122 | * @param callable $callback 123 | * @param callable|null $failure 124 | * @return mixed 125 | * 126 | * @throws \Illuminate\Contracts\Redis\LimiterTimeoutException 127 | */ 128 | public function then(callable $callback, callable $failure = null) 129 | { 130 | try { 131 | return (new DurationLimiter( 132 | $this->connection, $this->name, $this->maxLocks, $this->decay, $this->locks 133 | ))->block($this->timeout, $callback); 134 | } catch (LimiterTimeoutException $e) { 135 | if ($failure) { 136 | return $failure($e); 137 | } 138 | 139 | throw $e; 140 | } 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /src/Listeners/MonitorClusterClientMemory.php: -------------------------------------------------------------------------------- 1 | clusterClient; 12 | 13 | if ($clusterClient->memoryUsage() > $clusterClient->options->getMemory()) { 14 | $clusterClient->terminate(8); 15 | } 16 | } 17 | } -------------------------------------------------------------------------------- /src/Listeners/MonitorSupervisorMemory.php: -------------------------------------------------------------------------------- 1 | supervisor; 12 | 13 | if ($supervisor->memoryUsage() > config('tmi-cluster.memory_limit', 64)) { 14 | $supervisor->terminate(8); 15 | } 16 | } 17 | } -------------------------------------------------------------------------------- /src/Listeners/SendNotification.php: -------------------------------------------------------------------------------- 1 | toNotification(); 15 | 16 | if ($notification === null) { 17 | return; 18 | } 19 | 20 | /** @var Lock $lock */ 21 | $lock = app(Lock::class); 22 | 23 | if (!$lock->get('notification:' . $notification->signature(), 300)) { 24 | $event->skiped = true; 25 | return; 26 | } 27 | 28 | try { 29 | Notification::route('slack', TmiCluster::$slackWebhookUrl) 30 | ->route('nexmo', TmiCluster::$smsNumber) 31 | ->route('mail', TmiCluster::$email) 32 | ->notify($notification); 33 | 34 | $event->sent = true; 35 | } catch (Throwable $exception) { 36 | $event->sent = false; 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/ListensForSignals.php: -------------------------------------------------------------------------------- 1 | pendingSignals['terminate'] = 'terminate'; 27 | }); 28 | 29 | pcntl_signal(SIGTERM, function () { 30 | $this->pendingSignals['terminate'] = 'terminate'; 31 | }); 32 | 33 | pcntl_signal(SIGUSR1, function () { 34 | $this->pendingSignals['restart'] = 'restart'; 35 | }); 36 | 37 | pcntl_signal(SIGUSR2, function () { 38 | $this->pendingSignals['pause'] = 'pause'; 39 | }); 40 | 41 | pcntl_signal(SIGCONT, function () { 42 | $this->pendingSignals['continue'] = 'continue'; 43 | }); 44 | } 45 | 46 | /** 47 | * Process the pending signals. 48 | * 49 | * @return void 50 | */ 51 | protected function processPendingSignals(): void 52 | { 53 | while ($this->pendingSignals) { 54 | $signal = Arr::first($this->pendingSignals); 55 | 56 | $this->{$signal}(); 57 | 58 | unset($this->pendingSignals[$signal]); 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/Lock.php: -------------------------------------------------------------------------------- 1 | redis = $redis; 16 | } 17 | 18 | public function with(string $key, callable $callback, int $seconds = 60): void 19 | { 20 | if ($this->get($key, $seconds)) { 21 | try { 22 | $callback(); 23 | } finally { 24 | $this->release($key); 25 | } 26 | } 27 | } 28 | 29 | public function exists(string $key): bool 30 | { 31 | return $this->connection()->exists($key) === 1; 32 | } 33 | 34 | public function get(string $key, int $seconds = 60): bool 35 | { 36 | $result = $this->connection()->setnx($key, 1); 37 | 38 | if ($result === 1) { 39 | $this->connection()->expire($key, $seconds); 40 | } 41 | 42 | return $result === 1; 43 | } 44 | 45 | public function release(string $key): void 46 | { 47 | $this->connection()->del($key); 48 | } 49 | 50 | public function connection(): Connection 51 | { 52 | return $this->redis->connection('tmi-cluster'); 53 | } 54 | 55 | /** 56 | * Throttle a callback for a maximum number of executions over a given duration. 57 | * 58 | * @param string $name 59 | * @return DurationLimiterBuilder 60 | */ 61 | public function throttle($name) 62 | { 63 | return new DurationLimiterBuilder($this->connection(), $name); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/Models/Channel.php: -------------------------------------------------------------------------------- 1 | 'bool', 22 | 'reconnect' => 'bool', 23 | ]; 24 | } -------------------------------------------------------------------------------- /src/Models/Supervisor.php: -------------------------------------------------------------------------------- 1 | 'array', 32 | 'metrics' => 'array', 33 | 'last_ping_at' => 'datetime', 34 | ]; 35 | 36 | public function getIsStaleAttribute(): bool 37 | { 38 | return $this->last_ping_at->diffInSeconds() >= config('tmi-cluster.supervisor.stale', 300); 39 | } 40 | 41 | public function processes(): HasMany 42 | { 43 | return $this->hasMany(SupervisorProcess::class); 44 | } 45 | 46 | public function getPrometheusMetrics(): Collection 47 | { 48 | return Collection::make([ 49 | 'processes' => $this->processes()->count(), 50 | ])->merge($this->metrics ?? []); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/Models/SupervisorProcess.php: -------------------------------------------------------------------------------- 1 | 'array', 37 | 'metrics' => 'array', 38 | 'last_ping_at' => 'datetime', 39 | ]; 40 | 41 | public function supervisor(): BelongsTo 42 | { 43 | return $this->belongsTo(Supervisor::class); 44 | } 45 | 46 | public function getIsStaleAttribute(): bool 47 | { 48 | // we require at least 60 seconds for our restart cooldown 49 | return $this->last_ping_at->diffInSeconds() >= config('tmi-cluster.process.stale', 90); 50 | } 51 | 52 | public function getMemoryUsageAttribute(): string 53 | { 54 | return isset($this->metrics['memory_usage']) ? Str::convert($this->metrics['memory_usage']) : 'N/A'; 55 | } 56 | 57 | public function getPrometheusMetrics(): Collection 58 | { 59 | return Collection::make($this->metrics ?? []); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/Notifications/AutoScaleScaled.php: -------------------------------------------------------------------------------- 1 | message = $message; 21 | $this->description = $description; 22 | $this->event = $event; 23 | $this->cause = $cause; 24 | } 25 | 26 | /** 27 | * Get the mail representation of the notification. 28 | * 29 | * @param mixed $notifiable 30 | * @return MailMessage 31 | */ 32 | public function toMail($notifiable) 33 | { 34 | return (new MailMessage) 35 | ->error() 36 | ->subject('TMI Cluster: AutoScale Notification') 37 | ->greeting('Oh no! Something needs your attention.') 38 | ->line(sprintf('Message: %s', $this->message)) 39 | ->line(sprintf('Description: %s', $this->message)) 40 | ->line(sprintf('Event: %s', $this->message)) 41 | ->line(sprintf('Cause: %s', $this->message)); 42 | } 43 | 44 | /** 45 | * Get the Slack representation of the notification. 46 | * 47 | * @param mixed $notifiable 48 | * @return SlackMessage 49 | */ 50 | public function toSlack($notifiable) 51 | { 52 | return (new SlackMessage) 53 | ->from('TMI Cluster') 54 | ->to(TmiCluster::$slackChannel) 55 | //->image('... add tmi cluster image url') 56 | ->info() 57 | ->attachment(function ($attachment) { 58 | $attachment 59 | ->title('TMI Cluster: AutoScale Notification') 60 | ->fields([ 61 | 'Message' => $this->message, 62 | 'Description' => $this->description, 63 | 'Event' => $this->event, 64 | 'Cause' => $this->cause, 65 | ]); 66 | }); 67 | } 68 | 69 | /** 70 | * Get the Nexmo / SMS representation of the notification. 71 | * 72 | * @param mixed $notifiable 73 | * @return NexmoMessage 74 | */ 75 | public function toNexmo($notifiable) 76 | { 77 | return (new NexmoMessage)->content(sprintf( 78 | '[TMI Cluster] %s. %s.', $this->message, $this->description 79 | )); 80 | } 81 | 82 | public function signature(): string 83 | { 84 | return sha1($this->message . $this->description . $this->event . $this->cause); 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/Notifications/Notification.php: -------------------------------------------------------------------------------- 1 | getSupervisor(), 27 | $options->getMemory() 28 | ); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/Process/Process.php: -------------------------------------------------------------------------------- 1 | systemProcess = $systemProcess; 30 | $this->uuid = $uuid; 31 | $this->output = function () { 32 | // 33 | }; 34 | } 35 | 36 | public function getUuid(): string 37 | { 38 | return $this->uuid; 39 | } 40 | 41 | public function isRunning(): bool 42 | { 43 | return $this->systemProcess->isRunning(); 44 | } 45 | 46 | public function monitor(): void 47 | { 48 | if ($this->systemProcess->isRunning() || $this->coolingDown()) { 49 | return; 50 | } 51 | 52 | $this->restart(); 53 | } 54 | 55 | public function restart(): void 56 | { 57 | if ($this->systemProcess->isStarted()) { 58 | event(new WorkerProcessRestarting($this)); 59 | } 60 | 61 | $this->start(); 62 | } 63 | 64 | public function terminate(int $status = 0): void 65 | { 66 | $this->sendSignal(SIGINT); 67 | } 68 | 69 | public function stop(): void 70 | { 71 | if ($this->systemProcess->isRunning()) { 72 | $this->systemProcess->stop(); 73 | } 74 | } 75 | 76 | protected function sendSignal($signal): void 77 | { 78 | try { 79 | $this->systemProcess->signal($signal); 80 | } catch (ExceptionInterface $e) { 81 | if ($this->systemProcess->isRunning()) { 82 | throw $e; 83 | } 84 | } 85 | } 86 | 87 | public function start(): Process 88 | { 89 | $this->cooldown(); 90 | 91 | $this->systemProcess->start($this->output); 92 | 93 | return $this; 94 | } 95 | 96 | public function pause(): void 97 | { 98 | $this->sendSignal(SIGUSR2); 99 | } 100 | 101 | public function continue(): void 102 | { 103 | $this->sendSignal(SIGCONT); 104 | } 105 | 106 | protected function cooldown() 107 | { 108 | if ($this->coolingDown()) { 109 | return; 110 | } 111 | 112 | if ($this->restartAgainAt) { 113 | $this->restartAgainAt = !$this->systemProcess->isRunning() 114 | ? CarbonImmutable::now()->addMinute() 115 | : null; 116 | 117 | if (!$this->systemProcess->isRunning()) { 118 | event(new UnableToLaunchProcess($this)); 119 | } 120 | } else { 121 | $this->restartAgainAt = CarbonImmutable::now()->addSecond(); 122 | } 123 | } 124 | 125 | public function coolingDown(): bool 126 | { 127 | return isset($this->restartAgainAt) && 128 | CarbonImmutable::now()->lt($this->restartAgainAt); 129 | } 130 | 131 | public function handleOutputUsing(Closure $callback): self 132 | { 133 | $this->output = $callback; 134 | 135 | return $this; 136 | } 137 | 138 | 139 | public function shouldMarkForTermination(): bool 140 | { 141 | if ($exitCode = $this->systemProcess->getExitCode()) { 142 | return !in_array($this->systemProcess->getExitCode(), self::HEALTHY_EXIT_CODES); 143 | } 144 | 145 | return false; 146 | } 147 | } 148 | -------------------------------------------------------------------------------- /src/Process/ProcessOptions.php: -------------------------------------------------------------------------------- 1 | supervisor = $supervisor; 13 | $this->options = $options; 14 | } 15 | 16 | public function toWorkerCommand(string $uuid): string 17 | { 18 | return CommandString::fromOptions($this, $uuid); 19 | } 20 | 21 | public function getWorkingDirectory(): string 22 | { 23 | return app()->environment('testing') 24 | ? base_path('tests/laravel') 25 | : base_path(); 26 | } 27 | 28 | public function getSupervisor(): string 29 | { 30 | return $this->supervisor; 31 | } 32 | 33 | public function getTimeout(): int 34 | { 35 | return $this->options['timeout'] ?? config('tmi-cluster.process.timeout', 60); 36 | } 37 | 38 | public function getMemory(): int 39 | { 40 | return $this->options['memory'] ?? config('tmi-cluster.process.memory', 128); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/Process/ProcessPool.php: -------------------------------------------------------------------------------- 1 | options = $options; 34 | $this->output = $output; 35 | $this->supervisor = $supervisor; 36 | } 37 | 38 | public function count(): int 39 | { 40 | return count($this->processes); 41 | } 42 | 43 | public function monitor(): void 44 | { 45 | $this->processes()->each(function (Process $process, $key) { 46 | if ($process->shouldMarkForTermination()) { 47 | $this->markForTermination($process); 48 | $this->removeProcess($key); 49 | } else { 50 | $process->monitor(); 51 | } 52 | }); 53 | 54 | $this->pruneTerminatingProcesses(); 55 | $this->scale($this->requestedScale); 56 | } 57 | 58 | public function pruneTerminatingProcesses() 59 | { 60 | $this->stopTerminatingProcessesThatAreHanging(); 61 | $this->terminatingProcesses = collect( 62 | $this->terminatingProcesses 63 | )->filter(function ($process) { 64 | $filtered = $process['process']->isRunning(); 65 | 66 | if (!$filtered) { 67 | event(new ProcessScaled( 68 | sprintf('Auto Scaling: Process removed'), 69 | sprintf('Terminating process instance: %s', $process['process']->getUuid()), 70 | 'SUPERVISOR_PROCESS_TERMINATE', 71 | sprintf('At %s an process was removed in response of terminated process.', date('Y-m-d H:i:s')) 72 | )); 73 | } 74 | 75 | return $filtered; 76 | })->all(); 77 | } 78 | 79 | protected function stopTerminatingProcessesThatAreHanging() 80 | { 81 | foreach ($this->terminatingProcesses as $process) { 82 | $timeout = $this->options->getTimeout(); 83 | 84 | if ($process['terminatedAt']->addSeconds($timeout)->lte(CarbonImmutable::now())) { 85 | $process['process']->stop(); 86 | } 87 | } 88 | } 89 | 90 | public function processes(): Collection 91 | { 92 | return new Collection($this->processes); 93 | } 94 | 95 | public function runningProcesses(): Collection 96 | { 97 | return $this->processes()->filter(function (Process $process) { 98 | return $process->systemProcess->isRunning(); 99 | }); 100 | } 101 | 102 | public function scale($processes): void 103 | { 104 | $this->requestedScale = $processes; 105 | $processes = max(0, (int)$processes); 106 | 107 | if ($processes === count($this->processes)) { 108 | return; 109 | } 110 | 111 | if ($processes > count($this->processes)) { 112 | $this->scaleUp($processes); 113 | } else { 114 | $this->scaleDown($processes); 115 | } 116 | } 117 | 118 | protected function scaleUp($processes): void 119 | { 120 | $difference = $processes - count($this->processes); 121 | 122 | for ($i = 0; $i < $difference; $i++) { 123 | $this->start(); 124 | } 125 | } 126 | 127 | protected function scaleDown($processes): void 128 | { 129 | $difference = count($this->processes) - $processes; 130 | 131 | // Here we will slice off the correct number of processes that we need to terminate 132 | // and remove them from the active process array. We'll be adding them the array 133 | // of terminating processes where they'll run until they are fully terminated. 134 | $terminatingProcesses = array_slice( 135 | $this->processes, 0, $difference 136 | ); 137 | 138 | foreach ($terminatingProcesses as $process) { 139 | $this->markForTermination($process); 140 | } 141 | 142 | $this->removeProcesses($difference); 143 | 144 | // Finally we will call the terminate method on each of the processes that need get 145 | // terminated so they can start terminating. Terminating is a graceful operation 146 | // so any jobs they are already running will finish running before these quit. 147 | foreach ($this->terminatingProcesses as $process) { 148 | $process['process']->terminate(); 149 | 150 | event(new ProcessScaled( 151 | sprintf('Auto Scaling: Process terminated'), 152 | sprintf('Terminating process instance: %s', $process['process']->getUuid()), 153 | 'SUPERVISOR_PROCESS_TERMINATE', 154 | sprintf('At %s an process was taken out of service in response of a scale in.', date('Y-m-d H:i:s')) 155 | )); 156 | } 157 | } 158 | 159 | public function markForTermination(Process $process): void 160 | { 161 | $this->terminatingProcesses[] = [ 162 | 'process' => $process, 'terminatedAt' => CarbonImmutable::now(), 163 | ]; 164 | } 165 | 166 | protected function removeProcesses(int $count): void 167 | { 168 | array_splice($this->processes, 0, $count); 169 | 170 | $this->processes = array_values($this->processes); 171 | } 172 | 173 | protected function removeProcess($key): void 174 | { 175 | unset($this->processes[$key]); 176 | 177 | $this->processes = array_values($this->processes); 178 | } 179 | 180 | protected function start(): self 181 | { 182 | $process = $this->createProcess(); 183 | 184 | $process->handleOutputUsing(function ($type, $line) use ($process) { 185 | $shortUuid = explode('-', $process->getUuid())[0]; 186 | call_user_func($this->output, $type, sprintf('[%s] %s', $shortUuid, $line)); 187 | }); 188 | 189 | event(new ProcessScaled( 190 | sprintf('Auto Scaling: Process launched'), 191 | sprintf('Launching a new process instance: %s', $process->getUuid()), 192 | 'SUPERVISOR_PROCESS_LAUNCH', 193 | sprintf( 194 | 'At %s an instance was started in response to a difference between desired and actual capacity, increasing the capacity from %s to %s.', 195 | date('Y-m-d H:i:s'), 196 | $count = $this->count(), 197 | $count + 1, 198 | ) 199 | )); 200 | 201 | $this->processes[] = $process; 202 | 203 | return $this; 204 | } 205 | 206 | protected function createProcess(): Process 207 | { 208 | /** @var SystemProcess $class */ 209 | $class = config('tmi_cluster.fast_termination') 210 | ? BackgroundProcess::class 211 | : SystemProcess::class; 212 | 213 | $model = SupervisorProcess::query()->forceCreate([ 214 | 'id' => Str::uuid(), 215 | 'supervisor_id' => $this->supervisor->model->getKey(), 216 | 'state' => SupervisorProcess::STATE_INITIALIZE, 217 | 'channels' => [], 218 | 'last_ping_at' => now(), 219 | ]); 220 | 221 | return new Process($class::fromShellCommandline( 222 | $this->options->toWorkerCommand($model->getKey()), $this->options->getWorkingDirectory() 223 | )->setTimeout(null)->disableOutput(), $model->getKey()); 224 | } 225 | 226 | public function restart(): void 227 | { 228 | $count = count($this->processes); 229 | 230 | $this->scale(0); 231 | 232 | $this->scale($count); 233 | } 234 | 235 | public function pause(): void 236 | { 237 | $this->working = false; 238 | 239 | collect($this->processes)->each(fn(Process $x) => $x->stop()); 240 | } 241 | 242 | public function continue(): void 243 | { 244 | $this->working = true; 245 | 246 | collect($this->processes)->each(fn(Process $x) => $x->continue()); 247 | } 248 | 249 | public function terminate(int $status = 0): void 250 | { 251 | $this->working = false; 252 | 253 | collect($this->processes)->each(fn(Process $x) => $x->terminate($status)); 254 | } 255 | } 256 | -------------------------------------------------------------------------------- /src/Providers/TmiClusterServiceProvider.php: -------------------------------------------------------------------------------- 1 | configure(); 26 | $this->offerPublishing(); 27 | $this->registerEvents(); 28 | $this->registerServices(); 29 | $this->registerCommands(); 30 | $this->registerFrontend(); 31 | $this->defineAssetPublishing(); 32 | $this->registerLogger(); 33 | } 34 | 35 | protected function registerEvents(): void 36 | { 37 | $events = $this->app->make(Dispatcher::class); 38 | 39 | foreach ($this->events as $event => $listeners) { 40 | foreach ($listeners as $listener) { 41 | $events->listen($event, $listener); 42 | } 43 | } 44 | } 45 | 46 | /** 47 | * Setup the configuration for TmiCluster. 48 | * 49 | * @return void 50 | * @throws Exception 51 | */ 52 | protected function configure(): void 53 | { 54 | $this->mergeConfigFrom(__DIR__ . '/../../config/tmi-cluster.php', 'tmi-cluster'); 55 | $this->loadMigrationsFrom(__DIR__ . '/../../migrations'); 56 | 57 | TmiCluster::use(config('tmi-cluster.use')); 58 | } 59 | 60 | /** 61 | * Register TmiCluster's services in the container. 62 | * 63 | * @return void 64 | */ 65 | protected function registerServices(): void 66 | { 67 | foreach ($this->serviceBindings as $key => $value) { 68 | is_numeric($key) 69 | ? $this->app->singleton($value) 70 | : $this->app->singleton($key, $value); 71 | } 72 | 73 | $this->app->singleton(Contracts\ChannelManager::class, config('tmi-cluster.channel_manager.use')); 74 | } 75 | 76 | /** 77 | * Register the TmiCluster Artisan commands. 78 | * 79 | * @return void 80 | */ 81 | protected function registerCommands(): void 82 | { 83 | if ($this->app->runningInConsole()) { 84 | $this->commands([ 85 | Commands\TmiClusterAuthorizeCommand::class, 86 | Commands\TmiClusterCommand::class, 87 | Commands\TmiClusterFeaturesCommand::class, 88 | Commands\TmiClusterHealthCommand::class, 89 | Commands\TmiClusterListCommand::class, 90 | Commands\TmiClusterJoinCommand::class, 91 | Commands\TmiClusterProcessCommand::class, 92 | Commands\TmiClusterPurgeCommand::class, 93 | Commands\TmiClusterInstallCommand::class, 94 | Commands\TmiClusterPublishCommand::class, 95 | Commands\TmiClusterTerminateCommand::class, 96 | ]); 97 | } 98 | } 99 | 100 | /** 101 | * Register the TmiCluster frontend. 102 | * 103 | * @return void 104 | */ 105 | private function registerFrontend(): void 106 | { 107 | $this->loadViewsFrom(__DIR__ . '/../../resources/views', 'tmi-cluster'); 108 | } 109 | 110 | private function registerLogger(): void 111 | { 112 | } 113 | 114 | private function offerPublishing(): void 115 | { 116 | if ($this->app->runningInConsole()) { 117 | $this->publishes([ 118 | __DIR__ . '/../../config/tmi-cluster.php' => config_path('tmi-cluster.php'), 119 | ], 'tmi-cluster-config'); 120 | } 121 | } 122 | 123 | private function defineAssetPublishing(): void 124 | { 125 | $this->publishes([ 126 | __DIR__ . '/../../public' => public_path('vendor/tmi-cluster') 127 | ], 'tmi-cluster-assets'); 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /src/Repositories/DatabaseChannelManager.php: -------------------------------------------------------------------------------- 1 | updateOrCreate(['id' => $this->getKey($channel->getTwitchLogin())], array_merge([ 33 | 'revoked' => false, 34 | ], $options)); 35 | 36 | return $this; 37 | } 38 | 39 | /** 40 | * @inheritDoc 41 | */ 42 | public function revokeAuthorization(Invitable $channel): self 43 | { 44 | Channel::query() 45 | ->whereKey($this->getKey($channel->getTwitchLogin())) 46 | ->update([ 47 | 'revoked' => true, 48 | ]); 49 | 50 | return $this; 51 | } 52 | 53 | /** 54 | * @inheritDoc 55 | */ 56 | public function authorized(array $channels): array 57 | { 58 | return Collection::make($channels) 59 | ->chunk(self::MAX_CHANNELS_PER_EXECUTION) 60 | ->map(function (Collection $channels) { 61 | return Channel::query() 62 | ->whereIn('id', $channels->map(fn(string $channel) => $this->getKey($channel))) 63 | ->where(['revoked' => false]) 64 | ->pluck('id') 65 | ->toArray(); 66 | }) 67 | ->collapse() 68 | ->toArray(); 69 | } 70 | 71 | /** 72 | * @inheritDoc 73 | */ 74 | public function acknowledged(array $channels): self 75 | { 76 | Collection::make($channels) 77 | ->chunk(self::MAX_CHANNELS_PER_EXECUTION) 78 | ->each(function (Collection $channels) { 79 | Channel::query() 80 | ->whereIn('id', $channels->map(fn(string $channel) => $this->getKey($channel))) 81 | ->update([ 82 | 'acknowledged_at' => now(), 83 | ]); 84 | }); 85 | 86 | return $this; 87 | } 88 | 89 | private function getKey(string $channel): string 90 | { 91 | return TmiChannel::sanitize($channel); 92 | } 93 | 94 | /** 95 | * @inheritDoc 96 | */ 97 | public function stale(array $channels): array 98 | { 99 | return Channel::query() 100 | ->whereIn('id', array_map(fn($channel) => $this->getKey($channel), $channels)) 101 | ->where(function (Builder $query) { 102 | $query->where(['revoked' => true]); 103 | 104 | if (AutoCleanup::shouldCleanup()) { 105 | $staleAfter = Carbon::now() 106 | ->subHours(config('tmi-cluster.channel_manager.channel.stale', 168)) 107 | ->format('Y-m-d H:i:s'); 108 | 109 | $query->orWhere('acknowledged_at', '<', $staleAfter); 110 | } 111 | }) 112 | ->pluck('id') 113 | ->toArray(); 114 | } 115 | 116 | /** 117 | * Returns all channels that are not revoked and have reconnected equals true. 118 | * 119 | * @param array $channels channels that are currently connected 120 | */ 121 | public function disconnected(array $channels): array 122 | { 123 | $potentialChannels = Channel::query() 124 | ->where(['revoked' => false]) 125 | ->where(['reconnect' => true]) 126 | ->pluck('id') 127 | ->toArray(); 128 | 129 | return array_diff($potentialChannels, array_map(fn($channel) => $this->getKey($channel), $channels)); 130 | } 131 | } -------------------------------------------------------------------------------- /src/Repositories/DummyChannelManager.php: -------------------------------------------------------------------------------- 1 | commandQueue = app(CommandQueue::class); 29 | $this->lock = app(Lock::class); 30 | $this->channelManager = app(ChannelManager::class); 31 | } 32 | 33 | /** 34 | * @inheritdoc 35 | */ 36 | public function join(array $channels, array $staleIds = [], bool $acknowledge = true): void 37 | { 38 | $this->commandQueue->push(CommandQueue::NAME_JOIN_HANDLER, CommandQueue::COMMAND_TMI_JOIN, [ 39 | 'channels' => array_map(static fn($channel) => Channel::sanitize($channel), $channels), 40 | 'staleIds' => $staleIds, 41 | 'acknowledge' => $acknowledge, 42 | ]); 43 | } 44 | 45 | /** 46 | * @inheritdoc 47 | * @throws LimiterTimeoutException 48 | */ 49 | public function joinNow(array $channels, array $staleIds = [], bool $acknowledge = true): array 50 | { 51 | $channels = array_map(static fn($channel) => Channel::sanitize($channel), $channels); 52 | $commands = $this->commandQueue->pending(CommandQueue::NAME_JOIN_HANDLER); 53 | 54 | [$staleIds, $channels, $acknowledged] = Arr::unique($commands, $staleIds, $channels, $acknowledge); 55 | 56 | $this->channelManager->acknowledged($acknowledged); 57 | 58 | return $this->joinOrQueue($channels, $staleIds); 59 | } 60 | 61 | /** 62 | * @inheritdoc 63 | * @throws LimiterTimeoutException 64 | */ 65 | public function flushStale(array $channels, array $staleIds = []): array 66 | { 67 | $channels = array_map(static fn($channel) => Channel::sanitize($channel), $channels); 68 | $channels = $this->restoreQueuedChannelsFromStaleQueues($staleIds, $channels); 69 | 70 | return $this->joinNow($channels, $staleIds, false); 71 | } 72 | 73 | public function handle(Supervisor $supervisor, array $channels): void 74 | { 75 | $uuids = $supervisor->processes()->map(fn(Process $process) => $process->getUuid()); 76 | 77 | foreach ($channels as $channel) { 78 | $uuid = $uuids->shuffle()->shift(); 79 | $this->commandQueue->push(sprintf('%s-input', $uuid), CommandQueue::COMMAND_TMI_JOIN, [ 80 | 'channel' => $channel, 81 | ]); 82 | $this->lock->release($this->getKey($channel)); 83 | } 84 | } 85 | 86 | /** 87 | * This will be called if we need to join a channel. 88 | * 89 | * @param array $channels 90 | * @param array $staleIds 91 | * @return array 92 | * @throws LimiterTimeoutException 93 | */ 94 | private function joinOrQueue(array $channels, array $staleIds): array 95 | { 96 | $authorized = $this->channelManager->authorized($channels); 97 | $unauthorized = array_values(array_diff($channels, $authorized)); 98 | 99 | $result = ['rejected' => [], 'resolved' => [], 'ignored' => [], 'unauthorized' => $unauthorized]; 100 | 101 | $processes = Models\SupervisorProcess::query() 102 | ->whereTime('last_ping_at', '>', now()->subSeconds(3)) 103 | ->whereIn('state', [Models\SupervisorProcess::STATE_CONNECTED]) 104 | ->whereNotIn('id', $staleIds) 105 | ->get() 106 | ->map(function (Models\SupervisorProcess $process) { 107 | return (object)[ 108 | 'id' => $process->getKey(), 109 | 'channels' => $process->channels, 110 | 'channel_sum' => count($process->channels), 111 | ]; 112 | })->sortBy('channel_sum'); 113 | 114 | if ($processes->isEmpty()) { 115 | $result = $this->reject($result, $authorized, $staleIds); 116 | 117 | return $this->result($result); 118 | } 119 | 120 | $take = min( 121 | config('tmi-cluster.throttle.join.take', 100), 122 | config('tmi-cluster.throttle.join.allow', 2000) 123 | ); 124 | 125 | foreach (array_chunk($authorized, $take) as $chunk) { 126 | /** @var Lock $lock */ 127 | $lock = app(Lock::class); 128 | 129 | /** @noinspection PhpUnhandledExceptionInspection */ 130 | $result = $lock->throttle('throttle:join-handler') 131 | ->block(config('tmi-cluster.throttle.join.block', 0)) 132 | ->allow(config('tmi-cluster.throttle.join.allow', 2000)) 133 | ->every(config('tmi-cluster.throttle.join.every', 10)) 134 | ->take($take) 135 | ->then( 136 | fn() => $this->resolve($result, $chunk, $staleIds, $processes), 137 | fn() => $this->reject($result, $chunk, $staleIds) 138 | ); 139 | } 140 | 141 | return $this->result($result); 142 | } 143 | 144 | private function reject(array $result, array $channels, array $staleIds): array 145 | { 146 | // we didn't get any server, that is ready to join our channels 147 | // so we move them to our lost and found channel queue 148 | $this->commandQueue->push(CommandQueue::NAME_JOIN_HANDLER, CommandQueue::COMMAND_TMI_JOIN, [ 149 | 'channels' => $channels, 150 | 'staleIds' => $staleIds, 151 | ]); 152 | 153 | $result['rejected'][] = $channels; 154 | 155 | return $result; 156 | } 157 | 158 | private function resolve(array $result, array $channels, array $staleIds, Collection $processes): array 159 | { 160 | foreach ($channels as $channel) { 161 | if ($process = $this->getProcess($processes, $channel)) { 162 | if ($process instanceof stdClass) { 163 | $result['ignored'][$channel] = $this->increment($process, $channel); 164 | } else { 165 | $result['ignored'][$channel] = $process; 166 | } 167 | continue; 168 | } 169 | 170 | $nextProcess = $processes->sortBy('channel_sum')->shift(); 171 | $this->increment($nextProcess, $channel); 172 | $processes->push($nextProcess); 173 | 174 | // acquire lock to prevent double join 175 | $this->lock->connection()->set($this->getKey($channel), $nextProcess->id, 'EX', 60, 'NX'); 176 | 177 | $this->commandQueue->push(ClusterClient::getQueueName($nextProcess->id, ClusterClient::QUEUE_INPUT), CommandQueue::COMMAND_TMI_JOIN, [ 178 | 'channel' => $channel, 179 | 'staleIds' => $staleIds, 180 | ]); 181 | 182 | $result['resolved'][$channel] = $nextProcess->id; 183 | } 184 | 185 | return $result; 186 | } 187 | 188 | private function result(array $result): array 189 | { 190 | $result['rejected'] = array_merge(...$result['rejected']); 191 | 192 | return $result; 193 | } 194 | 195 | private function getKey(string $channel): string 196 | { 197 | return sprintf('channel-distributor:join-%s', $channel); 198 | } 199 | 200 | private function getProcess(Collection $processes, string $channel) 201 | { 202 | if ($id = $this->lock->connection()->get($this->getKey($channel))) { 203 | if ($process = $processes->where('id', '===', $id)->first()) { 204 | return $process; 205 | } 206 | 207 | return $id; 208 | } 209 | 210 | return $processes->filter(fn($x) => in_array($channel, $x->channels, true))->first(); 211 | } 212 | 213 | private function increment(stdClass $process, $channel): string 214 | { 215 | ++$process->channel_sum; 216 | $process->channels[] = $channel; 217 | 218 | return $process->id; 219 | } 220 | 221 | private function restoreQueuedChannelsFromStaleQueues(array $staleIds, array $channels): array 222 | { 223 | foreach ($staleIds as $staleId) { 224 | $commands = $this->commandQueue->pending($queueName = ClusterClient::getQueueName($staleId, ClusterClient::QUEUE_INPUT)); 225 | foreach ($commands as $command) { 226 | if ($command->command !== CommandQueue::COMMAND_TMI_JOIN) { 227 | $this->commandQueue->push($queueName, $command->command, (array)$command->options); 228 | continue; 229 | } 230 | 231 | $this->lock->release($this->getKey($command->options->channel)); 232 | $channels[] = $command->options->channel; 233 | } 234 | } 235 | 236 | return $channels; 237 | } 238 | 239 | public function forgetLocks(array $channels): void 240 | { 241 | foreach ($channels as $channel) { 242 | $this->lock->release($this->getKey($channel)); 243 | } 244 | } 245 | } -------------------------------------------------------------------------------- /src/Repositories/RedisCommandQueue.php: -------------------------------------------------------------------------------- 1 | redis = $redis; 29 | } 30 | 31 | /** 32 | * @inheritDoc 33 | * @noinspection PhpParamsInspection 34 | * @noinspection PhpUnhandledExceptionInspection 35 | */ 36 | public function push(string $name, string $command, array $options = []) 37 | { 38 | $this->connection()->rpush('commands:' . $name, json_encode([ 39 | 'command' => $command, 40 | 'options' => $options, 41 | 'time' => microtime(), 42 | ], JSON_THROW_ON_ERROR)); 43 | } 44 | 45 | /** 46 | * @inheritDoc 47 | */ 48 | public function pending(string $name): array 49 | { 50 | $length = $this->connection()->llen('commands:' . $name); 51 | 52 | if ($length < 1) { 53 | return []; 54 | } 55 | 56 | $results = $this->connection()->pipeline(function ($pipe) use ($name, $length) { 57 | $pipe->lrange('commands:' . $name, 0, $length - 1); 58 | 59 | $pipe->ltrim('commands:' . $name, $length, -1); 60 | }); 61 | 62 | return collect($results[0])->map(function ($result) { 63 | return json_decode($result, false, 512, JSON_THROW_ON_ERROR); 64 | })->all(); 65 | } 66 | 67 | /** 68 | * @inheritDoc 69 | */ 70 | public function flush(string $name): void 71 | { 72 | $this->connection()->del('commands:' . $name); 73 | } 74 | 75 | /** 76 | * Get the Redis connection instance. 77 | * 78 | * @return Connection|ClientInterface 79 | */ 80 | protected function connection() 81 | { 82 | return $this->redis->connection('tmi-cluster'); 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /src/Repositories/SupervisorRepository.php: -------------------------------------------------------------------------------- 1 | create(array_merge([ 22 | 'id' => $this->generateUniqueSupervisorKey(), 23 | 'last_ping_at' => now(), 24 | 'metrics' => [], 25 | 'options' => [ 26 | 'nice' => 0, 27 | ] 28 | ], $attributes)); 29 | 30 | return new Supervisor($supervisor); 31 | } 32 | 33 | /** 34 | * @inheritDoc 35 | */ 36 | public function all(array $columns = ['*']): Collection 37 | { 38 | return Models\Supervisor::all($columns)->collect(); 39 | } 40 | 41 | /** 42 | * @inheritDoc 43 | */ 44 | public function flushStale(): void 45 | { 46 | $channels = []; 47 | $staleIds = []; 48 | 49 | $this->all() 50 | ->filter(fn(Models\Supervisor $supervisor) => $supervisor->is_stale) 51 | ->each(fn(Models\Supervisor $supervisor) => $supervisor->delete()); 52 | 53 | Models\SupervisorProcess::query()->get() 54 | ->filter(fn(Models\SupervisorProcess $process) => $process->is_stale) 55 | ->each(function (Models\SupervisorProcess $process) use (&$channels, &$staleIds) { 56 | $staleIds[] = $process->getKey(); 57 | $channels[] = $process->channels; 58 | $process->delete(); 59 | }); 60 | 61 | app(ChannelDistributor::class)->flushStale(array_merge(...$channels), $staleIds); 62 | } 63 | 64 | private function generateUniqueSupervisorKey(int $length = 4): string 65 | { 66 | $key = sprintf('%s-%s', gethostname(), Str::random($length)); 67 | 68 | $exists = Models\Supervisor::query() 69 | ->whereKey($key) 70 | ->exists(); 71 | 72 | if ($exists) { 73 | throw new DomainException('A Supervisor with this name already exists.'); 74 | } 75 | 76 | return $key; 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /src/RouteRegistrar.php: -------------------------------------------------------------------------------- 1 | router = $router; 15 | } 16 | 17 | public function all(): void 18 | { 19 | $this->forHealth(); 20 | $this->forDashboard(); 21 | $this->forChannelManager(); 22 | } 23 | 24 | private function forHealth(): void 25 | { 26 | $this->router->get('health', [Controllers\DashboardController::class, 'health']); 27 | } 28 | 29 | private function forDashboard(): void 30 | { 31 | $this->router->get('', [Controllers\DashboardController::class, 'index']); 32 | $this->router->post('statistics', [Controllers\DashboardController::class, 'statistics']); 33 | $this->router->get('statistics', [Controllers\DashboardController::class, 'statistics']); 34 | $this->router->get('metrics', [Controllers\MetricsController::class, 'handle']); 35 | } 36 | 37 | private function forChannelManager(): void 38 | { 39 | $this->router->group(['middleware' => 'auth'], function (Router $router) { 40 | $router->resource('invite', Controllers\InviteController::class) 41 | ->only(['index', 'store']); 42 | }); 43 | } 44 | } -------------------------------------------------------------------------------- /src/ServiceBindings.php: -------------------------------------------------------------------------------- 1 | Repositories\SupervisorRepository::class, 23 | Contracts\CommandQueue::class => Repositories\RedisCommandQueue::class, 24 | Contracts\ChannelDistributor::class => Repositories\RedisChannelDistributor::class, 25 | Contracts\SupervisorJoinHandler::class => Repositories\RedisChannelDistributor::class, 26 | Contracts\ChannelManager::class => Repositories\DatabaseChannelManager::class, 27 | ]; 28 | } 29 | -------------------------------------------------------------------------------- /src/Supervisor.php: -------------------------------------------------------------------------------- 1 | model = $model; 34 | $this->processPools = $this->createProcessPools(); 35 | $this->autoScale = app(AutoScale::class); 36 | $this->autoReconnect = app(AutoReconnect::class); 37 | $this->commandQueue = app(CommandQueue::class); 38 | $this->commandQueue->flush($this->model->getKey()); 39 | $this->output = static function () { 40 | // 41 | }; 42 | } 43 | 44 | public function monitor(): void 45 | { 46 | $this->listenForSignals(); 47 | 48 | $this->model->save(); 49 | 50 | while (true) { 51 | sleep(1); 52 | 53 | $this->loop(); 54 | } 55 | } 56 | 57 | public function handleOutputUsing(Closure $output): void 58 | { 59 | $this->output = $output; 60 | } 61 | 62 | public function scale(int $int): void 63 | { 64 | $this->pools()->each(fn(ProcessPool $x) => $x->scale($int)); 65 | } 66 | 67 | public function loop(): void 68 | { 69 | try { 70 | $this->processPendingSignals(); 71 | 72 | $this->processPendingCommands(); 73 | 74 | // If the supervisor is working, we will perform any needed scaling operations and 75 | // monitor all of these underlying worker processes to make sure they are still 76 | // processing queued jobs. If they have died, we will restart them each here. 77 | if ($this->working) { 78 | $this->autoScale(); 79 | $this->autoReconnect(); 80 | 81 | $this->pools()->each(fn(ProcessPool $x) => $x->monitor()); 82 | } 83 | 84 | // Update all model data here 85 | $this->model->forceFill([ 86 | 'last_ping_at' => now(), 87 | 'metrics' => [ 88 | // todo add some fancy metrics 89 | ] 90 | ]); 91 | 92 | // Next, we'll persist the supervisor state to storage so that it can be read by a 93 | // user interface. This contains information on the specific options for it and 94 | // the current number of worker processes per queue for easy load monitoring. 95 | $this->model->save(); 96 | 97 | event(new SupervisorLooped($this)); 98 | } catch (Throwable $e) { 99 | $this->output(null, $e->getMessage()); 100 | $this->output(null, $e->getTraceAsString()); 101 | app(ExceptionHandler::class)->report($e); 102 | } 103 | } 104 | 105 | private function autoScale(): void 106 | { 107 | $this->autoScale->scale($this); 108 | } 109 | 110 | private function autoReconnect(): void 111 | { 112 | $this->autoReconnect->handle($this); 113 | } 114 | 115 | private function createProcessPools(): array 116 | { 117 | $options = new ProcessOptions($this->model->getKey(), $this->model->options); 118 | return [$this->createSingleProcessPool($options)]; 119 | } 120 | 121 | private function createSingleProcessPool(ProcessOptions $options): ProcessPool 122 | { 123 | return new ProcessPool($options, function ($type, $line) { 124 | $this->output($type, $line); 125 | }, $this); 126 | } 127 | 128 | public function output($type, $line): void 129 | { 130 | call_user_func($this->output, $type, $line); 131 | } 132 | 133 | private function pools(): Collection 134 | { 135 | return new Collection($this->processPools); 136 | } 137 | 138 | public function processes(): Collection 139 | { 140 | return $this->pools()->map(fn(ProcessPool $x) => $x->processes())->collapse(); 141 | } 142 | 143 | public function pause(): void 144 | { 145 | $this->working = false; 146 | 147 | $this->pools()->each(fn(ProcessPool $x) => $x->pause()); 148 | } 149 | 150 | public function continue(): void 151 | { 152 | $this->working = true; 153 | 154 | $this->pools()->each(fn(ProcessPool $x) => $x->continue()); 155 | } 156 | 157 | public function restart(): void 158 | { 159 | $this->working = true; 160 | 161 | $this->pools()->each(fn(ProcessPool $x) => $x->restart()); 162 | } 163 | 164 | public function terminate(int $status = 0): void 165 | { 166 | $this->working = false; 167 | 168 | // We will mark this supervisor as terminating so that any user interface can 169 | // correctly show the supervisor's status. Then, we will scale the process 170 | // pools down to zero workers to gracefully terminate them all out here. 171 | $this->pools()->each(function (ProcessPool $pool) { 172 | $pool->processes()->each(function (Process $process) { 173 | $process->terminate(); 174 | }); 175 | }); 176 | 177 | if ($this->shouldWait()) { 178 | while ($this->pools()->map(fn(ProcessPool $x) => $x->runningProcesses())->collapse()->count()) { 179 | sleep(1); 180 | } 181 | } 182 | 183 | // cleanup database before the clean stale process kills our models 184 | $this->model->processes()->forceDelete(); 185 | $this->model->forceDelete(); 186 | 187 | $this->exit($status); 188 | } 189 | 190 | protected function shouldWait(): bool 191 | { 192 | return !config('tmi-cluster.fast_termination'); 193 | } 194 | 195 | protected function exit($status = 0): void 196 | { 197 | $this->exitProcess($status); 198 | } 199 | 200 | protected function exitProcess($status = 0): void 201 | { 202 | exit((int)$status); 203 | } 204 | 205 | private function processPendingCommands(): void 206 | { 207 | $commands = $this->commandQueue->pending($this->model->getKey()); 208 | $channelsToJoin = []; 209 | 210 | foreach ($commands as $command) { 211 | switch ($command->command) { 212 | case CommandQueue::COMMAND_SUPERVISOR_SCALE_OUT: 213 | $this->autoScale->scaleOut($this); 214 | break; 215 | case CommandQueue::COMMAND_SUPERVISOR_SCALE_IN: 216 | $this->autoScale->scaleIn($this); 217 | break; 218 | case CommandQueue::COMMAND_SUPERVISOR_TERMINATE: 219 | $this->terminate(); 220 | break; 221 | case CommandQueue::COMMAND_TMI_JOIN: 222 | $channelsToJoin[] = $command->options->channel; 223 | break; 224 | } 225 | } 226 | 227 | if (!empty($channelsToJoin)) { 228 | app(SupervisorJoinHandler::class)->handle($this, $channelsToJoin); 229 | } 230 | } 231 | 232 | public function memoryUsage(): float 233 | { 234 | return memory_get_usage() / 1024 / 1024; 235 | } 236 | } 237 | -------------------------------------------------------------------------------- /src/Support/Arr.php: -------------------------------------------------------------------------------- 1 | command !== CommandQueue::COMMAND_TMI_JOIN) { 19 | continue; 20 | } 21 | 22 | $staleIds[] = $command->options->staleIds ?? []; 23 | $channels[] = $command->options->channels ?? []; 24 | $shouldAcknowledge = $command->options->acknowledge ?? false; 25 | 26 | if ($shouldAcknowledge) { 27 | $acknowledged[] = $command->options->channels ?? []; 28 | } 29 | } 30 | 31 | return [ 32 | array_unique(self::merge(...$staleIds)), 33 | array_unique(self::merge(...$channels)), 34 | array_unique(self::merge(...$acknowledged)), 35 | ]; 36 | } 37 | 38 | public static function merge(array|stdClass ...$arrays): array 39 | { 40 | return array_merge(...array_map(fn($array) => is_array($array) ? $array : (array)$array, $arrays)); 41 | } 42 | } -------------------------------------------------------------------------------- /src/Support/Composer.php: -------------------------------------------------------------------------------- 1 | =')) { 37 | return true; 38 | } 39 | } 40 | 41 | return false; 42 | } 43 | } -------------------------------------------------------------------------------- /src/Support/Health.php: -------------------------------------------------------------------------------- 1 | sum(function (Supervisor $supervisor) { 20 | return $supervisor->processes->filter(function (SupervisorProcess $process) { 21 | return self::isInConnectedState($process) || self::isInInitialState($process); 22 | })->count(); 23 | }); 24 | 25 | $count = $supervisors 26 | ->sum(function (Supervisor $supervisor) { 27 | return $supervisor->processes->count(); 28 | }); 29 | 30 | if ($atLeastOneProcess) { 31 | return $operational > 0; 32 | } 33 | 34 | return $count > 0 && $operational === $count; 35 | } 36 | 37 | /** 38 | * Consider a process in initial state if state is in initialize and ping is within the threshold. 39 | */ 40 | private static function isInInitialState(SupervisorProcess $process): bool 41 | { 42 | return $process->state === SupervisorProcess::STATE_INITIALIZE 43 | && $process->last_ping_at->diffInSeconds() < self::INRESPONSIVE_THRESHOLD; 44 | } 45 | 46 | /** 47 | * Consider a process in connected state if state is in connected and ping is within the threshold. 48 | */ 49 | private static function isInConnectedState(SupervisorProcess $process): bool 50 | { 51 | return $process->state === SupervisorProcess::STATE_CONNECTED 52 | && $process->last_ping_at->diffInSeconds() < self::INRESPONSIVE_THRESHOLD; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/Support/Os.php: -------------------------------------------------------------------------------- 1 | array_merge($config, [ 48 | 'options' => ['prefix' => config('tmi-cluster.prefix') ?: 'tmi-cluster:'], 49 | ])]); 50 | } 51 | 52 | public static function routeMailNotificationsTo(string $email): void 53 | { 54 | static::$email = $email; 55 | } 56 | 57 | public static function routeSlackNotificationsTo(string $url, string $channel = null): void 58 | { 59 | static::$slackWebhookUrl = $url; 60 | static::$slackChannel = $channel; 61 | } 62 | 63 | public static function routeSmsNotificationsTo(string $number): void 64 | { 65 | static::$smsNumber = $number; 66 | } 67 | 68 | public static function routes($callback = null, array $options = []): void 69 | { 70 | $callback = $callback ?: function ($router) { 71 | $router->all(); 72 | }; 73 | 74 | $defaultOptions = [ 75 | 'domain' => config('tmi-cluster.domain', null), 76 | 'prefix' => config('tmi-cluster.path', 'tmi-cluster'), 77 | 'middleware' => config('tmi-cluster.middleware', 'web'), 78 | ]; 79 | 80 | $options = array_merge($defaultOptions, $options); 81 | 82 | Route::group($options, function ($router) use($callback) { 83 | $callback(new RouteRegistrar($router)); 84 | }); 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/Traits/Lockable.php: -------------------------------------------------------------------------------- 1 | connection('tmi-cluster'); 20 | } 21 | 22 | protected function lock(string $name, int $seconds = 0, ?string $owner = null): RedisLock 23 | { 24 | return new RedisLock($this->connection(), $name, $seconds, $owner); 25 | } 26 | } -------------------------------------------------------------------------------- /src/Traits/TmiClusterHelpers.php: -------------------------------------------------------------------------------- 1 | authorized([$channel])) { 24 | return; 25 | } 26 | } 27 | 28 | $commandQueue->push(CommandQueue::NAME_ANY_SUPERVISOR, CommandQueue::COMMAND_TMI_WRITE, [ 29 | 'raw_command' => "PRIVMSG {$channel} :{$message}", 30 | ]); 31 | } 32 | 33 | /** 34 | * @param array $channels a list of channels that needs to be joined 35 | * @param array $staleIds a list of supervisors or processes ids, that needs to be avoid 36 | */ 37 | public static function joinNextServer(array $channels, array $staleIds = []): void 38 | { 39 | app(ChannelDistributor::class)->join($channels, $staleIds); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/TwitchLogin.php: -------------------------------------------------------------------------------- 1 | login = $login; 15 | } 16 | 17 | public function getTwitchLogin(): ?string 18 | { 19 | return $this->login; 20 | } 21 | } -------------------------------------------------------------------------------- /src/Watchable.php: -------------------------------------------------------------------------------- 1 | find('node'), 21 | 'file-watcher.js', 22 | json_encode(collect($paths)->map(fn($path) => base_path($path))), 23 | $this->option('poll'), 24 | ], realpath(__DIR__ . '/../bin'), null, null, null))->start(); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /tests/TestCase.php: -------------------------------------------------------------------------------- 1 | artisan('migrate')->run(); 24 | $this->flushRedis(); 25 | } 26 | 27 | protected function getPackageProviders($app) 28 | { 29 | return [ 30 | TmiClusterServiceProvider::class, 31 | ]; 32 | } 33 | 34 | protected function getPackageAliases($app) 35 | { 36 | return [ 37 | 'TmiCluster' => TmiCluster::class, 38 | ]; 39 | } 40 | 41 | /** 42 | * Define environment setup. 43 | * 44 | * @param Application $app 45 | * @return void 46 | */ 47 | protected function getEnvironmentSetUp($app) 48 | { 49 | // Setup default database to use sqlite :memory: 50 | $app['config']->set('database.default', 'testbench'); 51 | $app['config']->set('database.connections.testbench', [ 52 | 'driver' => 'sqlite', 53 | 'database' => ':memory:', 54 | 'prefix' => '', 55 | ]); 56 | } 57 | 58 | protected function flushRedis() 59 | { 60 | /** @var ClientInterface $connection */ 61 | $connection = app(Factory::class)->connection('tmi-cluster'); 62 | $connection->flushdb(); 63 | } 64 | 65 | protected function useChannelManager(string $concrete): ChannelManager 66 | { 67 | $this->app->singleton(ChannelManager::class, $concrete); 68 | 69 | return app(ChannelManager::class); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /tests/TestCases/ArrayTest.php: -------------------------------------------------------------------------------- 1 | fake([ 19 | 'channels' => ['channel-b'], 20 | 'staleIds' => ['stale-b'], 21 | 'acknowledge' => true, 22 | ]), 23 | $this->fake([ 24 | 'channels' => ['channel-b'], 25 | 'acknowledge' => true, 26 | ]), 27 | $this->fake([ 28 | 'channels' => ['channel-c'], 29 | 'staleIds' => ['stale-c'], 30 | 'acknowledge' => false, 31 | ]), 32 | ], ['stale-a'], ['channel-a'], true); 33 | 34 | self::assertEquals([ 35 | ['stale-a', 'stale-b', 'stale-c'], 36 | ['channel-a', 'channel-b', 3 => 'channel-c'], // index 2 got removed due unique 37 | ['channel-a', 'channel-b'] 38 | ], $result); 39 | } 40 | 41 | public function testCastArrayMerge(): void 42 | { 43 | $result = Arr::merge( 44 | ['test', 'test2'], 45 | json_decode(json_encode([0 => 'test3', 1 => 'test4'])) 46 | ); 47 | 48 | $this->assertEquals(['test', 'test2', 'test3', 'test4'], $result); 49 | } 50 | 51 | /** 52 | * @param array $options 53 | * @return mixed 54 | * @throws JsonException 55 | */ 56 | private function fake(array $options = []): mixed 57 | { 58 | // encode json string 59 | $encoded = json_encode([ 60 | 'command' => CommandQueue::COMMAND_TMI_JOIN, 61 | 'options' => $options, 62 | 'time' => microtime(), 63 | ], JSON_THROW_ON_ERROR); 64 | 65 | // decode json string 66 | return json_decode($encoded, false, 512, JSON_THROW_ON_ERROR); 67 | } 68 | } -------------------------------------------------------------------------------- /tests/TestCases/AutoScaleTest.php: -------------------------------------------------------------------------------- 1 | setRestore(false); 15 | self::assertFalse($this->getAutoScale()->shouldRestoreScale()); 16 | 17 | $this->setRestore(true); 18 | self::assertTrue($this->getAutoScale()->shouldRestoreScale()); 19 | } 20 | 21 | public function testRestoreScale() 22 | { 23 | $config = config('tmi-cluster.auto_scale.processes.min'); 24 | $this->getAutoScale()->setMinimumScale(7); 25 | 26 | // the value should be get from the config 27 | $this->setRestore(false); 28 | self::assertEquals($config, $this->getAutoScale()->getMinimumScale()); 29 | 30 | // the value should be get from the redis 31 | $this->setRestore(true); 32 | self::assertNotEquals($config, $scale = $this->getAutoScale()->getMinimumScale()); 33 | self::assertEquals(7, $scale); 34 | 35 | } 36 | 37 | private function setRestore(bool $restore) 38 | { 39 | app('config')->set('tmi-cluster.auto_scale.restore', $restore); 40 | } 41 | 42 | private function getAutoScale(): AutoScale 43 | { 44 | return app(AutoScale::class); 45 | } 46 | } -------------------------------------------------------------------------------- /tests/TestCases/ChannelDistributorTest.php: -------------------------------------------------------------------------------- 1 | useChannelManager(DatabaseChannelManager::class) 24 | ->authorize(new TwitchLogin('test1')) 25 | ->authorize(new TwitchLogin('test2')) 26 | ->authorize(new TwitchLogin('test3')) 27 | ->authorize(new TwitchLogin('test4')); 28 | 29 | $this->getChannelDistributor()->join(['test1', 'test2', 'test_unauthorized']); 30 | $this->getChannelDistributor()->join(['test3']); 31 | 32 | $uuid = $this->createSupervisor(now(), SupervisorProcess::STATE_CONNECTED); 33 | 34 | $result = $this->getChannelDistributor()->joinNow(['test4'], []); 35 | 36 | self::assertEquals([ 37 | 'rejected' => [], 38 | 'resolved' => [ 39 | '#test1' => $uuid, 40 | '#test2' => $uuid, 41 | '#test3' => $uuid, 42 | '#test4' => $uuid, 43 | ], 44 | 'ignored' => [], 45 | 'unauthorized' => [ 46 | '#test_unauthorized' 47 | ], 48 | ], $result); 49 | } 50 | 51 | public function testChannelGotRejectedDueMissingServers(): void 52 | { 53 | $this->useChannelManager(DummyChannelManager::class); 54 | 55 | $result = $this->getChannelDistributor()->joinNow(['ghostzero'], []); 56 | 57 | self::assertEquals([ 58 | 'rejected' => ['#ghostzero'], 59 | 'resolved' => [], 60 | 'ignored' => [], 61 | 'unauthorized' => [], 62 | ], $result); 63 | } 64 | 65 | public function testChannelGotRejectedWithUnhealthyServers(): void 66 | { 67 | $this->useChannelManager(DummyChannelManager::class); 68 | 69 | $this->createSupervisor(now()->subSeconds(5), SupervisorProcess::STATE_CONNECTED); 70 | $this->createSupervisor(now(), SupervisorProcess::STATE_INITIALIZE); 71 | 72 | $result = $this->getChannelDistributor()->joinNow(['ghostzero'], []); 73 | 74 | self::assertEquals([ 75 | 'rejected' => ['#ghostzero'], 76 | 'resolved' => [], 77 | 'ignored' => [], 78 | 'unauthorized' => [], 79 | ], $result); 80 | } 81 | 82 | public function testChannelGotResolvedDueActiveServer(): void 83 | { 84 | $this->useChannelManager(DummyChannelManager::class); 85 | 86 | $uuid = $this->createSupervisor(now(), SupervisorProcess::STATE_CONNECTED); 87 | 88 | $result = $this->getChannelDistributor()->joinNow(['ghostzero'], []); 89 | 90 | self::assertEquals([ 91 | 'rejected' => [], 92 | 'resolved' => ['#ghostzero' => $uuid], 93 | 'ignored' => [], 94 | 'unauthorized' => [], 95 | ], $result); 96 | } 97 | 98 | public function testChannelGotIgnoredDueAlreadyJoined(): void 99 | { 100 | $this->useChannelManager(DummyChannelManager::class); 101 | 102 | $uuid = $this->createSupervisor(now(), SupervisorProcess::STATE_CONNECTED); 103 | 104 | $result = $this->getChannelDistributor()->joinNow(['ghostzero'], []); 105 | 106 | self::assertEquals([ 107 | 'rejected' => [], 108 | 'resolved' => ['#ghostzero' => $uuid], 109 | 'ignored' => [], 110 | 'unauthorized' => [], 111 | ], $result); 112 | 113 | $uuid2 = $this->createSupervisor(now(), SupervisorProcess::STATE_CONNECTED); 114 | 115 | self::assertNotEquals($uuid, $uuid2); 116 | 117 | $result = $this->getChannelDistributor()->joinNow(['ghostzero', 'test', 'test2', 'test3', 'test4'], []); 118 | 119 | self::assertEquals([ 120 | 'rejected' => [], 121 | 'resolved' => [ 122 | '#test' => $uuid2, // because the second process has the lowest channels amount 123 | '#test2' => $uuid, // because the first process has the lowest channels amount 124 | '#test3' => $uuid2, // because the second process has the lowest channels amount 125 | '#test4' => $uuid, // because the first process has the lowest channels amount 126 | ], 127 | 'ignored' => [ 128 | '#ghostzero' => $uuid, // because they got already joined in the first server 129 | ], 130 | 'unauthorized' => [], 131 | ], $result); 132 | 133 | $this->assertGotQueued($result, 5); 134 | } 135 | 136 | public function testChannelGotResolvedAfterInactiveSupervisor(): void 137 | { 138 | $this->useChannelManager(DummyChannelManager::class); 139 | 140 | // join server normally 141 | $uuid = $this->createSupervisor(now()->subSeconds(2), SupervisorProcess::STATE_CONNECTED, [ 142 | 'ghostdemouser', // this channel got already connected 143 | ]); 144 | $result = $this->getChannelDistributor()->joinNow(['ghostzero'], []); 145 | self::assertEquals([ 146 | 'rejected' => [], 147 | 'resolved' => ['#ghostzero' => $uuid], 148 | 'ignored' => [], 149 | 'unauthorized' => [], 150 | ], $result); 151 | 152 | // let's kill all servers and re-join all lost channels into some fresh servers 153 | sleep(1); 154 | $newUuid = $this->createSupervisor(now(), SupervisorProcess::STATE_CONNECTED); 155 | $result = $this->getChannelDistributor()->flushStale([ 156 | '#ghostdemouser' 157 | ], [$uuid]); // this simulates our flush stale 158 | 159 | // check if our previous server got restored 160 | self::assertEquals([ 161 | 'rejected' => [], 162 | 'resolved' => [ 163 | '#ghostzero' => $newUuid, // got restored from our lost queue 164 | '#ghostdemouser' => $newUuid, // got restored from the flush stale 165 | ], 166 | 'ignored' => [], 167 | 'unauthorized' => [], 168 | ], $result); 169 | } 170 | 171 | private function getChannelDistributor(): ChannelDistributor 172 | { 173 | return app(RedisChannelDistributor::class); 174 | } 175 | 176 | private function assertGotQueued(array $result, int $expectedCount): void 177 | { 178 | $processIds = array_unique(array_merge( 179 | array_values($result['rejected']), 180 | array_values($result['resolved']), 181 | array_values($result['ignored']) 182 | )); 183 | 184 | self::assertNotEmpty($processIds); 185 | $commands = $this->getPendingCommands($processIds); 186 | self::assertCount($expectedCount, $commands); 187 | } 188 | 189 | private function getPendingCommands(array $processIds): array 190 | { 191 | /** @var CommandQueue $commandQueue */ 192 | $commandQueue = app(CommandQueue::class); 193 | 194 | $commands = []; 195 | foreach ($processIds as $processId) { 196 | $commands[] = $commandQueue->pending(ClusterClient::getQueueName($processId, ClusterClient::QUEUE_INPUT)); 197 | } 198 | $commands = array_merge(...$commands); 199 | return $commands; 200 | } 201 | } -------------------------------------------------------------------------------- /tests/TestCases/CommandQueueTest.php: -------------------------------------------------------------------------------- 1 | push('test', 'command'); 16 | 17 | self::assertCount(0, $commandQueue->pending('test2')); 18 | self::assertNotEmpty($data = $commandQueue->pending('test')); 19 | self::assertIsArray($data); 20 | self::assertEquals('command', $data[0]->command); 21 | self::assertIsArray($data[0]->options); 22 | self::assertCount(0, $commandQueue->pending('test')); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /tests/TestCases/ComposerTest.php: -------------------------------------------------------------------------------- 1 | assertEquals( 16 | 'unknown', 17 | Composer::detectTmiClusterVersion() 18 | ); 19 | } 20 | 21 | public function testIsSupportedVersion(): void 22 | { 23 | $this->assertFalse( 24 | Composer::isSupportedVersion('1.0.0') 25 | ); 26 | 27 | $this->assertFalse( 28 | Composer::isSupportedVersion('2.1.0') 29 | ); 30 | 31 | $this->assertTrue( 32 | Composer::isSupportedVersion('2.3.0') 33 | ); 34 | 35 | $this->assertFalse( 36 | Composer::isSupportedVersion('dev-master') 37 | ); 38 | 39 | $this->assertFalse( 40 | Composer::isSupportedVersion('dev-develop') 41 | ); 42 | 43 | $this->assertTrue( 44 | Composer::isSupportedVersion('3.0.0') 45 | ); 46 | 47 | $this->assertFalse( 48 | Composer::isSupportedVersion('dev-develop') 49 | ); 50 | 51 | } 52 | 53 | } -------------------------------------------------------------------------------- /tests/TestCases/DatabaseChannelManagerTest.php: -------------------------------------------------------------------------------- 1 | getChannelManager()->authorize($user, [ 20 | 'reconnect' => true, 21 | ]); 22 | 23 | $authorized = $this->getChannelManager()->authorized([ 24 | 'ghostzero', 25 | 'tmi_inspector', 26 | ]); 27 | 28 | self::assertContains('#ghostzero', $authorized); 29 | self::assertNotContains('#tmi_inspector', $authorized); 30 | } 31 | 32 | public function testChannelIsDisconnected(): void 33 | { 34 | $user = new TwitchLogin('ghostzero'); 35 | 36 | $this->getChannelManager()->authorize($user, [ 37 | 'reconnect' => true, 38 | ]); 39 | 40 | $user = new TwitchLogin('tmi_inspector'); 41 | 42 | $this->getChannelManager()->authorize($user, [ 43 | 'reconnect' => true, 44 | ]); 45 | 46 | $disconnected = $this->getChannelManager()->disconnected([ 47 | 'ghostzero', 48 | ]); 49 | 50 | self::assertNotContains('#ghostzero', $disconnected); 51 | self::assertContains('#tmi_inspector', $disconnected); 52 | } 53 | 54 | public function testChannelIsNotAuthorizedAfterRevoke(): void 55 | { 56 | $user = new TwitchLogin('ghostzero'); 57 | 58 | $this->getChannelManager()->authorize($user, [ 59 | 'reconnect' => true, 60 | ]); 61 | 62 | $authorized = $this->getChannelManager()->authorized(['ghostzero']); 63 | 64 | self::assertContains('#ghostzero', $authorized); 65 | 66 | $this->getChannelManager()->revokeAuthorization($user); 67 | 68 | $authorized = $this->getChannelManager()->authorized(['ghostzero']); 69 | 70 | self::assertNotContains('#ghostzero', $authorized); 71 | } 72 | 73 | private function getChannelManager(): DatabaseChannelManager 74 | { 75 | return app(DatabaseChannelManager::class); 76 | } 77 | } -------------------------------------------------------------------------------- /tests/TestCases/MetricsTest.php: -------------------------------------------------------------------------------- 1 | createdLoopedSupervisor(2); 17 | $this->createdLoopedSupervisor(2); 18 | 19 | $controller = new MetricsController(); 20 | $response = $controller->handle(); 21 | 22 | self::assertStringContainsString('TYPE tmi_cluster_avg_usage gauge', $response->getContent()); 23 | self::assertStringContainsString('TYPE tmi_cluster_avg_response_time gauge', $response->getContent()); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /tests/TestCases/MonitorClusterClientMemoryTest.php: -------------------------------------------------------------------------------- 1 | options = new ClusterClientOptions( 20 | 'uuid', 21 | 'supervisor', 22 | false, 23 | 64, 24 | false 25 | ); 26 | 27 | $clusterClient->shouldReceive('memoryUsage')->andReturn(192); 28 | $clusterClient->shouldReceive('terminate')->once()->with(8); 29 | 30 | $monitor->handle(new PeriodicTimerCalled($clusterClient)); 31 | } 32 | 33 | public function testTmiClusterIsNotTerminatedWhenUsingToMuchMemory(): void 34 | { 35 | $monitor = new MonitorClusterClientMemory(); 36 | 37 | $clusterClient = Mockery::mock(TmiClusterClient::class); 38 | $clusterClient->options = new ClusterClientOptions( 39 | 'uuid', 40 | 'supervisor', 41 | false, 42 | 64, 43 | false 44 | ); 45 | 46 | $clusterClient->shouldReceive('memoryUsage')->andReturn(64); 47 | $clusterClient->shouldReceive('terminate')->never(); 48 | 49 | $monitor->handle(new PeriodicTimerCalled($clusterClient)); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /tests/TestCases/MonitorSupervisorMemoryTest.php: -------------------------------------------------------------------------------- 1 | shouldReceive('memoryUsage')->andReturn(65); 20 | $supervisor->shouldReceive('terminate')->once()->with(8); 21 | 22 | $monitor->handle(new SupervisorLooped($supervisor)); 23 | } 24 | 25 | public function testSupervisorIsNotTerminatedWhenUsingToMuchMemory(): void 26 | { 27 | $monitor = new MonitorSupervisorMemory(); 28 | 29 | $supervisor = Mockery::mock(Supervisor::class); 30 | 31 | $supervisor->shouldReceive('memoryUsage')->andReturn(64); 32 | $supervisor->shouldReceive('terminate')->never(); 33 | 34 | $monitor->handle(new SupervisorLooped($supervisor)); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /tests/TestCases/NotificationTest.php: -------------------------------------------------------------------------------- 1 | handle($event); 23 | 24 | self::assertNotTrue($event->skipped); 25 | self::assertTrue($event->sent); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /tests/TestCases/SendMessageTest.php: -------------------------------------------------------------------------------- 1 | shouldRestrictMessages(); 19 | 20 | TmiCluster::sendMessage('ghostzero', 'Hello World!'); 21 | 22 | self::assertMessageNotSent(); 23 | } 24 | 25 | public function testCanSendMessageIfRestrictedAndAuthorized(): void 26 | { 27 | $this->shouldRestrictMessages(); 28 | 29 | $this->useChannelManager(DatabaseChannelManager::class) 30 | ->authorize(new TwitchLogin('ghostzero')); 31 | 32 | TmiCluster::sendMessage('ghostzero', 'Hello World!'); 33 | 34 | self::assertMessageSent('ghostzero', 'Hello World!'); 35 | } 36 | 37 | public function testCanSendMessageIfUnrestricted(): void 38 | { 39 | $this->shouldRestrictMessages(false); 40 | 41 | TmiCluster::sendMessage('ghostzero', 'Hello World!'); 42 | 43 | self::assertMessageSent('ghostzero', 'Hello World!'); 44 | } 45 | 46 | private static function assertMessageNotSent() 47 | { 48 | /** @var CommandQueue $commandQueue */ 49 | $commandQueue = app(CommandQueue::class); 50 | 51 | $commands = $commandQueue->pending(CommandQueue::NAME_ANY_SUPERVISOR); 52 | 53 | self::assertEmpty($commands); 54 | } 55 | 56 | private static function assertMessageSent(string $channel, string $message) 57 | { 58 | /** @var CommandQueue $commandQueue */ 59 | $commandQueue = app(CommandQueue::class); 60 | 61 | $commands = $commandQueue->pending(CommandQueue::NAME_ANY_SUPERVISOR); 62 | 63 | self::assertNotEmpty($commands); 64 | 65 | self::assertEquals( 66 | sprintf('PRIVMSG #%s :%s', $channel, $message), 67 | $commands[0]->options->raw_command 68 | ); 69 | } 70 | 71 | private function shouldRestrictMessages(bool $restrict = true): void 72 | { 73 | config()->set('tmi-cluster.channel_manager.channel.restrict_messages', $restrict); 74 | } 75 | } -------------------------------------------------------------------------------- /tests/TestCases/StaleTest.php: -------------------------------------------------------------------------------- 1 | createSupervisor(now()->subSeconds($staleSeconds), SupervisorProcess::STATE_CONNECTED); 23 | $this->createSupervisor(now()->subSeconds($staleSeconds), SupervisorProcess::STATE_DISCONNECTED); 24 | $this->createSupervisor(now()->subSeconds($staleSeconds - 10), SupervisorProcess::STATE_DISCONNECTED); 25 | $this->createSupervisor(now(), SupervisorProcess::STATE_CONNECTED); 26 | $this->createSupervisor(now(), SupervisorProcess::STATE_CONNECTED); 27 | $this->createSupervisor(now(), SupervisorProcess::STATE_CONNECTED); 28 | 29 | $stale = $this->repository()->all() 30 | ->filter(fn(Models\Supervisor $supervisor) => $supervisor->is_stale); 31 | 32 | $healthy = $this->repository()->all() 33 | ->filter(fn(Models\Supervisor $supervisor) => !$supervisor->is_stale); 34 | 35 | self::assertEquals(2, $stale->count()); 36 | self::assertEquals(4, $healthy->count()); 37 | } 38 | 39 | public function repository(): SupervisorRepository 40 | { 41 | return app(SupervisorRepository::class); 42 | } 43 | } -------------------------------------------------------------------------------- /tests/TestCases/TmiClusterTest.php: -------------------------------------------------------------------------------- 1 | create([]); 18 | 19 | self::assertStringStartsWith(gethostname(), $supervisor->model->getKey()); 20 | 21 | $supervisor->scale(2); 22 | $supervisor->loop(); 23 | 24 | self::assertCount(2, $supervisor->processes()); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /tests/Traits/CreatesSupervisors.php: -------------------------------------------------------------------------------- 1 | forceCreate([ 19 | 'id' => Str::uuid(), 20 | 'last_ping_at' => $lastPingAt, 21 | 'metrics' => [], 22 | 'options' => ['nice' => 0] 23 | ]); 24 | 25 | /** @var SupervisorProcess $process */ 26 | $process = $supervisor->processes()->forceCreate([ 27 | 'id' => Str::uuid(), 28 | 'supervisor_id' => $supervisor->getKey(), 29 | 'state' => $state, 30 | 'channels' => $channels, 31 | 'last_ping_at' => $lastPingAt, 32 | ]); 33 | 34 | return (string)$process->getKey(); 35 | } 36 | 37 | public function createdLoopedSupervisor(int $scale): \GhostZero\TmiCluster\Supervisor 38 | { 39 | /** @var \GhostZero\TmiCluster\Supervisor $supervisor */ 40 | $supervisor = app(SupervisorRepository::class)->create(); 41 | 42 | self::assertStringStartsWith(gethostname(), $supervisor->model->getKey()); 43 | 44 | $supervisor->scale($scale); 45 | $supervisor->loop(); 46 | 47 | self::assertCount($scale, $supervisor->processes()); 48 | 49 | return $supervisor; 50 | } 51 | } -------------------------------------------------------------------------------- /tests/laravel/artisan: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env php 2 |