├── .github
└── workflows
│ └── tests.yml
├── .gitignore
├── FUNDING.yml
├── LICENSE
├── README.md
├── SECURITY.md
├── UPGRADE.md
├── bin
└── file-watcher.js
├── composer.json
├── config
└── tmi-cluster.php
├── migrations
├── 2020_09_11_191542_create_supervisors_table.php
├── 2020_10_08_161019_create_supervisor_processes_table.php
├── 2021_03_21_172842_create_tmi_cluster_channels_table.php
└── 2021_04_06_073453_add_tmi_cluster_table_prefixes.php
├── package-lock.json
├── package.json
├── phpunit.xml
├── public
├── favicon-degraded.ico
├── favicon.ico
├── mix-manifest.json
├── tmi-cluster.css
├── tmi-cluster.js
└── tmi-cluster.js.LICENSE.txt
├── resources
├── cli
│ └── motd.txt
├── js
│ ├── components
│ │ ├── Charts.vue
│ │ └── TmiDashboard.vue
│ └── tmi-cluster.js
├── scss
│ ├── tmi-cluster.scss
│ └── variables.scss
└── views
│ ├── dashboard
│ └── index.blade.php
│ ├── invite
│ └── index.blade.php
│ └── layouts
│ ├── app.blade.php
│ └── astro.blade.php
├── src
├── AutoCleanup.php
├── AutoReconnect.php
├── AutoScale.php
├── BackgroundProcess.php
├── Commands
│ ├── TmiClusterAuthorizeCommand.php
│ ├── TmiClusterCommand.php
│ ├── TmiClusterFeaturesCommand.php
│ ├── TmiClusterHealthCommand.php
│ ├── TmiClusterInstallCommand.php
│ ├── TmiClusterJoinCommand.php
│ ├── TmiClusterListCommand.php
│ ├── TmiClusterProcessCommand.php
│ ├── TmiClusterPublishCommand.php
│ ├── TmiClusterPurgeCommand.php
│ └── TmiClusterTerminateCommand.php
├── Contracts
│ ├── ChannelDistributor.php
│ ├── ChannelManager.php
│ ├── ClusterClient.php
│ ├── ClusterClientOptions.php
│ ├── CommandQueue.php
│ ├── HasPrometheusMetrics.php
│ ├── Invitable.php
│ ├── Pausable.php
│ ├── Restartable.php
│ ├── SupervisorJoinHandler.php
│ ├── SupervisorRepository.php
│ └── Terminable.php
├── EventMap.php
├── Events
│ ├── ClusterClientBootstrap.php
│ ├── ClusterClientRegistered.php
│ ├── ClusterClientTerminated.php
│ ├── PeriodicTimerCalled.php
│ ├── ProcessScaled.php
│ ├── SupervisorBootstrap.php
│ ├── SupervisorLooped.php
│ ├── UnableToLaunchProcess.php
│ └── WorkerProcessRestarting.php
├── Exceptions
│ └── NotFoundSupervisorException.php
├── Facades
│ └── TmiCluster.php
├── Http
│ └── Controllers
│ │ ├── Controller.php
│ │ ├── DashboardController.php
│ │ ├── InviteController.php
│ │ └── MetricsController.php
├── Limiters
│ ├── DurationLimiter.php
│ └── DurationLimiterBuilder.php
├── Listeners
│ ├── MonitorClusterClientMemory.php
│ ├── MonitorSupervisorMemory.php
│ └── SendNotification.php
├── ListensForSignals.php
├── Lock.php
├── Models
│ ├── Channel.php
│ ├── Supervisor.php
│ └── SupervisorProcess.php
├── Notifications
│ ├── AutoScaleScaled.php
│ └── Notification.php
├── PhpBinary.php
├── Process
│ ├── BackgroundProcess.php
│ ├── CommandString.php
│ ├── Process.php
│ ├── ProcessOptions.php
│ └── ProcessPool.php
├── Providers
│ └── TmiClusterServiceProvider.php
├── Repositories
│ ├── DatabaseChannelManager.php
│ ├── DummyChannelManager.php
│ ├── RedisChannelDistributor.php
│ ├── RedisCommandQueue.php
│ └── SupervisorRepository.php
├── RouteRegistrar.php
├── ServiceBindings.php
├── Supervisor.php
├── Support
│ ├── Arr.php
│ ├── Composer.php
│ ├── Health.php
│ ├── Os.php
│ ├── Str.php
│ └── Url.php
├── TmiCluster.php
├── TmiClusterClient.php
├── Traits
│ ├── Lockable.php
│ └── TmiClusterHelpers.php
├── TwitchLogin.php
└── Watchable.php
├── tests
├── TestCase.php
├── TestCases
│ ├── ArrayTest.php
│ ├── AutoScaleTest.php
│ ├── ChannelDistributorTest.php
│ ├── CommandQueueTest.php
│ ├── ComposerTest.php
│ ├── DatabaseChannelManagerTest.php
│ ├── MetricsTest.php
│ ├── MonitorClusterClientMemoryTest.php
│ ├── MonitorSupervisorMemoryTest.php
│ ├── NotificationTest.php
│ ├── SendMessageTest.php
│ ├── StaleTest.php
│ └── TmiClusterTest.php
├── Traits
│ └── CreatesSupervisors.php
└── laravel
│ └── artisan
└── webpack.mix.js
/.github/workflows/tests.yml:
--------------------------------------------------------------------------------
1 | name: Tests
2 |
3 | on: [push, pull_request]
4 |
5 | jobs:
6 | tests:
7 |
8 | services:
9 | redis:
10 | image: redis
11 | ports:
12 | - 6379:6379
13 | options: --entrypoint redis-server
14 |
15 | strategy:
16 | fail-fast: false
17 | matrix:
18 | php: [8.0, 8.1, 8.2]
19 | composer-dependency: [prefer-stable, prefer-lowest]
20 |
21 | name: "PHP ${{ matrix.php }} - ${{ matrix.composer-dependency }}"
22 |
23 | runs-on: ubuntu-latest
24 |
25 | steps:
26 | - uses: actions/checkout@v2
27 |
28 | - name: Setup PHP
29 | uses: shivammathur/setup-php@v2
30 | with:
31 | php-version: ${{ matrix.php }}
32 | extensions: dom, curl, libxml, mbstring, redis, zip
33 | coverage: none
34 |
35 | - name: Install dependencies
36 | run: composer update --no-interaction --no-scripts --no-suggest --no-progress --prefer-dist --${{ matrix.composer-dependency }}
37 |
38 | - name: PHPUnit Tests
39 | run: vendor/bin/phpunit
40 | env:
41 | CLIENT_ID: ${{ secrets.TMI_IDENTITY_USERNAME }}
42 | CLIENT_SECRET: ${{ secrets.TMI_IDENTITY_PASSWORD }}
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | /vendor
2 | /node_modules
3 | /.idea
4 | composer.lock
5 | .phpunit.result.cache
6 | .env*
7 |
--------------------------------------------------------------------------------
/FUNDING.yml:
--------------------------------------------------------------------------------
1 | ko_fi: ene
2 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2020 René Preuß
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # TMI Cluster for Twitch Chatbots
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 | ## Introduction
15 |
16 | TMI Cluster enables a highly scalable IRC client cluster for Twitch. TMI Cluster consists of multiple supervisors that can be deployed on multiple hosts. The core is inspired by [Horizon](https://github.com/laravel/horizon), which handles the complex IRC process management. It is designed to work within the Laravel ecosystem.
17 |
18 | The cluster stores its data in the database and has a Redis Command Queue to send IRC commands and receive messages.
19 |
20 | ## Features
21 |
22 | - Supervisors can be deployed on multiple servers
23 | - Up-to-date Twitch IRC Client written in PHP 8
24 | - Scalable message input/output queue
25 | - Advanced cluster status dashboard
26 | - Channel management and invite screen
27 |
28 | ## PHP Twitch Messaging Interface
29 |
30 | The TMI Cluster is powered by the [PHP Twitch Messaging Interface](https://github.com/ghostzero/tmi) client to communicate with Twitch. It's a full featured, high performance Twitch IRC client written in PHP 8.
31 |
32 | ## Official Documentation
33 |
34 | You can view our official documentation [here](https://tmiphp.com/docs/tmi-cluster.html).
35 |
--------------------------------------------------------------------------------
/SECURITY.md:
--------------------------------------------------------------------------------
1 | # Security Policy
2 |
3 | ## Supported Versions
4 |
5 | Due to the fact that I do the project alone, I only support the latest major version of the project.
6 |
7 | | Version | Supported |
8 | | ------- | ------------------ |
9 | | 3.x | :white_check_mark: |
10 | | 2.x | :x: |
11 |
12 | ## Reporting a Vulnerability
13 |
14 | I ask you to report vulnerabilities as a direct message to `GhostZero#0001` in Discord.
15 | Alternatively, an email can be sent to rene@preuss.io, but I very rarely check my emails.
16 |
--------------------------------------------------------------------------------
/UPGRADE.md:
--------------------------------------------------------------------------------
1 | # Upgrade Guide
2 |
3 | When upgrading to any new TMI Cluster version, you should re-publish TMI Cluster's assets:
4 |
5 | ```bash
6 | php artisan tmi-cluster:publish
7 | ```
8 |
9 | To ensure TMI Cluster's assets are updated when a new version is downloaded, you may add a Composer hook inside your project's composer.json file to automatically publish TMI Cluster's latest assets:
10 |
11 | ```bash
12 | "scripts": {
13 | "post-update-cmd": [
14 | "@php artisan tmi-cluster:publish --ansi"
15 | ]
16 | }
17 | ```
18 |
19 | ## Upgrading To 3.0 From 2.x
20 |
21 | ### TMI Cluster Auto Cleanup
22 |
23 | PR: https://github.com/ghostzero/tmi-cluster/pull/14
24 |
25 | The ChannelManager now manages the channels and decides whether to remove channels from the TMI cluster. This was previously decided by the Twitch API. Because the API sometimes returns wrong values, we have completely removed the Twitch API.
26 |
27 | ### TMI Cluster Channel Manager
28 |
29 | PR: https://github.com/ghostzero/tmi-cluster/pull/14
30 |
31 | Channels must have given an authorization before the TMI cluster joins the channels. This prevents the TMI cluster from joining channels that have revoked authorization, for example.
32 |
33 | ### TMI Cluster Database
34 |
35 | PR: https://github.com/ghostzero/tmi-cluster/pull/14
36 |
37 | We added a prefix for all our tables with `tmi_cluster_` e.g. `tmi_cluster_channels`. Don't forget to update your database. We recommend that you turn off the TMI Custer for this migration.
38 |
39 | ```bash
40 | php artisan migrate
41 | ```
--------------------------------------------------------------------------------
/bin/file-watcher.js:
--------------------------------------------------------------------------------
1 | const chokidar = require('chokidar');
2 |
3 | const paths = JSON.parse(process.argv[2]);
4 | const poll = !!process.argv[3];
5 |
6 | const watcher = chokidar.watch(paths, {
7 | ignoreInitial: true,
8 | usePolling: poll,
9 | });
10 |
11 | watcher
12 | .on('add', () => console.log('File added...'))
13 | .on('change', () => console.log('File changed...'))
14 | .on('unlink', () => console.log('File deleted...'))
15 | .on('unlinkDir', () => console.log('Directory deleted...'));
--------------------------------------------------------------------------------
/composer.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "ghostzero/tmi-cluster",
3 | "description": "Laravel package to create a tmi cluster.",
4 | "type": "library",
5 | "license": "MIT",
6 | "authors": [
7 | {
8 | "name": "René Preuß",
9 | "email": "rene@preuss.io"
10 | }
11 | ],
12 | "require": {
13 | "php": "^8.0",
14 | "ext-json": "*",
15 | "ext-pcntl": "*",
16 | "ext-posix": "*",
17 | "ghostzero/tmi": "^2.1",
18 | "illuminate/console": "^v8.83|^9.0|^10.0",
19 | "illuminate/support": "^v8.83|^9.0|^10.0",
20 | "predis/predis": "^1.1",
21 | "nesbot/carbon": "^2.62.1"
22 | },
23 | "require-dev": {
24 | "mockery/mockery": "^1.3.2",
25 | "orchestra/testbench": "^6.0",
26 | "phpunit/phpunit": "^9.3",
27 | "romanzipp/laravel-twitch": "^3.0"
28 | },
29 | "autoload": {
30 | "psr-4": {
31 | "GhostZero\\TmiCluster\\": "src/"
32 | }
33 | },
34 | "autoload-dev": {
35 | "psr-4": {
36 | "GhostZero\\TmiCluster\\Tests\\": "tests/"
37 | }
38 | },
39 | "extra": {
40 | "laravel": {
41 | "providers": [
42 | "GhostZero\\TmiCluster\\Providers\\TmiClusterServiceProvider"
43 | ],
44 | "aliases": {
45 | "TmiCluster": "GhostZero\\TmiCluster\\Facades\\TmiCluster"
46 | }
47 | }
48 | },
49 | "scripts": {
50 | "lowest-test": [
51 | "Composer\\Config::disableProcessTimeout",
52 | "composer update --prefer-lowest --ignore-platform-req=ext-pcntl --ignore-platform-req=ext-posix"
53 | ]
54 | },
55 | "suggest": {
56 | "romanzipp/laravel-twitch": "Rquired to use the cluster auto cleanup feature (^3.0)."
57 | },
58 | "config": {
59 | "sort-packages": true
60 | }
61 | }
62 |
--------------------------------------------------------------------------------
/migrations/2020_09_11_191542_create_supervisors_table.php:
--------------------------------------------------------------------------------
1 | string('id')->primary();
18 | $table->json('options');
19 | $table->json('metrics')->nullable();
20 | $table->timestamp('last_ping_at');
21 | $table->timestamps();
22 | $table->softDeletes();
23 | });
24 | }
25 |
26 | /**
27 | * Reverse the migrations.
28 | *
29 | * @return void
30 | */
31 | public function down()
32 | {
33 | Schema::dropIfExists('supervisors');
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/migrations/2020_10_08_161019_create_supervisor_processes_table.php:
--------------------------------------------------------------------------------
1 | uuid('id')->primary();
18 | $table->string('supervisor_id')->nullable();
19 | $table->string('state');
20 | $table->json('channels');
21 | $table->json('metrics')->nullable();
22 | $table->timestamp('last_ping_at');
23 | $table->timestamps();
24 | });
25 | }
26 |
27 | /**
28 | * Reverse the migrations.
29 | *
30 | * @return void
31 | */
32 | public function down()
33 | {
34 | Schema::dropIfExists('supervisor_processes');
35 | }
36 | }
37 |
--------------------------------------------------------------------------------
/migrations/2021_03_21_172842_create_tmi_cluster_channels_table.php:
--------------------------------------------------------------------------------
1 | string('id', 25)->primary();
18 | $table->boolean('reconnect')->default(false);
19 | $table->boolean('revoked')->default(false);
20 | $table->timestamp('acknowledged_at')->nullable();
21 | $table->uuid('supervisor_process_id')->index()->nullable();
22 | $table->timestamps();
23 | });
24 | }
25 |
26 | /**
27 | * Reverse the migrations.
28 | *
29 | * @return void
30 | */
31 | public function down()
32 | {
33 | Schema::dropIfExists('tmi_cluster_channels');
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/migrations/2021_04_06_073453_add_tmi_cluster_table_prefixes.php:
--------------------------------------------------------------------------------
1 |
2 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 | ./tests/
16 |
17 |
18 |
19 |
--------------------------------------------------------------------------------
/public/favicon-degraded.ico:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ghostzero/tmi-cluster/6f0884974fb4aa29bd786171e1faf2d41a3917ed/public/favicon-degraded.ico
--------------------------------------------------------------------------------
/public/favicon.ico:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ghostzero/tmi-cluster/6f0884974fb4aa29bd786171e1faf2d41a3917ed/public/favicon.ico
--------------------------------------------------------------------------------
/public/mix-manifest.json:
--------------------------------------------------------------------------------
1 | {
2 | "/tmi-cluster.js": "/tmi-cluster.js?id=2fcff7006166774aa6cfa35c1af9443c",
3 | "/tmi-cluster.css": "/tmi-cluster.css?id=4b89ddcd445831ec010f57ae492e65ae"
4 | }
5 |
--------------------------------------------------------------------------------
/public/tmi-cluster.js.LICENSE.txt:
--------------------------------------------------------------------------------
1 | /*!
2 | * Vue.js v2.7.13
3 | * (c) 2014-2022 Evan You
4 | * Released under the MIT License.
5 | */
6 |
7 | /*! @preserve
8 | * numeral.js
9 | * version : 2.0.6
10 | * author : Adam Draper
11 | * license : MIT
12 | * http://adamwdraper.github.com/Numeral-js/
13 | */
14 |
--------------------------------------------------------------------------------
/resources/cli/motd.txt:
--------------------------------------------------------------------------------
1 | _ _ _ _
2 | | | (_) | | | |
3 | | |_ _ __ ___ _ ______ ___| |_ _ ___| |_ ___ _ __
4 | | __| '_ ` _ \| |______/ __| | | | / __| __/ _ \ '__|
5 | | |_| | | | | | | | (__| | |_| \__ \ || __/ |
6 | \__|_| |_| |_|_| \___|_|\__,_|___/\__\___|_|
7 |
8 | Thanks for using the tmi-cluster! If you have any questions, suggestions, or
9 | complaints, please contact me at: https://ghostzero.dev/discord
10 |
11 | Official Documentation: https://tmiphp.com/docs/tmi-cluster.html
12 |
--------------------------------------------------------------------------------
/resources/js/components/Charts.vue:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
93 |
94 |
--------------------------------------------------------------------------------
/resources/js/tmi-cluster.js:
--------------------------------------------------------------------------------
1 | const numeral = require('numeral');
2 |
3 | import Vue from 'vue';
4 |
5 | Vue.prototype.$http = require('axios');
6 |
7 | window.chartColors = {
8 | color_1: '#6d00ff',
9 | color_2: '#994fff',
10 | color_3: '#bb7fff',
11 | color_4: '#d6adff',
12 | };
13 |
14 | window.chartBackgroundColors = {
15 | color_1: 'rgba(109, 0, 255, .5)',
16 | color_2: 'rgba(153, 79, 255, .5)',
17 | color_3: 'rgba(187, 127, 255, .5)',
18 | color_4: 'rgba(214, 173, 255, .5)',
19 | };
20 |
21 | Vue.filter('formatNumber', function (value) {
22 | return numeral(value).format("0,0");
23 | });
24 |
25 |
26 | Vue.component('tmi-dashboard', require('./components/TmiDashboard').default);
27 |
28 | const app = new Vue({
29 | el: '#app',
30 | });
31 |
--------------------------------------------------------------------------------
/resources/scss/tmi-cluster.scss:
--------------------------------------------------------------------------------
1 | @use "sass:math";
2 |
3 | // Variables
4 | @import 'variables';
5 |
6 | // Bootstrap
7 | @import '~bootstrap/scss/bootstrap';
8 |
9 | html,
10 | body {
11 | height: 100%;
12 | }
13 | html,
14 | body {
15 | height: 100%;
16 | }
17 |
18 | .app-center {
19 | display: flex;
20 | align-items: center;
21 | padding-top: 40px;
22 | padding-bottom: 40px;
23 | background-color: #0a0117;
24 | height: 100%;
25 | }
26 |
27 | .form-signin {
28 | width: 100%;
29 | max-width: 400px;
30 | padding: 16px;
31 | margin: auto;
32 | }
33 | .form-signin .checkbox {
34 | font-weight: 400;
35 | }
36 | .form-signin .form-control {
37 | position: relative;
38 | box-sizing: border-box;
39 | height: auto;
40 | padding: 10px;
41 | font-size: 16px;
42 | }
43 | .form-signin .form-control:focus {
44 | z-index: 2;
45 | }
46 | .form-signin input[type="email"] {
47 | margin-bottom: -1px;
48 | border-bottom-right-radius: 0;
49 | border-bottom-left-radius: 0;
50 | }
51 | .form-signin input[type="password"] {
52 | margin-bottom: 10px;
53 | border-top-left-radius: 0;
54 | border-top-right-radius: 0;
55 | }
56 |
57 | @font-face {
58 | font-family: 'JetBrains Mono';
59 | font-weight: 500;
60 | font-style: normal;
61 | font-display: swap;
62 | /* Read next point */
63 | unicode-range: U+000-5FF; /* Download only latin glyphs */
64 | src: local('JetBrains Mono'),
65 | url('https://cdn.bitinflow.com/fonts/JetBrainsMono-Regular.woff2') format('woff2'),
66 | url('https://cdn.bitinflow.com/fonts/JetBrainsMono-Regular.woff') format('woff');
67 | }
68 |
69 | @font-face {
70 | font-family: 'JetBrains Mono Extra Bold';
71 | font-weight: 400;
72 | font-style: normal;
73 | font-display: swap;
74 | /* Read next point */
75 | unicode-range: U+000-5FF; /* Download only latin glyphs */
76 | src: local('JetBrains Mono Extra Bold'),
77 | url('https://cdn.bitinflow.com/fonts/JetBrainsMono-ExtraBold.woff2') format('woff2'),
78 | url('https://cdn.bitinflow.com/fonts/JetBrainsMono-ExtraBold.woff') format('woff');
79 | }
80 |
81 | body {
82 | font-family: "JetBrains Mono", sans-serif;
83 | }
84 |
85 | .bg-primary-light-1 {
86 | background-color: #994fff;
87 | }
88 |
89 | .bg-primary-light-2 {
90 | background-color: #bb7fff;
91 | }
92 |
93 | .bg-primary-light-3 {
94 | background-color: #d6adff;
95 | }
96 |
97 | .bg-gradient-tmi {
98 | background: linear-gradient(90deg, rgba(83,5,165,1) 0%, rgba(69,5,141,1) 60%, rgba(36,4,87,1) 100%);
99 | }
100 |
101 | // Utilities
102 |
103 | .opacity-0{
104 | opacity: 0;
105 | }
106 |
107 | @for $i from 1 through 9 {
108 | $opacity: math.div($i, 10);
109 | .opacity-#{($i*10)}{
110 | opacity: $opacity;
111 | }
112 | }
113 |
114 | // Bubbles
115 |
116 | .bg-bubbles {
117 | position: absolute;
118 | padding: 0 0 0 0;
119 | opacity: .5;
120 | // Fill the bubble background with the whole screen.
121 | top: 0;
122 | left: 0;
123 | width: 100%;
124 | height: 100%;
125 | // If the element content exceeds the given width and height, the overflow attribute can determine whether the scrollbar behavior is displayed or not.
126 | overflow: hidden;
127 | li {
128 | border-radius: 4px;
129 | position: absolute;
130 | // Bottom is set to create the effect of bubbles emerging from the bottom of the page.
131 | bottom: -160px;
132 | // Default bubble size;
133 | width: 40px;
134 | height: 40px;
135 | background-color: rgba(255, 255, 255, 0.15);
136 | list-style: none;
137 | // Use custom animation to make bubbles appear, rise and roll.
138 | animation: square 15s infinite;
139 | transition-timing-function: linear;
140 | // Different positions, sizes, transparency and velocities of each bubble are set to make it appear hierarchical.
141 | &:nth-child(1) {
142 | left: 10%;
143 | }
144 | &:nth-child(2) {
145 | left: 20%;
146 | width: 90px;
147 | height: 90px;
148 | animation-delay: 2s;
149 | animation-duration: 7s;
150 | }
151 | &:nth-child(3) {
152 | left: 25%;
153 | animation-delay: 4s;
154 | }
155 | &:nth-child(4) {
156 | left: 40%;
157 | width: 60px;
158 | height: 60px;
159 | animation-duration: 8s;
160 | background-color: rgba(255, 255, 255, 0.3);
161 | }
162 | &:nth-child(5) {
163 | left: 70%;
164 | }
165 | &:nth-child(6) {
166 | left: 80%;
167 | width: 120px;
168 | height: 120px;
169 | animation-delay: 3s;
170 | background-color: rgba(255, 255, 255, 0.2);
171 | }
172 | &:nth-child(7) {
173 | left: 32%;
174 | width: 160px;
175 | height: 160px;
176 | animation-delay: 2s;
177 | }
178 | &:nth-child(8) {
179 | left: 55%;
180 | width: 20px;
181 | height: 20px;
182 | animation-delay: 4s;
183 | animation-duration: 15s;
184 | }
185 | &:nth-child(9) {
186 | left: 25%;
187 | width: 20px;
188 | height: 20px;
189 | animation-delay: 2s;
190 | animation-duration: 12s;
191 | background-color: rgba(255, 255, 255, 0.3);
192 | }
193 | &:nth-child(10) {
194 | left: 85%;
195 | width: 160px;
196 | height: 160px;
197 | animation-delay: 5s;
198 | }
199 | }
200 | // Custom square animation;
201 | @keyframes square {
202 | 0% {
203 | opacity: 0.5;
204 | transform: translateY(0px) rotate(45deg);
205 | }
206 | 25% {
207 | opacity: 0.75;
208 | transform: translateY(-400px) rotate(90deg)
209 | }
210 | 50% {
211 | opacity: 1;
212 | transform: translateY(-600px) rotate(135deg);
213 | }
214 | 100% {
215 | opacity: 0;
216 | transform: translateY(-1000px) rotate(180deg);
217 | }
218 | }
219 | }
--------------------------------------------------------------------------------
/resources/scss/variables.scss:
--------------------------------------------------------------------------------
1 | $white: #fff !default;
2 | $gray-100: #f8f9fa !default;
3 | $gray-200: #ebebeb !default;
4 | $gray-300: #dee2e6 !default;
5 | $gray-400: #ced4da !default;
6 | $gray-500: #adb5bd !default;
7 | $gray-600: #888 !default;
8 | $gray-700: #181e23 !default;
9 | $gray-800: #141a1e !default;
10 | $gray-900: #0d1114 !default;
11 | $black: #000 !default;
12 |
13 | // Colors
14 | $primary: #6d00ff;
15 | $danger: #c00;
16 | $dark: #150130;
17 |
18 | // Body
19 | $body-bg: $white;
20 | $body-color: #150130;
21 |
22 | // Table
23 | .thead-primary {
24 | background-color: $primary;
25 | color: $white;
26 | }
27 |
28 | .thead-secondary {
29 | background-color: #250459;
30 | color: $white;
31 | }
32 |
33 | .small-notice {
34 | font-size: .8rem;
35 | }
36 |
37 | // Grid containers
38 |
39 | $container-max-widths: (
40 | sm: 540px,
41 | md: 720px,
42 | lg: 960px,
43 | xl: 1140px
44 | ) !default;
--------------------------------------------------------------------------------
/resources/views/dashboard/index.blade.php:
--------------------------------------------------------------------------------
1 | @extends('tmi-cluster::layouts.app')
2 |
3 | @section('content')
4 |
7 |
8 | @endsection
9 |
--------------------------------------------------------------------------------
/resources/views/invite/index.blade.php:
--------------------------------------------------------------------------------
1 | @extends('tmi-cluster::layouts.astro')
2 |
3 | @section('content')
4 |
5 |
55 |
56 | @endsection
57 |
--------------------------------------------------------------------------------
/resources/views/layouts/app.blade.php:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 | TMI Cluster
9 | @stack('tmi-cluster::meta')
10 |
11 |
12 |
14 |
15 |
16 |
18 |
20 |
23 |
26 |
27 |
29 |
30 |
31 |
32 |
33 |
34 |
35 | @yield('content')
36 |
37 |
42 |
43 |
44 |
47 |
48 |
49 |
50 |
--------------------------------------------------------------------------------
/resources/views/layouts/astro.blade.php:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 | TMI Cluster
9 | @stack('tmi-cluster::meta')
10 |
11 |
12 |
13 |
14 |
15 |
17 |
19 |
22 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
35 | @yield('content')
36 |
37 |
38 |
41 |
42 |
43 |
44 |
--------------------------------------------------------------------------------
/src/AutoCleanup.php:
--------------------------------------------------------------------------------
1 | channelManager = app(ChannelManager::class);
15 | }
16 |
17 | public static function shouldCleanup(): bool
18 | {
19 | return config('tmi-cluster.channel_manager.auto_cleanup.enabled');
20 | }
21 |
22 | public function register(TmiClusterClient $client)
23 | {
24 | $cleanupInterval = $this->getCleanupInterval();
25 | $client->log(sprintf('Register cleanup loop for every %s sec.', $cleanupInterval));
26 | $client->getTmiClient()->getLoop()->addPeriodicTimer($cleanupInterval, fn() => $this->handle($client));
27 | }
28 |
29 | private function handle(TmiClusterClient $client)
30 | {
31 | try {
32 | if (!self::shouldCleanup()) {
33 | return;
34 | }
35 |
36 | $channels = $this->channelManager->stale($client->getTmiClient()->getChannels());
37 |
38 | foreach ($channels as $channel) {
39 | $client->getTmiClient()->part($channel);
40 | }
41 |
42 | $client->log(sprintf('Cleanup complete! Parted: %s', count($channels)));
43 | } catch (Exception $e) {
44 | $client->log($e->getMessage());
45 | $client->log($e->getTraceAsString());
46 | }
47 | }
48 |
49 | private function getCleanupInterval(): int
50 | {
51 | return config('tmi-cluster.channel_manager.auto_cleanup.interval', 300)
52 | + random_int(0, max(0, config('tmi-cluster.channel_manager.auto_cleanup.max_delay', 600)));
53 | }
54 | }
55 |
--------------------------------------------------------------------------------
/src/AutoReconnect.php:
--------------------------------------------------------------------------------
1 | counter = random_int(0, 30);
29 | } catch (Exception $e) {
30 | $this->counter = 0;
31 | }
32 | }
33 |
34 | public function handle(Supervisor $supervisor): void
35 | {
36 | try {
37 | $this->counter++;
38 |
39 | $channelManager = app(ChannelManager::class);
40 |
41 | if ($channelManager instanceof DatabaseChannelManager) {
42 | $this->doReconnect($supervisor, $channelManager);
43 | }
44 | } catch (Throwable $exception) {
45 | $supervisor->output(null, $exception->getMessage());
46 | $supervisor->output(null, $exception->getTraceAsString());
47 | }
48 | }
49 |
50 | private function doReconnect(Supervisor $supervisor, DatabaseChannelManager $channelManager)
51 | {
52 | if ($this->counter % 30 !== 0) return;
53 | $lock = $this->lock('reconnect-disconnected-channels', 10);
54 |
55 | try {
56 | $lock->block(5);
57 | // Lock acquired after waiting maximum of 5 seconds...
58 | $connectedChannels = SupervisorProcess::all()
59 | ->filter(fn(SupervisorProcess $process) => !$process->is_stale)
60 | ->map(fn(SupervisorProcess $process) => $process->channels)
61 | ->collapse()
62 | ->toArray();
63 |
64 | $disconnectedChannels = $channelManager->disconnected($connectedChannels);
65 |
66 | $count = count($disconnectedChannels);
67 |
68 | if ($count > 0) {
69 | $supervisor->output(null, sprintf('Reconnecting %d disconnected channels...', $count));
70 | app(ChannelDistributor::class)->join($disconnectedChannels);
71 | }
72 | } catch (LockTimeoutException $e) {
73 | $supervisor->output(null, 'Unable to acquire reconnect lock...');
74 | } finally {
75 | optional($lock)->release();
76 | }
77 | }
78 |
79 |
80 | }
--------------------------------------------------------------------------------
/src/AutoScale.php:
--------------------------------------------------------------------------------
1 | counter = random_int(0, 10);
40 | } catch (Exception $e) {
41 | $this->counter = 0;
42 | }
43 | }
44 |
45 | public function scale(Supervisor $supervisor): void
46 | {
47 | try {
48 | $this->counter++;
49 |
50 | $channels = $this->getCurrentChannels($supervisor);
51 | $averageUsage = $this->getCurrentAverageChannelUsage($channels);
52 | $nextUsage = $this->getNextAverageChannelUsage($channels);
53 |
54 | if ($this->shouldScaleOut($averageUsage)) {
55 | $this->scaleOut($supervisor);
56 | } elseif ($this->shouldScaleIn($averageUsage)) {
57 | if (!$this->shouldScaleOut($nextUsage)) {
58 | $this->scaleIn($supervisor);
59 | }
60 | }
61 |
62 | $this->setMinimumScale($supervisor->processes()->count());
63 |
64 | $this->releaseStaleSupervisors($supervisor);
65 | } catch (Throwable $exception) {
66 | $supervisor->output(null, $exception->getMessage());
67 | $supervisor->output(null, $exception->getTraceAsString());
68 | }
69 | }
70 |
71 | private function releaseStaleSupervisors(Supervisor $supervisor): void
72 | {
73 | if ($this->counter % 10 !== 0) return;
74 | $lock = $this->lock('release-stale-supervisors', 10);
75 |
76 | try {
77 | $lock->block(5);
78 | // Lock acquired after waiting maximum of 5 seconds...
79 | app(SupervisorRepository::class)->flushStale();
80 | } catch (LockTimeoutException $e) {
81 | $supervisor->output(null, 'Unable to acquire flush stale lock...');
82 | } finally {
83 | optional($lock)->release();
84 | }
85 | }
86 |
87 | public function shouldScaleOut(float $usage): bool
88 | {
89 | return $usage > config('tmi-cluster.auto_scale.thresholds.scale_out', 70);
90 | }
91 |
92 | public function shouldRestoreScale(): bool
93 | {
94 | return config('tmi-cluster.auto_scale.restore', true);
95 | }
96 |
97 | public function shouldScaleIn(float $usage): bool
98 | {
99 | return $usage < config('tmi-cluster.auto_scale.thresholds.scale_in', 50);
100 | }
101 |
102 | private function getCurrentAverageChannelUsage(Collection $c)
103 | {
104 | $channelLimit = config('tmi-cluster.auto_scale.thresholds.channels', 50);
105 | $channelCount = $c->sum();
106 | $serverCount = $c->count() + 1;
107 |
108 | return (($channelCount / $serverCount) / $channelLimit) * 100;
109 | }
110 |
111 | private function getNextAverageChannelUsage(Collection $c)
112 | {
113 | $channelLimit = config('tmi-cluster.auto_scale.thresholds.channels', 50);
114 | $channelCount = $c->sum();
115 | $serverCount = $c->count();
116 |
117 | if ($serverCount <= 0) {
118 | return 0;
119 | }
120 |
121 | return (($channelCount / $serverCount) / $channelLimit) * 100;
122 | }
123 |
124 | public function scaleOut(Supervisor $supervisor): void
125 | {
126 | $count = $supervisor->processes()->count();
127 | $supervisor->output(null, 'Scale out: ' . ($count + 1));
128 |
129 | if ($count >= config('tmi-cluster.auto_scale.processes.max', 25)) {
130 | return; // skip scale out, keep a maximum of instance
131 | }
132 |
133 | $supervisor->scale($count + 1);
134 | }
135 |
136 | public function scaleIn(Supervisor $supervisor): void
137 | {
138 | $count = $supervisor->processes()->count();
139 |
140 | if ($count <= $this->getMinimumScale()) {
141 | return; // skip scale in, keep a minimum of instance
142 | }
143 |
144 | $supervisor->output(null, 'Scale in: ' . ($count - 1));
145 | $supervisor->scale($count - 1);
146 | }
147 |
148 | private function getCurrentChannels(Supervisor $supervisor): Collection
149 | {
150 | $c = collect();
151 | $supervisor->model->processes()
152 | ->each(function (SupervisorProcess $process) use ($c) {
153 | $c->push(count($process->channels));
154 | });
155 |
156 | return $c;
157 | }
158 |
159 | public function getMinimumScale(): int
160 | {
161 | $default = config('tmi-cluster.auto_scale.processes.min', 2);
162 | if (!$this->shouldRestoreScale()) {
163 | return $default;
164 | }
165 |
166 | return max($this->connection()->get('auto-scale:minimum-scale') ?? $default, $default);
167 | }
168 |
169 | public function setMinimumScale(int $scale)
170 | {
171 | return $this->connection()->set('auto-scale:minimum-scale', $scale, 'EX', 60);
172 | }
173 | }
174 |
--------------------------------------------------------------------------------
/src/BackgroundProcess.php:
--------------------------------------------------------------------------------
1 | argument('channel');
32 |
33 | if ($this->option('revoke')) {
34 | $this->revokeAuthorization($channels);
35 |
36 | $this->info('Channels authorization revoked successfully.');
37 | } else {
38 | $this->authorize($channels);
39 |
40 | $this->info('Channels authorized successfully.');
41 | }
42 |
43 | return 0;
44 | }
45 |
46 | private function authorize($channels): void
47 | {
48 | foreach ($channels as $channel) {
49 | $this->getChannelManager()->authorize(new TwitchLogin($channel));
50 | }
51 | }
52 |
53 | private function revokeAuthorization(array $channels)
54 | {
55 | foreach ($channels as $channel) {
56 | $this->getChannelManager()->revokeAuthorization(new TwitchLogin($channel));
57 | }
58 | }
59 |
60 | private function getChannelManager(): ChannelManager
61 | {
62 | return app(ChannelManager::class);
63 | }
64 | }
65 |
--------------------------------------------------------------------------------
/src/Commands/TmiClusterCommand.php:
--------------------------------------------------------------------------------
1 | printMotd();
43 |
44 | event(new SupervisorBootstrap());
45 |
46 | try {
47 | /** @var Supervisor $supervisor */
48 | $supervisor = app(SupervisorRepository::class)->create();
49 | } catch (DomainException) {
50 | $this->error('A supervisor with this name is already running.');
51 |
52 | return 13;
53 | }
54 |
55 | $this->info("Supervisor {$supervisor->model->getKey()} started successfully!");
56 | $this->info('Press Ctrl+C to stop the supervisor safely.');
57 |
58 | $this->getOutput()->newLine();
59 |
60 | return $this->start($supervisor);
61 | }
62 |
63 | private function start(Supervisor $supervisor): int
64 | {
65 | if ($supervisor->model->options['nice']) {
66 | proc_nice($supervisor->model->options['nice']);
67 | }
68 |
69 | $supervisor->handleOutputUsing(function ($type, $line) {
70 | $method = $this->matchOutputType($type);
71 | $time = date('Y-m-d H:i:s');
72 | $this->{$method}('[' . $time . '] ' . trim($line));
73 | });
74 |
75 | $supervisor->scale(config('tmi-cluster.auto_scale.processes.min'));
76 |
77 | if ($this->option('watch')) {
78 | $watcher = $this->startServerWatcher();
79 | Event::listen(function (SupervisorLooped $event) use ($watcher) {
80 | if ($watcher->isRunning() &&
81 | $watcher->getIncrementalOutput()) {
82 | $this->info('Application change detected. Restarting supervisor…');
83 | $event->supervisor->restart();
84 | } elseif ($watcher->isTerminated()) {
85 | $this->error(
86 | 'Watcher process has terminated. Please ensure Node and chokidar are installed.' . PHP_EOL .
87 | $watcher->getErrorOutput()
88 | );
89 | }
90 | });
91 | }
92 |
93 | $supervisor->monitor();
94 |
95 | return 0;
96 | }
97 |
98 | private function printMotd(): void
99 | {
100 | $lines = file_get_contents(__DIR__ . '/../../resources/cli/motd.txt');
101 | $lines = explode("\n", $lines);
102 | foreach ($lines as $line) {
103 | $this->comment($line);
104 | }
105 |
106 | $detectedTmiClusterVersion = Composer::detectTmiClusterVersion();
107 | $this->comment(sprintf('You are running version %s of tmi-cluster.', $detectedTmiClusterVersion));
108 |
109 | // print an unsupported message if the detected version is not within the supported range of dev-master or semantic version of ^2.3 or ^3.0
110 | if (!Composer::isSupportedVersion($detectedTmiClusterVersion)) {
111 | $this->warn('This version of tmi-cluster is not supported. Please upgrade to the latest version.');
112 | }
113 |
114 | $this->getOutput()->newLine();
115 | }
116 |
117 | private function matchOutputType(?string $type): string
118 | {
119 | return match ($type) {
120 | null, 'info', 'out' => 'info',
121 | 'error', 'err' => 'error',
122 | 'comment' => 'comment',
123 | 'question' => 'question',
124 | default => throw new DomainException(sprintf('Unknown output type "%s"', $type)),
125 | };
126 | }
127 | }
128 |
--------------------------------------------------------------------------------
/src/Commands/TmiClusterFeaturesCommand.php:
--------------------------------------------------------------------------------
1 | comment('TMI Cluster:');
33 | $this->info(sprintf('Framework: %s', config('tmi-cluster.framework')));
34 | $this->info(sprintf('Fast Termination: %s', (config('tmi-cluster.fast_termination')) ? 'Enabled' : 'Disabled'));
35 |
36 | $this->comment('Connections:');
37 | $this->info(sprintf('Database: %s', config('database.default')));
38 | $this->info(sprintf('Redis: %s', config('tmi-cluster.use')));
39 | $this->info(sprintf('Redis Prefix: %s', config('tmi-cluster.prefix')));
40 |
41 | $this->comment('TMI:');
42 | $this->info(sprintf('TMI Username: %s', config('tmi-cluster.tmi.identity.username') ?? 'Not set'));
43 | $this->info(sprintf('TMI Reconnect: %s', (config('tmi-cluster.tmi.connection.reconnect') ? 'Enabled' : 'Disabled')));
44 | $this->info(sprintf('TMI Rejoin: %s', (config('tmi-cluster.tmi.connection.rejoin') ? 'Enabled' : 'Disabled')));
45 |
46 | $this->comment('You will use the following implementations:');
47 |
48 | $channelManagerName = get_class(app(ChannelManager::class));
49 | $channelDistributorName = get_class(app(ChannelDistributor::class));
50 |
51 | $this->info(sprintf("Your channel manager is %s", $channelManagerName));
52 | $this->info(sprintf("Your channel distributor is %s", $channelDistributorName));
53 |
54 | $this->comment('Auto Reconnect:');
55 |
56 | if ($channelManagerName === DatabaseChannelManager::class) {
57 | $this->info('Your channel manager supports auto reconnect.');
58 | } else {
59 | $this->comment('Your channel manager does not support auto reconnect.');
60 | }
61 |
62 | $this->comment('Auto Cleanup:');
63 |
64 | if (AutoCleanup::shouldCleanup()) {
65 | $this->info('The auto cleanup feature is enabled.');
66 |
67 | $settings = config('tmi-cluster.channel_manager.auto_cleanup');
68 |
69 | $this->info(sprintf('The auto cleanup interval is %d seconds.', $settings['interval']));
70 | $this->info(sprintf('The auto cleanup will have a max start delay of %d seconds.', $settings['max_delay']));
71 | } else {
72 | $this->comment('The auto cleanup feature is disabled.');
73 | }
74 |
75 | return 0;
76 | }
77 | }
78 |
--------------------------------------------------------------------------------
/src/Commands/TmiClusterHealthCommand.php:
--------------------------------------------------------------------------------
1 | option('host-only')
39 | ? $this->getHostSupervisors($repository)
40 | : $repository->all();
41 |
42 | if (Health::isOperational($supervisors, $this->option('at-least-one'))) {
43 | $this->info('TMI cluster is healthy.');
44 | return 0;
45 | }
46 |
47 | $this->error('TMI cluster is unhealthy.');
48 | return 1;
49 | }
50 |
51 | private function getHostSupervisors(SupervisorRepository $repository): Collection
52 | {
53 | return $repository->all()->filter(function (Supervisor $supervisor) {
54 | return str_starts_with(sprintf('%s-', $supervisor->getKey()), $this->getHostname());
55 | });
56 | }
57 |
58 | private function getHostname(): string
59 | {
60 | return $this->option('hostname') ?? gethostname();
61 | }
62 | }
63 |
--------------------------------------------------------------------------------
/src/Commands/TmiClusterInstallCommand.php:
--------------------------------------------------------------------------------
1 | comment('Publishing TMI Cluster assets...');
29 | $this->callSilent('vendor:publish', ['--tag' => 'tmi-cluster-assets', '--force' => true]);
30 |
31 | $this->comment('Publishing TMI Cluster configuration...');
32 | $this->callSilent('vendor:publish', ['--tag' => 'tmi-cluster-config']);
33 |
34 | $this->info('TMI Cluster scaffolding installed successfully.');
35 |
36 | return 0;
37 | }
38 | }
39 |
--------------------------------------------------------------------------------
/src/Commands/TmiClusterJoinCommand.php:
--------------------------------------------------------------------------------
1 | argument('channel');
33 |
34 | $this->printChannelNames();
35 |
36 | if ($this->option('authorize')) {
37 | $result = Artisan::call('tmi-cluster:authorize', [
38 | 'channel' => $channels,
39 | ]);
40 |
41 | if ($result === 0) {
42 | $this->info(Artisan::output());
43 | } else {
44 | $this->error(Artisan::output());
45 | }
46 | }
47 |
48 | $results = $this->getChannelDistributor()->joinNow($channels);
49 |
50 | foreach ($results as $type => $channels) {
51 | if (empty($channels)) {
52 | continue;
53 | }
54 |
55 | $this->info(sprintf('%s:', ucfirst($type)));
56 | foreach ($channels as $channel) {
57 | $this->info(sprintf("\t%s", $channel));
58 | }
59 | }
60 |
61 | return 0;
62 | }
63 |
64 | private function getChannelDistributor(): ChannelDistributor
65 | {
66 | return app(ChannelDistributor::class);
67 | }
68 |
69 | private function getChannelManager(): ChannelManager
70 | {
71 | return app(ChannelManager::class);
72 | }
73 |
74 | private function printChannelNames(): void
75 | {
76 | $channelManagerName = get_class($this->getChannelManager());
77 | $channelDistributorName = get_class($this->getChannelDistributor());
78 |
79 | $this->info(sprintf("Your channel manager is %s", $channelManagerName));
80 | $this->info(sprintf("Your channel distributor is %s", $channelDistributorName));
81 |
82 | $this->getOutput()->newLine();
83 | }
84 | }
85 |
--------------------------------------------------------------------------------
/src/Commands/TmiClusterListCommand.php:
--------------------------------------------------------------------------------
1 | all()
36 | ->map(function (Supervisor $supervisor) {
37 | return [
38 | $supervisor->getKey(),
39 | $supervisor->getKey(),
40 | $supervisor->last_ping_at->diffInSeconds(),
41 | $supervisor->processes()->count(),
42 | ];
43 | });
44 |
45 | $this->table([
46 | 'ID', 'Name', 'Last Ping', 'Processes'
47 | ], $supervisors);
48 |
49 |
50 | $processes = SupervisorProcess::all()
51 | ->map(function (SupervisorProcess $process) {
52 | return [
53 | $this->option('uuid') ? $process->getKey() : explode('-', $process->getKey())[0],
54 | $process->supervisor ? $process->supervisor->getKey() : 'N/A',
55 | $process->state,
56 | $process->last_ping_at ? $process->last_ping_at->diffInSeconds() : 'N/A',
57 | count($process->channels),
58 | $process->memory_usage,
59 | ];
60 | });
61 |
62 | $this->table([
63 | 'ID', 'Supervisor', 'State', 'Last Ping', 'Channels', 'Memory Usage'
64 | ], $processes);
65 |
66 | return 0;
67 | }
68 | }
69 |
--------------------------------------------------------------------------------
/src/Commands/TmiClusterProcessCommand.php:
--------------------------------------------------------------------------------
1 | clusterClientOptions();
37 |
38 | event(new ClusterClientBootstrap($options));
39 |
40 | TmiClusterClient::make($options, function ($type, $line) {
41 | $this->info($line);
42 | })->connect();
43 |
44 | return 0;
45 | }
46 |
47 | /**
48 | * Returns the given cluster client configuration.
49 | *
50 | * @return ClusterClientOptions
51 | */
52 | private function clusterClientOptions(): ClusterClientOptions
53 | {
54 | return new ClusterClientOptions(
55 | $this->argument('uuid'),
56 | $this->option('supervisor'),
57 | $this->option('stop-when-empty'),
58 | $this->option('memory'),
59 | $this->option('force')
60 | );
61 | }
62 | }
63 |
--------------------------------------------------------------------------------
/src/Commands/TmiClusterPublishCommand.php:
--------------------------------------------------------------------------------
1 | comment('Publishing TMI Cluster assets...');
29 | $this->callSilent('vendor:publish', ['--tag' => 'tmi-cluster-assets', '--force' => true]);
30 |
31 | $this->info('TMI Cluster assets published successfully.');
32 |
33 | return 0;
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/src/Commands/TmiClusterPurgeCommand.php:
--------------------------------------------------------------------------------
1 | flushStale();
32 | } catch (Exception $e) {
33 | $this->error($e->getMessage());
34 |
35 | return 1;
36 | }
37 |
38 | $this->info('Stale supervisors have been purged.');
39 |
40 | return 0;
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/src/Commands/TmiClusterTerminateCommand.php:
--------------------------------------------------------------------------------
1 | argument('supervisor')) {
31 | $supervisors = $repository->all()->whereIn('id', [$id]);
32 | } else {
33 | $supervisors = $repository->all();
34 | }
35 |
36 | foreach ($supervisors as $supervisor) {
37 | $this->info(sprintf('Terminating %s...', $supervisor->getKey()));
38 | $commandQueue->push($supervisor->getKey(), CommandQueue::COMMAND_SUPERVISOR_TERMINATE);
39 | }
40 |
41 | return 0;
42 | }
43 | }
44 |
--------------------------------------------------------------------------------
/src/Contracts/ChannelDistributor.php:
--------------------------------------------------------------------------------
1 | options = $options;
17 | $this->output = $output;
18 | }
19 |
20 | /**
21 | * Starts a new TMI Client process.
22 | *
23 | * @param ClusterClientOptions $options
24 | * @param Closure $output
25 | * @return self
26 | */
27 | abstract public static function make(ClusterClientOptions $options, Closure $output): self;
28 |
29 | public static function getQueueName(string $id, string $name): string
30 | {
31 | return $id . '-' . $name;
32 | }
33 |
34 | abstract public function connect(): void;
35 |
36 | abstract public function pause(): void;
37 |
38 | abstract public function continue(): void;
39 |
40 | abstract public function restart(): void;
41 |
42 | abstract public function terminate(int $status = 0): void;
43 |
44 | abstract public function getTmiClient(): Client;
45 |
46 | abstract public function log(string $message): void;
47 |
48 | abstract public function getUuid();
49 |
50 | public function memoryUsage(): float
51 | {
52 | return memory_get_usage() / 1024 / 1024;
53 | }
54 | }
55 |
--------------------------------------------------------------------------------
/src/Contracts/ClusterClientOptions.php:
--------------------------------------------------------------------------------
1 | uuid = $uuid;
22 | $this->supervisor = $supervisor;
23 | $this->stopWhenEmpty = $stopWhenEmpty;
24 | $this->memory = $memory;
25 | $this->force = $force;
26 | }
27 |
28 | public function getUuid(): string
29 | {
30 | return $this->uuid;
31 | }
32 |
33 | public function getSupervisor(): string
34 | {
35 | return $this->supervisor;
36 | }
37 |
38 | public function isStopWhenEmpty(): bool
39 | {
40 | return $this->stopWhenEmpty;
41 | }
42 |
43 | public function getMemory(): int
44 | {
45 | return $this->memory;
46 | }
47 |
48 | public function isForce(): bool
49 | {
50 | return $this->force;
51 | }
52 | }
53 |
--------------------------------------------------------------------------------
/src/Contracts/CommandQueue.php:
--------------------------------------------------------------------------------
1 | #example]
11 | */
12 | public const COMMAND_TMI_WRITE = 'tmi:write';
13 |
14 | /**
15 | * Joins a irc channel.
16 | *
17 | * Payload: [channel => #example]
18 | */
19 | public const COMMAND_TMI_JOIN = 'tmi:join';
20 |
21 | /**
22 | * Leaves a irc channel.
23 | *
24 | * Payload: [channel => #example]
25 | */
26 | public const COMMAND_TMI_PART = 'tmi:part';
27 |
28 | /**
29 | * Executes a exit(6) within the tmi cluster process.
30 | */
31 | public const COMMAND_CLIENT_EXIT = 'client:exit';
32 |
33 | /**
34 | * Scales out the supervisor by one process.
35 | */
36 | public const COMMAND_SUPERVISOR_SCALE_OUT = 'supervisor:scale-out';
37 |
38 | /**
39 | * Scales in the supervisor by one process.
40 | */
41 | public const COMMAND_SUPERVISOR_SCALE_IN = 'supervisor:scale-in';
42 |
43 | /**
44 | * Executes a exit(0) within the tmi cluster supervisor.
45 | */
46 | public const COMMAND_SUPERVISOR_TERMINATE = 'supervisor:terminate';
47 |
48 | /**
49 | * Queue to handle lost and found cases. Eg. a channel cannot be joined.
50 | */
51 | public const NAME_JOIN_HANDLER = 'join-handler';
52 |
53 | /**
54 | * Supervisor queue: First-come, First-served.
55 | */
56 | public const NAME_ANY_SUPERVISOR = '*';
57 |
58 | /**
59 | * Push a command onto a queue.
60 | *
61 | * @param string $name
62 | * @param string $command
63 | * @param array $options
64 | */
65 | public function push(string $name, string $command, array $options = []);
66 |
67 | /**
68 | * Get the pending commands for a given queue name.
69 | *
70 | * @param string $name
71 | * @return array
72 | */
73 | public function pending(string $name): array;
74 |
75 | /**
76 | * Flush the command queue for a given queue name.
77 | *
78 | * @param string $name
79 | * @return void
80 | */
81 | public function flush(string $name): void;
82 | }
83 |
--------------------------------------------------------------------------------
/src/Contracts/HasPrometheusMetrics.php:
--------------------------------------------------------------------------------
1 |
13 | */
14 | public function getPrometheusMetrics(): Collection;
15 | }
--------------------------------------------------------------------------------
/src/Contracts/Invitable.php:
--------------------------------------------------------------------------------
1 | [
14 | Listeners\SendNotification::class,
15 | ],
16 |
17 | Events\SupervisorLooped::class => [
18 | Listeners\MonitorSupervisorMemory::class,
19 | ],
20 |
21 | Events\PeriodicTimerCalled::class => [
22 | Listeners\MonitorClusterClientMemory::class,
23 | ],
24 | ];
25 | }
26 |
--------------------------------------------------------------------------------
/src/Events/ClusterClientBootstrap.php:
--------------------------------------------------------------------------------
1 | options = $options;
14 | }
15 | }
16 |
--------------------------------------------------------------------------------
/src/Events/ClusterClientRegistered.php:
--------------------------------------------------------------------------------
1 | clusterClient = $clusterClient;
14 | }
15 | }
16 |
--------------------------------------------------------------------------------
/src/Events/ClusterClientTerminated.php:
--------------------------------------------------------------------------------
1 | clusterClient = $clusterClient;
14 | }
15 | }
16 |
--------------------------------------------------------------------------------
/src/Events/PeriodicTimerCalled.php:
--------------------------------------------------------------------------------
1 | clusterClient = $clusterClient;
14 | }
15 | }
16 |
--------------------------------------------------------------------------------
/src/Events/ProcessScaled.php:
--------------------------------------------------------------------------------
1 | message = $message;
19 | $this->description = $description;
20 | $this->event = $event;
21 | $this->cause = $cause;
22 | }
23 |
24 | public function toNotification(): ?AutoScaleScaledNotification
25 | {
26 | if (config('tmi-cluster.framework') === 'laravel-zero') {
27 | return null; // notifications are not supported
28 | }
29 |
30 | return new AutoScaleScaledNotification(
31 | $this->message, $this->description, $this->event, $this->cause
32 | );
33 | }
34 | }
35 |
--------------------------------------------------------------------------------
/src/Events/SupervisorBootstrap.php:
--------------------------------------------------------------------------------
1 | supervisor = $supervisor;
14 | }
15 | }
16 |
--------------------------------------------------------------------------------
/src/Events/UnableToLaunchProcess.php:
--------------------------------------------------------------------------------
1 | process = $process;
14 | }
15 | }
16 |
--------------------------------------------------------------------------------
/src/Events/WorkerProcessRestarting.php:
--------------------------------------------------------------------------------
1 | process = $process;
14 | }
15 | }
16 |
--------------------------------------------------------------------------------
/src/Exceptions/NotFoundSupervisorException.php:
--------------------------------------------------------------------------------
1 | get())) {
22 | return response('', 204);
23 | }
24 |
25 | return response('', 503);
26 | }
27 |
28 | public function statistics(Request $request): array
29 | {
30 | /** @var AutoScale $autoScale */
31 | $autoScale = app(AutoScale::class);
32 |
33 | $withChannels = !$request->has('time');
34 |
35 | $supervisors = Supervisor::query()
36 | ->with('processes:' . implode(',', $this->getProcessColumns($withChannels)))
37 | ->get();
38 |
39 | $ircMessages = $supervisors
40 | ->sum(function (Supervisor $supervisor) {
41 | return $supervisor->processes->sum(function (SupervisorProcess $process) {
42 | return $process->metrics['irc_messages'] ?? 0;
43 | });
44 | });
45 |
46 | $ircCommands = $supervisors
47 | ->sum(function (Supervisor $supervisor) {
48 | return $supervisor->processes->sum(function (SupervisorProcess $process) {
49 | return $process->metrics['irc_commands'] ?? 0;
50 | });
51 | });
52 |
53 | $supervisors->map(function (Supervisor $supervisor) {
54 | $tokens = explode('-', $supervisor->getKey());
55 | $supervisor->id_short = $tokens[count($tokens) - 1];
56 | $supervisor->processes->transform(function (SupervisorProcess $process) {
57 | $process->id_short = explode('-', $process->getKey())[0];
58 | $process->last_ping_at_in_seconds = $process->last_ping_at->diffInSeconds();
59 | return $process;
60 | });
61 | });
62 |
63 | $time = round(microtime(true) * 1000);
64 |
65 | return [
66 | 'time' => $time,
67 | 'operational' => Health::isOperational($supervisors),
68 | 'supervisors' => $supervisors,
69 | 'irc_messages' => $ircMessages,
70 | 'irc_commands' => $ircCommands,
71 | 'irc_messages_per_second' => $this->getDataPerSecond($request, 'irc_messages', $ircMessages, $time),
72 | 'irc_commands_per_second' => $this->getDataPerSecond($request, 'irc_commands', $ircCommands, $time),
73 | 'channels' => $supervisors
74 | ->sum(function (Supervisor $supervisor) {
75 | return $supervisor->processes->sum(function (SupervisorProcess $process) {
76 | return $process->metrics['channels'] ?? 0;
77 | });
78 | }),
79 | 'processes' => $supervisors->sum(fn(Supervisor $supervisor) => count($supervisor->processes)),
80 | 'auto_scale' => array_merge(config('tmi-cluster.auto_scale'), [
81 | 'minimum_scale' => $autoScale->getMinimumScale(),
82 | ]),
83 | ];
84 | }
85 |
86 | public function getDataPerSecond(Request $request, string $field, int $current, float $time): float
87 | {
88 | if ($request->has(['time', $field])) {
89 | $lastIrcMessages = $request->get($field, $current);
90 | $lastTime = (int)$request->get('time', $time);
91 | $timeDiff = $time - $lastTime;
92 | $diff = $current - $lastIrcMessages;
93 |
94 | return round($diff / ($timeDiff / 1000));
95 | }
96 |
97 | return 0;
98 | }
99 |
100 | private function getProcessColumns(bool $withChannels): array
101 | {
102 | $columns = [
103 | 'id',
104 | 'state',
105 | 'metrics',
106 | 'created_at',
107 | 'last_ping_at',
108 | 'supervisor_id',
109 | ];
110 |
111 | if ($withChannels) {
112 | $columns[] = 'channels';
113 | }
114 |
115 | return $columns;
116 | }
117 | }
118 |
--------------------------------------------------------------------------------
/src/Http/Controllers/InviteController.php:
--------------------------------------------------------------------------------
1 | getInvitableUser($request);
21 |
22 | return view('tmi-cluster::invite.index', [
23 | 'connections' => Channel::query()->count(),
24 | 'bot_username' => config('tmi-cluster.tmi.identity.username'),
25 | 'invited' => $channelManager->authorized([$user->getTwitchLogin()]),
26 | 'user_username' => $user->getTwitchLogin(),
27 | ]);
28 | }
29 |
30 | public function store(Request $request, ChannelManager $channelManager)
31 | {
32 | $user = $this->getInvitableUser($request);
33 |
34 | if ($channelManager->authorized([$user->getTwitchLogin()])) {
35 | $channelManager->revokeAuthorization($user);
36 | } else {
37 | $channelManager->authorize($user, [
38 | 'reconnect' => true,
39 | ]);
40 | }
41 |
42 | return back();
43 | }
44 |
45 | private function getInvitableUser(Request $request): Invitable
46 | {
47 | $user = $request->user();
48 |
49 | if (!($user instanceof Invitable) || !$user->getTwitchLogin()) {
50 | throw new HttpException(409, 'You cannot invite this bot into your channel.');
51 | }
52 |
53 | return $user;
54 | }
55 | }
56 |
--------------------------------------------------------------------------------
/src/Http/Controllers/MetricsController.php:
--------------------------------------------------------------------------------
1 | 'Number of processes of the supervisor',
15 | 'tmi_cluster_process_channels' => 'Memory usage of the process',
16 | 'tmi_cluster_process_irc_commands' => 'IRC commands processed by the process',
17 | 'tmi_cluster_process_irc_messages' => 'IRC messages processed by the process',
18 | 'tmi_cluster_process_memory_usage' => 'Memory usage of the process',
19 | 'tmi_cluster_process_command_queue_commands' => 'Commands in the command queue',
20 | 'tmi_cluster_avg_usage' => 'Average usage of all processes',
21 | 'tmi_cluster_avg_response_time' => 'Average response time of all processes',
22 | ];
23 |
24 | private const METRIC_TYPE_HINTS = [
25 | 'tmi_cluster_supervisor_processes' => 'gauge',
26 | 'tmi_cluster_process_channels' => 'gauge',
27 | 'tmi_cluster_process_irc_commands' => 'counter',
28 | 'tmi_cluster_process_irc_messages' => 'counter',
29 | 'tmi_cluster_process_memory_usage' => 'gauge',
30 | 'tmi_cluster_process_command_queue_commands' => 'gauge',
31 | 'tmi_cluster_avg_usage' => 'gauge',
32 | 'tmi_cluster_avg_response_time' => 'gauge',
33 | ];
34 |
35 | public function handle()
36 | {
37 | $metrics = collect();
38 |
39 | /** @var SupervisorRepository $repository */
40 | $repository = app(SupervisorRepository::class);
41 |
42 | $repository->all()->each(function (Supervisor $supervisor) use (&$metrics) {
43 | foreach ($supervisor->processes as $process) {
44 | $metrics->push($this->getProcessMetrics($supervisor, $process));
45 | }
46 |
47 | $metrics->push($this->getSupervisorMetrics($supervisor));
48 | });
49 |
50 | $metrics->push($this->getGlobalMetrics());
51 |
52 | return response($metrics->collapse()->join(PHP_EOL) . PHP_EOL)
53 | ->header('Content-Type', 'text/plain');
54 | }
55 |
56 | protected function getProcessMetrics(Supervisor $supervisor, SupervisorProcess $process): array
57 | {
58 | return $process->getPrometheusMetrics()
59 | ->map(function ($value, $key) use ($supervisor, $process) {
60 | return $this->ensureIsDocumented(sprintf('tmi_cluster_process_%s', $key), $value, [
61 | 'supervisor' => $supervisor->getKey(),
62 | 'process' => $process->getKey(),
63 | ]);
64 | })->values()->toArray();
65 | }
66 |
67 | protected function getSupervisorMetrics(Supervisor $supervisor): array
68 | {
69 | return $supervisor->getPrometheusMetrics()
70 | ->map(function ($value, $key) use ($supervisor) {
71 | return $this->ensureIsDocumented(sprintf('tmi_cluster_supervisor_%s', $key), $value, [
72 | 'supervisor' => $supervisor->getKey(),
73 | ]);
74 | })->values()->toArray();
75 | }
76 |
77 | private function getGlobalMetrics(): array
78 | {
79 | return [
80 | $this->ensureIsDocumented('tmi_cluster_avg_usage', 10),
81 | $this->ensureIsDocumented('tmi_cluster_avg_response_time', 700)
82 | ];
83 | }
84 |
85 | private function ensureIsDocumented(string $key, float $value, array $tags = []): string
86 | {
87 | $line = sprintf('%s%s %s', $key, $this->formatTags($tags), $value);
88 |
89 | if (isset($this->hintedMetrics[$key])) {
90 | return $line;
91 | }
92 |
93 | $this->hintedMetrics[$key] = true;
94 |
95 | return implode(PHP_EOL, [
96 | sprintf('# HELP %s %s', $key, self::METRIC_HELP_TEXT[$key] ?? 'N/A'),
97 | sprintf('# TYPE %s %s', $key, self::METRIC_TYPE_HINTS[$key] ?? 'gauge'),
98 | $line,
99 | ]);
100 | }
101 |
102 | /**
103 | * Format tags for prometheus.
104 | */
105 | private function formatTags(array $tags): string
106 | {
107 | $formatted = collect($tags)->map(function ($value, $key) {
108 | return sprintf('%s="%s"', $key, $value);
109 | })->join(',');
110 |
111 | return $formatted ? sprintf('{%s}', $formatted) : '';
112 | }
113 | }
114 |
--------------------------------------------------------------------------------
/src/Limiters/DurationLimiter.php:
--------------------------------------------------------------------------------
1 | name = $name;
70 | $this->decay = $decay;
71 | $this->redis = $redis;
72 | $this->maxLocks = $maxLocks;
73 | $this->locks = $locks;
74 | }
75 |
76 | /**
77 | * Attempt to acquire the lock for the given number of seconds.
78 | *
79 | * @param int $timeout
80 | * @param callable|null $callback
81 | * @return mixed
82 | *
83 | * @throws \Illuminate\Contracts\Redis\LimiterTimeoutException
84 | */
85 | public function block($timeout, $callback = null)
86 | {
87 | $starting = time();
88 |
89 | while (!$this->acquire()) {
90 | if (time() - $timeout >= $starting) {
91 | throw new LimiterTimeoutException;
92 | }
93 |
94 | usleep(750 * 1000);
95 | }
96 |
97 | if (is_callable($callback)) {
98 | return $callback();
99 | }
100 |
101 | return true;
102 | }
103 |
104 | /**
105 | * Attempt to acquire the lock.
106 | *
107 | * @return bool
108 | */
109 | public function acquire()
110 | {
111 | $results = $this->redis->eval(
112 | $this->luaScript(), 1, $this->name, microtime(true), time(), $this->decay, $this->maxLocks, $this->locks
113 | );
114 |
115 | $this->decaysAt = $results[1];
116 |
117 | $this->remaining = max(0, $results[2]);
118 |
119 | return (bool)$results[0];
120 | }
121 |
122 | /**
123 | * Get the Lua script for acquiring a lock.
124 | *
125 | * KEYS[1] - The limiter name
126 | * ARGV[1] - Current time in microseconds
127 | * ARGV[2] - Current time in seconds
128 | * ARGV[3] - Duration of the bucket
129 | * ARGV[4] - Allowed number of tasks
130 | * ARGV[5] - Tasks per execution
131 | *
132 | * @return string
133 | */
134 | protected function luaScript()
135 | {
136 | return <<<'LUA'
137 | local function reset()
138 | redis.call('HMSET', KEYS[1], 'start', ARGV[2], 'end', ARGV[2] + ARGV[3], 'count', ARGV[5])
139 | return redis.call('EXPIRE', KEYS[1], ARGV[3] * 2)
140 | end
141 |
142 | if redis.call('EXISTS', KEYS[1]) == 0 then
143 | return {reset(), ARGV[2] + ARGV[3], ARGV[4] - ARGV[5]}
144 | end
145 |
146 | if ARGV[1] >= redis.call('HGET', KEYS[1], 'start') and ARGV[1] <= redis.call('HGET', KEYS[1], 'end') then
147 | return {
148 | tonumber(redis.call('HINCRBY', KEYS[1], 'count', ARGV[5])) <= tonumber(ARGV[4]),
149 | redis.call('HGET', KEYS[1], 'end'),
150 | ARGV[4] - redis.call('HGET', KEYS[1], 'count')
151 | }
152 | end
153 |
154 | return {reset(), ARGV[2] + ARGV[3], ARGV[4] - ARGV[5]}
155 | LUA;
156 | }
157 | }
158 |
--------------------------------------------------------------------------------
/src/Limiters/DurationLimiterBuilder.php:
--------------------------------------------------------------------------------
1 | name = $name;
64 | $this->connection = $connection;
65 | }
66 |
67 | /**
68 | * Set the maximum number of locks that can obtained per time window.
69 | *
70 | * @param int $maxLocks
71 | * @return $this
72 | */
73 | public function allow($maxLocks)
74 | {
75 | $this->maxLocks = $maxLocks;
76 |
77 | return $this;
78 | }
79 |
80 | /**
81 | * Set the amount of time the lock window is maintained.
82 | *
83 | * @param \DateTimeInterface|\DateInterval|int $decay
84 | * @return $this
85 | */
86 | public function every($decay)
87 | {
88 | $this->decay = $this->secondsUntil($decay);
89 |
90 | return $this;
91 | }
92 |
93 | /**
94 | * Set the amount of time to block until a lock is available.
95 | *
96 | * @param int $timeout
97 | * @return $this
98 | */
99 | public function block($timeout)
100 | {
101 | $this->timeout = $timeout;
102 |
103 | return $this;
104 | }
105 |
106 | /**
107 | * Set the amount of locks, that should be taken per execution.
108 | *
109 | * @param int $locks
110 | * @return $this
111 | */
112 | public function take($locks)
113 | {
114 | $this->locks = $locks;
115 |
116 | return $this;
117 | }
118 |
119 | /**
120 | * Execute the given callback if a lock is obtained, otherwise call the failure callback.
121 | *
122 | * @param callable $callback
123 | * @param callable|null $failure
124 | * @return mixed
125 | *
126 | * @throws \Illuminate\Contracts\Redis\LimiterTimeoutException
127 | */
128 | public function then(callable $callback, callable $failure = null)
129 | {
130 | try {
131 | return (new DurationLimiter(
132 | $this->connection, $this->name, $this->maxLocks, $this->decay, $this->locks
133 | ))->block($this->timeout, $callback);
134 | } catch (LimiterTimeoutException $e) {
135 | if ($failure) {
136 | return $failure($e);
137 | }
138 |
139 | throw $e;
140 | }
141 | }
142 | }
143 |
--------------------------------------------------------------------------------
/src/Listeners/MonitorClusterClientMemory.php:
--------------------------------------------------------------------------------
1 | clusterClient;
12 |
13 | if ($clusterClient->memoryUsage() > $clusterClient->options->getMemory()) {
14 | $clusterClient->terminate(8);
15 | }
16 | }
17 | }
--------------------------------------------------------------------------------
/src/Listeners/MonitorSupervisorMemory.php:
--------------------------------------------------------------------------------
1 | supervisor;
12 |
13 | if ($supervisor->memoryUsage() > config('tmi-cluster.memory_limit', 64)) {
14 | $supervisor->terminate(8);
15 | }
16 | }
17 | }
--------------------------------------------------------------------------------
/src/Listeners/SendNotification.php:
--------------------------------------------------------------------------------
1 | toNotification();
15 |
16 | if ($notification === null) {
17 | return;
18 | }
19 |
20 | /** @var Lock $lock */
21 | $lock = app(Lock::class);
22 |
23 | if (!$lock->get('notification:' . $notification->signature(), 300)) {
24 | $event->skiped = true;
25 | return;
26 | }
27 |
28 | try {
29 | Notification::route('slack', TmiCluster::$slackWebhookUrl)
30 | ->route('nexmo', TmiCluster::$smsNumber)
31 | ->route('mail', TmiCluster::$email)
32 | ->notify($notification);
33 |
34 | $event->sent = true;
35 | } catch (Throwable $exception) {
36 | $event->sent = false;
37 | }
38 | }
39 | }
40 |
--------------------------------------------------------------------------------
/src/ListensForSignals.php:
--------------------------------------------------------------------------------
1 | pendingSignals['terminate'] = 'terminate';
27 | });
28 |
29 | pcntl_signal(SIGTERM, function () {
30 | $this->pendingSignals['terminate'] = 'terminate';
31 | });
32 |
33 | pcntl_signal(SIGUSR1, function () {
34 | $this->pendingSignals['restart'] = 'restart';
35 | });
36 |
37 | pcntl_signal(SIGUSR2, function () {
38 | $this->pendingSignals['pause'] = 'pause';
39 | });
40 |
41 | pcntl_signal(SIGCONT, function () {
42 | $this->pendingSignals['continue'] = 'continue';
43 | });
44 | }
45 |
46 | /**
47 | * Process the pending signals.
48 | *
49 | * @return void
50 | */
51 | protected function processPendingSignals(): void
52 | {
53 | while ($this->pendingSignals) {
54 | $signal = Arr::first($this->pendingSignals);
55 |
56 | $this->{$signal}();
57 |
58 | unset($this->pendingSignals[$signal]);
59 | }
60 | }
61 | }
62 |
--------------------------------------------------------------------------------
/src/Lock.php:
--------------------------------------------------------------------------------
1 | redis = $redis;
16 | }
17 |
18 | public function with(string $key, callable $callback, int $seconds = 60): void
19 | {
20 | if ($this->get($key, $seconds)) {
21 | try {
22 | $callback();
23 | } finally {
24 | $this->release($key);
25 | }
26 | }
27 | }
28 |
29 | public function exists(string $key): bool
30 | {
31 | return $this->connection()->exists($key) === 1;
32 | }
33 |
34 | public function get(string $key, int $seconds = 60): bool
35 | {
36 | $result = $this->connection()->setnx($key, 1);
37 |
38 | if ($result === 1) {
39 | $this->connection()->expire($key, $seconds);
40 | }
41 |
42 | return $result === 1;
43 | }
44 |
45 | public function release(string $key): void
46 | {
47 | $this->connection()->del($key);
48 | }
49 |
50 | public function connection(): Connection
51 | {
52 | return $this->redis->connection('tmi-cluster');
53 | }
54 |
55 | /**
56 | * Throttle a callback for a maximum number of executions over a given duration.
57 | *
58 | * @param string $name
59 | * @return DurationLimiterBuilder
60 | */
61 | public function throttle($name)
62 | {
63 | return new DurationLimiterBuilder($this->connection(), $name);
64 | }
65 | }
66 |
--------------------------------------------------------------------------------
/src/Models/Channel.php:
--------------------------------------------------------------------------------
1 | 'bool',
22 | 'reconnect' => 'bool',
23 | ];
24 | }
--------------------------------------------------------------------------------
/src/Models/Supervisor.php:
--------------------------------------------------------------------------------
1 | 'array',
32 | 'metrics' => 'array',
33 | 'last_ping_at' => 'datetime',
34 | ];
35 |
36 | public function getIsStaleAttribute(): bool
37 | {
38 | return $this->last_ping_at->diffInSeconds() >= config('tmi-cluster.supervisor.stale', 300);
39 | }
40 |
41 | public function processes(): HasMany
42 | {
43 | return $this->hasMany(SupervisorProcess::class);
44 | }
45 |
46 | public function getPrometheusMetrics(): Collection
47 | {
48 | return Collection::make([
49 | 'processes' => $this->processes()->count(),
50 | ])->merge($this->metrics ?? []);
51 | }
52 | }
53 |
--------------------------------------------------------------------------------
/src/Models/SupervisorProcess.php:
--------------------------------------------------------------------------------
1 | 'array',
37 | 'metrics' => 'array',
38 | 'last_ping_at' => 'datetime',
39 | ];
40 |
41 | public function supervisor(): BelongsTo
42 | {
43 | return $this->belongsTo(Supervisor::class);
44 | }
45 |
46 | public function getIsStaleAttribute(): bool
47 | {
48 | // we require at least 60 seconds for our restart cooldown
49 | return $this->last_ping_at->diffInSeconds() >= config('tmi-cluster.process.stale', 90);
50 | }
51 |
52 | public function getMemoryUsageAttribute(): string
53 | {
54 | return isset($this->metrics['memory_usage']) ? Str::convert($this->metrics['memory_usage']) : 'N/A';
55 | }
56 |
57 | public function getPrometheusMetrics(): Collection
58 | {
59 | return Collection::make($this->metrics ?? []);
60 | }
61 | }
62 |
--------------------------------------------------------------------------------
/src/Notifications/AutoScaleScaled.php:
--------------------------------------------------------------------------------
1 | message = $message;
21 | $this->description = $description;
22 | $this->event = $event;
23 | $this->cause = $cause;
24 | }
25 |
26 | /**
27 | * Get the mail representation of the notification.
28 | *
29 | * @param mixed $notifiable
30 | * @return MailMessage
31 | */
32 | public function toMail($notifiable)
33 | {
34 | return (new MailMessage)
35 | ->error()
36 | ->subject('TMI Cluster: AutoScale Notification')
37 | ->greeting('Oh no! Something needs your attention.')
38 | ->line(sprintf('Message: %s', $this->message))
39 | ->line(sprintf('Description: %s', $this->message))
40 | ->line(sprintf('Event: %s', $this->message))
41 | ->line(sprintf('Cause: %s', $this->message));
42 | }
43 |
44 | /**
45 | * Get the Slack representation of the notification.
46 | *
47 | * @param mixed $notifiable
48 | * @return SlackMessage
49 | */
50 | public function toSlack($notifiable)
51 | {
52 | return (new SlackMessage)
53 | ->from('TMI Cluster')
54 | ->to(TmiCluster::$slackChannel)
55 | //->image('... add tmi cluster image url')
56 | ->info()
57 | ->attachment(function ($attachment) {
58 | $attachment
59 | ->title('TMI Cluster: AutoScale Notification')
60 | ->fields([
61 | 'Message' => $this->message,
62 | 'Description' => $this->description,
63 | 'Event' => $this->event,
64 | 'Cause' => $this->cause,
65 | ]);
66 | });
67 | }
68 |
69 | /**
70 | * Get the Nexmo / SMS representation of the notification.
71 | *
72 | * @param mixed $notifiable
73 | * @return NexmoMessage
74 | */
75 | public function toNexmo($notifiable)
76 | {
77 | return (new NexmoMessage)->content(sprintf(
78 | '[TMI Cluster] %s. %s.', $this->message, $this->description
79 | ));
80 | }
81 |
82 | public function signature(): string
83 | {
84 | return sha1($this->message . $this->description . $this->event . $this->cause);
85 | }
86 | }
87 |
--------------------------------------------------------------------------------
/src/Notifications/Notification.php:
--------------------------------------------------------------------------------
1 | getSupervisor(),
27 | $options->getMemory()
28 | );
29 | }
30 | }
31 |
--------------------------------------------------------------------------------
/src/Process/Process.php:
--------------------------------------------------------------------------------
1 | systemProcess = $systemProcess;
30 | $this->uuid = $uuid;
31 | $this->output = function () {
32 | //
33 | };
34 | }
35 |
36 | public function getUuid(): string
37 | {
38 | return $this->uuid;
39 | }
40 |
41 | public function isRunning(): bool
42 | {
43 | return $this->systemProcess->isRunning();
44 | }
45 |
46 | public function monitor(): void
47 | {
48 | if ($this->systemProcess->isRunning() || $this->coolingDown()) {
49 | return;
50 | }
51 |
52 | $this->restart();
53 | }
54 |
55 | public function restart(): void
56 | {
57 | if ($this->systemProcess->isStarted()) {
58 | event(new WorkerProcessRestarting($this));
59 | }
60 |
61 | $this->start();
62 | }
63 |
64 | public function terminate(int $status = 0): void
65 | {
66 | $this->sendSignal(SIGINT);
67 | }
68 |
69 | public function stop(): void
70 | {
71 | if ($this->systemProcess->isRunning()) {
72 | $this->systemProcess->stop();
73 | }
74 | }
75 |
76 | protected function sendSignal($signal): void
77 | {
78 | try {
79 | $this->systemProcess->signal($signal);
80 | } catch (ExceptionInterface $e) {
81 | if ($this->systemProcess->isRunning()) {
82 | throw $e;
83 | }
84 | }
85 | }
86 |
87 | public function start(): Process
88 | {
89 | $this->cooldown();
90 |
91 | $this->systemProcess->start($this->output);
92 |
93 | return $this;
94 | }
95 |
96 | public function pause(): void
97 | {
98 | $this->sendSignal(SIGUSR2);
99 | }
100 |
101 | public function continue(): void
102 | {
103 | $this->sendSignal(SIGCONT);
104 | }
105 |
106 | protected function cooldown()
107 | {
108 | if ($this->coolingDown()) {
109 | return;
110 | }
111 |
112 | if ($this->restartAgainAt) {
113 | $this->restartAgainAt = !$this->systemProcess->isRunning()
114 | ? CarbonImmutable::now()->addMinute()
115 | : null;
116 |
117 | if (!$this->systemProcess->isRunning()) {
118 | event(new UnableToLaunchProcess($this));
119 | }
120 | } else {
121 | $this->restartAgainAt = CarbonImmutable::now()->addSecond();
122 | }
123 | }
124 |
125 | public function coolingDown(): bool
126 | {
127 | return isset($this->restartAgainAt) &&
128 | CarbonImmutable::now()->lt($this->restartAgainAt);
129 | }
130 |
131 | public function handleOutputUsing(Closure $callback): self
132 | {
133 | $this->output = $callback;
134 |
135 | return $this;
136 | }
137 |
138 |
139 | public function shouldMarkForTermination(): bool
140 | {
141 | if ($exitCode = $this->systemProcess->getExitCode()) {
142 | return !in_array($this->systemProcess->getExitCode(), self::HEALTHY_EXIT_CODES);
143 | }
144 |
145 | return false;
146 | }
147 | }
148 |
--------------------------------------------------------------------------------
/src/Process/ProcessOptions.php:
--------------------------------------------------------------------------------
1 | supervisor = $supervisor;
13 | $this->options = $options;
14 | }
15 |
16 | public function toWorkerCommand(string $uuid): string
17 | {
18 | return CommandString::fromOptions($this, $uuid);
19 | }
20 |
21 | public function getWorkingDirectory(): string
22 | {
23 | return app()->environment('testing')
24 | ? base_path('tests/laravel')
25 | : base_path();
26 | }
27 |
28 | public function getSupervisor(): string
29 | {
30 | return $this->supervisor;
31 | }
32 |
33 | public function getTimeout(): int
34 | {
35 | return $this->options['timeout'] ?? config('tmi-cluster.process.timeout', 60);
36 | }
37 |
38 | public function getMemory(): int
39 | {
40 | return $this->options['memory'] ?? config('tmi-cluster.process.memory', 128);
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/src/Process/ProcessPool.php:
--------------------------------------------------------------------------------
1 | options = $options;
34 | $this->output = $output;
35 | $this->supervisor = $supervisor;
36 | }
37 |
38 | public function count(): int
39 | {
40 | return count($this->processes);
41 | }
42 |
43 | public function monitor(): void
44 | {
45 | $this->processes()->each(function (Process $process, $key) {
46 | if ($process->shouldMarkForTermination()) {
47 | $this->markForTermination($process);
48 | $this->removeProcess($key);
49 | } else {
50 | $process->monitor();
51 | }
52 | });
53 |
54 | $this->pruneTerminatingProcesses();
55 | $this->scale($this->requestedScale);
56 | }
57 |
58 | public function pruneTerminatingProcesses()
59 | {
60 | $this->stopTerminatingProcessesThatAreHanging();
61 | $this->terminatingProcesses = collect(
62 | $this->terminatingProcesses
63 | )->filter(function ($process) {
64 | $filtered = $process['process']->isRunning();
65 |
66 | if (!$filtered) {
67 | event(new ProcessScaled(
68 | sprintf('Auto Scaling: Process removed'),
69 | sprintf('Terminating process instance: %s', $process['process']->getUuid()),
70 | 'SUPERVISOR_PROCESS_TERMINATE',
71 | sprintf('At %s an process was removed in response of terminated process.', date('Y-m-d H:i:s'))
72 | ));
73 | }
74 |
75 | return $filtered;
76 | })->all();
77 | }
78 |
79 | protected function stopTerminatingProcessesThatAreHanging()
80 | {
81 | foreach ($this->terminatingProcesses as $process) {
82 | $timeout = $this->options->getTimeout();
83 |
84 | if ($process['terminatedAt']->addSeconds($timeout)->lte(CarbonImmutable::now())) {
85 | $process['process']->stop();
86 | }
87 | }
88 | }
89 |
90 | public function processes(): Collection
91 | {
92 | return new Collection($this->processes);
93 | }
94 |
95 | public function runningProcesses(): Collection
96 | {
97 | return $this->processes()->filter(function (Process $process) {
98 | return $process->systemProcess->isRunning();
99 | });
100 | }
101 |
102 | public function scale($processes): void
103 | {
104 | $this->requestedScale = $processes;
105 | $processes = max(0, (int)$processes);
106 |
107 | if ($processes === count($this->processes)) {
108 | return;
109 | }
110 |
111 | if ($processes > count($this->processes)) {
112 | $this->scaleUp($processes);
113 | } else {
114 | $this->scaleDown($processes);
115 | }
116 | }
117 |
118 | protected function scaleUp($processes): void
119 | {
120 | $difference = $processes - count($this->processes);
121 |
122 | for ($i = 0; $i < $difference; $i++) {
123 | $this->start();
124 | }
125 | }
126 |
127 | protected function scaleDown($processes): void
128 | {
129 | $difference = count($this->processes) - $processes;
130 |
131 | // Here we will slice off the correct number of processes that we need to terminate
132 | // and remove them from the active process array. We'll be adding them the array
133 | // of terminating processes where they'll run until they are fully terminated.
134 | $terminatingProcesses = array_slice(
135 | $this->processes, 0, $difference
136 | );
137 |
138 | foreach ($terminatingProcesses as $process) {
139 | $this->markForTermination($process);
140 | }
141 |
142 | $this->removeProcesses($difference);
143 |
144 | // Finally we will call the terminate method on each of the processes that need get
145 | // terminated so they can start terminating. Terminating is a graceful operation
146 | // so any jobs they are already running will finish running before these quit.
147 | foreach ($this->terminatingProcesses as $process) {
148 | $process['process']->terminate();
149 |
150 | event(new ProcessScaled(
151 | sprintf('Auto Scaling: Process terminated'),
152 | sprintf('Terminating process instance: %s', $process['process']->getUuid()),
153 | 'SUPERVISOR_PROCESS_TERMINATE',
154 | sprintf('At %s an process was taken out of service in response of a scale in.', date('Y-m-d H:i:s'))
155 | ));
156 | }
157 | }
158 |
159 | public function markForTermination(Process $process): void
160 | {
161 | $this->terminatingProcesses[] = [
162 | 'process' => $process, 'terminatedAt' => CarbonImmutable::now(),
163 | ];
164 | }
165 |
166 | protected function removeProcesses(int $count): void
167 | {
168 | array_splice($this->processes, 0, $count);
169 |
170 | $this->processes = array_values($this->processes);
171 | }
172 |
173 | protected function removeProcess($key): void
174 | {
175 | unset($this->processes[$key]);
176 |
177 | $this->processes = array_values($this->processes);
178 | }
179 |
180 | protected function start(): self
181 | {
182 | $process = $this->createProcess();
183 |
184 | $process->handleOutputUsing(function ($type, $line) use ($process) {
185 | $shortUuid = explode('-', $process->getUuid())[0];
186 | call_user_func($this->output, $type, sprintf('[%s] %s', $shortUuid, $line));
187 | });
188 |
189 | event(new ProcessScaled(
190 | sprintf('Auto Scaling: Process launched'),
191 | sprintf('Launching a new process instance: %s', $process->getUuid()),
192 | 'SUPERVISOR_PROCESS_LAUNCH',
193 | sprintf(
194 | 'At %s an instance was started in response to a difference between desired and actual capacity, increasing the capacity from %s to %s.',
195 | date('Y-m-d H:i:s'),
196 | $count = $this->count(),
197 | $count + 1,
198 | )
199 | ));
200 |
201 | $this->processes[] = $process;
202 |
203 | return $this;
204 | }
205 |
206 | protected function createProcess(): Process
207 | {
208 | /** @var SystemProcess $class */
209 | $class = config('tmi_cluster.fast_termination')
210 | ? BackgroundProcess::class
211 | : SystemProcess::class;
212 |
213 | $model = SupervisorProcess::query()->forceCreate([
214 | 'id' => Str::uuid(),
215 | 'supervisor_id' => $this->supervisor->model->getKey(),
216 | 'state' => SupervisorProcess::STATE_INITIALIZE,
217 | 'channels' => [],
218 | 'last_ping_at' => now(),
219 | ]);
220 |
221 | return new Process($class::fromShellCommandline(
222 | $this->options->toWorkerCommand($model->getKey()), $this->options->getWorkingDirectory()
223 | )->setTimeout(null)->disableOutput(), $model->getKey());
224 | }
225 |
226 | public function restart(): void
227 | {
228 | $count = count($this->processes);
229 |
230 | $this->scale(0);
231 |
232 | $this->scale($count);
233 | }
234 |
235 | public function pause(): void
236 | {
237 | $this->working = false;
238 |
239 | collect($this->processes)->each(fn(Process $x) => $x->stop());
240 | }
241 |
242 | public function continue(): void
243 | {
244 | $this->working = true;
245 |
246 | collect($this->processes)->each(fn(Process $x) => $x->continue());
247 | }
248 |
249 | public function terminate(int $status = 0): void
250 | {
251 | $this->working = false;
252 |
253 | collect($this->processes)->each(fn(Process $x) => $x->terminate($status));
254 | }
255 | }
256 |
--------------------------------------------------------------------------------
/src/Providers/TmiClusterServiceProvider.php:
--------------------------------------------------------------------------------
1 | configure();
26 | $this->offerPublishing();
27 | $this->registerEvents();
28 | $this->registerServices();
29 | $this->registerCommands();
30 | $this->registerFrontend();
31 | $this->defineAssetPublishing();
32 | $this->registerLogger();
33 | }
34 |
35 | protected function registerEvents(): void
36 | {
37 | $events = $this->app->make(Dispatcher::class);
38 |
39 | foreach ($this->events as $event => $listeners) {
40 | foreach ($listeners as $listener) {
41 | $events->listen($event, $listener);
42 | }
43 | }
44 | }
45 |
46 | /**
47 | * Setup the configuration for TmiCluster.
48 | *
49 | * @return void
50 | * @throws Exception
51 | */
52 | protected function configure(): void
53 | {
54 | $this->mergeConfigFrom(__DIR__ . '/../../config/tmi-cluster.php', 'tmi-cluster');
55 | $this->loadMigrationsFrom(__DIR__ . '/../../migrations');
56 |
57 | TmiCluster::use(config('tmi-cluster.use'));
58 | }
59 |
60 | /**
61 | * Register TmiCluster's services in the container.
62 | *
63 | * @return void
64 | */
65 | protected function registerServices(): void
66 | {
67 | foreach ($this->serviceBindings as $key => $value) {
68 | is_numeric($key)
69 | ? $this->app->singleton($value)
70 | : $this->app->singleton($key, $value);
71 | }
72 |
73 | $this->app->singleton(Contracts\ChannelManager::class, config('tmi-cluster.channel_manager.use'));
74 | }
75 |
76 | /**
77 | * Register the TmiCluster Artisan commands.
78 | *
79 | * @return void
80 | */
81 | protected function registerCommands(): void
82 | {
83 | if ($this->app->runningInConsole()) {
84 | $this->commands([
85 | Commands\TmiClusterAuthorizeCommand::class,
86 | Commands\TmiClusterCommand::class,
87 | Commands\TmiClusterFeaturesCommand::class,
88 | Commands\TmiClusterHealthCommand::class,
89 | Commands\TmiClusterListCommand::class,
90 | Commands\TmiClusterJoinCommand::class,
91 | Commands\TmiClusterProcessCommand::class,
92 | Commands\TmiClusterPurgeCommand::class,
93 | Commands\TmiClusterInstallCommand::class,
94 | Commands\TmiClusterPublishCommand::class,
95 | Commands\TmiClusterTerminateCommand::class,
96 | ]);
97 | }
98 | }
99 |
100 | /**
101 | * Register the TmiCluster frontend.
102 | *
103 | * @return void
104 | */
105 | private function registerFrontend(): void
106 | {
107 | $this->loadViewsFrom(__DIR__ . '/../../resources/views', 'tmi-cluster');
108 | }
109 |
110 | private function registerLogger(): void
111 | {
112 | }
113 |
114 | private function offerPublishing(): void
115 | {
116 | if ($this->app->runningInConsole()) {
117 | $this->publishes([
118 | __DIR__ . '/../../config/tmi-cluster.php' => config_path('tmi-cluster.php'),
119 | ], 'tmi-cluster-config');
120 | }
121 | }
122 |
123 | private function defineAssetPublishing(): void
124 | {
125 | $this->publishes([
126 | __DIR__ . '/../../public' => public_path('vendor/tmi-cluster')
127 | ], 'tmi-cluster-assets');
128 | }
129 | }
130 |
--------------------------------------------------------------------------------
/src/Repositories/DatabaseChannelManager.php:
--------------------------------------------------------------------------------
1 | updateOrCreate(['id' => $this->getKey($channel->getTwitchLogin())], array_merge([
33 | 'revoked' => false,
34 | ], $options));
35 |
36 | return $this;
37 | }
38 |
39 | /**
40 | * @inheritDoc
41 | */
42 | public function revokeAuthorization(Invitable $channel): self
43 | {
44 | Channel::query()
45 | ->whereKey($this->getKey($channel->getTwitchLogin()))
46 | ->update([
47 | 'revoked' => true,
48 | ]);
49 |
50 | return $this;
51 | }
52 |
53 | /**
54 | * @inheritDoc
55 | */
56 | public function authorized(array $channels): array
57 | {
58 | return Collection::make($channels)
59 | ->chunk(self::MAX_CHANNELS_PER_EXECUTION)
60 | ->map(function (Collection $channels) {
61 | return Channel::query()
62 | ->whereIn('id', $channels->map(fn(string $channel) => $this->getKey($channel)))
63 | ->where(['revoked' => false])
64 | ->pluck('id')
65 | ->toArray();
66 | })
67 | ->collapse()
68 | ->toArray();
69 | }
70 |
71 | /**
72 | * @inheritDoc
73 | */
74 | public function acknowledged(array $channels): self
75 | {
76 | Collection::make($channels)
77 | ->chunk(self::MAX_CHANNELS_PER_EXECUTION)
78 | ->each(function (Collection $channels) {
79 | Channel::query()
80 | ->whereIn('id', $channels->map(fn(string $channel) => $this->getKey($channel)))
81 | ->update([
82 | 'acknowledged_at' => now(),
83 | ]);
84 | });
85 |
86 | return $this;
87 | }
88 |
89 | private function getKey(string $channel): string
90 | {
91 | return TmiChannel::sanitize($channel);
92 | }
93 |
94 | /**
95 | * @inheritDoc
96 | */
97 | public function stale(array $channels): array
98 | {
99 | return Channel::query()
100 | ->whereIn('id', array_map(fn($channel) => $this->getKey($channel), $channels))
101 | ->where(function (Builder $query) {
102 | $query->where(['revoked' => true]);
103 |
104 | if (AutoCleanup::shouldCleanup()) {
105 | $staleAfter = Carbon::now()
106 | ->subHours(config('tmi-cluster.channel_manager.channel.stale', 168))
107 | ->format('Y-m-d H:i:s');
108 |
109 | $query->orWhere('acknowledged_at', '<', $staleAfter);
110 | }
111 | })
112 | ->pluck('id')
113 | ->toArray();
114 | }
115 |
116 | /**
117 | * Returns all channels that are not revoked and have reconnected equals true.
118 | *
119 | * @param array $channels channels that are currently connected
120 | */
121 | public function disconnected(array $channels): array
122 | {
123 | $potentialChannels = Channel::query()
124 | ->where(['revoked' => false])
125 | ->where(['reconnect' => true])
126 | ->pluck('id')
127 | ->toArray();
128 |
129 | return array_diff($potentialChannels, array_map(fn($channel) => $this->getKey($channel), $channels));
130 | }
131 | }
--------------------------------------------------------------------------------
/src/Repositories/DummyChannelManager.php:
--------------------------------------------------------------------------------
1 | commandQueue = app(CommandQueue::class);
29 | $this->lock = app(Lock::class);
30 | $this->channelManager = app(ChannelManager::class);
31 | }
32 |
33 | /**
34 | * @inheritdoc
35 | */
36 | public function join(array $channels, array $staleIds = [], bool $acknowledge = true): void
37 | {
38 | $this->commandQueue->push(CommandQueue::NAME_JOIN_HANDLER, CommandQueue::COMMAND_TMI_JOIN, [
39 | 'channels' => array_map(static fn($channel) => Channel::sanitize($channel), $channels),
40 | 'staleIds' => $staleIds,
41 | 'acknowledge' => $acknowledge,
42 | ]);
43 | }
44 |
45 | /**
46 | * @inheritdoc
47 | * @throws LimiterTimeoutException
48 | */
49 | public function joinNow(array $channels, array $staleIds = [], bool $acknowledge = true): array
50 | {
51 | $channels = array_map(static fn($channel) => Channel::sanitize($channel), $channels);
52 | $commands = $this->commandQueue->pending(CommandQueue::NAME_JOIN_HANDLER);
53 |
54 | [$staleIds, $channels, $acknowledged] = Arr::unique($commands, $staleIds, $channels, $acknowledge);
55 |
56 | $this->channelManager->acknowledged($acknowledged);
57 |
58 | return $this->joinOrQueue($channels, $staleIds);
59 | }
60 |
61 | /**
62 | * @inheritdoc
63 | * @throws LimiterTimeoutException
64 | */
65 | public function flushStale(array $channels, array $staleIds = []): array
66 | {
67 | $channels = array_map(static fn($channel) => Channel::sanitize($channel), $channels);
68 | $channels = $this->restoreQueuedChannelsFromStaleQueues($staleIds, $channels);
69 |
70 | return $this->joinNow($channels, $staleIds, false);
71 | }
72 |
73 | public function handle(Supervisor $supervisor, array $channels): void
74 | {
75 | $uuids = $supervisor->processes()->map(fn(Process $process) => $process->getUuid());
76 |
77 | foreach ($channels as $channel) {
78 | $uuid = $uuids->shuffle()->shift();
79 | $this->commandQueue->push(sprintf('%s-input', $uuid), CommandQueue::COMMAND_TMI_JOIN, [
80 | 'channel' => $channel,
81 | ]);
82 | $this->lock->release($this->getKey($channel));
83 | }
84 | }
85 |
86 | /**
87 | * This will be called if we need to join a channel.
88 | *
89 | * @param array $channels
90 | * @param array $staleIds
91 | * @return array
92 | * @throws LimiterTimeoutException
93 | */
94 | private function joinOrQueue(array $channels, array $staleIds): array
95 | {
96 | $authorized = $this->channelManager->authorized($channels);
97 | $unauthorized = array_values(array_diff($channels, $authorized));
98 |
99 | $result = ['rejected' => [], 'resolved' => [], 'ignored' => [], 'unauthorized' => $unauthorized];
100 |
101 | $processes = Models\SupervisorProcess::query()
102 | ->whereTime('last_ping_at', '>', now()->subSeconds(3))
103 | ->whereIn('state', [Models\SupervisorProcess::STATE_CONNECTED])
104 | ->whereNotIn('id', $staleIds)
105 | ->get()
106 | ->map(function (Models\SupervisorProcess $process) {
107 | return (object)[
108 | 'id' => $process->getKey(),
109 | 'channels' => $process->channels,
110 | 'channel_sum' => count($process->channels),
111 | ];
112 | })->sortBy('channel_sum');
113 |
114 | if ($processes->isEmpty()) {
115 | $result = $this->reject($result, $authorized, $staleIds);
116 |
117 | return $this->result($result);
118 | }
119 |
120 | $take = min(
121 | config('tmi-cluster.throttle.join.take', 100),
122 | config('tmi-cluster.throttle.join.allow', 2000)
123 | );
124 |
125 | foreach (array_chunk($authorized, $take) as $chunk) {
126 | /** @var Lock $lock */
127 | $lock = app(Lock::class);
128 |
129 | /** @noinspection PhpUnhandledExceptionInspection */
130 | $result = $lock->throttle('throttle:join-handler')
131 | ->block(config('tmi-cluster.throttle.join.block', 0))
132 | ->allow(config('tmi-cluster.throttle.join.allow', 2000))
133 | ->every(config('tmi-cluster.throttle.join.every', 10))
134 | ->take($take)
135 | ->then(
136 | fn() => $this->resolve($result, $chunk, $staleIds, $processes),
137 | fn() => $this->reject($result, $chunk, $staleIds)
138 | );
139 | }
140 |
141 | return $this->result($result);
142 | }
143 |
144 | private function reject(array $result, array $channels, array $staleIds): array
145 | {
146 | // we didn't get any server, that is ready to join our channels
147 | // so we move them to our lost and found channel queue
148 | $this->commandQueue->push(CommandQueue::NAME_JOIN_HANDLER, CommandQueue::COMMAND_TMI_JOIN, [
149 | 'channels' => $channels,
150 | 'staleIds' => $staleIds,
151 | ]);
152 |
153 | $result['rejected'][] = $channels;
154 |
155 | return $result;
156 | }
157 |
158 | private function resolve(array $result, array $channels, array $staleIds, Collection $processes): array
159 | {
160 | foreach ($channels as $channel) {
161 | if ($process = $this->getProcess($processes, $channel)) {
162 | if ($process instanceof stdClass) {
163 | $result['ignored'][$channel] = $this->increment($process, $channel);
164 | } else {
165 | $result['ignored'][$channel] = $process;
166 | }
167 | continue;
168 | }
169 |
170 | $nextProcess = $processes->sortBy('channel_sum')->shift();
171 | $this->increment($nextProcess, $channel);
172 | $processes->push($nextProcess);
173 |
174 | // acquire lock to prevent double join
175 | $this->lock->connection()->set($this->getKey($channel), $nextProcess->id, 'EX', 60, 'NX');
176 |
177 | $this->commandQueue->push(ClusterClient::getQueueName($nextProcess->id, ClusterClient::QUEUE_INPUT), CommandQueue::COMMAND_TMI_JOIN, [
178 | 'channel' => $channel,
179 | 'staleIds' => $staleIds,
180 | ]);
181 |
182 | $result['resolved'][$channel] = $nextProcess->id;
183 | }
184 |
185 | return $result;
186 | }
187 |
188 | private function result(array $result): array
189 | {
190 | $result['rejected'] = array_merge(...$result['rejected']);
191 |
192 | return $result;
193 | }
194 |
195 | private function getKey(string $channel): string
196 | {
197 | return sprintf('channel-distributor:join-%s', $channel);
198 | }
199 |
200 | private function getProcess(Collection $processes, string $channel)
201 | {
202 | if ($id = $this->lock->connection()->get($this->getKey($channel))) {
203 | if ($process = $processes->where('id', '===', $id)->first()) {
204 | return $process;
205 | }
206 |
207 | return $id;
208 | }
209 |
210 | return $processes->filter(fn($x) => in_array($channel, $x->channels, true))->first();
211 | }
212 |
213 | private function increment(stdClass $process, $channel): string
214 | {
215 | ++$process->channel_sum;
216 | $process->channels[] = $channel;
217 |
218 | return $process->id;
219 | }
220 |
221 | private function restoreQueuedChannelsFromStaleQueues(array $staleIds, array $channels): array
222 | {
223 | foreach ($staleIds as $staleId) {
224 | $commands = $this->commandQueue->pending($queueName = ClusterClient::getQueueName($staleId, ClusterClient::QUEUE_INPUT));
225 | foreach ($commands as $command) {
226 | if ($command->command !== CommandQueue::COMMAND_TMI_JOIN) {
227 | $this->commandQueue->push($queueName, $command->command, (array)$command->options);
228 | continue;
229 | }
230 |
231 | $this->lock->release($this->getKey($command->options->channel));
232 | $channels[] = $command->options->channel;
233 | }
234 | }
235 |
236 | return $channels;
237 | }
238 |
239 | public function forgetLocks(array $channels): void
240 | {
241 | foreach ($channels as $channel) {
242 | $this->lock->release($this->getKey($channel));
243 | }
244 | }
245 | }
--------------------------------------------------------------------------------
/src/Repositories/RedisCommandQueue.php:
--------------------------------------------------------------------------------
1 | redis = $redis;
29 | }
30 |
31 | /**
32 | * @inheritDoc
33 | * @noinspection PhpParamsInspection
34 | * @noinspection PhpUnhandledExceptionInspection
35 | */
36 | public function push(string $name, string $command, array $options = [])
37 | {
38 | $this->connection()->rpush('commands:' . $name, json_encode([
39 | 'command' => $command,
40 | 'options' => $options,
41 | 'time' => microtime(),
42 | ], JSON_THROW_ON_ERROR));
43 | }
44 |
45 | /**
46 | * @inheritDoc
47 | */
48 | public function pending(string $name): array
49 | {
50 | $length = $this->connection()->llen('commands:' . $name);
51 |
52 | if ($length < 1) {
53 | return [];
54 | }
55 |
56 | $results = $this->connection()->pipeline(function ($pipe) use ($name, $length) {
57 | $pipe->lrange('commands:' . $name, 0, $length - 1);
58 |
59 | $pipe->ltrim('commands:' . $name, $length, -1);
60 | });
61 |
62 | return collect($results[0])->map(function ($result) {
63 | return json_decode($result, false, 512, JSON_THROW_ON_ERROR);
64 | })->all();
65 | }
66 |
67 | /**
68 | * @inheritDoc
69 | */
70 | public function flush(string $name): void
71 | {
72 | $this->connection()->del('commands:' . $name);
73 | }
74 |
75 | /**
76 | * Get the Redis connection instance.
77 | *
78 | * @return Connection|ClientInterface
79 | */
80 | protected function connection()
81 | {
82 | return $this->redis->connection('tmi-cluster');
83 | }
84 | }
85 |
--------------------------------------------------------------------------------
/src/Repositories/SupervisorRepository.php:
--------------------------------------------------------------------------------
1 | create(array_merge([
22 | 'id' => $this->generateUniqueSupervisorKey(),
23 | 'last_ping_at' => now(),
24 | 'metrics' => [],
25 | 'options' => [
26 | 'nice' => 0,
27 | ]
28 | ], $attributes));
29 |
30 | return new Supervisor($supervisor);
31 | }
32 |
33 | /**
34 | * @inheritDoc
35 | */
36 | public function all(array $columns = ['*']): Collection
37 | {
38 | return Models\Supervisor::all($columns)->collect();
39 | }
40 |
41 | /**
42 | * @inheritDoc
43 | */
44 | public function flushStale(): void
45 | {
46 | $channels = [];
47 | $staleIds = [];
48 |
49 | $this->all()
50 | ->filter(fn(Models\Supervisor $supervisor) => $supervisor->is_stale)
51 | ->each(fn(Models\Supervisor $supervisor) => $supervisor->delete());
52 |
53 | Models\SupervisorProcess::query()->get()
54 | ->filter(fn(Models\SupervisorProcess $process) => $process->is_stale)
55 | ->each(function (Models\SupervisorProcess $process) use (&$channels, &$staleIds) {
56 | $staleIds[] = $process->getKey();
57 | $channels[] = $process->channels;
58 | $process->delete();
59 | });
60 |
61 | app(ChannelDistributor::class)->flushStale(array_merge(...$channels), $staleIds);
62 | }
63 |
64 | private function generateUniqueSupervisorKey(int $length = 4): string
65 | {
66 | $key = sprintf('%s-%s', gethostname(), Str::random($length));
67 |
68 | $exists = Models\Supervisor::query()
69 | ->whereKey($key)
70 | ->exists();
71 |
72 | if ($exists) {
73 | throw new DomainException('A Supervisor with this name already exists.');
74 | }
75 |
76 | return $key;
77 | }
78 | }
79 |
--------------------------------------------------------------------------------
/src/RouteRegistrar.php:
--------------------------------------------------------------------------------
1 | router = $router;
15 | }
16 |
17 | public function all(): void
18 | {
19 | $this->forHealth();
20 | $this->forDashboard();
21 | $this->forChannelManager();
22 | }
23 |
24 | private function forHealth(): void
25 | {
26 | $this->router->get('health', [Controllers\DashboardController::class, 'health']);
27 | }
28 |
29 | private function forDashboard(): void
30 | {
31 | $this->router->get('', [Controllers\DashboardController::class, 'index']);
32 | $this->router->post('statistics', [Controllers\DashboardController::class, 'statistics']);
33 | $this->router->get('statistics', [Controllers\DashboardController::class, 'statistics']);
34 | $this->router->get('metrics', [Controllers\MetricsController::class, 'handle']);
35 | }
36 |
37 | private function forChannelManager(): void
38 | {
39 | $this->router->group(['middleware' => 'auth'], function (Router $router) {
40 | $router->resource('invite', Controllers\InviteController::class)
41 | ->only(['index', 'store']);
42 | });
43 | }
44 | }
--------------------------------------------------------------------------------
/src/ServiceBindings.php:
--------------------------------------------------------------------------------
1 | Repositories\SupervisorRepository::class,
23 | Contracts\CommandQueue::class => Repositories\RedisCommandQueue::class,
24 | Contracts\ChannelDistributor::class => Repositories\RedisChannelDistributor::class,
25 | Contracts\SupervisorJoinHandler::class => Repositories\RedisChannelDistributor::class,
26 | Contracts\ChannelManager::class => Repositories\DatabaseChannelManager::class,
27 | ];
28 | }
29 |
--------------------------------------------------------------------------------
/src/Supervisor.php:
--------------------------------------------------------------------------------
1 | model = $model;
34 | $this->processPools = $this->createProcessPools();
35 | $this->autoScale = app(AutoScale::class);
36 | $this->autoReconnect = app(AutoReconnect::class);
37 | $this->commandQueue = app(CommandQueue::class);
38 | $this->commandQueue->flush($this->model->getKey());
39 | $this->output = static function () {
40 | //
41 | };
42 | }
43 |
44 | public function monitor(): void
45 | {
46 | $this->listenForSignals();
47 |
48 | $this->model->save();
49 |
50 | while (true) {
51 | sleep(1);
52 |
53 | $this->loop();
54 | }
55 | }
56 |
57 | public function handleOutputUsing(Closure $output): void
58 | {
59 | $this->output = $output;
60 | }
61 |
62 | public function scale(int $int): void
63 | {
64 | $this->pools()->each(fn(ProcessPool $x) => $x->scale($int));
65 | }
66 |
67 | public function loop(): void
68 | {
69 | try {
70 | $this->processPendingSignals();
71 |
72 | $this->processPendingCommands();
73 |
74 | // If the supervisor is working, we will perform any needed scaling operations and
75 | // monitor all of these underlying worker processes to make sure they are still
76 | // processing queued jobs. If they have died, we will restart them each here.
77 | if ($this->working) {
78 | $this->autoScale();
79 | $this->autoReconnect();
80 |
81 | $this->pools()->each(fn(ProcessPool $x) => $x->monitor());
82 | }
83 |
84 | // Update all model data here
85 | $this->model->forceFill([
86 | 'last_ping_at' => now(),
87 | 'metrics' => [
88 | // todo add some fancy metrics
89 | ]
90 | ]);
91 |
92 | // Next, we'll persist the supervisor state to storage so that it can be read by a
93 | // user interface. This contains information on the specific options for it and
94 | // the current number of worker processes per queue for easy load monitoring.
95 | $this->model->save();
96 |
97 | event(new SupervisorLooped($this));
98 | } catch (Throwable $e) {
99 | $this->output(null, $e->getMessage());
100 | $this->output(null, $e->getTraceAsString());
101 | app(ExceptionHandler::class)->report($e);
102 | }
103 | }
104 |
105 | private function autoScale(): void
106 | {
107 | $this->autoScale->scale($this);
108 | }
109 |
110 | private function autoReconnect(): void
111 | {
112 | $this->autoReconnect->handle($this);
113 | }
114 |
115 | private function createProcessPools(): array
116 | {
117 | $options = new ProcessOptions($this->model->getKey(), $this->model->options);
118 | return [$this->createSingleProcessPool($options)];
119 | }
120 |
121 | private function createSingleProcessPool(ProcessOptions $options): ProcessPool
122 | {
123 | return new ProcessPool($options, function ($type, $line) {
124 | $this->output($type, $line);
125 | }, $this);
126 | }
127 |
128 | public function output($type, $line): void
129 | {
130 | call_user_func($this->output, $type, $line);
131 | }
132 |
133 | private function pools(): Collection
134 | {
135 | return new Collection($this->processPools);
136 | }
137 |
138 | public function processes(): Collection
139 | {
140 | return $this->pools()->map(fn(ProcessPool $x) => $x->processes())->collapse();
141 | }
142 |
143 | public function pause(): void
144 | {
145 | $this->working = false;
146 |
147 | $this->pools()->each(fn(ProcessPool $x) => $x->pause());
148 | }
149 |
150 | public function continue(): void
151 | {
152 | $this->working = true;
153 |
154 | $this->pools()->each(fn(ProcessPool $x) => $x->continue());
155 | }
156 |
157 | public function restart(): void
158 | {
159 | $this->working = true;
160 |
161 | $this->pools()->each(fn(ProcessPool $x) => $x->restart());
162 | }
163 |
164 | public function terminate(int $status = 0): void
165 | {
166 | $this->working = false;
167 |
168 | // We will mark this supervisor as terminating so that any user interface can
169 | // correctly show the supervisor's status. Then, we will scale the process
170 | // pools down to zero workers to gracefully terminate them all out here.
171 | $this->pools()->each(function (ProcessPool $pool) {
172 | $pool->processes()->each(function (Process $process) {
173 | $process->terminate();
174 | });
175 | });
176 |
177 | if ($this->shouldWait()) {
178 | while ($this->pools()->map(fn(ProcessPool $x) => $x->runningProcesses())->collapse()->count()) {
179 | sleep(1);
180 | }
181 | }
182 |
183 | // cleanup database before the clean stale process kills our models
184 | $this->model->processes()->forceDelete();
185 | $this->model->forceDelete();
186 |
187 | $this->exit($status);
188 | }
189 |
190 | protected function shouldWait(): bool
191 | {
192 | return !config('tmi-cluster.fast_termination');
193 | }
194 |
195 | protected function exit($status = 0): void
196 | {
197 | $this->exitProcess($status);
198 | }
199 |
200 | protected function exitProcess($status = 0): void
201 | {
202 | exit((int)$status);
203 | }
204 |
205 | private function processPendingCommands(): void
206 | {
207 | $commands = $this->commandQueue->pending($this->model->getKey());
208 | $channelsToJoin = [];
209 |
210 | foreach ($commands as $command) {
211 | switch ($command->command) {
212 | case CommandQueue::COMMAND_SUPERVISOR_SCALE_OUT:
213 | $this->autoScale->scaleOut($this);
214 | break;
215 | case CommandQueue::COMMAND_SUPERVISOR_SCALE_IN:
216 | $this->autoScale->scaleIn($this);
217 | break;
218 | case CommandQueue::COMMAND_SUPERVISOR_TERMINATE:
219 | $this->terminate();
220 | break;
221 | case CommandQueue::COMMAND_TMI_JOIN:
222 | $channelsToJoin[] = $command->options->channel;
223 | break;
224 | }
225 | }
226 |
227 | if (!empty($channelsToJoin)) {
228 | app(SupervisorJoinHandler::class)->handle($this, $channelsToJoin);
229 | }
230 | }
231 |
232 | public function memoryUsage(): float
233 | {
234 | return memory_get_usage() / 1024 / 1024;
235 | }
236 | }
237 |
--------------------------------------------------------------------------------
/src/Support/Arr.php:
--------------------------------------------------------------------------------
1 | command !== CommandQueue::COMMAND_TMI_JOIN) {
19 | continue;
20 | }
21 |
22 | $staleIds[] = $command->options->staleIds ?? [];
23 | $channels[] = $command->options->channels ?? [];
24 | $shouldAcknowledge = $command->options->acknowledge ?? false;
25 |
26 | if ($shouldAcknowledge) {
27 | $acknowledged[] = $command->options->channels ?? [];
28 | }
29 | }
30 |
31 | return [
32 | array_unique(self::merge(...$staleIds)),
33 | array_unique(self::merge(...$channels)),
34 | array_unique(self::merge(...$acknowledged)),
35 | ];
36 | }
37 |
38 | public static function merge(array|stdClass ...$arrays): array
39 | {
40 | return array_merge(...array_map(fn($array) => is_array($array) ? $array : (array)$array, $arrays));
41 | }
42 | }
--------------------------------------------------------------------------------
/src/Support/Composer.php:
--------------------------------------------------------------------------------
1 | =')) {
37 | return true;
38 | }
39 | }
40 |
41 | return false;
42 | }
43 | }
--------------------------------------------------------------------------------
/src/Support/Health.php:
--------------------------------------------------------------------------------
1 | sum(function (Supervisor $supervisor) {
20 | return $supervisor->processes->filter(function (SupervisorProcess $process) {
21 | return self::isInConnectedState($process) || self::isInInitialState($process);
22 | })->count();
23 | });
24 |
25 | $count = $supervisors
26 | ->sum(function (Supervisor $supervisor) {
27 | return $supervisor->processes->count();
28 | });
29 |
30 | if ($atLeastOneProcess) {
31 | return $operational > 0;
32 | }
33 |
34 | return $count > 0 && $operational === $count;
35 | }
36 |
37 | /**
38 | * Consider a process in initial state if state is in initialize and ping is within the threshold.
39 | */
40 | private static function isInInitialState(SupervisorProcess $process): bool
41 | {
42 | return $process->state === SupervisorProcess::STATE_INITIALIZE
43 | && $process->last_ping_at->diffInSeconds() < self::INRESPONSIVE_THRESHOLD;
44 | }
45 |
46 | /**
47 | * Consider a process in connected state if state is in connected and ping is within the threshold.
48 | */
49 | private static function isInConnectedState(SupervisorProcess $process): bool
50 | {
51 | return $process->state === SupervisorProcess::STATE_CONNECTED
52 | && $process->last_ping_at->diffInSeconds() < self::INRESPONSIVE_THRESHOLD;
53 | }
54 | }
55 |
--------------------------------------------------------------------------------
/src/Support/Os.php:
--------------------------------------------------------------------------------
1 | array_merge($config, [
48 | 'options' => ['prefix' => config('tmi-cluster.prefix') ?: 'tmi-cluster:'],
49 | ])]);
50 | }
51 |
52 | public static function routeMailNotificationsTo(string $email): void
53 | {
54 | static::$email = $email;
55 | }
56 |
57 | public static function routeSlackNotificationsTo(string $url, string $channel = null): void
58 | {
59 | static::$slackWebhookUrl = $url;
60 | static::$slackChannel = $channel;
61 | }
62 |
63 | public static function routeSmsNotificationsTo(string $number): void
64 | {
65 | static::$smsNumber = $number;
66 | }
67 |
68 | public static function routes($callback = null, array $options = []): void
69 | {
70 | $callback = $callback ?: function ($router) {
71 | $router->all();
72 | };
73 |
74 | $defaultOptions = [
75 | 'domain' => config('tmi-cluster.domain', null),
76 | 'prefix' => config('tmi-cluster.path', 'tmi-cluster'),
77 | 'middleware' => config('tmi-cluster.middleware', 'web'),
78 | ];
79 |
80 | $options = array_merge($defaultOptions, $options);
81 |
82 | Route::group($options, function ($router) use($callback) {
83 | $callback(new RouteRegistrar($router));
84 | });
85 | }
86 | }
87 |
--------------------------------------------------------------------------------
/src/Traits/Lockable.php:
--------------------------------------------------------------------------------
1 | connection('tmi-cluster');
20 | }
21 |
22 | protected function lock(string $name, int $seconds = 0, ?string $owner = null): RedisLock
23 | {
24 | return new RedisLock($this->connection(), $name, $seconds, $owner);
25 | }
26 | }
--------------------------------------------------------------------------------
/src/Traits/TmiClusterHelpers.php:
--------------------------------------------------------------------------------
1 | authorized([$channel])) {
24 | return;
25 | }
26 | }
27 |
28 | $commandQueue->push(CommandQueue::NAME_ANY_SUPERVISOR, CommandQueue::COMMAND_TMI_WRITE, [
29 | 'raw_command' => "PRIVMSG {$channel} :{$message}",
30 | ]);
31 | }
32 |
33 | /**
34 | * @param array $channels a list of channels that needs to be joined
35 | * @param array $staleIds a list of supervisors or processes ids, that needs to be avoid
36 | */
37 | public static function joinNextServer(array $channels, array $staleIds = []): void
38 | {
39 | app(ChannelDistributor::class)->join($channels, $staleIds);
40 | }
41 | }
42 |
--------------------------------------------------------------------------------
/src/TwitchLogin.php:
--------------------------------------------------------------------------------
1 | login = $login;
15 | }
16 |
17 | public function getTwitchLogin(): ?string
18 | {
19 | return $this->login;
20 | }
21 | }
--------------------------------------------------------------------------------
/src/Watchable.php:
--------------------------------------------------------------------------------
1 | find('node'),
21 | 'file-watcher.js',
22 | json_encode(collect($paths)->map(fn($path) => base_path($path))),
23 | $this->option('poll'),
24 | ], realpath(__DIR__ . '/../bin'), null, null, null))->start();
25 | }
26 | }
27 |
--------------------------------------------------------------------------------
/tests/TestCase.php:
--------------------------------------------------------------------------------
1 | artisan('migrate')->run();
24 | $this->flushRedis();
25 | }
26 |
27 | protected function getPackageProviders($app)
28 | {
29 | return [
30 | TmiClusterServiceProvider::class,
31 | ];
32 | }
33 |
34 | protected function getPackageAliases($app)
35 | {
36 | return [
37 | 'TmiCluster' => TmiCluster::class,
38 | ];
39 | }
40 |
41 | /**
42 | * Define environment setup.
43 | *
44 | * @param Application $app
45 | * @return void
46 | */
47 | protected function getEnvironmentSetUp($app)
48 | {
49 | // Setup default database to use sqlite :memory:
50 | $app['config']->set('database.default', 'testbench');
51 | $app['config']->set('database.connections.testbench', [
52 | 'driver' => 'sqlite',
53 | 'database' => ':memory:',
54 | 'prefix' => '',
55 | ]);
56 | }
57 |
58 | protected function flushRedis()
59 | {
60 | /** @var ClientInterface $connection */
61 | $connection = app(Factory::class)->connection('tmi-cluster');
62 | $connection->flushdb();
63 | }
64 |
65 | protected function useChannelManager(string $concrete): ChannelManager
66 | {
67 | $this->app->singleton(ChannelManager::class, $concrete);
68 |
69 | return app(ChannelManager::class);
70 | }
71 | }
72 |
--------------------------------------------------------------------------------
/tests/TestCases/ArrayTest.php:
--------------------------------------------------------------------------------
1 | fake([
19 | 'channels' => ['channel-b'],
20 | 'staleIds' => ['stale-b'],
21 | 'acknowledge' => true,
22 | ]),
23 | $this->fake([
24 | 'channels' => ['channel-b'],
25 | 'acknowledge' => true,
26 | ]),
27 | $this->fake([
28 | 'channels' => ['channel-c'],
29 | 'staleIds' => ['stale-c'],
30 | 'acknowledge' => false,
31 | ]),
32 | ], ['stale-a'], ['channel-a'], true);
33 |
34 | self::assertEquals([
35 | ['stale-a', 'stale-b', 'stale-c'],
36 | ['channel-a', 'channel-b', 3 => 'channel-c'], // index 2 got removed due unique
37 | ['channel-a', 'channel-b']
38 | ], $result);
39 | }
40 |
41 | public function testCastArrayMerge(): void
42 | {
43 | $result = Arr::merge(
44 | ['test', 'test2'],
45 | json_decode(json_encode([0 => 'test3', 1 => 'test4']))
46 | );
47 |
48 | $this->assertEquals(['test', 'test2', 'test3', 'test4'], $result);
49 | }
50 |
51 | /**
52 | * @param array $options
53 | * @return mixed
54 | * @throws JsonException
55 | */
56 | private function fake(array $options = []): mixed
57 | {
58 | // encode json string
59 | $encoded = json_encode([
60 | 'command' => CommandQueue::COMMAND_TMI_JOIN,
61 | 'options' => $options,
62 | 'time' => microtime(),
63 | ], JSON_THROW_ON_ERROR);
64 |
65 | // decode json string
66 | return json_decode($encoded, false, 512, JSON_THROW_ON_ERROR);
67 | }
68 | }
--------------------------------------------------------------------------------
/tests/TestCases/AutoScaleTest.php:
--------------------------------------------------------------------------------
1 | setRestore(false);
15 | self::assertFalse($this->getAutoScale()->shouldRestoreScale());
16 |
17 | $this->setRestore(true);
18 | self::assertTrue($this->getAutoScale()->shouldRestoreScale());
19 | }
20 |
21 | public function testRestoreScale()
22 | {
23 | $config = config('tmi-cluster.auto_scale.processes.min');
24 | $this->getAutoScale()->setMinimumScale(7);
25 |
26 | // the value should be get from the config
27 | $this->setRestore(false);
28 | self::assertEquals($config, $this->getAutoScale()->getMinimumScale());
29 |
30 | // the value should be get from the redis
31 | $this->setRestore(true);
32 | self::assertNotEquals($config, $scale = $this->getAutoScale()->getMinimumScale());
33 | self::assertEquals(7, $scale);
34 |
35 | }
36 |
37 | private function setRestore(bool $restore)
38 | {
39 | app('config')->set('tmi-cluster.auto_scale.restore', $restore);
40 | }
41 |
42 | private function getAutoScale(): AutoScale
43 | {
44 | return app(AutoScale::class);
45 | }
46 | }
--------------------------------------------------------------------------------
/tests/TestCases/ChannelDistributorTest.php:
--------------------------------------------------------------------------------
1 | useChannelManager(DatabaseChannelManager::class)
24 | ->authorize(new TwitchLogin('test1'))
25 | ->authorize(new TwitchLogin('test2'))
26 | ->authorize(new TwitchLogin('test3'))
27 | ->authorize(new TwitchLogin('test4'));
28 |
29 | $this->getChannelDistributor()->join(['test1', 'test2', 'test_unauthorized']);
30 | $this->getChannelDistributor()->join(['test3']);
31 |
32 | $uuid = $this->createSupervisor(now(), SupervisorProcess::STATE_CONNECTED);
33 |
34 | $result = $this->getChannelDistributor()->joinNow(['test4'], []);
35 |
36 | self::assertEquals([
37 | 'rejected' => [],
38 | 'resolved' => [
39 | '#test1' => $uuid,
40 | '#test2' => $uuid,
41 | '#test3' => $uuid,
42 | '#test4' => $uuid,
43 | ],
44 | 'ignored' => [],
45 | 'unauthorized' => [
46 | '#test_unauthorized'
47 | ],
48 | ], $result);
49 | }
50 |
51 | public function testChannelGotRejectedDueMissingServers(): void
52 | {
53 | $this->useChannelManager(DummyChannelManager::class);
54 |
55 | $result = $this->getChannelDistributor()->joinNow(['ghostzero'], []);
56 |
57 | self::assertEquals([
58 | 'rejected' => ['#ghostzero'],
59 | 'resolved' => [],
60 | 'ignored' => [],
61 | 'unauthorized' => [],
62 | ], $result);
63 | }
64 |
65 | public function testChannelGotRejectedWithUnhealthyServers(): void
66 | {
67 | $this->useChannelManager(DummyChannelManager::class);
68 |
69 | $this->createSupervisor(now()->subSeconds(5), SupervisorProcess::STATE_CONNECTED);
70 | $this->createSupervisor(now(), SupervisorProcess::STATE_INITIALIZE);
71 |
72 | $result = $this->getChannelDistributor()->joinNow(['ghostzero'], []);
73 |
74 | self::assertEquals([
75 | 'rejected' => ['#ghostzero'],
76 | 'resolved' => [],
77 | 'ignored' => [],
78 | 'unauthorized' => [],
79 | ], $result);
80 | }
81 |
82 | public function testChannelGotResolvedDueActiveServer(): void
83 | {
84 | $this->useChannelManager(DummyChannelManager::class);
85 |
86 | $uuid = $this->createSupervisor(now(), SupervisorProcess::STATE_CONNECTED);
87 |
88 | $result = $this->getChannelDistributor()->joinNow(['ghostzero'], []);
89 |
90 | self::assertEquals([
91 | 'rejected' => [],
92 | 'resolved' => ['#ghostzero' => $uuid],
93 | 'ignored' => [],
94 | 'unauthorized' => [],
95 | ], $result);
96 | }
97 |
98 | public function testChannelGotIgnoredDueAlreadyJoined(): void
99 | {
100 | $this->useChannelManager(DummyChannelManager::class);
101 |
102 | $uuid = $this->createSupervisor(now(), SupervisorProcess::STATE_CONNECTED);
103 |
104 | $result = $this->getChannelDistributor()->joinNow(['ghostzero'], []);
105 |
106 | self::assertEquals([
107 | 'rejected' => [],
108 | 'resolved' => ['#ghostzero' => $uuid],
109 | 'ignored' => [],
110 | 'unauthorized' => [],
111 | ], $result);
112 |
113 | $uuid2 = $this->createSupervisor(now(), SupervisorProcess::STATE_CONNECTED);
114 |
115 | self::assertNotEquals($uuid, $uuid2);
116 |
117 | $result = $this->getChannelDistributor()->joinNow(['ghostzero', 'test', 'test2', 'test3', 'test4'], []);
118 |
119 | self::assertEquals([
120 | 'rejected' => [],
121 | 'resolved' => [
122 | '#test' => $uuid2, // because the second process has the lowest channels amount
123 | '#test2' => $uuid, // because the first process has the lowest channels amount
124 | '#test3' => $uuid2, // because the second process has the lowest channels amount
125 | '#test4' => $uuid, // because the first process has the lowest channels amount
126 | ],
127 | 'ignored' => [
128 | '#ghostzero' => $uuid, // because they got already joined in the first server
129 | ],
130 | 'unauthorized' => [],
131 | ], $result);
132 |
133 | $this->assertGotQueued($result, 5);
134 | }
135 |
136 | public function testChannelGotResolvedAfterInactiveSupervisor(): void
137 | {
138 | $this->useChannelManager(DummyChannelManager::class);
139 |
140 | // join server normally
141 | $uuid = $this->createSupervisor(now()->subSeconds(2), SupervisorProcess::STATE_CONNECTED, [
142 | 'ghostdemouser', // this channel got already connected
143 | ]);
144 | $result = $this->getChannelDistributor()->joinNow(['ghostzero'], []);
145 | self::assertEquals([
146 | 'rejected' => [],
147 | 'resolved' => ['#ghostzero' => $uuid],
148 | 'ignored' => [],
149 | 'unauthorized' => [],
150 | ], $result);
151 |
152 | // let's kill all servers and re-join all lost channels into some fresh servers
153 | sleep(1);
154 | $newUuid = $this->createSupervisor(now(), SupervisorProcess::STATE_CONNECTED);
155 | $result = $this->getChannelDistributor()->flushStale([
156 | '#ghostdemouser'
157 | ], [$uuid]); // this simulates our flush stale
158 |
159 | // check if our previous server got restored
160 | self::assertEquals([
161 | 'rejected' => [],
162 | 'resolved' => [
163 | '#ghostzero' => $newUuid, // got restored from our lost queue
164 | '#ghostdemouser' => $newUuid, // got restored from the flush stale
165 | ],
166 | 'ignored' => [],
167 | 'unauthorized' => [],
168 | ], $result);
169 | }
170 |
171 | private function getChannelDistributor(): ChannelDistributor
172 | {
173 | return app(RedisChannelDistributor::class);
174 | }
175 |
176 | private function assertGotQueued(array $result, int $expectedCount): void
177 | {
178 | $processIds = array_unique(array_merge(
179 | array_values($result['rejected']),
180 | array_values($result['resolved']),
181 | array_values($result['ignored'])
182 | ));
183 |
184 | self::assertNotEmpty($processIds);
185 | $commands = $this->getPendingCommands($processIds);
186 | self::assertCount($expectedCount, $commands);
187 | }
188 |
189 | private function getPendingCommands(array $processIds): array
190 | {
191 | /** @var CommandQueue $commandQueue */
192 | $commandQueue = app(CommandQueue::class);
193 |
194 | $commands = [];
195 | foreach ($processIds as $processId) {
196 | $commands[] = $commandQueue->pending(ClusterClient::getQueueName($processId, ClusterClient::QUEUE_INPUT));
197 | }
198 | $commands = array_merge(...$commands);
199 | return $commands;
200 | }
201 | }
--------------------------------------------------------------------------------
/tests/TestCases/CommandQueueTest.php:
--------------------------------------------------------------------------------
1 | push('test', 'command');
16 |
17 | self::assertCount(0, $commandQueue->pending('test2'));
18 | self::assertNotEmpty($data = $commandQueue->pending('test'));
19 | self::assertIsArray($data);
20 | self::assertEquals('command', $data[0]->command);
21 | self::assertIsArray($data[0]->options);
22 | self::assertCount(0, $commandQueue->pending('test'));
23 | }
24 | }
25 |
--------------------------------------------------------------------------------
/tests/TestCases/ComposerTest.php:
--------------------------------------------------------------------------------
1 | assertEquals(
16 | 'unknown',
17 | Composer::detectTmiClusterVersion()
18 | );
19 | }
20 |
21 | public function testIsSupportedVersion(): void
22 | {
23 | $this->assertFalse(
24 | Composer::isSupportedVersion('1.0.0')
25 | );
26 |
27 | $this->assertFalse(
28 | Composer::isSupportedVersion('2.1.0')
29 | );
30 |
31 | $this->assertTrue(
32 | Composer::isSupportedVersion('2.3.0')
33 | );
34 |
35 | $this->assertFalse(
36 | Composer::isSupportedVersion('dev-master')
37 | );
38 |
39 | $this->assertFalse(
40 | Composer::isSupportedVersion('dev-develop')
41 | );
42 |
43 | $this->assertTrue(
44 | Composer::isSupportedVersion('3.0.0')
45 | );
46 |
47 | $this->assertFalse(
48 | Composer::isSupportedVersion('dev-develop')
49 | );
50 |
51 | }
52 |
53 | }
--------------------------------------------------------------------------------
/tests/TestCases/DatabaseChannelManagerTest.php:
--------------------------------------------------------------------------------
1 | getChannelManager()->authorize($user, [
20 | 'reconnect' => true,
21 | ]);
22 |
23 | $authorized = $this->getChannelManager()->authorized([
24 | 'ghostzero',
25 | 'tmi_inspector',
26 | ]);
27 |
28 | self::assertContains('#ghostzero', $authorized);
29 | self::assertNotContains('#tmi_inspector', $authorized);
30 | }
31 |
32 | public function testChannelIsDisconnected(): void
33 | {
34 | $user = new TwitchLogin('ghostzero');
35 |
36 | $this->getChannelManager()->authorize($user, [
37 | 'reconnect' => true,
38 | ]);
39 |
40 | $user = new TwitchLogin('tmi_inspector');
41 |
42 | $this->getChannelManager()->authorize($user, [
43 | 'reconnect' => true,
44 | ]);
45 |
46 | $disconnected = $this->getChannelManager()->disconnected([
47 | 'ghostzero',
48 | ]);
49 |
50 | self::assertNotContains('#ghostzero', $disconnected);
51 | self::assertContains('#tmi_inspector', $disconnected);
52 | }
53 |
54 | public function testChannelIsNotAuthorizedAfterRevoke(): void
55 | {
56 | $user = new TwitchLogin('ghostzero');
57 |
58 | $this->getChannelManager()->authorize($user, [
59 | 'reconnect' => true,
60 | ]);
61 |
62 | $authorized = $this->getChannelManager()->authorized(['ghostzero']);
63 |
64 | self::assertContains('#ghostzero', $authorized);
65 |
66 | $this->getChannelManager()->revokeAuthorization($user);
67 |
68 | $authorized = $this->getChannelManager()->authorized(['ghostzero']);
69 |
70 | self::assertNotContains('#ghostzero', $authorized);
71 | }
72 |
73 | private function getChannelManager(): DatabaseChannelManager
74 | {
75 | return app(DatabaseChannelManager::class);
76 | }
77 | }
--------------------------------------------------------------------------------
/tests/TestCases/MetricsTest.php:
--------------------------------------------------------------------------------
1 | createdLoopedSupervisor(2);
17 | $this->createdLoopedSupervisor(2);
18 |
19 | $controller = new MetricsController();
20 | $response = $controller->handle();
21 |
22 | self::assertStringContainsString('TYPE tmi_cluster_avg_usage gauge', $response->getContent());
23 | self::assertStringContainsString('TYPE tmi_cluster_avg_response_time gauge', $response->getContent());
24 | }
25 | }
26 |
--------------------------------------------------------------------------------
/tests/TestCases/MonitorClusterClientMemoryTest.php:
--------------------------------------------------------------------------------
1 | options = new ClusterClientOptions(
20 | 'uuid',
21 | 'supervisor',
22 | false,
23 | 64,
24 | false
25 | );
26 |
27 | $clusterClient->shouldReceive('memoryUsage')->andReturn(192);
28 | $clusterClient->shouldReceive('terminate')->once()->with(8);
29 |
30 | $monitor->handle(new PeriodicTimerCalled($clusterClient));
31 | }
32 |
33 | public function testTmiClusterIsNotTerminatedWhenUsingToMuchMemory(): void
34 | {
35 | $monitor = new MonitorClusterClientMemory();
36 |
37 | $clusterClient = Mockery::mock(TmiClusterClient::class);
38 | $clusterClient->options = new ClusterClientOptions(
39 | 'uuid',
40 | 'supervisor',
41 | false,
42 | 64,
43 | false
44 | );
45 |
46 | $clusterClient->shouldReceive('memoryUsage')->andReturn(64);
47 | $clusterClient->shouldReceive('terminate')->never();
48 |
49 | $monitor->handle(new PeriodicTimerCalled($clusterClient));
50 | }
51 | }
52 |
--------------------------------------------------------------------------------
/tests/TestCases/MonitorSupervisorMemoryTest.php:
--------------------------------------------------------------------------------
1 | shouldReceive('memoryUsage')->andReturn(65);
20 | $supervisor->shouldReceive('terminate')->once()->with(8);
21 |
22 | $monitor->handle(new SupervisorLooped($supervisor));
23 | }
24 |
25 | public function testSupervisorIsNotTerminatedWhenUsingToMuchMemory(): void
26 | {
27 | $monitor = new MonitorSupervisorMemory();
28 |
29 | $supervisor = Mockery::mock(Supervisor::class);
30 |
31 | $supervisor->shouldReceive('memoryUsage')->andReturn(64);
32 | $supervisor->shouldReceive('terminate')->never();
33 |
34 | $monitor->handle(new SupervisorLooped($supervisor));
35 | }
36 | }
37 |
--------------------------------------------------------------------------------
/tests/TestCases/NotificationTest.php:
--------------------------------------------------------------------------------
1 | handle($event);
23 |
24 | self::assertNotTrue($event->skipped);
25 | self::assertTrue($event->sent);
26 | }
27 | }
28 |
--------------------------------------------------------------------------------
/tests/TestCases/SendMessageTest.php:
--------------------------------------------------------------------------------
1 | shouldRestrictMessages();
19 |
20 | TmiCluster::sendMessage('ghostzero', 'Hello World!');
21 |
22 | self::assertMessageNotSent();
23 | }
24 |
25 | public function testCanSendMessageIfRestrictedAndAuthorized(): void
26 | {
27 | $this->shouldRestrictMessages();
28 |
29 | $this->useChannelManager(DatabaseChannelManager::class)
30 | ->authorize(new TwitchLogin('ghostzero'));
31 |
32 | TmiCluster::sendMessage('ghostzero', 'Hello World!');
33 |
34 | self::assertMessageSent('ghostzero', 'Hello World!');
35 | }
36 |
37 | public function testCanSendMessageIfUnrestricted(): void
38 | {
39 | $this->shouldRestrictMessages(false);
40 |
41 | TmiCluster::sendMessage('ghostzero', 'Hello World!');
42 |
43 | self::assertMessageSent('ghostzero', 'Hello World!');
44 | }
45 |
46 | private static function assertMessageNotSent()
47 | {
48 | /** @var CommandQueue $commandQueue */
49 | $commandQueue = app(CommandQueue::class);
50 |
51 | $commands = $commandQueue->pending(CommandQueue::NAME_ANY_SUPERVISOR);
52 |
53 | self::assertEmpty($commands);
54 | }
55 |
56 | private static function assertMessageSent(string $channel, string $message)
57 | {
58 | /** @var CommandQueue $commandQueue */
59 | $commandQueue = app(CommandQueue::class);
60 |
61 | $commands = $commandQueue->pending(CommandQueue::NAME_ANY_SUPERVISOR);
62 |
63 | self::assertNotEmpty($commands);
64 |
65 | self::assertEquals(
66 | sprintf('PRIVMSG #%s :%s', $channel, $message),
67 | $commands[0]->options->raw_command
68 | );
69 | }
70 |
71 | private function shouldRestrictMessages(bool $restrict = true): void
72 | {
73 | config()->set('tmi-cluster.channel_manager.channel.restrict_messages', $restrict);
74 | }
75 | }
--------------------------------------------------------------------------------
/tests/TestCases/StaleTest.php:
--------------------------------------------------------------------------------
1 | createSupervisor(now()->subSeconds($staleSeconds), SupervisorProcess::STATE_CONNECTED);
23 | $this->createSupervisor(now()->subSeconds($staleSeconds), SupervisorProcess::STATE_DISCONNECTED);
24 | $this->createSupervisor(now()->subSeconds($staleSeconds - 10), SupervisorProcess::STATE_DISCONNECTED);
25 | $this->createSupervisor(now(), SupervisorProcess::STATE_CONNECTED);
26 | $this->createSupervisor(now(), SupervisorProcess::STATE_CONNECTED);
27 | $this->createSupervisor(now(), SupervisorProcess::STATE_CONNECTED);
28 |
29 | $stale = $this->repository()->all()
30 | ->filter(fn(Models\Supervisor $supervisor) => $supervisor->is_stale);
31 |
32 | $healthy = $this->repository()->all()
33 | ->filter(fn(Models\Supervisor $supervisor) => !$supervisor->is_stale);
34 |
35 | self::assertEquals(2, $stale->count());
36 | self::assertEquals(4, $healthy->count());
37 | }
38 |
39 | public function repository(): SupervisorRepository
40 | {
41 | return app(SupervisorRepository::class);
42 | }
43 | }
--------------------------------------------------------------------------------
/tests/TestCases/TmiClusterTest.php:
--------------------------------------------------------------------------------
1 | create([]);
18 |
19 | self::assertStringStartsWith(gethostname(), $supervisor->model->getKey());
20 |
21 | $supervisor->scale(2);
22 | $supervisor->loop();
23 |
24 | self::assertCount(2, $supervisor->processes());
25 | }
26 | }
27 |
--------------------------------------------------------------------------------
/tests/Traits/CreatesSupervisors.php:
--------------------------------------------------------------------------------
1 | forceCreate([
19 | 'id' => Str::uuid(),
20 | 'last_ping_at' => $lastPingAt,
21 | 'metrics' => [],
22 | 'options' => ['nice' => 0]
23 | ]);
24 |
25 | /** @var SupervisorProcess $process */
26 | $process = $supervisor->processes()->forceCreate([
27 | 'id' => Str::uuid(),
28 | 'supervisor_id' => $supervisor->getKey(),
29 | 'state' => $state,
30 | 'channels' => $channels,
31 | 'last_ping_at' => $lastPingAt,
32 | ]);
33 |
34 | return (string)$process->getKey();
35 | }
36 |
37 | public function createdLoopedSupervisor(int $scale): \GhostZero\TmiCluster\Supervisor
38 | {
39 | /** @var \GhostZero\TmiCluster\Supervisor $supervisor */
40 | $supervisor = app(SupervisorRepository::class)->create();
41 |
42 | self::assertStringStartsWith(gethostname(), $supervisor->model->getKey());
43 |
44 | $supervisor->scale($scale);
45 | $supervisor->loop();
46 |
47 | self::assertCount($scale, $supervisor->processes());
48 |
49 | return $supervisor;
50 | }
51 | }
--------------------------------------------------------------------------------
/tests/laravel/artisan:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env php
2 |