├── .config └── dotnet-tools.json ├── .github └── workflows │ ├── ci.yml │ └── pack.yml ├── .gitignore ├── LICENCE ├── README.md ├── docker-compose.yml ├── global.json ├── osu.Server.QueueProcessor.Tests ├── BatchProcessorTests.cs ├── BeatmapStatusWatcherTests.cs ├── FakeData.cs ├── InputOnlyQueueTests.cs ├── TestBatchProcessor.cs ├── TestProcessor.cs └── osu.Server.QueueProcessor.Tests.csproj ├── osu.Server.QueueProcessor.sln ├── osu.Server.QueueProcessor.sln.DotSettings └── osu.Server.QueueProcessor ├── BeatmapStatusWatcher.cs ├── BeatmapUpdates.cs ├── ConnectionMultiplexerExtensions.cs ├── DatabaseAccess.cs ├── GracefulShutdownSource.cs ├── QueueConfiguration.cs ├── QueueItem.cs ├── QueueProcessor.cs ├── RedisAccess.cs └── osu.Server.QueueProcessor.csproj /.config/dotnet-tools.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 1, 3 | "isRoot": true, 4 | "tools": { 5 | "jetbrains.resharper.globaltools": { 6 | "version": "2023.3.3", 7 | "commands": [ 8 | "jb" 9 | ] 10 | }, 11 | "nvika": { 12 | "version": "4.0.0", 13 | "commands": [ 14 | "nvika" 15 | ] 16 | }, 17 | "codefilesanity": { 18 | "version": "0.0.36", 19 | "commands": [ 20 | "CodeFileSanity" 21 | ] 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | on: [push, pull_request] 2 | name: Continuous Integration 3 | concurrency: 4 | group: ${{ github.workflow }}-${{ github.ref }} 5 | cancel-in-progress: true 6 | 7 | jobs: 8 | inspect-code: 9 | name: Code Quality 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Checkout 13 | uses: actions/checkout@v4 14 | 15 | - name: Install .NET 8.0.x 16 | uses: actions/setup-dotnet@v4 17 | with: 18 | dotnet-version: "8.0.x" 19 | 20 | - name: Restore Tools 21 | run: dotnet tool restore 22 | 23 | - name: Restore Packages 24 | run: dotnet restore 25 | 26 | - name: CodeFileSanity 27 | run: | 28 | # TODO: Add ignore filters and GitHub Workflow Command Reporting in CFS. That way we don't have to do this workaround. 29 | # FIXME: Suppress warnings from templates project 30 | exit_code=0 31 | while read -r line; do 32 | if [[ ! -z "$line" ]]; then 33 | echo "::error::$line" 34 | exit_code=1 35 | fi 36 | done <<< $(dotnet codefilesanity) 37 | exit $exit_code 38 | 39 | - name: InspectCode 40 | run: dotnet jb inspectcode $(pwd)/osu.Server.QueueProcessor.sln --build --output="inspectcodereport.xml" --caches-home="inspectcode" --verbosity=WARN 41 | 42 | - name: NVika 43 | run: dotnet nvika parsereport "${{github.workspace}}/inspectcodereport.xml" --treatwarningsaserrors 44 | 45 | test: 46 | name: Test 47 | runs-on: ubuntu-latest 48 | steps: 49 | - name: Checkout 50 | uses: actions/checkout@v4 51 | 52 | - name: Install .NET 8.0.x 53 | uses: actions/setup-dotnet@v4 54 | with: 55 | dotnet-version: "8.0.x" 56 | 57 | - name: Docker compose 58 | run: docker compose up -d 59 | 60 | - name: Test 61 | run: dotnet test 62 | -------------------------------------------------------------------------------- /.github/workflows/pack.yml: -------------------------------------------------------------------------------- 1 | name: Pack 2 | 3 | on: 4 | push: 5 | tags: '*' 6 | 7 | jobs: 8 | pack: 9 | runs-on: ubuntu-latest 10 | 11 | steps: 12 | - name: Checkout 13 | uses: actions/checkout@v4 14 | - name: Install .NET 8.0.x 15 | uses: actions/setup-dotnet@v4 16 | with: 17 | dotnet-version: "8.0.x" 18 | env: 19 | NUGET_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} 20 | - name: Get the version 21 | id: get_version 22 | run: echo ::set-output name=VERSION::${GITHUB_REF#refs/tags/} 23 | 24 | - name: Create package 25 | run: dotnet pack --configuration Release /p:Version=${{ steps.get_version.outputs.VERSION }} 26 | 27 | - name: Publish the package to nuget.org 28 | run: dotnet nuget push osu.Server.QueueProcessor/bin/Release/*.nupkg --api-key ${{ secrets.NUGET_API_KEY }} --source https://api.nuget.org/v3/index.json 29 | 30 | - name: Create Release 31 | id: create_release 32 | uses: actions/create-release@v1 33 | env: 34 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 35 | with: 36 | tag_name: ${{ steps.get_version.outputs.VERSION }} 37 | release_name: ${{ steps.get_version.outputs.VERSION }} 38 | draft: false 39 | prerelease: false 40 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | 4 | # User-specific files 5 | *.suo 6 | *.user 7 | *.userosscache 8 | *.sln.docstates 9 | 10 | # User-specific files (MonoDevelop/Xamarin Studio) 11 | *.userprefs 12 | 13 | # Build results 14 | bin/[Dd]ebug/ 15 | [Dd]ebugPublic/ 16 | [Rr]elease/ 17 | [Rr]eleases/ 18 | bld/ 19 | [Bb]in/ 20 | [Oo]bj/ 21 | [Ll]og/ 22 | 23 | # Visual Studio 2015 cache/options directory 24 | .vs/ 25 | # Uncomment if you have tasks that create the project's static files in wwwroot 26 | #wwwroot/ 27 | 28 | # MSTest test Results 29 | [Tt]est[Rr]esult*/ 30 | [Bb]uild[Ll]og.* 31 | 32 | # NUNIT 33 | *.VisualState.xml 34 | TestResult.xml 35 | 36 | # Build Results of an ATL Project 37 | [Dd]ebugPS/ 38 | [Rr]eleasePS/ 39 | dlldata.c 40 | 41 | # DNX 42 | project.lock.json 43 | project.fragment.lock.json 44 | artifacts/ 45 | 46 | *_i.c 47 | *_p.c 48 | *_i.h 49 | *.ilk 50 | *.meta 51 | *.obj 52 | *.pch 53 | *.pdb 54 | *.pgc 55 | *.pgd 56 | *.rsp 57 | *.sbr 58 | *.tlb 59 | *.tli 60 | *.tlh 61 | *.tmp 62 | *.tmp_proj 63 | *.log 64 | *.vspscc 65 | *.vssscc 66 | .builds 67 | *.pidb 68 | *.svclog 69 | *.scc 70 | 71 | # Chutzpah Test files 72 | _Chutzpah* 73 | 74 | # Visual C++ cache files 75 | ipch/ 76 | *.aps 77 | *.ncb 78 | *.opendb 79 | *.opensdf 80 | *.sdf 81 | *.cachefile 82 | *.VC.db 83 | *.VC.VC.opendb 84 | 85 | # Visual Studio profiler 86 | *.psess 87 | *.vsp 88 | *.vspx 89 | *.sap 90 | 91 | # TFS 2012 Local Workspace 92 | $tf/ 93 | 94 | # Guidance Automation Toolkit 95 | *.gpState 96 | 97 | # ReSharper is a .NET coding add-in 98 | _ReSharper*/ 99 | *.[Rr]e[Ss]harper 100 | *.DotSettings.user 101 | 102 | # JustCode is a .NET coding add-in 103 | .JustCode 104 | 105 | # TeamCity is a build add-in 106 | _TeamCity* 107 | 108 | # DotCover is a Code Coverage Tool 109 | *.dotCover 110 | 111 | # NCrunch 112 | _NCrunch_* 113 | .*crunch*.local.xml 114 | nCrunchTemp_* 115 | 116 | # MightyMoose 117 | *.mm.* 118 | AutoTest.Net/ 119 | 120 | # Web workbench (sass) 121 | .sass-cache/ 122 | 123 | # Installshield output folder 124 | [Ee]xpress/ 125 | 126 | # DocProject is a documentation generator add-in 127 | DocProject/buildhelp/ 128 | DocProject/Help/*.HxT 129 | DocProject/Help/*.HxC 130 | DocProject/Help/*.hhc 131 | DocProject/Help/*.hhk 132 | DocProject/Help/*.hhp 133 | DocProject/Help/Html2 134 | DocProject/Help/html 135 | 136 | # Click-Once directory 137 | publish/ 138 | 139 | # Publish Web Output 140 | *.[Pp]ublish.xml 141 | *.azurePubxml 142 | # TODO: Comment the next line if you want to checkin your web deploy settings 143 | # but database connection strings (with potential passwords) will be unencrypted 144 | *.pubxml 145 | *.publishproj 146 | 147 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 148 | # checkin your Azure Web App publish settings, but sensitive information contained 149 | # in these scripts will be unencrypted 150 | PublishScripts/ 151 | 152 | # NuGet Packages 153 | *.nupkg 154 | # The packages folder can be ignored because of Package Restore 155 | **/packages/* 156 | # except build/, which is used as an MSBuild target. 157 | !**/packages/build/ 158 | # Uncomment if necessary however generally it will be regenerated when needed 159 | #!**/packages/repositories.config 160 | # NuGet v3's project.json files produces more ignoreable files 161 | *.nuget.props 162 | *.nuget.targets 163 | 164 | # Microsoft Azure Build Output 165 | csx/ 166 | *.build.csdef 167 | 168 | # Microsoft Azure Emulator 169 | ecf/ 170 | rcf/ 171 | 172 | # Windows Store app package directories and files 173 | AppPackages/ 174 | BundleArtifacts/ 175 | Package.StoreAssociation.xml 176 | _pkginfo.txt 177 | 178 | # Visual Studio cache files 179 | # files ending in .cache can be ignored 180 | *.[Cc]ache 181 | # but keep track of directories ending in .cache 182 | !*.[Cc]ache/ 183 | 184 | # Others 185 | ClientBin/ 186 | ~$* 187 | *~ 188 | *.dbmdl 189 | *.dbproj.schemaview 190 | *.pfx 191 | *.publishsettings 192 | node_modules/ 193 | orleans.codegen.cs 194 | 195 | # Since there are multiple workflows, uncomment next line to ignore bower_components 196 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 197 | #bower_components/ 198 | 199 | # RIA/Silverlight projects 200 | Generated_Code/ 201 | 202 | # Backup & report files from converting an old project file 203 | # to a newer Visual Studio version. Backup files are not needed, 204 | # because we have git ;-) 205 | _UpgradeReport_Files/ 206 | Backup*/ 207 | UpgradeLog*.XML 208 | UpgradeLog*.htm 209 | 210 | # SQL Server files 211 | *.mdf 212 | *.ldf 213 | 214 | # Business Intelligence projects 215 | *.rdl.data 216 | *.bim.layout 217 | *.bim_*.settings 218 | 219 | # Microsoft Fakes 220 | FakesAssemblies/ 221 | 222 | # GhostDoc plugin setting file 223 | *.GhostDoc.xml 224 | 225 | # Node.js Tools for Visual Studio 226 | .ntvs_analysis.dat 227 | 228 | # Visual Studio 6 build log 229 | *.plg 230 | 231 | # Visual Studio 6 workspace options file 232 | *.opt 233 | 234 | # Visual Studio LightSwitch build output 235 | **/*.HTMLClient/GeneratedArtifacts 236 | **/*.DesktopClient/GeneratedArtifacts 237 | **/*.DesktopClient/ModelManifest.xml 238 | **/*.Server/GeneratedArtifacts 239 | **/*.Server/ModelManifest.xml 240 | _Pvt_Extensions 241 | 242 | # Paket dependency manager 243 | .paket/paket.exe 244 | paket-files/ 245 | 246 | # FAKE - F# Make 247 | .fake/ 248 | 249 | # JetBrains Rider 250 | .idea/ 251 | *.sln.iml 252 | 253 | # CodeRush 254 | .cr/ 255 | 256 | # Python Tools for Visual Studio (PTVS) 257 | __pycache__/ 258 | *.pyc 259 | Staging/ 260 | -------------------------------------------------------------------------------- /LICENCE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2020 ppy Pty Ltd . 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # osu-queue-processor [![dev chat](https://discordapp.com/api/guilds/188630481301012481/widget.png?style=shield)](https://discord.gg/ppy) 2 | 3 | A lightweight redis-backed message queue processor 4 | 5 | # Contributing 6 | 7 | Contributions can be made via pull requests to this repository. If you're unsure of what you can help with, check out the [list of open issues](https://github.com/ppy/osu-queue-processor/issues). 8 | 9 | Note that while we already have certain standards in place, nothing is set in stone. If you have an issue with the way code is structured; with any libraries we are using; with any processes involved with contributing, *please* bring it up. I welcome all feedback so we can make contributing to this project as pain-free as possible. 10 | 11 | We love to reward quality contributions. If you have made a large contribution, or are a regular contributor, you are welcome to [submit an expense via opencollective](https://opencollective.com/ppy/expenses/new). If you have any questions, feel free to [reach out to peppy](mailto:pe@ppy.sh) before doing so. 12 | 13 | # Licence 14 | 15 | The osu! client code, framework, and server-side components are licensed under the [MIT licence](https://opensource.org/licenses/MIT). Please see [the licence file](LICENCE) for more information. [tl;dr](https://tldrlegal.com/license/mit-license) you can do whatever you want as long as you include the original copyright and license notice in any copy of the software/source. 16 | 17 | Please note that this *does not cover* the usage of the "osu!" or "ppy" branding in any software, resources, advertising or promotion, as this is protected by trademark law. 18 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.9' 2 | 3 | x-env: &x-env 4 | DB_CONNECTION_STRING: Server=db;Database=osu;Uid=osuweb; 5 | DB_HOST: db 6 | DB_USERNAME: 'root' 7 | APP_ENV: 'local' 8 | GITHUB_TOKEN: "${GITHUB_TOKEN}" 9 | BROADCAST_DRIVER: redis 10 | CACHE_DRIVER: redis 11 | NOTIFICATION_REDIS_HOST: redis 12 | REDIS_HOST: redis 13 | SESSION_DRIVER: redis 14 | MYSQL_DATABASE: 'osu' 15 | MYSQL_ALLOW_EMPTY_PASSWORD: "yes" 16 | MYSQL_ROOT_HOST: '%' 17 | 18 | services: 19 | # just a placeholder service to ensure we wait for migrator to complete successfully. 20 | ready_for_use: 21 | image: hello-world:latest 22 | depends_on: 23 | migrator: 24 | condition: service_completed_successfully 25 | 26 | migrator: 27 | image: pppy/osu-web:latest-dev 28 | command: ['artisan', 'db:setup'] 29 | depends_on: 30 | db: 31 | condition: service_healthy 32 | redis: 33 | condition: service_healthy 34 | environment: 35 | <<: *x-env 36 | 37 | db: 38 | image: mysql/mysql-server:8.0 39 | environment: 40 | <<: *x-env 41 | volumes: 42 | - database:/var/lib/mysql 43 | ports: 44 | - "${MYSQL_EXTERNAL_PORT:-3306}:3306" 45 | command: --default-authentication-plugin=mysql_native_password 46 | healthcheck: 47 | # important to use 127.0.0.1 instead of localhost as mysql starts twice. 48 | # the first time it listens on sockets but isn't actually ready 49 | # see https://github.com/docker-library/mysql/issues/663 50 | test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1"] 51 | interval: 1s 52 | timeout: 60s 53 | start_period: 60s 54 | 55 | redis: 56 | image: redis:latest 57 | ports: 58 | - "${REDIS_EXTERNAL_PORT:-6379}:6379" 59 | healthcheck: 60 | test: ["CMD", "redis-cli", "--raw", "incr", "ping"] 61 | interval: 1s 62 | timeout: 60s 63 | start_period: 60s 64 | 65 | volumes: 66 | database: 67 | -------------------------------------------------------------------------------- /global.json: -------------------------------------------------------------------------------- 1 | { 2 | "sdk": { 3 | "version": "8.0.100", 4 | "rollForward": "latestFeature", 5 | "allowPrerelease": false 6 | } 7 | } -------------------------------------------------------------------------------- /osu.Server.QueueProcessor.Tests/BatchProcessorTests.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. 2 | // See the LICENCE file in the repository root for full licence text. 3 | 4 | using System; 5 | using System.Collections.Generic; 6 | using System.Diagnostics.CodeAnalysis; 7 | using System.Threading; 8 | using System.Threading.Tasks; 9 | using Xunit; 10 | using Xunit.Abstractions; 11 | 12 | namespace osu.Server.QueueProcessor.Tests 13 | { 14 | public class BatchProcessorTests 15 | { 16 | private readonly ITestOutputHelper output; 17 | private readonly TestBatchProcessor processor; 18 | 19 | public BatchProcessorTests(ITestOutputHelper output) 20 | { 21 | this.output = output; 22 | 23 | processor = new TestBatchProcessor(); 24 | processor.ClearQueue(); 25 | } 26 | 27 | /// 28 | /// Checking that processing an empty queue works as expected. 29 | /// 30 | [Fact] 31 | public void ProcessEmptyQueue() 32 | { 33 | processor.Run(new CancellationTokenSource(1000).Token); 34 | } 35 | 36 | [Fact] 37 | public void SendThenReceive_Single() 38 | { 39 | var cts = new CancellationTokenSource(10000); 40 | 41 | var obj = FakeData.New(); 42 | 43 | FakeData? receivedObject = null; 44 | 45 | processor.PushToQueue(obj); 46 | 47 | processor.Received += o => 48 | { 49 | receivedObject = o; 50 | cts.Cancel(); 51 | }; 52 | 53 | processor.Run(cts.Token); 54 | 55 | Assert.Equal(obj, receivedObject); 56 | } 57 | 58 | [Fact] 59 | public void SendThenReceive_Multiple() 60 | { 61 | const int send_count = 20; 62 | 63 | var cts = new CancellationTokenSource(10000); 64 | 65 | var objects = new HashSet(); 66 | for (int i = 0; i < send_count; i++) 67 | objects.Add(FakeData.New()); 68 | 69 | var receivedObjects = new HashSet(); 70 | 71 | foreach (var obj in objects) 72 | processor.PushToQueue(obj); 73 | 74 | processor.Received += o => 75 | { 76 | lock (receivedObjects) 77 | receivedObjects.Add(o); 78 | 79 | if (receivedObjects.Count == send_count) 80 | cts.Cancel(); 81 | }; 82 | 83 | processor.Run(cts.Token); 84 | 85 | Assert.Equal(objects, receivedObjects); 86 | } 87 | 88 | /// 89 | /// If the processor is cancelled mid-operation, every item should either be processed or still in the queue. 90 | /// 91 | [Fact] 92 | [SuppressMessage("Usage", "xUnit1031:Do not use blocking task operations in test method")] // For simplicity. 93 | public void EnsureCancellingDoesNotLoseItems() 94 | { 95 | var inFlightObjects = new List(); 96 | 97 | int processed = 0; 98 | int sent = 0; 99 | 100 | processor.Received += o => 101 | { 102 | lock (inFlightObjects) 103 | { 104 | inFlightObjects.Remove(o); 105 | Interlocked.Increment(ref processed); 106 | } 107 | }; 108 | 109 | const int run_count = 5; 110 | 111 | // start and stop processing multiple times, checking items are in a good state each time. 112 | 113 | for (int i = 0; i < run_count; i++) 114 | { 115 | var cts = new CancellationTokenSource(); 116 | 117 | var sendTask = Task.Run(() => 118 | { 119 | while (!cts.IsCancellationRequested) 120 | { 121 | var obj = FakeData.New(); 122 | 123 | lock (inFlightObjects) 124 | { 125 | processor.PushToQueue(obj); 126 | inFlightObjects.Add(obj); 127 | } 128 | 129 | Interlocked.Increment(ref sent); 130 | } 131 | }, CancellationToken.None); 132 | 133 | // Ensure there are some items in the queue before starting the processor. 134 | while (inFlightObjects.Count < 1000) 135 | Thread.Sleep(100); 136 | 137 | var receiveTask = Task.Run(() => processor.Run(cts.Token), CancellationToken.None); 138 | 139 | Thread.Sleep(1000); 140 | 141 | cts.Cancel(); 142 | 143 | sendTask.Wait(10000); 144 | receiveTask.Wait(10000); 145 | 146 | output.WriteLine($"Sent: {sent} In-flight: {inFlightObjects.Count} Processed: {processed}"); 147 | } 148 | 149 | var finalCts = new CancellationTokenSource(10000); 150 | 151 | processor.Received += _ => 152 | { 153 | if (inFlightObjects.Count == 0) 154 | // early cancel once the list is emptied. 155 | finalCts.Cancel(); 156 | }; 157 | 158 | // process all remaining items 159 | processor.Run(finalCts.Token); 160 | 161 | Assert.Empty(inFlightObjects); 162 | Assert.Equal(0, processor.GetQueueSize()); 163 | 164 | output.WriteLine($"Sent: {sent} In-flight: {inFlightObjects.Count} Processed: {processed}"); 165 | } 166 | 167 | [Fact] 168 | public void SendThenErrorDoesRetry() 169 | { 170 | var cts = new CancellationTokenSource(10000); 171 | 172 | var obj = FakeData.New(); 173 | 174 | FakeData? receivedObject = null; 175 | 176 | bool didThrowOnce = false; 177 | 178 | processor.PushToQueue(obj); 179 | 180 | processor.Received += o => 181 | { 182 | if (o.TotalRetries == 0) 183 | { 184 | didThrowOnce = true; 185 | throw new Exception(); 186 | } 187 | 188 | receivedObject = o; 189 | cts.Cancel(); 190 | }; 191 | 192 | processor.Run(cts.Token); 193 | 194 | Assert.True(didThrowOnce); 195 | Assert.Equal(obj, receivedObject); 196 | } 197 | 198 | [Fact] 199 | public void MultipleErrorsAttachedToCorrectItems() 200 | { 201 | var cts = new CancellationTokenSource(10000); 202 | 203 | var obj1 = FakeData.New(); 204 | var obj2 = FakeData.New(); 205 | 206 | bool gotCorrectExceptionForItem1 = false; 207 | bool gotCorrectExceptionForItem2 = false; 208 | 209 | processor.Error += (exception, item) => 210 | { 211 | Assert.NotNull(exception); 212 | Assert.Equal(exception, item.Exception); 213 | 214 | gotCorrectExceptionForItem1 |= Equals(item.Data, obj1.Data) && exception.Message == "1"; 215 | gotCorrectExceptionForItem2 |= Equals(item.Data, obj2.Data) && exception.Message == "2"; 216 | }; 217 | 218 | processor.PushToQueue(new[] { obj1, obj2 }); 219 | 220 | processor.Received += o => 221 | { 222 | if (Equals(o.Data, obj1.Data)) throw new Exception("1"); 223 | if (Equals(o.Data, obj2.Data)) throw new Exception("2"); 224 | }; 225 | 226 | processor.Run(cts.Token); 227 | 228 | Assert.Equal(0, processor.GetQueueSize()); 229 | Assert.True(gotCorrectExceptionForItem1); 230 | Assert.True(gotCorrectExceptionForItem2); 231 | } 232 | 233 | [Fact] 234 | public void SendThenErrorForeverDoesDrop() 235 | { 236 | var cts = new CancellationTokenSource(10000); 237 | 238 | var obj = FakeData.New(); 239 | 240 | int attemptCount = 0; 241 | 242 | processor.PushToQueue(obj); 243 | 244 | processor.Received += o => 245 | { 246 | attemptCount++; 247 | if (attemptCount > 3) 248 | cts.Cancel(); 249 | 250 | throw new Exception(); 251 | }; 252 | 253 | processor.Run(cts.Token); 254 | 255 | Assert.Equal(4, attemptCount); 256 | Assert.Equal(0, processor.GetQueueSize()); 257 | } 258 | 259 | [Fact] 260 | public void ExitOnErrorThresholdHit() 261 | { 262 | var cts = new CancellationTokenSource(10000); 263 | 264 | int attemptCount = 0; 265 | 266 | // 3 retries for each, so at least one should remain in queue. 267 | processor.PushToQueue(FakeData.New()); 268 | processor.PushToQueue(FakeData.New()); 269 | processor.PushToQueue(FakeData.New()); 270 | processor.PushToQueue(FakeData.New()); 271 | 272 | processor.Received += o => 273 | { 274 | o.Failed = true; 275 | attemptCount++; 276 | }; 277 | 278 | Assert.Throws(() => processor.Run(cts.Token)); 279 | 280 | Assert.True(attemptCount >= 10, "attemptCount >= 10"); 281 | Assert.NotEqual(0, processor.GetQueueSize()); 282 | } 283 | } 284 | } 285 | -------------------------------------------------------------------------------- /osu.Server.QueueProcessor.Tests/BeatmapStatusWatcherTests.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. 2 | // See the LICENCE file in the repository root for full licence text. 3 | 4 | using System; 5 | using System.Threading; 6 | using System.Threading.Tasks; 7 | using Dapper; 8 | using Xunit; 9 | 10 | namespace osu.Server.QueueProcessor.Tests 11 | { 12 | public class BeatmapStatusWatcherTests 13 | { 14 | /// 15 | /// Checking that processing an empty queue works as expected. 16 | /// 17 | [Fact] 18 | public async Task TestBasic() 19 | { 20 | var cts = new CancellationTokenSource(10000); 21 | 22 | TaskCompletionSource tcs = new TaskCompletionSource(); 23 | using var db = await DatabaseAccess.GetConnectionAsync(cts.Token); 24 | 25 | // just a safety measure for now to ensure we don't hit production. since i was running on production until now. 26 | // will throw if not on test database. 27 | if (db.QueryFirstOrDefault("SELECT `count` FROM `osu_counts` WHERE `name` = 'is_production'") != null) 28 | throw new InvalidOperationException("You are trying to do something very silly."); 29 | 30 | await db.ExecuteAsync("TRUNCATE TABLE `bss_process_queue`"); 31 | 32 | using var poller = await BeatmapStatusWatcher.StartPollingAsync(updates => { tcs.SetResult(updates); }, pollMilliseconds: 100); 33 | 34 | await db.ExecuteAsync("INSERT INTO `bss_process_queue` (beatmapset_id) VALUES (1)"); 35 | 36 | var updates = await tcs.Task.WaitAsync(cts.Token); 37 | 38 | Assert.Equal(new[] { 1 }, updates.BeatmapSetIDs); 39 | Assert.Equal(1, updates.LastProcessedQueueID); 40 | 41 | tcs = new TaskCompletionSource(); 42 | 43 | await db.ExecuteAsync("INSERT INTO `bss_process_queue` (beatmapset_id) VALUES (2), (3)"); 44 | 45 | updates = await tcs.Task.WaitAsync(cts.Token); 46 | 47 | Assert.Equal(new[] { 2, 3 }, updates.BeatmapSetIDs); 48 | Assert.Equal(3, updates.LastProcessedQueueID); 49 | } 50 | 51 | /// 52 | /// Checking that processing an empty queue works as expected. 53 | /// 54 | [Fact] 55 | public async Task TestLimit() 56 | { 57 | var cts = new CancellationTokenSource(10000); 58 | 59 | TaskCompletionSource tcs = new TaskCompletionSource(); 60 | using var db = await DatabaseAccess.GetConnectionAsync(cts.Token); 61 | 62 | // just a safety measure for now to ensure we don't hit production. since i was running on production until now. 63 | // will throw if not on test database. 64 | if (db.QueryFirstOrDefault("SELECT `count` FROM `osu_counts` WHERE `name` = 'is_production'") != null) 65 | throw new InvalidOperationException("You are trying to do something very silly."); 66 | 67 | await db.ExecuteAsync("TRUNCATE TABLE `bss_process_queue`"); 68 | 69 | using var poller = await BeatmapStatusWatcher.StartPollingAsync(updates => { tcs.SetResult(updates); }, limit: 1, pollMilliseconds: 100); 70 | 71 | await db.ExecuteAsync("INSERT INTO `bss_process_queue` (beatmapset_id) VALUES (1)"); 72 | 73 | var updates = await tcs.Task.WaitAsync(cts.Token); 74 | tcs = new TaskCompletionSource(); 75 | 76 | Assert.Equal(new[] { 1 }, updates.BeatmapSetIDs); 77 | Assert.Equal(1, updates.LastProcessedQueueID); 78 | 79 | await db.ExecuteAsync("INSERT INTO `bss_process_queue` (beatmapset_id) VALUES (2), (3)"); 80 | 81 | updates = await tcs.Task.WaitAsync(cts.Token); 82 | tcs = new TaskCompletionSource(); 83 | 84 | Assert.Equal(new[] { 2 }, updates.BeatmapSetIDs); 85 | Assert.Equal(2, updates.LastProcessedQueueID); 86 | 87 | updates = await tcs.Task.WaitAsync(cts.Token); 88 | 89 | Assert.Equal(new[] { 3 }, updates.BeatmapSetIDs); 90 | Assert.Equal(3, updates.LastProcessedQueueID); 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /osu.Server.QueueProcessor.Tests/FakeData.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace osu.Server.QueueProcessor.Tests 4 | { 5 | public class FakeData : QueueItem, IEquatable 6 | { 7 | public readonly Guid Data; 8 | 9 | public FakeData(Guid data) 10 | { 11 | Data = data; 12 | } 13 | 14 | public static FakeData New() => new FakeData(Guid.NewGuid()); 15 | 16 | public override string ToString() => Data.ToString(); 17 | 18 | public bool Equals(FakeData? other) 19 | { 20 | if (ReferenceEquals(null, other)) return false; 21 | if (ReferenceEquals(this, other)) return true; 22 | 23 | return Data.Equals(other.Data); 24 | } 25 | 26 | public override bool Equals(object? obj) 27 | { 28 | if (ReferenceEquals(null, obj)) return false; 29 | if (ReferenceEquals(this, obj)) return true; 30 | if (obj.GetType() != this.GetType()) return false; 31 | 32 | return Equals((FakeData)obj); 33 | } 34 | 35 | public override int GetHashCode() 36 | { 37 | return Data.GetHashCode(); 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /osu.Server.QueueProcessor.Tests/InputOnlyQueueTests.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Diagnostics.CodeAnalysis; 4 | using System.Threading; 5 | using System.Threading.Tasks; 6 | using Xunit; 7 | using Xunit.Abstractions; 8 | using Assert = Xunit.Assert; 9 | 10 | namespace osu.Server.QueueProcessor.Tests 11 | { 12 | public class InputOnlyQueueTests 13 | { 14 | private readonly ITestOutputHelper output; 15 | private readonly TestProcessor processor; 16 | 17 | public InputOnlyQueueTests(ITestOutputHelper output) 18 | { 19 | this.output = output; 20 | 21 | processor = new TestProcessor(); 22 | processor.ClearQueue(); 23 | } 24 | 25 | [Fact] 26 | public void SendThenReceive_Single() 27 | { 28 | var cts = new CancellationTokenSource(10000); 29 | 30 | var obj = FakeData.New(); 31 | 32 | FakeData? receivedObject = null; 33 | 34 | processor.PushToQueue(obj); 35 | 36 | processor.Received += o => 37 | { 38 | receivedObject = o; 39 | cts.Cancel(); 40 | }; 41 | 42 | processor.Run(cts.Token); 43 | 44 | Assert.Equal(obj, receivedObject); 45 | } 46 | 47 | [Fact] 48 | public void SendThenReceive_Multiple() 49 | { 50 | const int send_count = 20; 51 | 52 | var cts = new CancellationTokenSource(10000); 53 | 54 | var objects = new HashSet(); 55 | for (int i = 0; i < send_count; i++) 56 | objects.Add(FakeData.New()); 57 | 58 | var receivedObjects = new HashSet(); 59 | 60 | foreach (var obj in objects) 61 | processor.PushToQueue(obj); 62 | 63 | processor.Received += o => 64 | { 65 | lock (receivedObjects) 66 | receivedObjects.Add(o); 67 | 68 | if (receivedObjects.Count == send_count) 69 | cts.Cancel(); 70 | }; 71 | 72 | processor.Run(cts.Token); 73 | 74 | Assert.Equal(objects, receivedObjects); 75 | } 76 | 77 | [Fact] 78 | public void SendThenReceive_MultipleUsingSingleCall() 79 | { 80 | const int send_count = 10000; 81 | 82 | var cts = new CancellationTokenSource(10000); 83 | 84 | var objects = new HashSet(); 85 | for (int i = 0; i < send_count; i++) 86 | objects.Add(FakeData.New()); 87 | 88 | var receivedObjects = new HashSet(); 89 | 90 | processor.PushToQueue(objects); 91 | 92 | processor.Received += o => 93 | { 94 | lock (receivedObjects) 95 | receivedObjects.Add(o); 96 | 97 | if (receivedObjects.Count == send_count) 98 | cts.Cancel(); 99 | }; 100 | 101 | processor.Run(cts.Token); 102 | 103 | Assert.Equal(objects, receivedObjects); 104 | } 105 | 106 | /// 107 | /// If the processor is cancelled mid-operation, every item should either be processed or still in the queue. 108 | /// 109 | [Fact] 110 | [SuppressMessage("Usage", "xUnit1031:Do not use blocking task operations in test method")] // For simplicity. 111 | public void EnsureCancellingDoesNotLoseItems() 112 | { 113 | var inFlightObjects = new List(); 114 | 115 | int processed = 0; 116 | int sent = 0; 117 | 118 | processor.Received += o => 119 | { 120 | lock (inFlightObjects) 121 | { 122 | inFlightObjects.Remove(o); 123 | Interlocked.Increment(ref processed); 124 | } 125 | }; 126 | 127 | const int run_count = 5; 128 | 129 | // start and stop processing multiple times, checking items are in a good state each time. 130 | 131 | for (int i = 0; i < run_count; i++) 132 | { 133 | var cts = new CancellationTokenSource(); 134 | 135 | var sendTask = Task.Run(() => 136 | { 137 | while (!cts.IsCancellationRequested) 138 | { 139 | var obj = FakeData.New(); 140 | 141 | lock (inFlightObjects) 142 | { 143 | processor.PushToQueue(obj); 144 | inFlightObjects.Add(obj); 145 | } 146 | 147 | Interlocked.Increment(ref sent); 148 | } 149 | }, CancellationToken.None); 150 | 151 | // Ensure there are some items in the queue before starting the processor. 152 | while (inFlightObjects.Count < 1000) 153 | Thread.Sleep(100); 154 | 155 | var receiveTask = Task.Run(() => processor.Run(cts.Token), CancellationToken.None); 156 | 157 | Thread.Sleep(1000); 158 | 159 | cts.Cancel(); 160 | 161 | sendTask.Wait(10000); 162 | receiveTask.Wait(10000); 163 | 164 | output.WriteLine($"Sent: {sent} In-flight: {inFlightObjects.Count} Processed: {processed}"); 165 | } 166 | 167 | var finalCts = new CancellationTokenSource(10000); 168 | 169 | processor.Received += _ => 170 | { 171 | if (inFlightObjects.Count == 0) 172 | // early cancel once the list is emptied. 173 | finalCts.Cancel(); 174 | }; 175 | 176 | // process all remaining items 177 | processor.Run(finalCts.Token); 178 | 179 | Assert.Empty(inFlightObjects); 180 | Assert.Equal(0, processor.GetQueueSize()); 181 | 182 | output.WriteLine($"Sent: {sent} In-flight: {inFlightObjects.Count} Processed: {processed}"); 183 | } 184 | 185 | [Fact] 186 | public void SendThenErrorDoesRetry() 187 | { 188 | var cts = new CancellationTokenSource(10000); 189 | 190 | var obj = FakeData.New(); 191 | 192 | FakeData? receivedObject = null; 193 | 194 | bool didThrowOnce = false; 195 | 196 | processor.PushToQueue(obj); 197 | 198 | processor.Received += o => 199 | { 200 | if (o.TotalRetries == 0) 201 | { 202 | didThrowOnce = true; 203 | throw new Exception(); 204 | } 205 | 206 | receivedObject = o; 207 | cts.Cancel(); 208 | }; 209 | 210 | processor.Run(cts.Token); 211 | 212 | Assert.True(didThrowOnce); 213 | Assert.Equal(obj, receivedObject); 214 | } 215 | 216 | [Fact] 217 | public void SendThenErrorForeverDoesDrop() 218 | { 219 | var cts = new CancellationTokenSource(10000); 220 | 221 | var obj = FakeData.New(); 222 | 223 | int attemptCount = 0; 224 | 225 | processor.PushToQueue(obj); 226 | 227 | processor.Received += o => 228 | { 229 | attemptCount++; 230 | if (attemptCount > 3) 231 | cts.Cancel(); 232 | 233 | throw new Exception(); 234 | }; 235 | 236 | processor.Run(cts.Token); 237 | 238 | Assert.Equal(4, attemptCount); 239 | Assert.Equal(0, processor.GetQueueSize()); 240 | } 241 | 242 | [Fact] 243 | public void ExitOnErrorThresholdHit() 244 | { 245 | var cts = new CancellationTokenSource(10000); 246 | 247 | int attemptCount = 0; 248 | 249 | // 3 retries for each, so at least one should remain in queue. 250 | processor.PushToQueue(FakeData.New()); 251 | processor.PushToQueue(FakeData.New()); 252 | processor.PushToQueue(FakeData.New()); 253 | processor.PushToQueue(FakeData.New()); 254 | 255 | processor.Received += o => 256 | { 257 | attemptCount++; 258 | throw new Exception(); 259 | }; 260 | 261 | Assert.Throws(() => processor.Run(cts.Token)); 262 | 263 | Assert.True(attemptCount >= 10, "attemptCount >= 10"); 264 | Assert.NotEqual(0, processor.GetQueueSize()); 265 | } 266 | } 267 | } 268 | -------------------------------------------------------------------------------- /osu.Server.QueueProcessor.Tests/TestBatchProcessor.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. 2 | // See the LICENCE file in the repository root for full licence text. 3 | 4 | using System; 5 | using System.Collections.Generic; 6 | 7 | namespace osu.Server.QueueProcessor.Tests 8 | { 9 | public class TestBatchProcessor : QueueProcessor 10 | { 11 | public TestBatchProcessor() 12 | : base(new QueueConfiguration 13 | { 14 | InputQueueName = "test-batch", 15 | BatchSize = 5, 16 | }) 17 | { 18 | } 19 | 20 | protected override void ProcessResults(IEnumerable items) 21 | { 22 | foreach (var item in items) 23 | { 24 | try 25 | { 26 | Received?.Invoke(item); 27 | } 28 | catch (Exception e) 29 | { 30 | item.Exception = e; 31 | } 32 | } 33 | } 34 | 35 | public Action? Received; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /osu.Server.QueueProcessor.Tests/TestProcessor.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace osu.Server.QueueProcessor.Tests 4 | { 5 | public class TestProcessor : QueueProcessor 6 | { 7 | public TestProcessor() 8 | : base(new QueueConfiguration 9 | { 10 | InputQueueName = "test" 11 | }) 12 | { 13 | } 14 | 15 | protected override void ProcessResult(FakeData result) => Received?.Invoke(result); 16 | 17 | public Action? Received; 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /osu.Server.QueueProcessor.Tests/osu.Server.QueueProcessor.Tests.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net8.0 5 | enable 6 | false 7 | 8 | 9 | 10 | 11 | 12 | 13 | all 14 | runtime; build; native; contentfiles; analyzers; buildtransitive 15 | 16 | 17 | all 18 | runtime; build; native; contentfiles; analyzers; buildtransitive 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /osu.Server.QueueProcessor.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio Version 16 4 | VisualStudioVersion = 16.0.30804.86 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "osu.Server.QueueProcessor", "osu.Server.QueueProcessor\osu.Server.QueueProcessor.csproj", "{2601BC23-DAD3-46CA-861B-5D2386631939}" 7 | EndProject 8 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "osu.Server.QueueProcessor.Tests", "osu.Server.QueueProcessor.Tests\osu.Server.QueueProcessor.Tests.csproj", "{844442F7-C058-42A6-9217-433D9663C9F6}" 9 | EndProject 10 | Global 11 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 12 | Debug|Any CPU = Debug|Any CPU 13 | Release|Any CPU = Release|Any CPU 14 | EndGlobalSection 15 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 16 | {2601BC23-DAD3-46CA-861B-5D2386631939}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 17 | {2601BC23-DAD3-46CA-861B-5D2386631939}.Debug|Any CPU.Build.0 = Debug|Any CPU 18 | {2601BC23-DAD3-46CA-861B-5D2386631939}.Release|Any CPU.ActiveCfg = Release|Any CPU 19 | {2601BC23-DAD3-46CA-861B-5D2386631939}.Release|Any CPU.Build.0 = Release|Any CPU 20 | {844442F7-C058-42A6-9217-433D9663C9F6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 21 | {844442F7-C058-42A6-9217-433D9663C9F6}.Debug|Any CPU.Build.0 = Debug|Any CPU 22 | {844442F7-C058-42A6-9217-433D9663C9F6}.Release|Any CPU.ActiveCfg = Release|Any CPU 23 | {844442F7-C058-42A6-9217-433D9663C9F6}.Release|Any CPU.Build.0 = Release|Any CPU 24 | EndGlobalSection 25 | GlobalSection(SolutionProperties) = preSolution 26 | HideSolutionNode = FALSE 27 | EndGlobalSection 28 | GlobalSection(ExtensibilityGlobals) = postSolution 29 | SolutionGuid = {5085A430-FAF4-4FFF-86C2-119794B67075} 30 | EndGlobalSection 31 | EndGlobal 32 | -------------------------------------------------------------------------------- /osu.Server.QueueProcessor.sln.DotSettings: -------------------------------------------------------------------------------- 1 |  2 | True 3 | True 4 | True 5 | True 6 | ExplicitlyExcluded 7 | ExplicitlyExcluded 8 | g_*.cs 9 | SOLUTION 10 | WARNING 11 | WARNING 12 | WARNING 13 | WARNING 14 | WARNING 15 | HINT 16 | HINT 17 | WARNING 18 | WARNING 19 | WARNING 20 | WARNING 21 | WARNING 22 | True 23 | WARNING 24 | WARNING 25 | HINT 26 | DO_NOT_SHOW 27 | WARNING 28 | WARNING 29 | HINT 30 | HINT 31 | WARNING 32 | WARNING 33 | WARNING 34 | WARNING 35 | WARNING 36 | WARNING 37 | WARNING 38 | WARNING 39 | WARNING 40 | WARNING 41 | WARNING 42 | WARNING 43 | WARNING 44 | WARNING 45 | WARNING 46 | WARNING 47 | WARNING 48 | WARNING 49 | WARNING 50 | WARNING 51 | WARNING 52 | WARNING 53 | WARNING 54 | WARNING 55 | WARNING 56 | WARNING 57 | WARNING 58 | WARNING 59 | WARNING 60 | HINT 61 | WARNING 62 | HINT 63 | SUGGESTION 64 | HINT 65 | HINT 66 | HINT 67 | WARNING 68 | DO_NOT_SHOW 69 | WARNING 70 | WARNING 71 | WARNING 72 | WARNING 73 | DO_NOT_SHOW 74 | WARNING 75 | WARNING 76 | WARNING 77 | DO_NOT_SHOW 78 | WARNING 79 | WARNING 80 | WARNING 81 | WARNING 82 | WARNING 83 | HINT 84 | WARNING 85 | HINT 86 | DO_NOT_SHOW 87 | WARNING 88 | DO_NOT_SHOW 89 | WARNING 90 | WARNING 91 | WARNING 92 | WARNING 93 | WARNING 94 | WARNING 95 | WARNING 96 | WARNING 97 | HINT 98 | WARNING 99 | DO_NOT_SHOW 100 | WARNING 101 | HINT 102 | DO_NOT_SHOW 103 | HINT 104 | HINT 105 | HINT 106 | ERROR 107 | WARNING 108 | HINT 109 | HINT 110 | HINT 111 | WARNING 112 | WARNING 113 | HINT 114 | DO_NOT_SHOW 115 | HINT 116 | HINT 117 | HINT 118 | HINT 119 | WARNING 120 | DO_NOT_SHOW 121 | DO_NOT_SHOW 122 | WARNING 123 | WARNING 124 | WARNING 125 | WARNING 126 | WARNING 127 | WARNING 128 | WARNING 129 | WARNING 130 | WARNING 131 | HINT 132 | WARNING 133 | WARNING 134 | WARNING 135 | WARNING 136 | HINT 137 | HINT 138 | WARNING 139 | HINT 140 | HINT 141 | HINT 142 | HINT 143 | HINT 144 | HINT 145 | HINT 146 | DO_NOT_SHOW 147 | HINT 148 | HINT 149 | WARNING 150 | HINT 151 | HINT 152 | WARNING 153 | HINT 154 | WARNING 155 | WARNING 156 | WARNING 157 | WARNING 158 | HINT 159 | DO_NOT_SHOW 160 | DO_NOT_SHOW 161 | DO_NOT_SHOW 162 | WARNING 163 | WARNING 164 | WARNING 165 | WARNING 166 | WARNING 167 | WARNING 168 | WARNING 169 | WARNING 170 | WARNING 171 | ERROR 172 | WARNING 173 | WARNING 174 | DO_NOT_SHOW 175 | WARNING 176 | WARNING 177 | WARNING 178 | WARNING 179 | WARNING 180 | WARNING 181 | WARNING 182 | WARNING 183 | WARNING 184 | WARNING 185 | WARNING 186 | WARNING 187 | WARNING 188 | WARNING 189 | WARNING 190 | WARNING 191 | WARNING 192 | WARNING 193 | WARNING 194 | WARNING 195 | WARNING 196 | WARNING 197 | WARNING 198 | WARNING 199 | WARNING 200 | WARNING 201 | WARNING 202 | WARNING 203 | WARNING 204 | WARNING 205 | WARNING 206 | WARNING 207 | WARNING 208 | WARNING 209 | WARNING 210 | WARNING 211 | WARNING 212 | WARNING 213 | WARNING 214 | WARNING 215 | WARNING 216 | WARNING 217 | WARNING 218 | WARNING 219 | WARNING 220 | WARNING 221 | WARNING 222 | WARNING 223 | WARNING 224 | WARNING 225 | WARNING 226 | WARNING 227 | HINT 228 | WARNING 229 | WARNING 230 | SUGGESTION 231 | DO_NOT_SHOW 232 | 233 | True 234 | DO_NOT_SHOW 235 | WARNING 236 | WARNING 237 | WARNING 238 | WARNING 239 | WARNING 240 | HINT 241 | WARNING 242 | HINT 243 | HINT 244 | HINT 245 | HINT 246 | HINT 247 | HINT 248 | HINT 249 | HINT 250 | HINT 251 | HINT 252 | DO_NOT_SHOW 253 | WARNING 254 | WARNING 255 | WARNING 256 | WARNING 257 | WARNING 258 | WARNING 259 | WARNING 260 | 261 | True 262 | WARNING 263 | WARNING 264 | WARNING 265 | WARNING 266 | WARNING 267 | HINT 268 | HINT 269 | WARNING 270 | WARNING 271 | <?xml version="1.0" encoding="utf-16"?><Profile name="Code Cleanup (peppy)"><CSArrangeThisQualifier>True</CSArrangeThisQualifier><CSUseVar><BehavourStyle>CAN_CHANGE_TO_EXPLICIT</BehavourStyle><LocalVariableStyle>ALWAYS_EXPLICIT</LocalVariableStyle><ForeachVariableStyle>ALWAYS_EXPLICIT</ForeachVariableStyle></CSUseVar><CSOptimizeUsings><OptimizeUsings>True</OptimizeUsings><EmbraceInRegion>False</EmbraceInRegion><RegionName></RegionName></CSOptimizeUsings><CSShortenReferences>True</CSShortenReferences><CSReformatCode>True</CSReformatCode><CSUpdateFileHeader>True</CSUpdateFileHeader><CSCodeStyleAttributes ArrangeTypeAccessModifier="False" ArrangeTypeMemberAccessModifier="False" SortModifiers="True" RemoveRedundantParentheses="True" AddMissingParentheses="False" ArrangeBraces="False" ArrangeAttributes="False" ArrangeArgumentsStyle="False" /><XAMLCollapseEmptyTags>False</XAMLCollapseEmptyTags><CSFixBuiltinTypeReferences>True</CSFixBuiltinTypeReferences><CSArrangeQualifiers>True</CSArrangeQualifiers></Profile> 272 | Code Cleanup (peppy) 273 | RequiredForMultiline 274 | RequiredForMultiline 275 | RequiredForMultiline 276 | RequiredForMultiline 277 | RequiredForMultiline 278 | RequiredForMultiline 279 | RequiredForMultiline 280 | RequiredForMultiline 281 | Explicit 282 | ExpressionBody 283 | BlockBody 284 | BlockScoped 285 | ExplicitlyTyped 286 | True 287 | NEXT_LINE 288 | True 289 | True 290 | True 291 | True 292 | True 293 | True 294 | True 295 | True 296 | NEXT_LINE 297 | 1 298 | 1 299 | NEXT_LINE 300 | MULTILINE 301 | True 302 | True 303 | True 304 | True 305 | NEXT_LINE 306 | 1 307 | 1 308 | True 309 | NEXT_LINE 310 | NEVER 311 | NEVER 312 | True 313 | False 314 | True 315 | NEVER 316 | False 317 | False 318 | True 319 | False 320 | False 321 | True 322 | True 323 | False 324 | False 325 | CHOP_IF_LONG 326 | True 327 | 200 328 | CHOP_IF_LONG 329 | UseExplicitType 330 | UseVarWhenEvident 331 | UseVarWhenEvident 332 | False 333 | False 334 | AABB 335 | API 336 | ARGB 337 | BPM 338 | EF 339 | FPS 340 | GC 341 | GL 342 | GLSL 343 | HID 344 | HSL 345 | HSPA 346 | HSV 347 | HTML 348 | HUD 349 | ID 350 | IL 351 | IOS 352 | IP 353 | IPC 354 | JIT 355 | LTRB 356 | MD5 357 | NS 358 | OS 359 | PM 360 | RGB 361 | RGBA 362 | RNG 363 | SDL 364 | SHA 365 | SRGB 366 | TK 367 | SS 368 | PP 369 | GMT 370 | QAT 371 | BNG 372 | UI 373 | False 374 | HINT 375 | <?xml version="1.0" encoding="utf-16"?> 376 | <Patterns xmlns="urn:schemas-jetbrains-com:member-reordering-patterns"> 377 | <TypePattern DisplayName="COM interfaces or structs"> 378 | <TypePattern.Match> 379 | <Or> 380 | <And> 381 | <Kind Is="Interface" /> 382 | <Or> 383 | <HasAttribute Name="System.Runtime.InteropServices.InterfaceTypeAttribute" /> 384 | <HasAttribute Name="System.Runtime.InteropServices.ComImport" /> 385 | </Or> 386 | </And> 387 | <Kind Is="Struct" /> 388 | </Or> 389 | </TypePattern.Match> 390 | </TypePattern> 391 | <TypePattern DisplayName="NUnit Test Fixtures" RemoveRegions="All"> 392 | <TypePattern.Match> 393 | <And> 394 | <Kind Is="Class" /> 395 | <HasAttribute Name="NUnit.Framework.TestFixtureAttribute" Inherited="True" /> 396 | </And> 397 | </TypePattern.Match> 398 | <Entry DisplayName="Setup/Teardown Methods"> 399 | <Entry.Match> 400 | <And> 401 | <Kind Is="Method" /> 402 | <Or> 403 | <HasAttribute Name="NUnit.Framework.SetUpAttribute" Inherited="True" /> 404 | <HasAttribute Name="NUnit.Framework.TearDownAttribute" Inherited="True" /> 405 | <HasAttribute Name="NUnit.Framework.FixtureSetUpAttribute" Inherited="True" /> 406 | <HasAttribute Name="NUnit.Framework.FixtureTearDownAttribute" Inherited="True" /> 407 | </Or> 408 | </And> 409 | </Entry.Match> 410 | </Entry> 411 | <Entry DisplayName="All other members" /> 412 | <Entry Priority="100" DisplayName="Test Methods"> 413 | <Entry.Match> 414 | <And> 415 | <Kind Is="Method" /> 416 | <HasAttribute Name="NUnit.Framework.TestAttribute" /> 417 | </And> 418 | </Entry.Match> 419 | <Entry.SortBy> 420 | <Name /> 421 | </Entry.SortBy> 422 | </Entry> 423 | </TypePattern> 424 | <TypePattern DisplayName="Default Pattern"> 425 | <Group DisplayName="Fields/Properties"> 426 | <Group DisplayName="Public Fields"> 427 | <Entry DisplayName="Constant Fields"> 428 | <Entry.Match> 429 | <And> 430 | <Access Is="Public" /> 431 | <Or> 432 | <Kind Is="Constant" /> 433 | <Readonly /> 434 | <And> 435 | <Static /> 436 | <Readonly /> 437 | </And> 438 | </Or> 439 | </And> 440 | </Entry.Match> 441 | </Entry> 442 | <Entry DisplayName="Static Fields"> 443 | <Entry.Match> 444 | <And> 445 | <Access Is="Public" /> 446 | <Static /> 447 | <Not> 448 | <Readonly /> 449 | </Not> 450 | <Kind Is="Field" /> 451 | </And> 452 | </Entry.Match> 453 | </Entry> 454 | <Entry DisplayName="Normal Fields"> 455 | <Entry.Match> 456 | <And> 457 | <Access Is="Public" /> 458 | <Not> 459 | <Or> 460 | <Static /> 461 | <Readonly /> 462 | </Or> 463 | </Not> 464 | <Kind Is="Field" /> 465 | </And> 466 | </Entry.Match> 467 | </Entry> 468 | </Group> 469 | <Entry DisplayName="Public Properties"> 470 | <Entry.Match> 471 | <And> 472 | <Access Is="Public" /> 473 | <Kind Is="Property" /> 474 | </And> 475 | </Entry.Match> 476 | </Entry> 477 | <Group DisplayName="Internal Fields"> 478 | <Entry DisplayName="Constant Fields"> 479 | <Entry.Match> 480 | <And> 481 | <Access Is="Internal" /> 482 | <Or> 483 | <Kind Is="Constant" /> 484 | <Readonly /> 485 | <And> 486 | <Static /> 487 | <Readonly /> 488 | </And> 489 | </Or> 490 | </And> 491 | </Entry.Match> 492 | </Entry> 493 | <Entry DisplayName="Static Fields"> 494 | <Entry.Match> 495 | <And> 496 | <Access Is="Internal" /> 497 | <Static /> 498 | <Not> 499 | <Readonly /> 500 | </Not> 501 | <Kind Is="Field" /> 502 | </And> 503 | </Entry.Match> 504 | </Entry> 505 | <Entry DisplayName="Normal Fields"> 506 | <Entry.Match> 507 | <And> 508 | <Access Is="Internal" /> 509 | <Not> 510 | <Or> 511 | <Static /> 512 | <Readonly /> 513 | </Or> 514 | </Not> 515 | <Kind Is="Field" /> 516 | </And> 517 | </Entry.Match> 518 | </Entry> 519 | </Group> 520 | <Entry DisplayName="Internal Properties"> 521 | <Entry.Match> 522 | <And> 523 | <Access Is="Internal" /> 524 | <Kind Is="Property" /> 525 | </And> 526 | </Entry.Match> 527 | </Entry> 528 | <Group DisplayName="Protected Fields"> 529 | <Entry DisplayName="Constant Fields"> 530 | <Entry.Match> 531 | <And> 532 | <Access Is="Protected" /> 533 | <Or> 534 | <Kind Is="Constant" /> 535 | <Readonly /> 536 | <And> 537 | <Static /> 538 | <Readonly /> 539 | </And> 540 | </Or> 541 | </And> 542 | </Entry.Match> 543 | </Entry> 544 | <Entry DisplayName="Static Fields"> 545 | <Entry.Match> 546 | <And> 547 | <Access Is="Protected" /> 548 | <Static /> 549 | <Not> 550 | <Readonly /> 551 | </Not> 552 | <Kind Is="Field" /> 553 | </And> 554 | </Entry.Match> 555 | </Entry> 556 | <Entry DisplayName="Normal Fields"> 557 | <Entry.Match> 558 | <And> 559 | <Access Is="Protected" /> 560 | <Not> 561 | <Or> 562 | <Static /> 563 | <Readonly /> 564 | </Or> 565 | </Not> 566 | <Kind Is="Field" /> 567 | </And> 568 | </Entry.Match> 569 | </Entry> 570 | </Group> 571 | <Entry DisplayName="Protected Properties"> 572 | <Entry.Match> 573 | <And> 574 | <Access Is="Protected" /> 575 | <Kind Is="Property" /> 576 | </And> 577 | </Entry.Match> 578 | </Entry> 579 | <Group DisplayName="Private Fields"> 580 | <Entry DisplayName="Constant Fields"> 581 | <Entry.Match> 582 | <And> 583 | <Access Is="Private" /> 584 | <Or> 585 | <Kind Is="Constant" /> 586 | <Readonly /> 587 | <And> 588 | <Static /> 589 | <Readonly /> 590 | </And> 591 | </Or> 592 | </And> 593 | </Entry.Match> 594 | </Entry> 595 | <Entry DisplayName="Static Fields"> 596 | <Entry.Match> 597 | <And> 598 | <Access Is="Private" /> 599 | <Static /> 600 | <Not> 601 | <Readonly /> 602 | </Not> 603 | <Kind Is="Field" /> 604 | </And> 605 | </Entry.Match> 606 | </Entry> 607 | <Entry DisplayName="Normal Fields"> 608 | <Entry.Match> 609 | <And> 610 | <Access Is="Private" /> 611 | <Not> 612 | <Or> 613 | <Static /> 614 | <Readonly /> 615 | </Or> 616 | </Not> 617 | <Kind Is="Field" /> 618 | </And> 619 | </Entry.Match> 620 | </Entry> 621 | </Group> 622 | <Entry DisplayName="Private Properties"> 623 | <Entry.Match> 624 | <And> 625 | <Access Is="Private" /> 626 | <Kind Is="Property" /> 627 | </And> 628 | </Entry.Match> 629 | </Entry> 630 | </Group> 631 | <Group DisplayName="Constructor/Destructor"> 632 | <Entry DisplayName="Ctor"> 633 | <Entry.Match> 634 | <Kind Is="Constructor" /> 635 | </Entry.Match> 636 | </Entry> 637 | <Region Name="Disposal"> 638 | <Entry DisplayName="Dtor"> 639 | <Entry.Match> 640 | <Kind Is="Destructor" /> 641 | </Entry.Match> 642 | </Entry> 643 | <Entry DisplayName="Dispose()"> 644 | <Entry.Match> 645 | <And> 646 | <Access Is="Public" /> 647 | <Kind Is="Method" /> 648 | <Name Is="Dispose" /> 649 | </And> 650 | </Entry.Match> 651 | </Entry> 652 | <Entry DisplayName="Dispose(true)"> 653 | <Entry.Match> 654 | <And> 655 | <Access Is="Protected" /> 656 | <Or> 657 | <Virtual /> 658 | <Override /> 659 | </Or> 660 | <Kind Is="Method" /> 661 | <Name Is="Dispose" /> 662 | </And> 663 | </Entry.Match> 664 | </Entry> 665 | </Region> 666 | </Group> 667 | <Group DisplayName="Methods"> 668 | <Group DisplayName="Public"> 669 | <Entry DisplayName="Static Methods"> 670 | <Entry.Match> 671 | <And> 672 | <Access Is="Public" /> 673 | <Static /> 674 | <Kind Is="Method" /> 675 | </And> 676 | </Entry.Match> 677 | </Entry> 678 | <Entry DisplayName="Methods"> 679 | <Entry.Match> 680 | <And> 681 | <Access Is="Public" /> 682 | <Not> 683 | <Static /> 684 | </Not> 685 | <Kind Is="Method" /> 686 | </And> 687 | </Entry.Match> 688 | </Entry> 689 | </Group> 690 | <Group DisplayName="Internal"> 691 | <Entry DisplayName="Static Methods"> 692 | <Entry.Match> 693 | <And> 694 | <Access Is="Internal" /> 695 | <Static /> 696 | <Kind Is="Method" /> 697 | </And> 698 | </Entry.Match> 699 | </Entry> 700 | <Entry DisplayName="Methods"> 701 | <Entry.Match> 702 | <And> 703 | <Access Is="Internal" /> 704 | <Not> 705 | <Static /> 706 | </Not> 707 | <Kind Is="Method" /> 708 | </And> 709 | </Entry.Match> 710 | </Entry> 711 | </Group> 712 | <Group DisplayName="Protected"> 713 | <Entry DisplayName="Static Methods"> 714 | <Entry.Match> 715 | <And> 716 | <Access Is="Protected" /> 717 | <Static /> 718 | <Kind Is="Method" /> 719 | </And> 720 | </Entry.Match> 721 | </Entry> 722 | <Entry DisplayName="Methods"> 723 | <Entry.Match> 724 | <And> 725 | <Access Is="Protected" /> 726 | <Not> 727 | <Static /> 728 | </Not> 729 | <Kind Is="Method" /> 730 | </And> 731 | </Entry.Match> 732 | </Entry> 733 | </Group> 734 | <Group DisplayName="Private"> 735 | <Entry DisplayName="Static Methods"> 736 | <Entry.Match> 737 | <And> 738 | <Access Is="Private" /> 739 | <Static /> 740 | <Kind Is="Method" /> 741 | </And> 742 | </Entry.Match> 743 | </Entry> 744 | <Entry DisplayName="Methods"> 745 | <Entry.Match> 746 | <And> 747 | <Access Is="Private" /> 748 | <Not> 749 | <Static /> 750 | </Not> 751 | <Kind Is="Method" /> 752 | </And> 753 | </Entry.Match> 754 | </Entry> 755 | </Group> 756 | </Group> 757 | </TypePattern> 758 | </Patterns> 759 | Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence. 760 | See the LICENCE file in the repository root for full licence text. 761 | 762 | <Policy Inspect="True" Prefix="" Suffix="" Style="AA_BB" /> 763 | <Policy Inspect="False" Prefix="" Suffix="" Style="AaBb" /> 764 | <Policy Inspect="True" Prefix="" Suffix="" Style="aa_bb" /> 765 | <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> 766 | <Policy Inspect="True" Prefix="" Suffix="" Style="aa_bb" /> 767 | <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb"><ExtraRule Prefix="_" Suffix="" Style="aaBb" /></Policy> 768 | <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> 769 | <Policy Inspect="True" Prefix="" Suffix="" Style="aa_bb" /> 770 | <Policy Inspect="True" Prefix="" Suffix="" Style="AA_BB" /> 771 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 772 | <Policy><Descriptor Staticness="Static" AccessRightKinds="Private" Description="Static readonly fields (private)"><ElementKinds><Kind Name="READONLY_FIELD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="aa_bb" /></Policy> 773 | <Policy><Descriptor Staticness="Any" AccessRightKinds="Private" Description="Constant fields (private)"><ElementKinds><Kind Name="CONSTANT_FIELD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="aa_bb" /></Policy> 774 | <Policy><Descriptor Staticness="Any" AccessRightKinds="Any" Description="Type parameters"><ElementKinds><Kind Name="TYPE_PARAMETER" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /></Policy> 775 | <Policy><Descriptor Staticness="Instance" AccessRightKinds="Private" Description="Instance fields (private)"><ElementKinds><Kind Name="FIELD" /><Kind Name="READONLY_FIELD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="aaBb"><ExtraRule Prefix="_" Suffix="" Style="aaBb" /></Policy></Policy> 776 | <Policy><Descriptor Staticness="Any" AccessRightKinds="Protected, ProtectedInternal, Internal, Public, PrivateProtected" Description="Constant fields (not private)"><ElementKinds><Kind Name="CONSTANT_FIELD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="AA_BB" /></Policy> 777 | <Policy><Descriptor Staticness="Any" AccessRightKinds="Any" Description="Local functions"><ElementKinds><Kind Name="LOCAL_FUNCTION" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /></Policy> 778 | <Policy><Descriptor Staticness="Any" AccessRightKinds="Any" Description="Enum members"><ElementKinds><Kind Name="ENUM_MEMBER" /></ElementKinds></Descriptor><Policy Inspect="False" Prefix="" Suffix="" Style="AaBb" /></Policy> 779 | <Policy><Descriptor Staticness="Static, Instance" AccessRightKinds="Private" Description="private methods"><ElementKinds><Kind Name="ASYNC_METHOD" /><Kind Name="METHOD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /></Policy> 780 | <Policy><Descriptor Staticness="Static, Instance" AccessRightKinds="Protected, ProtectedInternal, Internal, Public" Description="internal/protected/public methods"><ElementKinds><Kind Name="ASYNC_METHOD" /><Kind Name="METHOD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /></Policy> 781 | <Policy><Descriptor Staticness="Static, Instance" AccessRightKinds="Private" Description="private properties"><ElementKinds><Kind Name="PROPERTY" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /></Policy> 782 | <Policy><Descriptor Staticness="Any" AccessRightKinds="Any" Description="Local constants"><ElementKinds><Kind Name="LOCAL_CONSTANT" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="aa_bb" /></Policy> 783 | <Policy><Descriptor Staticness="Static" AccessRightKinds="Protected, ProtectedInternal, Internal, Public, PrivateProtected" Description="Static readonly fields (not private)"><ElementKinds><Kind Name="READONLY_FIELD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="AA_BB" /></Policy> 784 | <Policy><Descriptor Staticness="Static" AccessRightKinds="Private" Description="Static fields (private)"><ElementKinds><Kind Name="FIELD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /></Policy> 785 | <Policy><Descriptor Staticness="Static, Instance" AccessRightKinds="Protected, ProtectedInternal, Internal, Public" Description="internal/protected/public properties"><ElementKinds><Kind Name="PROPERTY" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /></Policy> 786 | <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> 787 | <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> 788 | <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> 789 | <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> 790 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 791 | <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> 792 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 793 | <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> 794 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 795 | <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> 796 | <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> 797 | <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> 798 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 799 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 800 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 801 | <Policy Inspect="True" Prefix="I" Suffix="" Style="AaBb" /> 802 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 803 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 804 | <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> 805 | <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> 806 | <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> 807 | <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> 808 | <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> 809 | <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> 810 | <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> 811 | <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> 812 | <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> 813 | <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> 814 | <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> 815 | <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> 816 | <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> 817 | <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> 818 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 819 | <Policy Inspect="True" Prefix="T" Suffix="" Style="AaBb" /> 820 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 821 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 822 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 823 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 824 | <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> 825 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 826 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 827 | 828 | True 829 | ||||||| 848eaa8 830 | True 831 | True 832 | True 833 | True 834 | True 835 | True 836 | True 837 | True 838 | True 839 | True 840 | True 841 | True 842 | True 843 | True 844 | True 845 | True 846 | True 847 | True 848 | True 849 | True 850 | True 851 | True 852 | True 853 | TestFolder 854 | True 855 | True 856 | o!f – Object Initializer: Anchor&Origin 857 | True 858 | constant("Centre") 859 | 0 860 | True 861 | True 862 | 2.0 863 | InCSharpFile 864 | ofao 865 | True 866 | Anchor = Anchor.$anchor$, 867 | Origin = Anchor.$anchor$, 868 | True 869 | True 870 | o!f – InternalChildren = [] 871 | True 872 | True 873 | 2.0 874 | InCSharpFile 875 | ofic 876 | True 877 | InternalChildren = new Drawable[] 878 | { 879 | $END$ 880 | }; 881 | True 882 | True 883 | o!f – new GridContainer { .. } 884 | True 885 | True 886 | 2.0 887 | InCSharpFile 888 | ofgc 889 | True 890 | new GridContainer 891 | { 892 | RelativeSizeAxes = Axes.Both, 893 | Content = new[] 894 | { 895 | new Drawable[] { $END$ }, 896 | new Drawable[] { } 897 | } 898 | }; 899 | True 900 | True 901 | o!f – new FillFlowContainer { .. } 902 | True 903 | True 904 | 2.0 905 | InCSharpFile 906 | offf 907 | True 908 | new FillFlowContainer 909 | { 910 | RelativeSizeAxes = Axes.Both, 911 | Direction = FillDirection.Vertical, 912 | Children = new Drawable[] 913 | { 914 | $END$ 915 | } 916 | }, 917 | True 918 | True 919 | o!f – new Container { .. } 920 | True 921 | True 922 | 2.0 923 | InCSharpFile 924 | ofcont 925 | True 926 | new Container 927 | { 928 | RelativeSizeAxes = Axes.Both, 929 | Children = new Drawable[] 930 | { 931 | $END$ 932 | } 933 | }, 934 | True 935 | True 936 | o!f – BackgroundDependencyLoader load() 937 | True 938 | True 939 | 2.0 940 | InCSharpFile 941 | ofbdl 942 | True 943 | [BackgroundDependencyLoader] 944 | private void load() 945 | { 946 | $END$ 947 | } 948 | True 949 | True 950 | o!f – new Box { .. } 951 | True 952 | True 953 | 2.0 954 | InCSharpFile 955 | ofbox 956 | True 957 | new Box 958 | { 959 | Colour = Color4.Black, 960 | RelativeSizeAxes = Axes.Both, 961 | }, 962 | True 963 | True 964 | o!f – Children = [] 965 | True 966 | True 967 | 2.0 968 | InCSharpFile 969 | ofc 970 | True 971 | Children = new Drawable[] 972 | { 973 | $END$ 974 | }; 975 | True 976 | True 977 | True 978 | True 979 | True 980 | True 981 | True 982 | True 983 | True 984 | True 985 | True 986 | True 987 | True 988 | True 989 | True 990 | True 991 | True 992 | True 993 | True 994 | True 995 | True 996 | True 997 | True 998 | True 999 | True 1000 | True 1001 | True 1002 | True 1003 | True 1004 | True 1005 | True 1006 | True 1007 | True 1008 | True 1009 | True 1010 | True 1011 | True 1012 | True 1013 | True 1014 | True 1015 | True 1016 | True 1017 | True 1018 | True 1019 | True 1020 | True 1021 | True 1022 | True 1023 | True 1024 | True 1025 | True 1026 | True 1027 | True 1028 | True 1029 | True 1030 | True 1031 | True 1032 | True 1033 | True 1034 | True 1035 | True 1036 | True 1037 | True 1038 | True 1039 | True 1040 | True 1041 | True 1042 | True 1043 | True 1044 | True 1045 | True 1046 | True 1047 | True 1048 | True 1049 | True 1050 | True 1051 | True 1052 | True 1053 | True 1054 | -------------------------------------------------------------------------------- /osu.Server.QueueProcessor/BeatmapStatusWatcher.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. 2 | // See the LICENCE file in the repository root for full licence text. 3 | 4 | using System; 5 | using System.Linq; 6 | using System.Threading; 7 | using System.Threading.Tasks; 8 | using Dapper; 9 | using MySqlConnector; 10 | 11 | namespace osu.Server.QueueProcessor 12 | { 13 | /// 14 | /// Provides insight into whenever a beatmap has changed status based on a user or system update. 15 | /// 16 | public static class BeatmapStatusWatcher 17 | { 18 | /// 19 | /// Start a background task which will poll for beatmap sets with updates. 20 | /// 21 | /// 22 | /// The general flow of usage should be: 23 | /// 24 | /// // before doing anything else, start polling. 25 | /// // it's important to await the completion of this operation to ensure no updates are missed. 26 | /// using var pollingOperation = await StartPollingAsync(updates, callback); 27 | /// 28 | /// void callback(BeatmapUpdates u) 29 | /// { 30 | /// foreach (int beatmapSetId in u.BeatmapSetIDs) 31 | /// { 32 | /// // invalidate anything related to `beatmapSetId` 33 | /// } 34 | /// } 35 | /// 36 | /// A callback to receive information about any updated beatmap sets. 37 | /// The number of milliseconds to wait between polls. Starts counting from response of previous poll. 38 | /// The maximum number of beatmap sets to return in a single response. 39 | /// An that should be disposed of to stop polling. 40 | public static async Task StartPollingAsync(Action callback, int pollMilliseconds = 10000, int limit = 50) 41 | { 42 | var initialUpdates = await GetUpdatedBeatmapSetsAsync(limit: limit); 43 | return new PollingBeatmapStatusWatcher(initialUpdates.LastProcessedQueueID, callback, pollMilliseconds, limit); 44 | } 45 | 46 | /// 47 | /// Check for any beatmap sets with updates since the provided queue ID. 48 | /// Should be called on a regular basis. See for automatic polling after the first call. 49 | /// 50 | /// The last checked queue ID, ie . 51 | /// The maximum number of beatmap sets to return in a single response. 52 | /// A response containing information about any updated beatmap sets. 53 | public static async Task GetUpdatedBeatmapSetsAsync(int? lastQueueId = null, int limit = 50) 54 | { 55 | using MySqlConnection connection = await DatabaseAccess.GetConnectionAsync(); 56 | 57 | if (lastQueueId.HasValue) 58 | { 59 | var items = (await connection.QueryAsync("SELECT * FROM bss_process_queue WHERE queue_id > @lastQueueId ORDER BY queue_id LIMIT @limit", new 60 | { 61 | lastQueueId, 62 | limit 63 | })).ToArray(); 64 | 65 | return new BeatmapUpdates 66 | { 67 | BeatmapSetIDs = items.Select(i => i.beatmapset_id).ToArray(), 68 | LastProcessedQueueID = items.LastOrDefault()?.queue_id ?? lastQueueId.Value 69 | }; 70 | } 71 | 72 | var lastEntry = await connection.QueryFirstOrDefaultAsync("SELECT * FROM bss_process_queue ORDER BY queue_id DESC LIMIT 1"); 73 | 74 | return new BeatmapUpdates 75 | { 76 | BeatmapSetIDs = [], 77 | LastProcessedQueueID = lastEntry?.queue_id ?? 0 78 | }; 79 | } 80 | 81 | // ReSharper disable InconsistentNaming (matches database table) 82 | [Serializable] 83 | public class bss_process_queue_item 84 | { 85 | public int queue_id; 86 | public int beatmapset_id; 87 | } 88 | 89 | private class PollingBeatmapStatusWatcher : IDisposable 90 | { 91 | private readonly Action callback; 92 | 93 | private readonly int pollMilliseconds; 94 | private readonly int limit; 95 | 96 | private int lastQueueId; 97 | private readonly CancellationTokenSource cts; 98 | 99 | public PollingBeatmapStatusWatcher(int initialQueueId, Action callback, int pollMilliseconds, int limit = 50) 100 | { 101 | lastQueueId = initialQueueId; 102 | this.pollMilliseconds = pollMilliseconds; 103 | this.limit = limit; 104 | this.callback = callback; 105 | 106 | cts = new CancellationTokenSource(); 107 | 108 | _ = Task.Factory.StartNew(poll, TaskCreationOptions.LongRunning); 109 | } 110 | 111 | private async Task poll() 112 | { 113 | while (!cts.Token.IsCancellationRequested) 114 | { 115 | try 116 | { 117 | var result = await GetUpdatedBeatmapSetsAsync(lastQueueId, limit); 118 | 119 | lastQueueId = result.LastProcessedQueueID; 120 | if (result.BeatmapSetIDs.Length > 0) 121 | callback(result); 122 | } 123 | catch (Exception e) 124 | { 125 | Console.WriteLine($"Poll failed with {e}."); 126 | await Task.Delay(1000, cts.Token); 127 | } 128 | 129 | await Task.Delay(pollMilliseconds, cts.Token); 130 | } 131 | } 132 | 133 | public void Dispose() 134 | { 135 | cts.Cancel(); 136 | } 137 | } 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /osu.Server.QueueProcessor/BeatmapUpdates.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. 2 | // See the LICENCE file in the repository root for full licence text. 3 | 4 | namespace osu.Server.QueueProcessor 5 | { 6 | public record BeatmapUpdates 7 | { 8 | public required int[] BeatmapSetIDs { get; init; } 9 | public required int LastProcessedQueueID { get; init; } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /osu.Server.QueueProcessor/ConnectionMultiplexerExtensions.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. 2 | // See the LICENCE file in the repository root for full licence text. 3 | 4 | using System; 5 | using System.Linq; 6 | using StackExchange.Redis; 7 | 8 | namespace osu.Server.QueueProcessor 9 | { 10 | /// 11 | /// Extensions to manage schema deployments on redis. 12 | /// 13 | public static class ConnectionMultiplexerExtensions 14 | { 15 | private static string allActiveSchemasKey => $"osu-queue:score-index:{Environment.GetEnvironmentVariable("ES_INDEX_PREFIX") ?? string.Empty}active-schemas"; 16 | private static string mainSchemaKey => $"osu-queue:score-index:{Environment.GetEnvironmentVariable("ES_INDEX_PREFIX") ?? string.Empty}schema"; 17 | 18 | /// 19 | /// Add a new schema version to the active list. Note that it will not be set to the current schema (call for that). 20 | /// 21 | public static bool AddActiveSchema(this ConnectionMultiplexer connection, string value) 22 | { 23 | return connection.GetDatabase().SetAdd(allActiveSchemasKey, value); 24 | } 25 | 26 | /// 27 | /// Clears the current live schema. 28 | /// 29 | public static void ClearCurrentSchema(this ConnectionMultiplexer connection) 30 | { 31 | connection.GetDatabase().KeyDelete(mainSchemaKey); 32 | } 33 | 34 | /// 35 | /// Get all active schemas (including past or future). 36 | /// 37 | public static string?[] GetActiveSchemas(this ConnectionMultiplexer connection) 38 | { 39 | return connection.GetDatabase().SetMembers(allActiveSchemasKey).ToStringArray(); 40 | } 41 | 42 | /// 43 | /// Get the current (live) schema version. 44 | /// 45 | public static string GetCurrentSchema(this ConnectionMultiplexer connection) 46 | { 47 | return connection.GetDatabase().StringGet(mainSchemaKey).ToString(); 48 | } 49 | 50 | /// 51 | /// Removes a specified schema from the active list. 52 | /// 53 | public static bool RemoveActiveSchema(this ConnectionMultiplexer connection, string value) 54 | { 55 | if (connection.GetCurrentSchema() == value) 56 | throw new InvalidOperationException($"Specified schema is current. Call {nameof(ClearCurrentSchema)} first"); 57 | 58 | return connection.GetDatabase().SetRemove(allActiveSchemasKey, value); 59 | } 60 | 61 | /// 62 | /// Set the current (live) schema version. 63 | /// 64 | /// 65 | /// 66 | public static void SetCurrentSchema(this ConnectionMultiplexer connection, string value) 67 | { 68 | IDatabase database = connection.GetDatabase(); 69 | 70 | if (connection.GetActiveSchemas().All(s => s != value)) 71 | throw new InvalidOperationException($"Attempted to set current schema without schema being in active list. Call {nameof(AddActiveSchema)} first"); 72 | 73 | database.StringSet(mainSchemaKey, value); 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /osu.Server.QueueProcessor/DatabaseAccess.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. 2 | // See the LICENCE file in the repository root for full licence text. 3 | 4 | using System; 5 | using System.Threading; 6 | using System.Threading.Tasks; 7 | using MySqlConnector; 8 | 9 | namespace osu.Server.QueueProcessor 10 | { 11 | /// 12 | /// Provides access to a MySQL database. 13 | /// 14 | public static class DatabaseAccess 15 | { 16 | /// 17 | /// Retrieve a fresh MySQL connection. Should be disposed after use. 18 | /// 19 | public static MySqlConnection GetConnection() 20 | { 21 | var connection = new MySqlConnection(getConnectionString()); 22 | 23 | connection.Open(); 24 | 25 | // TODO: remove this when we have set a saner time zone server-side. 26 | using (var cmd = connection.CreateCommand()) 27 | { 28 | cmd.CommandText = "SET time_zone = '+00:00';"; 29 | cmd.ExecuteNonQuery(); 30 | } 31 | 32 | return connection; 33 | } 34 | 35 | /// 36 | /// Retrieve a fresh MySQL connection. Should be disposed after use. 37 | /// 38 | public static async Task GetConnectionAsync(CancellationToken cancellationToken = default) 39 | { 40 | var connection = new MySqlConnection(getConnectionString()); 41 | 42 | await connection.OpenAsync(cancellationToken); 43 | 44 | // TODO: remove this when we have set a saner time zone server-side. 45 | using (var cmd = connection.CreateCommand()) 46 | { 47 | cmd.CommandText = "SET time_zone = '+00:00';"; 48 | await cmd.ExecuteNonQueryAsync(cancellationToken); 49 | } 50 | 51 | return connection; 52 | } 53 | 54 | private static string getConnectionString() 55 | { 56 | string connectionString = Environment.GetEnvironmentVariable("DB_CONNECTION_STRING") ?? String.Empty; 57 | 58 | if (string.IsNullOrEmpty(connectionString)) 59 | { 60 | string host = (Environment.GetEnvironmentVariable("DB_HOST") ?? "localhost"); 61 | string port = (Environment.GetEnvironmentVariable("DB_PORT") ?? "3306"); 62 | string user = (Environment.GetEnvironmentVariable("DB_USER") ?? "root"); 63 | string password = (Environment.GetEnvironmentVariable("DB_PASS") ?? string.Empty); 64 | string name = (Environment.GetEnvironmentVariable("DB_NAME") ?? "osu"); 65 | bool pooling = Environment.GetEnvironmentVariable("DB_POOLING") != "0"; 66 | int maxPoolSize = int.Parse(Environment.GetEnvironmentVariable("DB_MAX_POOL_SIZE") ?? "100"); 67 | 68 | string passwordString = string.IsNullOrEmpty(password) ? string.Empty : $"Password={password};"; 69 | 70 | // Pipelining disabled because ProxySQL no like. 71 | connectionString = 72 | $"Server={host};Port={port};Database={name};User ID={user};{passwordString}ConnectionTimeout=5;ConnectionReset=false;Pooling={pooling};Max Pool Size={maxPoolSize}; Pipelining=false"; 73 | } 74 | 75 | return connectionString; 76 | } 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /osu.Server.QueueProcessor/GracefulShutdownSource.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Runtime.Loader; 3 | using System.Threading; 4 | 5 | namespace osu.Server.QueueProcessor 6 | { 7 | public class GracefulShutdownSource : IDisposable 8 | { 9 | public CancellationToken Token => cts.Token; 10 | 11 | private readonly ManualResetEventSlim shutdownComplete; 12 | 13 | private readonly CancellationTokenSource cts; 14 | 15 | public void Cancel() => cts.Cancel(); 16 | 17 | public GracefulShutdownSource(in CancellationToken cancellation = default) 18 | { 19 | cts = CancellationTokenSource.CreateLinkedTokenSource(cancellation); 20 | 21 | shutdownComplete = new ManualResetEventSlim(); 22 | 23 | AssemblyLoadContext.Default.Unloading += onUnloading; 24 | Console.CancelKeyPress += onCancelKeyPress; 25 | } 26 | 27 | private void onCancelKeyPress(object? sender, ConsoleCancelEventArgs args) 28 | { 29 | args.Cancel = true; 30 | cts.Cancel(); 31 | } 32 | 33 | private void onUnloading(AssemblyLoadContext _) 34 | { 35 | cts.Cancel(); 36 | shutdownComplete.Wait(CancellationToken.None); 37 | } 38 | 39 | public void Dispose() 40 | { 41 | shutdownComplete.Set(); 42 | Console.CancelKeyPress -= onCancelKeyPress; 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /osu.Server.QueueProcessor/QueueConfiguration.cs: -------------------------------------------------------------------------------- 1 | namespace osu.Server.QueueProcessor 2 | { 3 | public class QueueConfiguration 4 | { 5 | /// 6 | /// The queue to read from. 7 | /// 8 | public string InputQueueName { get; set; } = "default"; 9 | 10 | /// 11 | /// The time between polls (in the case a poll returns no items). 12 | /// 13 | public int TimeBetweenPolls { get; set; } = 100; 14 | 15 | /// 16 | /// The number of items allowed to be dequeued but not processed at one time. 17 | /// 18 | public int MaxInFlightItems { get; set; } = 100; 19 | 20 | /// 21 | /// The number of times to re-queue a failed item for another attempt. 22 | /// 23 | public int MaxRetries { get; set; } = 3; 24 | 25 | /// 26 | /// The maximum number of recent errors before exiting with an error. 27 | /// 28 | /// 29 | /// Every error will increment an internal count, while every success will decrement it. 30 | /// 31 | public int ErrorThreshold { get; set; } = 10; 32 | 33 | /// 34 | /// Setting above 1 will allow processing in batches (see ). 35 | /// 36 | public int BatchSize { get; set; } = 1; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /osu.Server.QueueProcessor/QueueItem.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. 2 | // See the LICENCE file in the repository root for full licence text. 3 | 4 | using System; 5 | using System.Runtime.Serialization; 6 | 7 | namespace osu.Server.QueueProcessor 8 | { 9 | /// 10 | /// An item to be managed by a . 11 | /// 12 | [Serializable] 13 | public abstract class QueueItem 14 | { 15 | [IgnoreDataMember] 16 | private bool failed; 17 | 18 | /// 19 | /// Set to true to mark this item is failed. This will cause it to be retried. 20 | /// 21 | [IgnoreDataMember] 22 | public bool Failed 23 | { 24 | get => failed || Exception != null; 25 | set => failed = value; 26 | } 27 | 28 | [IgnoreDataMember] 29 | public Exception? Exception { get; set; } 30 | 31 | /// 32 | /// The number of times processing this item has been retried. Handled internally by . 33 | /// 34 | public int TotalRetries { get; set; } 35 | 36 | /// 37 | /// Tags which will be used for tracking a processed item. 38 | /// 39 | public string[]? Tags { get; set; } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /osu.Server.QueueProcessor/QueueProcessor.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading; 5 | using System.Threading.Tasks; 6 | using MySqlConnector; 7 | using Newtonsoft.Json; 8 | using osu.Framework.Threading; 9 | using Sentry; 10 | using StackExchange.Redis; 11 | using StatsdClient; 12 | 13 | namespace osu.Server.QueueProcessor 14 | { 15 | public abstract class QueueProcessor where T : QueueItem 16 | { 17 | /// 18 | /// The total queue items processed since startup. 19 | /// 20 | public long TotalProcessed => totalProcessed; 21 | 22 | /// 23 | /// The total queue items dequeued since startup. 24 | /// 25 | public long TotalDequeued => totalDequeued; 26 | 27 | /// 28 | /// The total errors encountered processing items since startup. 29 | /// 30 | /// 31 | /// Note that this may include more than one error from the same queue item failing multiple times. 32 | /// 33 | public long TotalErrors => totalErrors; 34 | 35 | public event Action? Error; 36 | 37 | /// 38 | /// The name of this queue, as provided by . 39 | /// 40 | public string QueueName { get; } 41 | 42 | /// 43 | /// Report statistics about this queue via datadog. 44 | /// 45 | protected DogStatsdService DogStatsd { get; } 46 | 47 | private readonly QueueConfiguration config; 48 | 49 | private readonly Lazy redis = new Lazy(RedisAccess.GetConnection); 50 | 51 | private IDatabase getRedisDatabase() => redis.Value.GetDatabase(); 52 | 53 | private long totalProcessed; 54 | 55 | private long totalDequeued; 56 | 57 | private long totalErrors; 58 | 59 | private int consecutiveErrors; 60 | 61 | private long totalInFlight => totalDequeued - totalProcessed - totalErrors; 62 | 63 | protected QueueProcessor(QueueConfiguration config) 64 | { 65 | this.config = config; 66 | 67 | const string queue_prefix = "osu-queue:"; 68 | 69 | QueueName = $"{queue_prefix}{config.InputQueueName}"; 70 | 71 | DogStatsd = new DogStatsdService(); 72 | DogStatsd.Configure(new StatsdConfig 73 | { 74 | StatsdServerName = Environment.GetEnvironmentVariable("DD_AGENT_HOST") ?? "localhost", 75 | Prefix = $"osu.server.queues.{config.InputQueueName}" 76 | }); 77 | } 78 | 79 | /// 80 | /// Start running the queue. 81 | /// 82 | /// An optional cancellation token. 83 | public void Run(CancellationToken cancellation = default) 84 | { 85 | using (SentrySdk.Init(setupSentry)) 86 | using (new Timer(_ => outputStats(), null, TimeSpan.Zero, TimeSpan.FromSeconds(5))) 87 | using (var cts = new GracefulShutdownSource(cancellation)) 88 | { 89 | Console.WriteLine($"Starting queue processing (Backlog of {GetQueueSize()}).."); 90 | 91 | using (var threadPool = new ThreadedTaskScheduler(Environment.ProcessorCount, "workers")) 92 | { 93 | IDatabase database = getRedisDatabase(); 94 | 95 | while (!cts.Token.IsCancellationRequested) 96 | { 97 | if (consecutiveErrors > config.ErrorThreshold) 98 | throw new Exception("Error threshold exceeded, shutting down"); 99 | 100 | try 101 | { 102 | if (totalInFlight >= config.MaxInFlightItems || consecutiveErrors > config.ErrorThreshold) 103 | { 104 | Thread.Sleep(config.TimeBetweenPolls); 105 | continue; 106 | } 107 | 108 | var redisItems = database.ListRightPop(QueueName, config.BatchSize); 109 | 110 | // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract (https://github.com/StackExchange/StackExchange.Redis/issues/2697) 111 | // queue doesn't exist. 112 | if (redisItems == null) 113 | { 114 | Thread.Sleep(config.TimeBetweenPolls); 115 | continue; 116 | } 117 | 118 | List items = new List(); 119 | 120 | // null or empty check is required for redis 6.x. 7.x reports `null` instead. 121 | foreach (var redisItem in redisItems.Where(i => !i.IsNullOrEmpty)) 122 | items.Add(JsonConvert.DeserializeObject(redisItem!) ?? throw new InvalidOperationException("Dequeued item could not be deserialised.")); 123 | 124 | if (items.Count == 0) 125 | { 126 | Thread.Sleep(config.TimeBetweenPolls); 127 | continue; 128 | } 129 | 130 | Interlocked.Add(ref totalDequeued, items.Count); 131 | DogStatsd.Increment("total_dequeued", items.Count); 132 | 133 | // individual processing should not be cancelled as we have already grabbed from the queue. 134 | Task.Factory.StartNew(() => { ProcessResults(items); }, CancellationToken.None, TaskCreationOptions.HideScheduler, threadPool) 135 | .ContinueWith(_ => 136 | { 137 | foreach (var item in items) 138 | { 139 | if (item.Failed) 140 | { 141 | Interlocked.Increment(ref totalErrors); 142 | 143 | // ReSharper disable once AccessToDisposedClosure 144 | DogStatsd.Increment("total_errors", tags: item.Tags); 145 | 146 | Interlocked.Increment(ref consecutiveErrors); 147 | 148 | try 149 | { 150 | Error?.Invoke(item.Exception, item); 151 | } 152 | catch 153 | { 154 | } 155 | 156 | if (item.Exception != null) 157 | SentrySdk.CaptureException(item.Exception); 158 | 159 | Console.WriteLine($"Error processing {item}: {item.Exception}"); 160 | attemptRetry(item); 161 | } 162 | else 163 | { 164 | Interlocked.Increment(ref totalProcessed); 165 | 166 | // ReSharper disable once AccessToDisposedClosure 167 | DogStatsd.Increment("total_processed", tags: item.Tags); 168 | 169 | Interlocked.Exchange(ref consecutiveErrors, 0); 170 | } 171 | } 172 | }, CancellationToken.None); 173 | } 174 | catch (Exception e) 175 | { 176 | Interlocked.Increment(ref consecutiveErrors); 177 | Console.WriteLine($"Error dequeueing from queue: {e}"); 178 | SentrySdk.CaptureException(e); 179 | } 180 | } 181 | 182 | Console.WriteLine("Shutting down.."); 183 | 184 | while (totalInFlight > 0) 185 | { 186 | Console.WriteLine($"Waiting for remaining {totalInFlight} in-flight items..."); 187 | Thread.Sleep(5000); 188 | } 189 | 190 | Console.WriteLine("Bye!"); 191 | } 192 | } 193 | 194 | DogStatsd.Dispose(); 195 | outputStats(); 196 | } 197 | 198 | private void setupSentry(SentryOptions options) 199 | { 200 | options.Dsn = Environment.GetEnvironmentVariable("SENTRY_DSN") ?? string.Empty; 201 | options.DefaultTags["queue"] = QueueName; 202 | } 203 | 204 | private void attemptRetry(T item) 205 | { 206 | if (item.TotalRetries++ < config.MaxRetries) 207 | { 208 | Console.WriteLine($"Re-queueing for attempt {item.TotalRetries} / {config.MaxRetries}"); 209 | PushToQueue(item); 210 | } 211 | else 212 | { 213 | Console.WriteLine("Attempts exhausted; dropping item"); 214 | } 215 | } 216 | 217 | private void outputStats() 218 | { 219 | try 220 | { 221 | DogStatsd.Gauge("in_flight", totalInFlight); 222 | Console.WriteLine($"stats: queue:{GetQueueSize()} inflight:{totalInFlight} dequeued:{totalDequeued} processed:{totalProcessed} errors:{totalErrors}"); 223 | } 224 | catch (Exception e) 225 | { 226 | Console.WriteLine($"Error outputting stats: {e}"); 227 | } 228 | } 229 | 230 | /// 231 | /// Push a single item to the queue. 232 | /// 233 | /// 234 | public void PushToQueue(T item) => 235 | getRedisDatabase().ListLeftPush(QueueName, JsonConvert.SerializeObject(item)); 236 | 237 | /// 238 | /// Push multiple items to the queue. 239 | /// 240 | /// 241 | public void PushToQueue(IEnumerable items) => 242 | getRedisDatabase().ListLeftPush(QueueName, items.Select(obj => new RedisValue(JsonConvert.SerializeObject(obj))).ToArray()); 243 | 244 | public long GetQueueSize() => 245 | getRedisDatabase().ListLength(QueueName); 246 | 247 | public void ClearQueue() => getRedisDatabase().KeyDelete(QueueName); 248 | 249 | /// 250 | /// Publishes a message to a Redis channel with the supplied . 251 | /// 252 | /// 253 | /// The message will be serialised using JSON. 254 | /// Successful publications are tracked in Datadog, using the and the 's full type name as a tag. 255 | /// 256 | /// The name of the Redis channel to publish to. 257 | /// The message to publish to the channel. 258 | /// The type of message to be published. 259 | public void PublishMessage(string channelName, TMessage message) 260 | { 261 | getRedisDatabase().Publish(new RedisChannel(channelName, RedisChannel.PatternMode.Auto), JsonConvert.SerializeObject(message)); 262 | DogStatsd.Increment("messages_published", tags: new[] { $"channel:{channelName}", $"type:{typeof(TMessage).FullName}" }); 263 | } 264 | 265 | /// 266 | /// Retrieve a database connection. 267 | /// 268 | public MySqlConnection GetDatabaseConnection() => DatabaseAccess.GetConnection(); 269 | 270 | /// 271 | /// Implement to process a single item from the queue. Will only be invoked if is not implemented. 272 | /// 273 | /// The item to process. 274 | protected virtual void ProcessResult(T item) 275 | { 276 | } 277 | 278 | /// 279 | /// Implement to process batches of items from the queue. 280 | /// 281 | /// 282 | /// In most cases, you should only need to override and implement . 283 | /// Only override this if you need more efficient batch processing. 284 | /// 285 | /// If overriding this method, you should try-catch for exceptions, and set any exception against 286 | /// the relevant . If this is not done, failures will not be handled correctly. 287 | /// The items to process. 288 | protected virtual void ProcessResults(IEnumerable items) 289 | { 290 | foreach (var item in items) 291 | { 292 | try 293 | { 294 | ProcessResult(item); 295 | } 296 | catch (Exception e) 297 | { 298 | item.Exception = e; 299 | } 300 | } 301 | } 302 | } 303 | } 304 | -------------------------------------------------------------------------------- /osu.Server.QueueProcessor/RedisAccess.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. 2 | // See the LICENCE file in the repository root for full licence text. 3 | 4 | using System; 5 | using StackExchange.Redis; 6 | 7 | namespace osu.Server.QueueProcessor 8 | { 9 | /// 10 | /// Provides access to a Redis database. 11 | /// 12 | public static class RedisAccess 13 | { 14 | private static readonly ConfigurationOptions redis_config = ConfigurationOptions.Parse(Environment.GetEnvironmentVariable("REDIS_HOST") ?? "localhost"); 15 | 16 | /// 17 | /// Retrieve a fresh Redis connection. Should be disposed after use. 18 | /// 19 | public static ConnectionMultiplexer GetConnection() => ConnectionMultiplexer.Connect(redis_config); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /osu.Server.QueueProcessor/osu.Server.QueueProcessor.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net8.0 5 | OsuQueueProcessor 6 | ppy.osu.Server.OsuQueueProcessor 7 | Dean Herbert 8 | ppy Pty Ltd 9 | https://github.com/ppy/osu-queue-processor 10 | enable 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | --------------------------------------------------------------------------------