├── .editorconfig ├── .github └── FUNDING.yml ├── .gitignore ├── LICENSE ├── README.md ├── Sqs Toolbox.sln ├── samples ├── ConsoleAppSample │ ├── ConsoleAppSample.csproj │ └── Program.cs └── WorkerServiceSample │ ├── CustomExceptionHandler.cs │ ├── DiagnosticsMonitorService.cs │ ├── Program.cs │ ├── Properties │ └── launchSettings.json │ ├── QueueProcessor.cs │ ├── WorkerServiceSample.csproj │ ├── appsettings.Development.json │ └── appsettings.json ├── sqs-toolbox.png ├── src ├── DotNetCloud.SqsToolbox.Core │ ├── Abstractions │ │ ├── IExceptionHandler.cs │ │ ├── IFailedDeletionEntryHandler.cs │ │ ├── ISqsBatchDeleteQueue.cs │ │ ├── ISqsBatchDeleter.cs │ │ ├── ISqsMessageChannelSource.cs │ │ ├── ISqsPollingQueueReader.cs │ │ ├── ISqsReceiveDelayCalculator.cs │ │ └── SqsMessageChannelSource.cs │ ├── DefaultExceptionHandler.cs │ ├── Delete │ │ ├── DefaultFailedDeletionEntryHandler.cs │ │ ├── SqsBatchDeleteQueue.cs │ │ ├── SqsBatchDeleter.cs │ │ ├── SqsBatchDeleterBuilder.cs │ │ └── SqsBatchDeletionOptions.cs │ ├── Diagnostics │ │ ├── BeginReceiveRequestPayload.cs │ │ ├── DeletionBatchCreatedPayload.cs │ │ ├── DiagnosticEvents.cs │ │ ├── EndDeletionBatchPayload.cs │ │ ├── EndReceiveRequestPayload.cs │ │ └── ExceptionPayload.cs │ ├── DotNetCloud.SqsToolbox.Core.csproj │ ├── ILogicalQueueNameGenerator.cs │ └── Receive │ │ ├── SqsPollingQueueReader.cs │ │ ├── SqsPollingQueueReaderOptions.cs │ │ ├── SqsPollingQueueReaderOptionsExtensions.cs │ │ └── SqsReceiveDelayCalculator.cs └── DotNetCloud.SqsToolbox │ ├── ConfigurationExtensions.cs │ ├── DefaultChannelReaderAccessor.cs │ ├── DefaultSqsPollingQueueReaderBuilder.cs │ ├── DependencyInjection │ ├── ISqsBatchDeletionBuilder.cs │ ├── ISqsPollingReaderBuilder.cs │ ├── SqsBatchDeleterServiceCollectionExtensions.cs │ ├── SqsBatchDeletionBuilder.cs │ └── SqsPollingReaderServiceCollectionExtensions.cs │ ├── Diagnostics │ └── DiagnosticsMonitoringService.cs │ ├── DotNetCloud.SqsToolbox.csproj │ ├── Hosting │ ├── MessageProcessorService.cs │ ├── SqsBatchDeleteBackgroundService.cs │ ├── SqsMessageProcessingBackgroundService.cs │ └── SqsPollingBackgroundService.cs │ ├── IChannelReaderAccessor.cs │ ├── ISqsMessageChannelFactory.cs │ ├── ISqsPollingQueueReaderFactory.cs │ ├── SqsPollingQueueReaderFactory.cs │ ├── SqsPollingQueueReaderFactoryOptions.cs │ └── StopApplicationExceptionHandler.cs ├── templates └── SqsWorkerService │ ├── .template.config │ └── template.json │ ├── DiagnosticsMonitorService.cs │ ├── DotNetCloud.SqsWorkerService.csproj │ ├── MessageProcessingService.cs │ ├── Program.cs │ ├── Properties │ └── launchSettings.json │ ├── appsettings.Development.json │ └── appsettings.json └── test ├── DotNetCloud.SqsToolbox.Core.Tests ├── DefaultExceptionHandlerTests.cs ├── DefaultLogicalQueueNameGeneratorTests.cs ├── Delete │ └── SqsBatchDeleterTests.cs ├── DotNetCloud.SqsToolbox.Core.Tests.csproj └── SqsPollingDelayerTests.cs └── DotNetCloud.SqsToolbox.Tests ├── DependencyInjection └── SqsBatchDeleterServiceCollectionExtensionsTests.cs └── DotNetCloud.SqsToolbox.Tests.csproj /.editorconfig: -------------------------------------------------------------------------------- 1 | # editorconfig.org 2 | 3 | # top-most EditorConfig file 4 | root = true 5 | 6 | # Default settings: 7 | # A newline ending every file 8 | # Use 4 spaces as indentation 9 | [*] 10 | insert_final_newline = true 11 | indent_style = space 12 | indent_size = 4 13 | 14 | [*.json] 15 | indent_size = 2 16 | 17 | # C# files 18 | [*.cs] 19 | # New line preferences 20 | csharp_new_line_before_open_brace = all 21 | csharp_new_line_before_else = true 22 | csharp_new_line_before_catch = true 23 | csharp_new_line_before_finally = true 24 | csharp_new_line_before_members_in_object_initializers = true 25 | csharp_new_line_before_members_in_anonymous_types = true 26 | csharp_new_line_between_query_expression_clauses = true 27 | 28 | # Indentation preferences 29 | csharp_indent_block_contents = true 30 | csharp_indent_braces = false 31 | csharp_indent_case_contents = true 32 | csharp_indent_switch_labels = true 33 | csharp_indent_labels = one_less_than_current 34 | 35 | # avoid this. unless absolutely necessary 36 | dotnet_style_qualification_for_field = false:suggestion 37 | dotnet_style_qualification_for_property = false:suggestion 38 | dotnet_style_qualification_for_method = false:suggestion 39 | dotnet_style_qualification_for_event = false:suggestion 40 | 41 | # use language keywords instead of BCL types 42 | dotnet_style_predefined_type_for_locals_parameters_members = true:suggestion 43 | dotnet_style_predefined_type_for_member_access = true:suggestion 44 | 45 | # name all constant fields using PascalCase 46 | dotnet_naming_rule.constant_fields_should_be_pascal_case.severity = suggestion 47 | dotnet_naming_rule.constant_fields_should_be_pascal_case.symbols = constant_fields 48 | dotnet_naming_rule.constant_fields_should_be_pascal_case.style = pascal_case_style 49 | 50 | dotnet_naming_symbols.constant_fields.applicable_kinds = field 51 | dotnet_naming_symbols.constant_fields.required_modifiers = const 52 | 53 | dotnet_naming_style.pascal_case_style.capitalization = pascal_case 54 | 55 | # internal and private fields should be _camelCase 56 | dotnet_naming_rule.camel_case_for_private_internal_fields.severity = suggestion 57 | dotnet_naming_rule.camel_case_for_private_internal_fields.symbols = private_internal_fields 58 | dotnet_naming_rule.camel_case_for_private_internal_fields.style = camel_case_underscore_style 59 | 60 | dotnet_naming_symbols.private_internal_fields.applicable_kinds = field 61 | dotnet_naming_symbols.private_internal_fields.applicable_accessibilities = private, internal 62 | 63 | dotnet_naming_style.camel_case_underscore_style.required_prefix = _ 64 | dotnet_naming_style.camel_case_underscore_style.capitalization = camel_case 65 | 66 | # Code style defaults 67 | dotnet_sort_system_directives_first = true 68 | csharp_preserve_single_line_blocks = true 69 | csharp_preserve_single_line_statements = false 70 | 71 | # Expression-level preferences 72 | dotnet_style_object_initializer = true:suggestion 73 | dotnet_style_collection_initializer = true:suggestion 74 | dotnet_style_explicit_tuple_names = true:suggestion 75 | dotnet_style_coalesce_expression = true:suggestion 76 | dotnet_style_null_propagation = true:suggestion 77 | 78 | # Pattern matching 79 | csharp_style_pattern_matching_over_is_with_cast_check = true:suggestion 80 | csharp_style_pattern_matching_over_as_with_null_check = true:suggestion 81 | csharp_style_inlined_variable_declaration = true:suggestion 82 | 83 | # Null checking preferences 84 | csharp_style_throw_expression = true:suggestion 85 | csharp_style_conditional_delegate_call = true:suggestion 86 | 87 | # Space preferences 88 | csharp_space_after_cast = false 89 | csharp_space_after_colon_in_inheritance_clause = true 90 | csharp_space_after_comma = true 91 | csharp_space_after_dot = false 92 | csharp_space_after_keywords_in_control_flow_statements = true 93 | csharp_space_after_semicolon_in_for_statement = true 94 | csharp_space_around_binary_operators = before_and_after 95 | csharp_space_around_declaration_statements = do_not_ignore 96 | csharp_space_before_colon_in_inheritance_clause = true 97 | csharp_space_before_comma = false 98 | csharp_space_before_dot = false 99 | csharp_space_before_open_square_brackets = false 100 | csharp_space_before_semicolon_in_for_statement = false 101 | csharp_space_between_empty_square_brackets = false 102 | csharp_space_between_method_call_empty_parameter_list_parentheses = false 103 | csharp_space_between_method_call_name_and_opening_parenthesis = false 104 | csharp_space_between_method_call_parameter_list_parentheses = false 105 | csharp_space_between_method_declaration_empty_parameter_list_parentheses = false 106 | csharp_space_between_method_declaration_name_and_open_parenthesis = false 107 | csharp_space_between_method_declaration_parameter_list_parentheses = false 108 | csharp_space_between_parentheses = false 109 | csharp_space_between_square_brackets = false -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: [stevejgordon] 2 | custom: ["https://www.buymeacoffee.com/stevejgordon"] 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | ## 4 | ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore 5 | 6 | # User-specific files 7 | *.rsuser 8 | *.suo 9 | *.user 10 | *.userosscache 11 | *.sln.docstates 12 | 13 | # User-specific files (MonoDevelop/Xamarin Studio) 14 | *.userprefs 15 | 16 | # Rider 17 | .idea 18 | 19 | # Mono auto generated files 20 | mono_crash.* 21 | 22 | # Build results 23 | [Dd]ebug/ 24 | [Dd]ebugPublic/ 25 | [Rr]elease/ 26 | [Rr]eleases/ 27 | x64/ 28 | x86/ 29 | [Aa][Rr][Mm]/ 30 | [Aa][Rr][Mm]64/ 31 | bld/ 32 | [Bb]in/ 33 | [Oo]bj/ 34 | [Ll]og/ 35 | [Ll]ogs/ 36 | 37 | # Visual Studio 2015/2017 cache/options directory 38 | .vs/ 39 | # Uncomment if you have tasks that create the project's static files in wwwroot 40 | #wwwroot/ 41 | 42 | # Visual Studio 2017 auto generated files 43 | Generated\ Files/ 44 | 45 | # MSTest test Results 46 | [Tt]est[Rr]esult*/ 47 | [Bb]uild[Ll]og.* 48 | 49 | # NUnit 50 | *.VisualState.xml 51 | TestResult.xml 52 | nunit-*.xml 53 | 54 | # Build Results of an ATL Project 55 | [Dd]ebugPS/ 56 | [Rr]eleasePS/ 57 | dlldata.c 58 | 59 | # Benchmark Results 60 | BenchmarkDotNet.Artifacts/ 61 | 62 | # .NET Core 63 | project.lock.json 64 | project.fragment.lock.json 65 | artifacts/ 66 | 67 | # StyleCop 68 | StyleCopReport.xml 69 | 70 | # Files built by Visual Studio 71 | *_i.c 72 | *_p.c 73 | *_h.h 74 | *.ilk 75 | *.meta 76 | *.obj 77 | *.iobj 78 | *.pch 79 | *.pdb 80 | *.ipdb 81 | *.pgc 82 | *.pgd 83 | *.rsp 84 | *.sbr 85 | *.tlb 86 | *.tli 87 | *.tlh 88 | *.tmp 89 | *.tmp_proj 90 | *_wpftmp.csproj 91 | *.log 92 | *.vspscc 93 | *.vssscc 94 | .builds 95 | *.pidb 96 | *.svclog 97 | *.scc 98 | 99 | # Chutzpah Test files 100 | _Chutzpah* 101 | 102 | # Visual C++ cache files 103 | ipch/ 104 | *.aps 105 | *.ncb 106 | *.opendb 107 | *.opensdf 108 | *.sdf 109 | *.cachefile 110 | *.VC.db 111 | *.VC.VC.opendb 112 | 113 | # Visual Studio profiler 114 | *.psess 115 | *.vsp 116 | *.vspx 117 | *.sap 118 | 119 | # Visual Studio Trace Files 120 | *.e2e 121 | 122 | # TFS 2012 Local Workspace 123 | $tf/ 124 | 125 | # Guidance Automation Toolkit 126 | *.gpState 127 | 128 | # ReSharper is a .NET coding add-in 129 | _ReSharper*/ 130 | *.[Rr]e[Ss]harper 131 | *.DotSettings.user 132 | 133 | # TeamCity is a build add-in 134 | _TeamCity* 135 | 136 | # DotCover is a Code Coverage Tool 137 | *.dotCover 138 | 139 | # AxoCover is a Code Coverage Tool 140 | .axoCover/* 141 | !.axoCover/settings.json 142 | 143 | # Coverlet is a free, cross platform Code Coverage Tool 144 | coverage*[.json, .xml, .info] 145 | 146 | # Visual Studio code coverage results 147 | *.coverage 148 | *.coveragexml 149 | 150 | # NCrunch 151 | _NCrunch_* 152 | .*crunch*.local.xml 153 | nCrunchTemp_* 154 | 155 | # MightyMoose 156 | *.mm.* 157 | AutoTest.Net/ 158 | 159 | # Web workbench (sass) 160 | .sass-cache/ 161 | 162 | # Installshield output folder 163 | [Ee]xpress/ 164 | 165 | # DocProject is a documentation generator add-in 166 | DocProject/buildhelp/ 167 | DocProject/Help/*.HxT 168 | DocProject/Help/*.HxC 169 | DocProject/Help/*.hhc 170 | DocProject/Help/*.hhk 171 | DocProject/Help/*.hhp 172 | DocProject/Help/Html2 173 | DocProject/Help/html 174 | 175 | # Click-Once directory 176 | publish/ 177 | 178 | # Publish Web Output 179 | *.[Pp]ublish.xml 180 | *.azurePubxml 181 | # Note: Comment the next line if you want to checkin your web deploy settings, 182 | # but database connection strings (with potential passwords) will be unencrypted 183 | *.pubxml 184 | *.publishproj 185 | 186 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 187 | # checkin your Azure Web App publish settings, but sensitive information contained 188 | # in these scripts will be unencrypted 189 | PublishScripts/ 190 | 191 | # NuGet Packages 192 | *.nupkg 193 | # NuGet Symbol Packages 194 | *.snupkg 195 | # The packages folder can be ignored because of Package Restore 196 | **/[Pp]ackages/* 197 | # except build/, which is used as an MSBuild target. 198 | !**/[Pp]ackages/build/ 199 | # Uncomment if necessary however generally it will be regenerated when needed 200 | #!**/[Pp]ackages/repositories.config 201 | # NuGet v3's project.json files produces more ignorable files 202 | *.nuget.props 203 | *.nuget.targets 204 | 205 | # Microsoft Azure Build Output 206 | csx/ 207 | *.build.csdef 208 | 209 | # Microsoft Azure Emulator 210 | ecf/ 211 | rcf/ 212 | 213 | # Windows Store app package directories and files 214 | AppPackages/ 215 | BundleArtifacts/ 216 | Package.StoreAssociation.xml 217 | _pkginfo.txt 218 | *.appx 219 | *.appxbundle 220 | *.appxupload 221 | 222 | # Visual Studio cache files 223 | # files ending in .cache can be ignored 224 | *.[Cc]ache 225 | # but keep track of directories ending in .cache 226 | !?*.[Cc]ache/ 227 | 228 | # Others 229 | ClientBin/ 230 | ~$* 231 | *~ 232 | *.dbmdl 233 | *.dbproj.schemaview 234 | *.jfm 235 | *.pfx 236 | *.publishsettings 237 | orleans.codegen.cs 238 | 239 | # Including strong name files can present a security risk 240 | # (https://github.com/github/gitignore/pull/2483#issue-259490424) 241 | #*.snk 242 | 243 | # Since there are multiple workflows, uncomment next line to ignore bower_components 244 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 245 | #bower_components/ 246 | 247 | # RIA/Silverlight projects 248 | Generated_Code/ 249 | 250 | # Backup & report files from converting an old project file 251 | # to a newer Visual Studio version. Backup files are not needed, 252 | # because we have git ;-) 253 | _UpgradeReport_Files/ 254 | Backup*/ 255 | UpgradeLog*.XML 256 | UpgradeLog*.htm 257 | ServiceFabricBackup/ 258 | *.rptproj.bak 259 | 260 | # SQL Server files 261 | *.mdf 262 | *.ldf 263 | *.ndf 264 | 265 | # Business Intelligence projects 266 | *.rdl.data 267 | *.bim.layout 268 | *.bim_*.settings 269 | *.rptproj.rsuser 270 | *- [Bb]ackup.rdl 271 | *- [Bb]ackup ([0-9]).rdl 272 | *- [Bb]ackup ([0-9][0-9]).rdl 273 | 274 | # Microsoft Fakes 275 | FakesAssemblies/ 276 | 277 | # GhostDoc plugin setting file 278 | *.GhostDoc.xml 279 | 280 | # Node.js Tools for Visual Studio 281 | .ntvs_analysis.dat 282 | node_modules/ 283 | 284 | # Visual Studio 6 build log 285 | *.plg 286 | 287 | # Visual Studio 6 workspace options file 288 | *.opt 289 | 290 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) 291 | *.vbw 292 | 293 | # Visual Studio LightSwitch build output 294 | **/*.HTMLClient/GeneratedArtifacts 295 | **/*.DesktopClient/GeneratedArtifacts 296 | **/*.DesktopClient/ModelManifest.xml 297 | **/*.Server/GeneratedArtifacts 298 | **/*.Server/ModelManifest.xml 299 | _Pvt_Extensions 300 | 301 | # Paket dependency manager 302 | .paket/paket.exe 303 | paket-files/ 304 | 305 | # FAKE - F# Make 306 | .fake/ 307 | 308 | # CodeRush personal settings 309 | .cr/personal 310 | 311 | # Python Tools for Visual Studio (PTVS) 312 | __pycache__/ 313 | *.pyc 314 | 315 | # Cake - Uncomment if you are using it 316 | # tools/** 317 | # !tools/packages.config 318 | 319 | # Tabs Studio 320 | *.tss 321 | 322 | # Telerik's JustMock configuration file 323 | *.jmconfig 324 | 325 | # BizTalk build output 326 | *.btp.cs 327 | *.btm.cs 328 | *.odx.cs 329 | *.xsd.cs 330 | 331 | # OpenCover UI analysis results 332 | OpenCover/ 333 | 334 | # Azure Stream Analytics local run output 335 | ASALocalRun/ 336 | 337 | # MSBuild Binary and Structured Log 338 | *.binlog 339 | 340 | # NVidia Nsight GPU debugger configuration file 341 | *.nvuser 342 | 343 | # MFractors (Xamarin productivity tool) working folder 344 | .mfractor/ 345 | 346 | # Local History for Visual Studio 347 | .localhistory/ 348 | 349 | # BeatPulse healthcheck temp database 350 | healthchecksdb 351 | 352 | # Backup folder for Package Reference Convert tool in Visual Studio 2017 353 | MigrationBackup/ 354 | 355 | # Ionide (cross platform F# VS Code tools) working folder 356 | .ionide/ 357 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Steve Gordon 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SQS Toolbox 2 | 3 | This is a work-in-progress repository for a set of libraries, extensions and helpers to support working with AWS Simple Queue Service from .NET applications. 4 | 5 |

6 | 7 |

8 | 9 | ## Packages 10 | 11 | **IN PREVIEW 6 THE NAMESPACES AND PACKAGE NAMES WILL CHANGE** 12 | 13 | | Package | NuGet Stable | NuGet Pre-release | Downloads | 14 | | ------- | ------------ | ----------------- | --------- | 15 | | [DotNetCloud.SqsToolbox](https://www.nuget.org/packages/DotNetCloud.SqsToolbox) | [![NuGet](https://img.shields.io/nuget/v/DotNetCloud.SqsToolbox.svg)](https://www.nuget.org/packages/DotNetCloud.SqsToolbox) | [![NuGet](https://img.shields.io/nuget/vpre/DotNetCloud.SqsToolbox.svg)](https://www.nuget.org/packages/DotNetCloud.SqsToolbox) | [![Nuget](https://img.shields.io/nuget/dt/DotNetCloud.SqsToolbox.svg)](https://www.nuget.org/packages/DotNetCloud.SqsToolbox) | 16 | | [DotNetCloud.SqsToolbox.Extensions](https://www.nuget.org/packages/DotNetCloud.SqsToolbox.Extensions) | [![NuGet](https://img.shields.io/nuget/v/DotNetCloud.SqsToolbox.Extensions.svg)](https://www.nuget.org/packages/DotNetCloud.SqsToolbox.Extensions) | [![NuGet](https://img.shields.io/nuget/vpre/DotNetCloud.SqsToolbox.Extensions.svg)](https://www.nuget.org/packages/DotNetCloud.SqsToolbox.Extensions) | [![Nuget](https://img.shields.io/nuget/dt/DotNetCloud.SqsToolbox.Extensions.svg)](https://www.nuget.org/packages/DotNetCloud.SqsToolbox.Extensions) | 17 | 18 | # Features 19 | 20 | ## SQS Polling Queue Reader 21 | 22 | **Status**: Work in progress, released in alpha. 23 | 24 | Supporting types for polling an SQS queue repeatedly for messages in a background `Task`. 25 | 26 | ### Design Goals 27 | 28 | Support minimal boilerplate code required for the common scenario in a queue processing worker service. Most of the code is provided by the library, with extension points for customisation of the default behaviours. When used in a `Host` based worker service or ASP.NET Core app, service registrations support registration of readers for multiple, logically named queues. 29 | 30 | ### Quick Start 31 | 32 | **WARNING** 33 | These packages are considered alpha quality. They are not fully tested and the public API is likely to change during development and based on feedback. I encourage you to try the packages to provide your thoughts and requirements, but perhaps be wary of using this in production! 34 | 35 | The most convenient consumption pattern is to utilise the DotNetCloud.SqsToolbox.Extensions package which provides extensions to integration with the Microsoft dependency injection and configuration libraries. 36 | 37 | **IN PREVIEW 6 THE NAMESPACES AND PACKAGE NAMES WILL CHANGE** 38 | 39 | Add the latest alpha NuGet package from [nuget.org](https://www.nuget.org/packages/DotNetCloud.SqsToolbox.Extensions). 40 | 41 | Inside an ASP.NET Core worker service you may register a polling queue reader as follows: 42 | 43 | ```csharp 44 | services.AddPollingSqs(hostContext.Configuration.GetSection("TestQueue")) 45 | .Configure(c => c.UseExponentialBackoff = true) 46 | .WithBackgroundService() 47 | .WithMessageProcessor() 48 | .WithDefaultExceptionHandler(); 49 | ``` 50 | 51 | Various builder extension methods exist to customise the queue reader and message consumption. These are optional and provide convenience use for common scenarios. 52 | 53 | The above code registers the polling queue reader, loading it's logical name and URL from an `IConfigurationSection`. 54 | 55 | Additional configuration can be provided by calling the `Configure` method on the `ISqsPollingReaderBuilder`. 56 | 57 | `WithBackgroundService` registers an `IHostedService` which will start and stop the queue reader for the `IHost`. 58 | 59 | `WithMessageProcessor` allows you to register a special kind of `IHostedService` which consumes messages from the channel. You must derive from the abstract `SqsMessageProcessingBackgroundService` class to provide the basic message handling functionality you require. 60 | 61 | An abstract class `MessageProcessorService`, which inherits from `SqsMessageProcessingBackgroundService`, may also be used to simplify the code you need to implement. When deriving from this class, you implement the `ProcessMessage` to handle each message. 62 | 63 | For example: 64 | 65 | ```csharp 66 | public class QueueProcessor : MessageProcessorService 67 | { 68 | private readonly ILogger _logger; 69 | 70 | public QueueProcessor(IChannelReaderAccessor channelReaderAccessor, ILogger logger) 71 | : base(channelReaderAccessor) 72 | { 73 | _logger = logger; 74 | } 75 | 76 | public override Task ProcessMessageAsync(Message message, CancellationToken cancellationToken = default) 77 | { 78 | _logger.LogInformation(message.Body); 79 | 80 | foreach (var (key, value) in message.Attributes) 81 | { 82 | _logger.LogInformation($"{key} = {value}"); 83 | } 84 | 85 | // more processing / deletion etc. 86 | 87 | return Task.CompletedTask; 88 | } 89 | } 90 | ``` 91 | 92 | Back to the builder, `WithDefaultExceptionHandler()` registered a simple exception handler which logs major failures, such as lack of queue permissions and then gracefully shuts down the host. You may provide a custom `IExceptionHandler` for this if you require different behaviour. 93 | 94 | For more usage ideas, see the sample project. 95 | 96 | #### Future 97 | 98 | Not yet in the alpha package, but available in the source is a simplified extension method for register the reader in common cases. 99 | 100 | ```csharp 101 | services.AddDefaultPollingSqs(hostContext.Configuration.GetSection("TestQueue")); 102 | ``` 103 | 104 | This is similar to the earlier example and will register the reader background service + the QueueProcessor service, along with default exception handling. Configure is not called in this example, but can be. 105 | 106 | # Diagnostics 107 | 108 | I've started plumbing in some `DiagnosticListener` logging for activty tracing. This is available but not documented yet. 109 | 110 | # Planned Features 111 | 112 | ## SQS Batch Deleter 113 | 114 | Support for registering messages for deletion in batches, with an optional timer that triggers the batch if the batch size has not been met. 115 | 116 | Status: Work in progress. 117 | 118 | This is made internal currently as there will be API breaking changes to support running multiple batch deleters against multiple queues. Current code assumed a single queue use case which is a bit restrictive. This work will be available in a future alpha release. 119 | 120 | # Support 121 | 122 | If this library has helped you, feel free to [buy me a coffee](https://www.buymeacoffee.com/stevejgordon) or see the "Sponsor" link [at the top of the GitHub page](https://github.com/stevejgordon/CorrelationId). 123 | 124 | ## Feedback 125 | 126 | I welcome ideas for features and improvements to be raised as issues which I will respond to as soon as I can. This is a hobby project so it may not be immediate! 127 | -------------------------------------------------------------------------------- /Sqs Toolbox.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio Version 16 4 | VisualStudioVersion = 16.0.29613.14 5 | MinimumVisualStudioVersion = 15.0.26124.0 6 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{08E9DCCD-57A4-4FDF-B43D-8CE57A53E014}" 7 | EndProject 8 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{2E9465F8-DCF0-49DA-9880-F285E99E476D}" 9 | EndProject 10 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "samples", "samples", "{3DB02190-40D2-4553-B732-2DE6718A777D}" 11 | EndProject 12 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ConsoleAppSample", "samples\ConsoleAppSample\ConsoleAppSample.csproj", "{593CB3AE-6F26-43FD-A368-C0C2A07A8BB4}" 13 | EndProject 14 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WorkerServiceSample", "samples\WorkerServiceSample\WorkerServiceSample.csproj", "{40FF8B10-B763-4BDB-B1BC-864B0A2046AF}" 15 | EndProject 16 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "docs", "docs", "{C3A385C2-F5E6-469D-8B15-627406CFA096}" 17 | ProjectSection(SolutionItems) = preProject 18 | README.md = README.md 19 | EndProjectSection 20 | EndProject 21 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "templates", "templates", "{7B33C7CD-7079-4A40-BD57-8DC3B7CB23A2}" 22 | EndProject 23 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DotNetCloud.SqsWorkerService", "templates\SqsWorkerService\DotNetCloud.SqsWorkerService.csproj", "{5075B9C6-4945-4E12-B9BF-85C97BA095C5}" 24 | EndProject 25 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Sqs", "Sqs", "{26E1F2F4-B50F-4287-9FFE-C6AD020E46BD}" 26 | ProjectSection(SolutionItems) = preProject 27 | templates\SqsMessageProcessingWorkerService\.template.config\template.json = templates\SqsMessageProcessingWorkerService\.template.config\template.json 28 | EndProjectSection 29 | EndProject 30 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{D6F3761E-5991-4020-A192-D46823F2F2E0}" 31 | ProjectSection(SolutionItems) = preProject 32 | .editorconfig = .editorconfig 33 | EndProjectSection 34 | EndProject 35 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DotNetCloud.SqsToolbox.Core", "src\DotNetCloud.SqsToolbox.Core\DotNetCloud.SqsToolbox.Core.csproj", "{95250BCD-CDDA-4FC8-BD92-48589F3E79BB}" 36 | EndProject 37 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DotNetCloud.SqsToolbox.Core.Tests", "test\DotNetCloud.SqsToolbox.Core.Tests\DotNetCloud.SqsToolbox.Core.Tests.csproj", "{2CC21051-02BE-43AB-BE77-B935D9DC707A}" 38 | EndProject 39 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DotNetCloud.SqsToolbox", "src\DotNetCloud.SqsToolbox\DotNetCloud.SqsToolbox.csproj", "{1434FBC8-0B06-4FF5-B79F-E6D2DD5AF1AA}" 40 | EndProject 41 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DotNetCloud.SqsToolbox.Tests", "test\DotNetCloud.SqsToolbox.Tests\DotNetCloud.SqsToolbox.Tests.csproj", "{CB1E1326-C09E-4794-95F2-B9151EF5C175}" 42 | EndProject 43 | Global 44 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 45 | Debug|Any CPU = Debug|Any CPU 46 | Debug|x64 = Debug|x64 47 | Debug|x86 = Debug|x86 48 | Release|Any CPU = Release|Any CPU 49 | Release|x64 = Release|x64 50 | Release|x86 = Release|x86 51 | EndGlobalSection 52 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 53 | {593CB3AE-6F26-43FD-A368-C0C2A07A8BB4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 54 | {593CB3AE-6F26-43FD-A368-C0C2A07A8BB4}.Debug|Any CPU.Build.0 = Debug|Any CPU 55 | {593CB3AE-6F26-43FD-A368-C0C2A07A8BB4}.Debug|x64.ActiveCfg = Debug|Any CPU 56 | {593CB3AE-6F26-43FD-A368-C0C2A07A8BB4}.Debug|x64.Build.0 = Debug|Any CPU 57 | {593CB3AE-6F26-43FD-A368-C0C2A07A8BB4}.Debug|x86.ActiveCfg = Debug|Any CPU 58 | {593CB3AE-6F26-43FD-A368-C0C2A07A8BB4}.Debug|x86.Build.0 = Debug|Any CPU 59 | {593CB3AE-6F26-43FD-A368-C0C2A07A8BB4}.Release|Any CPU.ActiveCfg = Release|Any CPU 60 | {593CB3AE-6F26-43FD-A368-C0C2A07A8BB4}.Release|Any CPU.Build.0 = Release|Any CPU 61 | {593CB3AE-6F26-43FD-A368-C0C2A07A8BB4}.Release|x64.ActiveCfg = Release|Any CPU 62 | {593CB3AE-6F26-43FD-A368-C0C2A07A8BB4}.Release|x64.Build.0 = Release|Any CPU 63 | {593CB3AE-6F26-43FD-A368-C0C2A07A8BB4}.Release|x86.ActiveCfg = Release|Any CPU 64 | {593CB3AE-6F26-43FD-A368-C0C2A07A8BB4}.Release|x86.Build.0 = Release|Any CPU 65 | {40FF8B10-B763-4BDB-B1BC-864B0A2046AF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 66 | {40FF8B10-B763-4BDB-B1BC-864B0A2046AF}.Debug|Any CPU.Build.0 = Debug|Any CPU 67 | {40FF8B10-B763-4BDB-B1BC-864B0A2046AF}.Debug|x64.ActiveCfg = Debug|Any CPU 68 | {40FF8B10-B763-4BDB-B1BC-864B0A2046AF}.Debug|x64.Build.0 = Debug|Any CPU 69 | {40FF8B10-B763-4BDB-B1BC-864B0A2046AF}.Debug|x86.ActiveCfg = Debug|Any CPU 70 | {40FF8B10-B763-4BDB-B1BC-864B0A2046AF}.Debug|x86.Build.0 = Debug|Any CPU 71 | {40FF8B10-B763-4BDB-B1BC-864B0A2046AF}.Release|Any CPU.ActiveCfg = Release|Any CPU 72 | {40FF8B10-B763-4BDB-B1BC-864B0A2046AF}.Release|Any CPU.Build.0 = Release|Any CPU 73 | {40FF8B10-B763-4BDB-B1BC-864B0A2046AF}.Release|x64.ActiveCfg = Release|Any CPU 74 | {40FF8B10-B763-4BDB-B1BC-864B0A2046AF}.Release|x64.Build.0 = Release|Any CPU 75 | {40FF8B10-B763-4BDB-B1BC-864B0A2046AF}.Release|x86.ActiveCfg = Release|Any CPU 76 | {40FF8B10-B763-4BDB-B1BC-864B0A2046AF}.Release|x86.Build.0 = Release|Any CPU 77 | {5075B9C6-4945-4E12-B9BF-85C97BA095C5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 78 | {5075B9C6-4945-4E12-B9BF-85C97BA095C5}.Debug|Any CPU.Build.0 = Debug|Any CPU 79 | {5075B9C6-4945-4E12-B9BF-85C97BA095C5}.Debug|x64.ActiveCfg = Debug|Any CPU 80 | {5075B9C6-4945-4E12-B9BF-85C97BA095C5}.Debug|x64.Build.0 = Debug|Any CPU 81 | {5075B9C6-4945-4E12-B9BF-85C97BA095C5}.Debug|x86.ActiveCfg = Debug|Any CPU 82 | {5075B9C6-4945-4E12-B9BF-85C97BA095C5}.Debug|x86.Build.0 = Debug|Any CPU 83 | {5075B9C6-4945-4E12-B9BF-85C97BA095C5}.Release|Any CPU.ActiveCfg = Release|Any CPU 84 | {5075B9C6-4945-4E12-B9BF-85C97BA095C5}.Release|Any CPU.Build.0 = Release|Any CPU 85 | {5075B9C6-4945-4E12-B9BF-85C97BA095C5}.Release|x64.ActiveCfg = Release|Any CPU 86 | {5075B9C6-4945-4E12-B9BF-85C97BA095C5}.Release|x64.Build.0 = Release|Any CPU 87 | {5075B9C6-4945-4E12-B9BF-85C97BA095C5}.Release|x86.ActiveCfg = Release|Any CPU 88 | {5075B9C6-4945-4E12-B9BF-85C97BA095C5}.Release|x86.Build.0 = Release|Any CPU 89 | {95250BCD-CDDA-4FC8-BD92-48589F3E79BB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 90 | {95250BCD-CDDA-4FC8-BD92-48589F3E79BB}.Debug|Any CPU.Build.0 = Debug|Any CPU 91 | {95250BCD-CDDA-4FC8-BD92-48589F3E79BB}.Debug|x64.ActiveCfg = Debug|Any CPU 92 | {95250BCD-CDDA-4FC8-BD92-48589F3E79BB}.Debug|x64.Build.0 = Debug|Any CPU 93 | {95250BCD-CDDA-4FC8-BD92-48589F3E79BB}.Debug|x86.ActiveCfg = Debug|Any CPU 94 | {95250BCD-CDDA-4FC8-BD92-48589F3E79BB}.Debug|x86.Build.0 = Debug|Any CPU 95 | {95250BCD-CDDA-4FC8-BD92-48589F3E79BB}.Release|Any CPU.ActiveCfg = Release|Any CPU 96 | {95250BCD-CDDA-4FC8-BD92-48589F3E79BB}.Release|Any CPU.Build.0 = Release|Any CPU 97 | {95250BCD-CDDA-4FC8-BD92-48589F3E79BB}.Release|x64.ActiveCfg = Release|Any CPU 98 | {95250BCD-CDDA-4FC8-BD92-48589F3E79BB}.Release|x64.Build.0 = Release|Any CPU 99 | {95250BCD-CDDA-4FC8-BD92-48589F3E79BB}.Release|x86.ActiveCfg = Release|Any CPU 100 | {95250BCD-CDDA-4FC8-BD92-48589F3E79BB}.Release|x86.Build.0 = Release|Any CPU 101 | {2CC21051-02BE-43AB-BE77-B935D9DC707A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 102 | {2CC21051-02BE-43AB-BE77-B935D9DC707A}.Debug|Any CPU.Build.0 = Debug|Any CPU 103 | {2CC21051-02BE-43AB-BE77-B935D9DC707A}.Debug|x64.ActiveCfg = Debug|Any CPU 104 | {2CC21051-02BE-43AB-BE77-B935D9DC707A}.Debug|x64.Build.0 = Debug|Any CPU 105 | {2CC21051-02BE-43AB-BE77-B935D9DC707A}.Debug|x86.ActiveCfg = Debug|Any CPU 106 | {2CC21051-02BE-43AB-BE77-B935D9DC707A}.Debug|x86.Build.0 = Debug|Any CPU 107 | {2CC21051-02BE-43AB-BE77-B935D9DC707A}.Release|Any CPU.ActiveCfg = Release|Any CPU 108 | {2CC21051-02BE-43AB-BE77-B935D9DC707A}.Release|Any CPU.Build.0 = Release|Any CPU 109 | {2CC21051-02BE-43AB-BE77-B935D9DC707A}.Release|x64.ActiveCfg = Release|Any CPU 110 | {2CC21051-02BE-43AB-BE77-B935D9DC707A}.Release|x64.Build.0 = Release|Any CPU 111 | {2CC21051-02BE-43AB-BE77-B935D9DC707A}.Release|x86.ActiveCfg = Release|Any CPU 112 | {2CC21051-02BE-43AB-BE77-B935D9DC707A}.Release|x86.Build.0 = Release|Any CPU 113 | {1434FBC8-0B06-4FF5-B79F-E6D2DD5AF1AA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 114 | {1434FBC8-0B06-4FF5-B79F-E6D2DD5AF1AA}.Debug|Any CPU.Build.0 = Debug|Any CPU 115 | {1434FBC8-0B06-4FF5-B79F-E6D2DD5AF1AA}.Debug|x64.ActiveCfg = Debug|Any CPU 116 | {1434FBC8-0B06-4FF5-B79F-E6D2DD5AF1AA}.Debug|x64.Build.0 = Debug|Any CPU 117 | {1434FBC8-0B06-4FF5-B79F-E6D2DD5AF1AA}.Debug|x86.ActiveCfg = Debug|Any CPU 118 | {1434FBC8-0B06-4FF5-B79F-E6D2DD5AF1AA}.Debug|x86.Build.0 = Debug|Any CPU 119 | {1434FBC8-0B06-4FF5-B79F-E6D2DD5AF1AA}.Release|Any CPU.ActiveCfg = Release|Any CPU 120 | {1434FBC8-0B06-4FF5-B79F-E6D2DD5AF1AA}.Release|Any CPU.Build.0 = Release|Any CPU 121 | {1434FBC8-0B06-4FF5-B79F-E6D2DD5AF1AA}.Release|x64.ActiveCfg = Release|Any CPU 122 | {1434FBC8-0B06-4FF5-B79F-E6D2DD5AF1AA}.Release|x64.Build.0 = Release|Any CPU 123 | {1434FBC8-0B06-4FF5-B79F-E6D2DD5AF1AA}.Release|x86.ActiveCfg = Release|Any CPU 124 | {1434FBC8-0B06-4FF5-B79F-E6D2DD5AF1AA}.Release|x86.Build.0 = Release|Any CPU 125 | {CB1E1326-C09E-4794-95F2-B9151EF5C175}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 126 | {CB1E1326-C09E-4794-95F2-B9151EF5C175}.Debug|Any CPU.Build.0 = Debug|Any CPU 127 | {CB1E1326-C09E-4794-95F2-B9151EF5C175}.Debug|x64.ActiveCfg = Debug|Any CPU 128 | {CB1E1326-C09E-4794-95F2-B9151EF5C175}.Debug|x64.Build.0 = Debug|Any CPU 129 | {CB1E1326-C09E-4794-95F2-B9151EF5C175}.Debug|x86.ActiveCfg = Debug|Any CPU 130 | {CB1E1326-C09E-4794-95F2-B9151EF5C175}.Debug|x86.Build.0 = Debug|Any CPU 131 | {CB1E1326-C09E-4794-95F2-B9151EF5C175}.Release|Any CPU.ActiveCfg = Release|Any CPU 132 | {CB1E1326-C09E-4794-95F2-B9151EF5C175}.Release|Any CPU.Build.0 = Release|Any CPU 133 | {CB1E1326-C09E-4794-95F2-B9151EF5C175}.Release|x64.ActiveCfg = Release|Any CPU 134 | {CB1E1326-C09E-4794-95F2-B9151EF5C175}.Release|x64.Build.0 = Release|Any CPU 135 | {CB1E1326-C09E-4794-95F2-B9151EF5C175}.Release|x86.ActiveCfg = Release|Any CPU 136 | {CB1E1326-C09E-4794-95F2-B9151EF5C175}.Release|x86.Build.0 = Release|Any CPU 137 | EndGlobalSection 138 | GlobalSection(SolutionProperties) = preSolution 139 | HideSolutionNode = FALSE 140 | EndGlobalSection 141 | GlobalSection(NestedProjects) = preSolution 142 | {593CB3AE-6F26-43FD-A368-C0C2A07A8BB4} = {3DB02190-40D2-4553-B732-2DE6718A777D} 143 | {40FF8B10-B763-4BDB-B1BC-864B0A2046AF} = {3DB02190-40D2-4553-B732-2DE6718A777D} 144 | {5075B9C6-4945-4E12-B9BF-85C97BA095C5} = {26E1F2F4-B50F-4287-9FFE-C6AD020E46BD} 145 | {26E1F2F4-B50F-4287-9FFE-C6AD020E46BD} = {7B33C7CD-7079-4A40-BD57-8DC3B7CB23A2} 146 | {95250BCD-CDDA-4FC8-BD92-48589F3E79BB} = {08E9DCCD-57A4-4FDF-B43D-8CE57A53E014} 147 | {2CC21051-02BE-43AB-BE77-B935D9DC707A} = {2E9465F8-DCF0-49DA-9880-F285E99E476D} 148 | {1434FBC8-0B06-4FF5-B79F-E6D2DD5AF1AA} = {08E9DCCD-57A4-4FDF-B43D-8CE57A53E014} 149 | {CB1E1326-C09E-4794-95F2-B9151EF5C175} = {2E9465F8-DCF0-49DA-9880-F285E99E476D} 150 | EndGlobalSection 151 | GlobalSection(ExtensibilityGlobals) = postSolution 152 | SolutionGuid = {87DA5DC6-C5DA-417A-AC7F-B6C4492D9CD0} 153 | EndGlobalSection 154 | EndGlobal 155 | -------------------------------------------------------------------------------- /samples/ConsoleAppSample/ConsoleAppSample.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | netcoreapp3.1 6 | 8 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /samples/ConsoleAppSample/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Diagnostics; 4 | using System.Threading.Tasks; 5 | using Amazon; 6 | using Amazon.Runtime.CredentialManagement; 7 | using Amazon.SQS; 8 | using DotNetCloud.SqsToolbox.Core.Receive; 9 | 10 | namespace ConsoleAppSample 11 | { 12 | public sealed class ExampleDiagnosticObserver : IObserver, IObserver> 13 | { 14 | private readonly List _subscriptions = new List(); 15 | 16 | void IObserver.OnNext(DiagnosticListener diagnosticListener) 17 | { 18 | if (diagnosticListener.Name == SqsPollingQueueReader.DiagnosticListenerName) 19 | { 20 | var subscription = diagnosticListener.Subscribe(this); 21 | _subscriptions.Add(subscription); 22 | } 23 | } 24 | 25 | public void OnCompleted() 26 | { 27 | throw new NotImplementedException(); 28 | } 29 | 30 | public void OnError(Exception error) 31 | { 32 | throw new NotImplementedException(); 33 | } 34 | 35 | public void OnNext(KeyValuePair value) 36 | { 37 | var (key, payload) = value; 38 | 39 | Console.WriteLine($"Event: {key} ActivityName: {Activity.Current.OperationName} Id: {Activity.Current.Id} Payload: {payload}"); 40 | } 41 | 42 | void IObserver.OnError(Exception error) 43 | { 44 | } 45 | 46 | void IObserver.OnCompleted() 47 | { 48 | _subscriptions.ForEach(x => x.Dispose()); 49 | _subscriptions.Clear(); 50 | } 51 | } 52 | 53 | internal class Program 54 | { 55 | private static async Task Main(string[] args) 56 | { 57 | //var observer = new ExampleDiagnosticObserver(); 58 | 59 | //DiagnosticListener.AllListeners.Subscribe(observer); 60 | 61 | var f = new SharedCredentialsFile(SharedCredentialsFile.DefaultFilePath); 62 | 63 | f.TryGetProfile("default", out var profile); 64 | 65 | var credentials = profile.GetAWSCredentials(null); 66 | 67 | var client = new AmazonSQSClient(credentials, RegionEndpoint.EUWest2); 68 | 69 | var options = new SqsPollingQueueReaderOptions { QueueUrl = "https://sqs.eu-west-1.amazonaws.com/123456789012/test-queue" }; 70 | 71 | //using var pollingReader = new SqsPollingQueueReader(options, client, new SqsReceiveDelayCalculator(options), null); 72 | 73 | //using var deleter = new SqsBatchDeleter(new SqsBatchDeletionOptions { MaxWaitForFullBatch = TimeSpan.FromSeconds(10), DrainOnStop = true, QueueUrl = "https://sqs.eu-west-1.amazonaws.com/123456789012/test-queue" }, client); 74 | 75 | //using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(5)); 76 | 77 | //var readingTask = ReadFromChannelAsync(pollingReader.ChannelReader, deleter, cts.Token); 78 | 79 | //deleter.Start(cts.Token); 80 | //pollingReader.Start(cts.Token); 81 | 82 | //await readingTask; 83 | 84 | //for (var i = 0; i < 26; i++) 85 | //{ 86 | // await deleter.AddMessageAsync(new Message{ MessageId = Guid.NewGuid().ToString() }, cts.Token); 87 | //} 88 | 89 | //deleter.Start(cts.Token); 90 | 91 | //await Task.Delay(TimeSpan.FromSeconds(8), cts.Token); 92 | 93 | //await deleter.AddMessageAsync(new Message { MessageId = Guid.NewGuid().ToString() }, cts.Token); 94 | 95 | //await Task.Delay(TimeSpan.FromSeconds(15), cts.Token); 96 | 97 | //for (var i = 0; i < 11; i++) 98 | //{ 99 | // await deleter.AddMessageAsync(new Message { MessageId = Guid.NewGuid().ToString() }, cts.Token); 100 | //} 101 | 102 | //var messages = Enumerable.Range(0, 57).Select(x => new Message {MessageId = Guid.NewGuid().ToString()}).ToArray(); 103 | 104 | //await deleter.AddMessagesAsync(messages, cts.Token); 105 | 106 | //await Task.Delay(TimeSpan.FromSeconds(10), cts.Token); 107 | 108 | //for (var i = 0; i < 2; i++) 109 | //{ 110 | // await deleter.AddMessageAsync(new Message{ MessageId = "ABC" }, cts.Token); 111 | //} 112 | 113 | //await Task.Delay(TimeSpan.FromSeconds(10), cts.Token); 114 | 115 | //await deleter.StopAsync(); 116 | 117 | //await Task.Delay(Timeout.Infinite, cts.Token); 118 | } 119 | 120 | //private static async Task ReadFromChannelAsync(ChannelReader reader, SqsBatchDeleter deleter, CancellationToken cancellationToken) 121 | //{ 122 | // await foreach (var message in reader.ReadAllAsync(cancellationToken)) 123 | // { 124 | // Console.WriteLine(message.MessageId); 125 | 126 | // await deleter.AddMessageAsync(message, cancellationToken); 127 | // } 128 | //} 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /samples/WorkerServiceSample/CustomExceptionHandler.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using DotNetCloud.SqsToolbox.Core.Abstractions; 3 | 4 | namespace WorkerServiceSample 5 | { 6 | public class CustomExceptionHandler : IExceptionHandler 7 | { 8 | public void OnException(T1 exception, T2 source) where T1 : Exception where T2 : class 9 | { 10 | Console.WriteLine("An exception occurred!!!!!!!!"); 11 | } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /samples/WorkerServiceSample/DiagnosticsMonitorService.cs: -------------------------------------------------------------------------------- 1 | using Amazon.SQS.Model; 2 | using DotNetCloud.SqsToolbox.Diagnostics; 3 | using Microsoft.Extensions.Logging; 4 | 5 | namespace WorkerServiceSample 6 | { 7 | public class DiagnosticsMonitorService : DiagnosticsMonitoringService 8 | { 9 | private readonly ILogger _logger; 10 | 11 | public DiagnosticsMonitorService(ILogger logger) => _logger = logger; 12 | 13 | public override void OnBegin(string queueUrl) => _logger.LogInformation("Polling for messages"); 14 | 15 | public override void OnReceived(string queueUrl, in int messageCount) => _logger.LogInformation($"Completed polling for messages. Received {messageCount}"); 16 | 17 | public override void OnDeleteBatchCreated(in int messageCount, in long millisecondsTaken) => _logger.LogInformation($"Batch with {messageCount} message(s) created in {millisecondsTaken}ms."); 18 | 19 | public override void OnBatchDeleted(DeleteMessageBatchResponse deleteMessageBatchResponse, in long millisecondsTaken) => _logger.LogInformation($"Batch with {deleteMessageBatchResponse.Successful.Count} successful message(s) and {deleteMessageBatchResponse.Failed.Count} failed message(s), deleted in {millisecondsTaken}ms."); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /samples/WorkerServiceSample/Program.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Extensions.DependencyInjection; 2 | using Microsoft.Extensions.Hosting; 3 | 4 | namespace WorkerServiceSample 5 | { 6 | public class Program 7 | { 8 | public static void Main() => CreateHostBuilder().Build().Run(); 9 | 10 | public static IHostBuilder CreateHostBuilder() => 11 | Host.CreateDefaultBuilder() 12 | .ConfigureServices((hostContext, services) => 13 | { 14 | services.AddPollingSqs(hostContext.Configuration.GetSection("TestQueue")) 15 | .Configure(c => c.UseExponentialBackoff = true) 16 | .WithBackgroundService() 17 | .WithMessageProcessor() 18 | .WithDefaultExceptionHandler(); 19 | 20 | // the above can be simplified to: 21 | services.AddDefaultPollingSqs(hostContext.Configuration.GetSection("TestQueue2")); // This snippet does not call configure, but can do if required. 22 | 23 | services.AddSqsToolboxDiagnosticsMonitoring(); 24 | }); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /samples/WorkerServiceSample/Properties/launchSettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "profiles": { 3 | "WorkerServiceSample": { 4 | "commandName": "Project", 5 | "environmentVariables": { 6 | "DOTNET_ENVIRONMENT": "Development" 7 | } 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /samples/WorkerServiceSample/QueueProcessor.cs: -------------------------------------------------------------------------------- 1 | using System.Threading; 2 | using System.Threading.Tasks; 3 | using Amazon.SQS.Model; 4 | using DotNetCloud.SqsToolbox; 5 | using DotNetCloud.SqsToolbox.Hosting; 6 | using Microsoft.Extensions.Logging; 7 | 8 | namespace WorkerServiceSample 9 | { 10 | public class QueueProcessor : MessageProcessorService 11 | { 12 | private readonly ILogger _logger; 13 | 14 | public QueueProcessor(IChannelReaderAccessor channelReaderAccessor, ILogger logger) : base(channelReaderAccessor) 15 | { 16 | _logger = logger; 17 | } 18 | 19 | public override Task ProcessMessageAsync(Message message, CancellationToken cancellationToken = default) 20 | { 21 | _logger.LogInformation(message.Body); 22 | 23 | foreach (var (key, value) in message.Attributes) 24 | { 25 | _logger.LogInformation($"{key} = {value}"); 26 | } 27 | 28 | // more processing / deletion etc. 29 | 30 | return Task.CompletedTask; 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /samples/WorkerServiceSample/WorkerServiceSample.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netcoreapp3.1 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /samples/WorkerServiceSample/appsettings.Development.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Information", 5 | "Microsoft": "Warning", 6 | "Microsoft.Hosting.Lifetime": "Information" 7 | } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /samples/WorkerServiceSample/appsettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "AWS": { 3 | "Region": "eu-west-2" 4 | }, 5 | 6 | "Logging": { 7 | "LogLevel": { 8 | "Default": "Information", 9 | "Microsoft": "Warning", 10 | "Microsoft.Hosting.Lifetime": "Information" 11 | } 12 | }, 13 | 14 | "TestQueue": { 15 | "QueueName": "QueueOne", 16 | "QueueUrl": "https://sqs.eu-west-2.amazonaws.com/865288682694/TestQueue" 17 | }, 18 | "TestQueue2": { 19 | "QueueName": "QueueTwo", 20 | "QueueUrl": "https://sqs.eu-west-2.amazonaws.com/865288682694/TestQueue" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /sqs-toolbox.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dotnetcloud/SqsToolbox/bc27d1f4762c063f94a80ca45bec4bf804ff283c/sqs-toolbox.png -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox.Core/Abstractions/IExceptionHandler.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace DotNetCloud.SqsToolbox.Core.Abstractions 4 | { 5 | /// 6 | /// Handler for exceptions which occur within critical paths. 7 | /// 8 | public interface IExceptionHandler 9 | { 10 | /// 11 | /// Handler for an exception. 12 | /// 13 | /// The type of the 14 | /// The type of the source for the exception. 15 | /// The exception being handled. 16 | /// The source of the exception. 17 | void OnException(T1 exception, T2 source) 18 | where T1 : Exception 19 | where T2 : class; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox.Core/Abstractions/IFailedDeletionEntryHandler.cs: -------------------------------------------------------------------------------- 1 | using System.Threading; 2 | using System.Threading.Tasks; 3 | using Amazon.SQS.Model; 4 | 5 | namespace DotNetCloud.SqsToolbox.Core.Abstractions 6 | { 7 | internal interface IFailedDeletionEntryHandler 8 | { 9 | Task OnFailureAsync(BatchResultErrorEntry batchResultErrorEntry, CancellationToken cancellationToken = default); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox.Core/Abstractions/ISqsBatchDeleteQueue.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Threading; 3 | using System.Threading.Tasks; 4 | using Amazon.SQS.Model; 5 | 6 | namespace DotNetCloud.SqsToolbox.Core.Abstractions 7 | { 8 | internal interface ISqsBatchDeleteQueue 9 | { 10 | Task AddMessageAsync(Message message, CancellationToken cancellationToken = default); 11 | Task AddMessagesAsync(IList messages, CancellationToken cancellationToken = default); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox.Core/Abstractions/ISqsBatchDeleter.cs: -------------------------------------------------------------------------------- 1 | using System.Threading; 2 | using System.Threading.Tasks; 3 | 4 | namespace DotNetCloud.SqsToolbox.Core.Abstractions 5 | { 6 | internal interface ISqsBatchDeleter : ISqsBatchDeleteQueue 7 | { 8 | void Start(CancellationToken cancellationToken = default); 9 | Task StopAsync(); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox.Core/Abstractions/ISqsMessageChannelSource.cs: -------------------------------------------------------------------------------- 1 | namespace DotNetCloud.SqsToolbox.Core.Abstractions 2 | { 3 | //public interface ISqsMessageChannelSource 4 | //{ 5 | // /// 6 | // /// Get an instance of a of . 7 | // /// 8 | // /// A of . 9 | // Channel GetChannel(); 10 | //} 11 | } 12 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox.Core/Abstractions/ISqsPollingQueueReader.cs: -------------------------------------------------------------------------------- 1 | using System.Threading; 2 | using System.Threading.Channels; 3 | using System.Threading.Tasks; 4 | using Amazon.SQS.Model; 5 | 6 | namespace DotNetCloud.SqsToolbox.Core.Abstractions 7 | { 8 | /// 9 | /// Once started, polls a queue for messages, writing them to a channel, until stopped. 10 | /// 11 | public interface ISqsPollingQueueReader 12 | { 13 | /// 14 | /// The from which received messages can be read for processing. 15 | /// 16 | ChannelReader ChannelReader { get; } 17 | 18 | /// 19 | /// Start polling the queue for messages. 20 | /// 21 | /// A which can be used to cancel the polling of the queue. 22 | void Start(CancellationToken cancellationToken = default); 23 | 24 | /// 25 | /// Stop polling the queue for messages. 26 | /// 27 | Task StopAsync(); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox.Core/Abstractions/ISqsReceiveDelayCalculator.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using Amazon.SQS.Model; 4 | 5 | namespace DotNetCloud.SqsToolbox.Core.Abstractions 6 | { 7 | /// 8 | /// Calculates the next delay for a polling receive message attempt. 9 | /// 10 | public interface ISqsReceiveDelayCalculator 11 | { 12 | /// 13 | /// Calculates a delay between the previous and next polling receive attempt. 14 | /// 15 | /// The of from the last receive attempt. 16 | /// A representing the delay to apply before the next polling attempt. 17 | TimeSpan CalculateSecondsToDelay(IEnumerable messages); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox.Core/Abstractions/SqsMessageChannelSource.cs: -------------------------------------------------------------------------------- 1 | namespace DotNetCloud.SqsToolbox.Core.Abstractions 2 | { 3 | ///// 4 | ///// Base class for implementing a source of a of . 5 | ///// 6 | //public abstract class SqsMessageChannelSource : ISqsMessageChannelSource 7 | //{ 8 | // private static readonly object _lock = new object(); 9 | 10 | // private Channel _messageChannel; 11 | 12 | // /// 13 | // /// Get an instance of a of . 14 | // /// 15 | // /// A of . 16 | // public Channel GetChannel() 17 | // { 18 | // if (_messageChannel is object) return _messageChannel; 19 | 20 | // lock (_lock) 21 | // { 22 | // if (_messageChannel is object) return _messageChannel; 23 | 24 | // _messageChannel = InitialiseChannel(); 25 | // } 26 | 27 | // return _messageChannel; 28 | // } 29 | 30 | // /// 31 | // /// Initialises a of . 32 | // /// 33 | // /// The initialised . 34 | // protected abstract Channel InitialiseChannel(); 35 | //} 36 | } 37 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox.Core/DefaultExceptionHandler.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using DotNetCloud.SqsToolbox.Core.Abstractions; 3 | 4 | namespace DotNetCloud.SqsToolbox.Core 5 | { 6 | /// 7 | /// A default exception handler which no-ops for all exceptions. 8 | /// 9 | public sealed class DefaultExceptionHandler : IExceptionHandler 10 | { 11 | /// 12 | /// A reusable instance of a shared . 13 | /// 14 | public static readonly DefaultExceptionHandler Instance = new DefaultExceptionHandler(); 15 | 16 | /// 17 | public void OnException(T1 exception, T2 source) where T1 : Exception where T2 : class 18 | { 19 | // No-op 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox.Core/Delete/DefaultFailedDeletionEntryHandler.cs: -------------------------------------------------------------------------------- 1 | using System.Threading; 2 | using System.Threading.Tasks; 3 | using Amazon.SQS.Model; 4 | using DotNetCloud.SqsToolbox.Core.Abstractions; 5 | 6 | namespace DotNetCloud.SqsToolbox.Core.Delete 7 | { 8 | internal sealed class DefaultFailedDeletionEntryHandler : IFailedDeletionEntryHandler 9 | { 10 | public static DefaultFailedDeletionEntryHandler Instance = new DefaultFailedDeletionEntryHandler(); 11 | 12 | public Task OnFailureAsync(BatchResultErrorEntry batchResultErrorEntry, CancellationToken cancellationToken = default) => Task.CompletedTask; 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox.Core/Delete/SqsBatchDeleteQueue.cs: -------------------------------------------------------------------------------- 1 | namespace DotNetCloud.SqsToolbox.Core.Delete 2 | { 3 | //public class SqsBatchDeleteQueue : ISqsBatchDeleteQueue 4 | //{ 5 | // private readonly ISqsBatchDeleter _batchDeleter; 6 | 7 | // public SqsBatchDeleteQueue(ISqsBatchDeleter batchDeleter) => _batchDeleter = batchDeleter; 8 | 9 | // public Task AddMessageAsync(Message message, CancellationToken cancellationToken = default) => 10 | // _batchDeleter.AddMessageAsync(message, cancellationToken); 11 | 12 | // public Task AddMessagesAsync(IList messages, CancellationToken cancellationToken = default) => 13 | // _batchDeleter.AddMessagesAsync(messages, cancellationToken); 14 | //} 15 | } 16 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox.Core/Delete/SqsBatchDeleter.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Diagnostics; 4 | using System.Linq; 5 | using System.Net; 6 | using System.Threading; 7 | using System.Threading.Channels; 8 | using System.Threading.Tasks; 9 | using Amazon.SQS; 10 | using Amazon.SQS.Model; 11 | using DotNetCloud.SqsToolbox.Core.Abstractions; 12 | using DotNetCloud.SqsToolbox.Core.Diagnostics; 13 | 14 | namespace DotNetCloud.SqsToolbox.Core.Delete 15 | { 16 | internal class SqsBatchDeleter : ISqsBatchDeleter, IDisposable 17 | { 18 | public const string DiagnosticListenerName = "DotNetCloud.SqsToolbox.SqsBatchDeleter"; 19 | 20 | private readonly SqsBatchDeletionOptions _sqsBatchDeletionOptions; 21 | private readonly IAmazonSQS _amazonSqs; 22 | private readonly IFailedDeletionEntryHandler _failedDeletionEntryHandler; 23 | private readonly IExceptionHandler _exceptionHandler; 24 | private readonly Channel _channel; 25 | 26 | private CancellationTokenSource _cancellationTokenSource; 27 | 28 | private Task _batchingTask; 29 | private bool _disposed; 30 | private bool _isStarted; 31 | 32 | private readonly Dictionary _currentBatch; 33 | private readonly DeleteMessageBatchRequest _deleteMessageBatchRequest; 34 | private readonly object _startLock = new object(); 35 | 36 | private static readonly DiagnosticListener _diagnostics = new DiagnosticListener(DiagnosticListenerName); 37 | 38 | public SqsBatchDeleter(SqsBatchDeletionOptions sqsBatchDeletionOptions, IAmazonSQS amazonSqs, IExceptionHandler exceptionHandler, IFailedDeletionEntryHandler failedDeletionEntryHandler) 39 | : this(sqsBatchDeletionOptions, amazonSqs, exceptionHandler, failedDeletionEntryHandler, null) 40 | { 41 | } 42 | 43 | public SqsBatchDeleter(SqsBatchDeletionOptions sqsBatchDeletionOptions, IAmazonSQS amazonSqs, IExceptionHandler exceptionHandler, IFailedDeletionEntryHandler failedDeletionEntryHandler, Channel channel) 44 | { 45 | _ = sqsBatchDeletionOptions ?? throw new ArgumentNullException(nameof(sqsBatchDeletionOptions)); 46 | 47 | _sqsBatchDeletionOptions = sqsBatchDeletionOptions.Clone(); 48 | _amazonSqs = amazonSqs ?? throw new ArgumentNullException(nameof(amazonSqs)); 49 | _failedDeletionEntryHandler = failedDeletionEntryHandler ?? DefaultFailedDeletionEntryHandler.Instance; 50 | _exceptionHandler = exceptionHandler ?? DefaultExceptionHandler.Instance; 51 | 52 | _channel = channel ?? Channel.CreateBounded(new BoundedChannelOptions(_sqsBatchDeletionOptions.ChannelCapacity) 53 | { 54 | SingleReader = true 55 | }); 56 | 57 | _currentBatch = new Dictionary(sqsBatchDeletionOptions.BatchSize); 58 | 59 | _deleteMessageBatchRequest = new DeleteMessageBatchRequest 60 | { 61 | QueueUrl = sqsBatchDeletionOptions.QueueUrl 62 | }; 63 | } 64 | 65 | public void Start(CancellationToken cancellationToken = default) 66 | { 67 | if (_isStarted) 68 | throw new InvalidOperationException("The batch deleter is already started."); 69 | 70 | lock (_startLock) 71 | { 72 | if (_isStarted) 73 | throw new InvalidOperationException("The batch deleter is already started."); 74 | 75 | _cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); 76 | 77 | _batchingTask = Task.Run(BatchAsync, cancellationToken); 78 | 79 | _isStarted = true; 80 | } 81 | } 82 | 83 | public async Task StopAsync() 84 | { 85 | if (!_isStarted) 86 | return; 87 | 88 | _channel.Writer.TryComplete(); // nothing more will be written 89 | 90 | if (!_sqsBatchDeletionOptions.DrainOnStop) 91 | { 92 | _cancellationTokenSource?.Cancel(); 93 | } 94 | 95 | await _batchingTask.ConfigureAwait(false); 96 | } 97 | 98 | public async Task AddMessageAsync(Message message, CancellationToken cancellationToken = default) 99 | { 100 | _ = message ?? throw new ArgumentNullException(nameof(message)); 101 | 102 | await _channel.Writer.WriteAsync(message, cancellationToken).ConfigureAwait(false); 103 | } 104 | 105 | public async Task AddMessagesAsync(IList messages, CancellationToken cancellationToken = default) 106 | { 107 | _ = messages ?? throw new ArgumentNullException(nameof(messages)); 108 | 109 | var i = 0; 110 | 111 | while (i < messages.Count && await _channel.Writer.WaitToWriteAsync(cancellationToken).ConfigureAwait(false)) 112 | { 113 | while (i < messages.Count && _channel.Writer.TryWrite(messages[i])) 114 | i++; 115 | } 116 | } 117 | 118 | private async Task BatchAsync() 119 | { 120 | var cancellationToken = _cancellationTokenSource.Token; 121 | 122 | try 123 | { 124 | while (await _channel.Reader.WaitToReadAsync(cancellationToken).ConfigureAwait(false)) // wait until there are messages in the channel before we try to batch 125 | { 126 | using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); 127 | linkedCts.CancelAfter(_sqsBatchDeletionOptions.MaxWaitForFullBatch); 128 | 129 | await CreateBatchAsync(linkedCts.Token).ConfigureAwait(false); 130 | 131 | _deleteMessageBatchRequest.Entries = _currentBatch.Select(m => new DeleteMessageBatchRequestEntry(m.Key, m.Value)).ToList(); 132 | 133 | cancellationToken.ThrowIfCancellationRequested(); 134 | 135 | var sw = Stopwatch.StartNew(); 136 | 137 | var sqsDeleteBatchResponse = await _amazonSqs.DeleteMessageBatchAsync(_deleteMessageBatchRequest, cancellationToken).ConfigureAwait(false); 138 | 139 | sw.Stop(); 140 | 141 | if (sqsDeleteBatchResponse.HttpStatusCode == HttpStatusCode.OK) 142 | { 143 | BatchRequestCompletedDiagnostics(sqsDeleteBatchResponse, sw); 144 | 145 | var failureTasks = sqsDeleteBatchResponse.Failed.Select(entry => 146 | _failedDeletionEntryHandler.OnFailureAsync(entry, cancellationToken)).ToArray(); 147 | 148 | await Task.WhenAll(failureTasks); 149 | } 150 | else 151 | { 152 | // TODO - Handle non-success status code 153 | } 154 | } 155 | } 156 | catch (Exception ex) 157 | { 158 | _exceptionHandler.OnException(ex, this); 159 | } 160 | } 161 | 162 | private async Task CreateBatchAsync(CancellationToken cancellationToken) 163 | { 164 | var sw = Stopwatch.StartNew(); 165 | 166 | _currentBatch.Clear(); 167 | 168 | try 169 | { 170 | while (_currentBatch.Count < 10 && await _channel.Reader.WaitToReadAsync(cancellationToken).ConfigureAwait(false)) 171 | { 172 | var exitBatchCreation = !_channel.Reader.TryRead(out var message) || cancellationToken.IsCancellationRequested; 173 | 174 | if (exitBatchCreation) 175 | continue; 176 | 177 | _currentBatch[message.MessageId] = message.ReceiptHandle; // only add each message ID once, using latest receipt handle 178 | } 179 | } 180 | catch (OperationCanceledException) 181 | { 182 | // swallow this as expected when batch is not full within timeout period 183 | } 184 | 185 | sw.Stop(); 186 | 187 | BatchCreatedDiagnostics(_currentBatch.Count, sw); 188 | } 189 | 190 | private static void BatchCreatedDiagnostics(int messageCount, Stopwatch stopwatch) 191 | { 192 | if (_diagnostics.IsEnabled(DiagnosticEvents.DeletionBatchCreated)) 193 | _diagnostics.Write(DiagnosticEvents.DeletionBatchCreated, 194 | new DeletionBatchCreatedPayload(messageCount, stopwatch.ElapsedMilliseconds)); 195 | } 196 | 197 | private static void BatchRequestCompletedDiagnostics(DeleteMessageBatchResponse response, Stopwatch stopwatch) 198 | { 199 | if (_diagnostics.IsEnabled(DiagnosticEvents.DeleteBatchRequestComplete)) 200 | _diagnostics.Write(DiagnosticEvents.DeleteBatchRequestComplete, 201 | new EndDeletionBatchPayload(response, stopwatch.ElapsedMilliseconds)); 202 | } 203 | 204 | public void Dispose() 205 | { 206 | Dispose(true); 207 | GC.SuppressFinalize(this); 208 | } 209 | 210 | protected virtual void Dispose(bool disposing) 211 | { 212 | if (_disposed) 213 | return; 214 | 215 | if (disposing) 216 | { 217 | _cancellationTokenSource.Dispose(); 218 | _batchingTask.Dispose(); 219 | } 220 | 221 | _disposed = true; 222 | } 223 | } 224 | } 225 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox.Core/Delete/SqsBatchDeleterBuilder.cs: -------------------------------------------------------------------------------- 1 | namespace DotNetCloud.SqsToolbox.Core.Delete 2 | { 3 | //internal class SqsBatchDeleterBuilder 4 | //{ 5 | // private readonly IAmazonSQS _amazonSqs; 6 | // private readonly SqsBatchDeletionOptions _sqsBatchDeletionOptions; 7 | 8 | // private IExceptionHandler _exceptionHandler; 9 | // private IFailedDeletionEntryHandler _failedDeletionEntryHandler; 10 | // private SqsMessageChannelSource _channelSource; 11 | // private Channel _channel; 12 | 13 | // public SqsBatchDeleterBuilder(IAmazonSQS amazonSqs, SqsBatchDeletionOptions sqsBatchDeletionOptions) 14 | // { 15 | // _amazonSqs = amazonSqs ?? throw new ArgumentNullException(nameof(amazonSqs)); 16 | // _sqsBatchDeletionOptions = sqsBatchDeletionOptions ?? throw new ArgumentNullException(nameof(sqsBatchDeletionOptions)); 17 | // } 18 | 19 | // public SqsBatchDeleterBuilder WithExceptionHandler(IExceptionHandler exceptionHandler) 20 | // { 21 | // _exceptionHandler = exceptionHandler ?? throw new ArgumentNullException(nameof(exceptionHandler)); 22 | 23 | // return this; 24 | // } 25 | 26 | // public SqsBatchDeleterBuilder WithFailedDeletionEntryHandler(IFailedDeletionEntryHandler failedDeletionEntryHandler) 27 | // { 28 | // _failedDeletionEntryHandler = failedDeletionEntryHandler ?? throw new ArgumentNullException(nameof(failedDeletionEntryHandler)); 29 | 30 | // return this; 31 | // } 32 | 33 | // public SqsBatchDeleterBuilder WithCustomChannel(SqsMessageChannelSource channelSource) 34 | // { 35 | // _channelSource = channelSource ?? throw new ArgumentNullException(nameof(channelSource)); 36 | 37 | // return this; 38 | // } 39 | 40 | // public SqsBatchDeleterBuilder WithCustomChannel(Channel channel) 41 | // { 42 | // _channel = channel ?? throw new ArgumentNullException(nameof(channel)); 43 | 44 | // return this; 45 | // } 46 | 47 | // public SqsBatchDeleter Build() 48 | // { 49 | // Channel channel; 50 | 51 | // if (_channelSource is object) 52 | // { 53 | // channel = _channelSource.GetChannel(); 54 | // } 55 | // else if (_channel is object) 56 | // { 57 | // channel = _channel; 58 | // } 59 | // else 60 | // { 61 | // channel = Channel.CreateBounded(new BoundedChannelOptions(_sqsBatchDeletionOptions.ChannelCapacity) 62 | // { 63 | // SingleReader = true 64 | // }); 65 | // } 66 | 67 | // return new SqsBatchDeleter(_sqsBatchDeletionOptions, _amazonSqs, _exceptionHandler ?? DefaultExceptionHandler.Instance, _failedDeletionEntryHandler ?? DefaultFailedDeletionEntryHandler.Instance, channel); 68 | // } 69 | //} 70 | } 71 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox.Core/Delete/SqsBatchDeletionOptions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Diagnostics; 3 | 4 | namespace DotNetCloud.SqsToolbox.Core.Delete 5 | { 6 | /// 7 | /// Provides options used to configure the processing performed by an . 8 | /// 9 | [DebuggerDisplay("QueueUrl = {QueueUrl}")] 10 | internal class SqsBatchDeletionOptions 11 | { 12 | private string _queueUrl; 13 | private int _batchSize = 10; 14 | 15 | /// 16 | /// The URL of the SQS queue from which to delete messages. 17 | /// 18 | public string QueueUrl 19 | { 20 | get => _queueUrl; 21 | set 22 | { 23 | if (!Uri.TryCreate(value, UriKind.Absolute, out _)) 24 | { 25 | throw new ArgumentException("The value must be a valid URI", nameof(value)); 26 | } 27 | 28 | _queueUrl = value; 29 | } 30 | } 31 | 32 | /// 33 | /// The capacity of the channel which controls back-pressure in cases where producer(s) outpace the . 34 | /// 35 | public int ChannelCapacity { get; set; } = 100; 36 | 37 | /// 38 | /// The number of messages to include in each batch deletion request. 39 | /// 40 | public int BatchSize 41 | { 42 | get => _batchSize; 43 | set 44 | { 45 | if (value < 1 || value > 10) 46 | { 47 | throw new ArgumentOutOfRangeException(nameof(value), "The value must be between 1 and 10 inclusive."); 48 | } 49 | 50 | _batchSize = value; 51 | } 52 | } 53 | 54 | /// 55 | /// The maximum to wait for before forcing a batch deletion request despite the required batch size not being reached. 56 | /// 57 | public TimeSpan MaxWaitForFullBatch { get; set; } = TimeSpan.FromSeconds(60); 58 | 59 | /// 60 | /// When stopping, should any queued messages be deleted until the internal channel is deleted. 61 | /// 62 | public bool DrainOnStop { get; set; } 63 | 64 | /// A default instance of . 65 | /// 66 | /// Do not change the values of this instance. It is shared by all of our blocks when no options are provided by the user. 67 | /// 68 | internal static readonly SqsBatchDeletionOptions Default = new SqsBatchDeletionOptions(); 69 | 70 | /// 71 | /// Returns a cloned instance of this . 72 | /// 73 | /// 74 | /// An instance of the options that may be cached by the . 75 | /// 76 | internal SqsBatchDeletionOptions Clone() => 77 | new SqsBatchDeletionOptions 78 | { 79 | QueueUrl = QueueUrl, 80 | ChannelCapacity = ChannelCapacity, 81 | BatchSize = BatchSize, 82 | MaxWaitForFullBatch = MaxWaitForFullBatch, 83 | DrainOnStop = DrainOnStop 84 | }; 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox.Core/Diagnostics/BeginReceiveRequestPayload.cs: -------------------------------------------------------------------------------- 1 | namespace DotNetCloud.SqsToolbox.Core.Diagnostics 2 | { 3 | public sealed class BeginReceiveRequestPayload 4 | { 5 | internal BeginReceiveRequestPayload(string queueUrl) 6 | { 7 | QueueUrl = queueUrl; 8 | } 9 | 10 | public string QueueUrl { get; } 11 | 12 | public override string ToString() => $"QueueUrl = {QueueUrl}"; 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox.Core/Diagnostics/DeletionBatchCreatedPayload.cs: -------------------------------------------------------------------------------- 1 | namespace DotNetCloud.SqsToolbox.Core.Diagnostics 2 | { 3 | internal sealed class DeletionBatchCreatedPayload 4 | { 5 | internal DeletionBatchCreatedPayload(int messageCount, long millisecondsTaken) 6 | { 7 | MessageCount = messageCount; 8 | MillisecondsTaken = millisecondsTaken; 9 | } 10 | 11 | public int MessageCount { get; } 12 | 13 | public long MillisecondsTaken { get; } 14 | 15 | public override string ToString() => $"Created batch with {MessageCount} items, in {MillisecondsTaken} milliseconds"; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox.Core/Diagnostics/DiagnosticEvents.cs: -------------------------------------------------------------------------------- 1 | namespace DotNetCloud.SqsToolbox.Core.Diagnostics 2 | { 3 | public static class DiagnosticEvents 4 | { 5 | public const string PollingForMessages = "DotNetCloud.SqsToolbox.PollingForMessages"; 6 | 7 | public const string PollingForMessagesStart = "DotNetCloud.SqsToolbox.PollingForMessages.Start"; 8 | 9 | public const string ReceiveMessagesBeginRequest = "DotNetCloud.SqsToolbox.ReceiveMessagesBeginRequest"; 10 | 11 | public const string ReceiveMessagesRequestComplete = "DotNetCloud.SqsToolbox.ReceiveMessagesRequestComplete"; 12 | 13 | public const string OverLimitException = "DotNetCloud.SqsToolbox.OverLimitException"; 14 | 15 | public const string AmazonSqsException = "DotNetCloud.SqsToolbox.AmazonSQSException"; 16 | 17 | public const string Exception = "DotNetCloud.SqsToolbox.Exception"; 18 | 19 | public const string DeletionBatchCreated = "DotNetCloud.SqsToolbox.DeletionBatchCreated"; 20 | 21 | public const string DeleteBatchRequestComplete = "DotNetCloud.SqsToolbox.DeleteBatchRequestComplete"; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox.Core/Diagnostics/EndDeletionBatchPayload.cs: -------------------------------------------------------------------------------- 1 | using Amazon.SQS.Model; 2 | 3 | namespace DotNetCloud.SqsToolbox.Core.Diagnostics 4 | { 5 | internal sealed class EndDeletionBatchPayload 6 | { 7 | internal EndDeletionBatchPayload(DeleteMessageBatchResponse response, long millisecondsTaken) 8 | { 9 | DeleteMessageBatchResponse = response; 10 | MillisecondsTaken = millisecondsTaken; 11 | } 12 | 13 | public DeleteMessageBatchResponse DeleteMessageBatchResponse { get; } 14 | 15 | public long MillisecondsTaken { get; } 16 | 17 | public override string ToString() => $"Deleted batch with {DeleteMessageBatchResponse.Successful} items and {DeleteMessageBatchResponse.Failed} items, in {MillisecondsTaken} milliseconds"; 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox.Core/Diagnostics/EndReceiveRequestPayload.cs: -------------------------------------------------------------------------------- 1 | namespace DotNetCloud.SqsToolbox.Core.Diagnostics 2 | { 3 | public sealed class EndReceiveRequestPayload 4 | { 5 | internal EndReceiveRequestPayload(string queueUrl, int messageCount) 6 | { 7 | QueueUrl = queueUrl; 8 | MessageCount = messageCount; 9 | } 10 | 11 | public string QueueUrl { get; } 12 | 13 | public int MessageCount { get; } 14 | 15 | public override string ToString() => $"Received {MessageCount} messages from {QueueUrl}"; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox.Core/Diagnostics/ExceptionPayload.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Amazon.SQS.Model; 3 | 4 | namespace DotNetCloud.SqsToolbox.Core.Diagnostics 5 | { 6 | public sealed class ExceptionPayload 7 | { 8 | internal ExceptionPayload(Exception exception, ReceiveMessageRequest request) 9 | { 10 | Exception = exception; 11 | Request = request; 12 | } 13 | 14 | public Exception Exception { get; } 15 | public ReceiveMessageRequest Request { get; } 16 | 17 | public override string ToString() => $"{{ {nameof(Exception)} = {Exception}, {nameof(Request)} = {Request} }}"; 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox.Core/DotNetCloud.SqsToolbox.Core.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | A collection of tools, extensions and helpers for working with AWS Simple Queue Service (SQS) from .NET applications. 5 | Copyright © 2020, Steve Gordon 6 | netstandard2.0 7 | 1.0.0-alpha.5 8 | 1.0.0.$([System.DateTime]::UtcNow.ToString(mmff)) 9 | 0.0.0.1 10 | $(VersionSuffix) 11 | 1.0.0 Alpha 5 12 | 1.0.0-alpha.5 13 | DotNetCloud.SqsToolbox.Core 14 | sqs;aws;toolbox;tools 15 | https://github.com/dotnetcloud/SqsToolbox 16 | false 17 | MIT 18 | git 19 | git://github.com/dotnetcloud/SqsToolbox 20 | Steve Gordon 21 | 8 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | <_Parameter1>DotNetCloud.SqsToolbox.Core.Tests 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox.Core/ILogicalQueueNameGenerator.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Buffers; 3 | 4 | namespace DotNetCloud.SqsToolbox.Core 5 | { 6 | /// 7 | /// Can be used to generate a logical name for a queue based on properties of the queue URL. 8 | /// 9 | public interface ILogicalQueueNameGenerator 10 | { 11 | /// 12 | /// Generate a logical name for the queue reader and channel registrations. 13 | /// 14 | /// The AWS queue URL. 15 | /// A logical name for the queue. 16 | public string GenerateName(string queueUrl); 17 | } 18 | 19 | internal sealed class DefaultLogicalQueueNameGenerator : ILogicalQueueNameGenerator 20 | { 21 | public string GenerateName(string queueUrl) 22 | { 23 | if (!Uri.TryCreate(queueUrl, UriKind.Absolute, out var uri)) 24 | throw new InvalidOperationException("The queue URL was not valid"); 25 | 26 | var hostSpan = uri.Host.AsSpan(); 27 | var pathSpan = uri.LocalPath.AsSpan(); 28 | 29 | // todo - handle localstack 30 | // todo - handle a URL which is not a queue URL (i.e. no path) 31 | 32 | var nameLength = pathSpan.Length - pathSpan.LastIndexOf('/') - 1; 33 | var hostStart = hostSpan.IndexOf('.') + 1; 34 | var hostLength = hostSpan.Slice(hostStart).IndexOf('.'); 35 | 36 | var totalLength = nameLength + hostLength + 1; 37 | 38 | if (totalLength <= 64) 39 | { 40 | Span nameChars = stackalloc char[totalLength]; 41 | 42 | hostSpan.Slice(hostStart, hostLength).CopyTo(nameChars); 43 | nameChars[hostLength] = '_'; 44 | pathSpan.Slice(pathSpan.LastIndexOf('/') + 1, nameLength).CopyTo(nameChars.Slice(hostLength + 1)); 45 | 46 | return nameChars.ToString(); 47 | } 48 | else 49 | { 50 | var nameChars = ArrayPool.Shared.Rent(totalLength); 51 | var nameCharsSpan = nameChars.AsSpan(); 52 | 53 | try 54 | { 55 | hostSpan.Slice(hostStart, hostLength).CopyTo(nameCharsSpan); 56 | nameChars[hostLength] = '_'; 57 | pathSpan.Slice(pathSpan.LastIndexOf('/') + 1, nameLength).CopyTo(nameCharsSpan.Slice(hostLength + 1)); 58 | 59 | return nameChars.ToString(); 60 | } 61 | finally 62 | { 63 | ArrayPool.Shared.Return(nameChars); 64 | } 65 | } 66 | } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox.Core/Receive/SqsPollingQueueReader.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Diagnostics; 4 | using System.Linq; 5 | using System.Net; 6 | using System.Threading; 7 | using System.Threading.Channels; 8 | using System.Threading.Tasks; 9 | using Amazon.SQS; 10 | using Amazon.SQS.Model; 11 | using DotNetCloud.SqsToolbox.Core.Abstractions; 12 | using DotNetCloud.SqsToolbox.Core.Diagnostics; 13 | 14 | namespace DotNetCloud.SqsToolbox.Core.Receive 15 | { 16 | public class SqsPollingQueueReader : ISqsPollingQueueReader, IDisposable 17 | { 18 | public const string DiagnosticListenerName = "DotNetCloud.SqsToolbox.SqsSqsPollingQueueReader"; 19 | 20 | private readonly SqsPollingQueueReaderOptions _queueReaderOptions; 21 | private readonly IAmazonSQS _amazonSqs; 22 | private readonly ISqsReceiveDelayCalculator _pollingDelayer; 23 | private readonly Channel _channel; 24 | private readonly ReceiveMessageRequest _receiveMessageRequest; 25 | private readonly IExceptionHandler _exceptionHandler; 26 | 27 | private CancellationTokenSource _cancellationTokenSource; 28 | private Task _pollingTask; 29 | private bool _disposed; 30 | private bool _isStarted; 31 | 32 | private readonly object _startLock = new object(); 33 | private static readonly DiagnosticListener _diagnostics = new DiagnosticListener(DiagnosticListenerName); 34 | 35 | public SqsPollingQueueReader(SqsPollingQueueReaderOptions queueReaderOptions, IAmazonSQS amazonSqs, ISqsReceiveDelayCalculator pollingDelayer, IExceptionHandler exceptionHandler, Channel channel = null) 36 | { 37 | _queueReaderOptions = queueReaderOptions ?? throw new ArgumentNullException(nameof(queueReaderOptions)); 38 | _amazonSqs = amazonSqs ?? throw new ArgumentNullException(nameof(amazonSqs)); 39 | _pollingDelayer = pollingDelayer ?? throw new ArgumentNullException(nameof(pollingDelayer)); 40 | 41 | if (queueReaderOptions.ReceiveMessageRequest is object) 42 | { 43 | _receiveMessageRequest = queueReaderOptions.ReceiveMessageRequest; 44 | } 45 | else 46 | { 47 | _receiveMessageRequest = new ReceiveMessageRequest 48 | { 49 | QueueUrl = queueReaderOptions.QueueUrl ?? throw new ArgumentNullException(nameof(queueReaderOptions), "A queue URL is required for the polling queue reader to be created"), 50 | MaxNumberOfMessages = queueReaderOptions.MaxMessages, 51 | WaitTimeSeconds = queueReaderOptions.PollTimeInSeconds 52 | }; 53 | } 54 | 55 | _exceptionHandler = exceptionHandler ?? DefaultExceptionHandler.Instance; 56 | _channel = channel ?? Channel.CreateBounded(new BoundedChannelOptions(queueReaderOptions.ChannelCapacity) 57 | { 58 | SingleWriter = true 59 | }); 60 | } 61 | 62 | public ChannelReader ChannelReader => _channel.Reader; 63 | 64 | /// 65 | public void Start(CancellationToken cancellationToken = default) 66 | { 67 | if (_isStarted) 68 | throw new InvalidOperationException("The queue reader is already started."); 69 | 70 | lock (_startLock) 71 | { 72 | if (_isStarted) 73 | throw new InvalidOperationException("The queue reader is already started."); 74 | 75 | _cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); 76 | 77 | _pollingTask = Task.Run(PollForMessagesAsync, cancellationToken); 78 | 79 | _isStarted = true; 80 | } 81 | } 82 | 83 | /// 84 | public async Task StopAsync() 85 | { 86 | if (!_isStarted) 87 | return; 88 | 89 | _cancellationTokenSource?.Cancel(); 90 | 91 | await _pollingTask.ConfigureAwait(false); 92 | } 93 | 94 | private async Task PollForMessagesAsync() 95 | { 96 | var writer = _channel.Writer; 97 | 98 | try 99 | { 100 | while (!_cancellationTokenSource.IsCancellationRequested && await writer.WaitToWriteAsync(_cancellationTokenSource.Token).ConfigureAwait(false)) 101 | { 102 | var activity = StartActivity(); 103 | 104 | ReceiveMessageResponse response = null; 105 | 106 | try 107 | { 108 | DiagnosticsStart(); 109 | 110 | response = await _amazonSqs.ReceiveMessageAsync(_receiveMessageRequest, _cancellationTokenSource.Token).ConfigureAwait(false); 111 | 112 | DiagnosticsEnd(response); 113 | } 114 | catch (OverLimitException ex) // May be the case if the maximum number of in-flight messages is reached 115 | { 116 | DiagnosticsOverLimit(ex, activity); 117 | 118 | await Task.Delay(_queueReaderOptions.DelayWhenOverLimit).ConfigureAwait(false); 119 | 120 | continue; 121 | } 122 | catch (AmazonSQSException ex) 123 | { 124 | DiagnosticsSqsException(ex, activity); 125 | 126 | _exceptionHandler.OnException(ex, this); 127 | 128 | break; 129 | } 130 | catch (Exception ex) 131 | { 132 | DiagnosticsException(ex, activity); 133 | 134 | _exceptionHandler.OnException(ex, this); 135 | 136 | break; 137 | } 138 | finally 139 | { 140 | if (activity is object) 141 | { 142 | _diagnostics.StopActivity(activity, new { response }); 143 | } 144 | } 145 | 146 | if (response is null || response.HttpStatusCode != HttpStatusCode.OK) 147 | { 148 | continue; // Something went wrong 149 | } 150 | 151 | // Status code was 200-OK 152 | 153 | await PublishMessagesAsync(response.Messages).ConfigureAwait(false); 154 | 155 | var delayTimeSpan = _pollingDelayer.CalculateSecondsToDelay(response.Messages); 156 | 157 | await Task.Delay(delayTimeSpan).ConfigureAwait(false); 158 | } 159 | } 160 | finally 161 | { 162 | writer.TryComplete(); 163 | } 164 | } 165 | 166 | private void DiagnosticsException(Exception ex, Activity activity) 167 | { 168 | if (_diagnostics.IsEnabled(DiagnosticEvents.Exception)) 169 | _diagnostics.Write(DiagnosticEvents.Exception, new ExceptionPayload(ex, _receiveMessageRequest)); 170 | 171 | activity?.AddTag("error", "true"); 172 | } 173 | 174 | private void DiagnosticsSqsException(AmazonSQSException ex, Activity activity) 175 | { 176 | if (_diagnostics.IsEnabled(DiagnosticEvents.AmazonSqsException)) 177 | _diagnostics.Write(DiagnosticEvents.AmazonSqsException, new ExceptionPayload(ex, _receiveMessageRequest)); 178 | 179 | activity?.AddTag("error", "true"); 180 | } 181 | 182 | private void DiagnosticsOverLimit(OverLimitException ex, Activity activity) 183 | { 184 | if (_diagnostics.IsEnabled(DiagnosticEvents.OverLimitException)) 185 | _diagnostics.Write(DiagnosticEvents.OverLimitException, new ExceptionPayload(ex, _receiveMessageRequest)); 186 | 187 | activity?.AddTag("error", "true"); 188 | } 189 | 190 | private Activity StartActivity() 191 | { 192 | Activity activity = null; 193 | 194 | if (_diagnostics.IsEnabled() && _diagnostics.IsEnabled(DiagnosticEvents.PollingForMessages)) 195 | { 196 | activity = new Activity(DiagnosticEvents.PollingForMessages); 197 | 198 | if (_diagnostics.IsEnabled(DiagnosticEvents.PollingForMessagesStart)) 199 | { 200 | _diagnostics.StartActivity(activity, new { _receiveMessageRequest }); 201 | } 202 | else 203 | { 204 | activity.Start(); 205 | } 206 | } 207 | 208 | return activity; 209 | } 210 | 211 | private void DiagnosticsEnd(ReceiveMessageResponse response) 212 | { 213 | if (_diagnostics.IsEnabled(DiagnosticEvents.ReceiveMessagesRequestComplete)) 214 | _diagnostics.Write(DiagnosticEvents.ReceiveMessagesRequestComplete, 215 | new EndReceiveRequestPayload(_queueReaderOptions.QueueUrl, response.Messages.Count)); 216 | } 217 | 218 | private void DiagnosticsStart() 219 | { 220 | if (_diagnostics.IsEnabled(DiagnosticEvents.ReceiveMessagesBeginRequest)) 221 | _diagnostics.Write(DiagnosticEvents.ReceiveMessagesBeginRequest, 222 | new BeginReceiveRequestPayload(_queueReaderOptions.QueueUrl)); 223 | } 224 | 225 | private async ValueTask PublishMessagesAsync(IReadOnlyList messages) 226 | { 227 | if (!messages.Any()) 228 | return; 229 | 230 | var writer = _channel.Writer; 231 | 232 | var index = 0; 233 | 234 | while (index < messages.Count && await writer.WaitToWriteAsync(_cancellationTokenSource.Token).ConfigureAwait(false)) 235 | { 236 | while (index < messages.Count && writer.TryWrite(messages[index])) 237 | { 238 | index++; 239 | } 240 | } 241 | } 242 | 243 | public void Dispose() 244 | { 245 | Dispose(true); 246 | GC.SuppressFinalize(this); 247 | } 248 | 249 | protected virtual void Dispose(bool disposing) 250 | { 251 | if (!_disposed) 252 | { 253 | if (disposing) 254 | { 255 | _cancellationTokenSource.Dispose(); 256 | _pollingTask.Dispose(); 257 | _diagnostics.Dispose(); 258 | } 259 | 260 | _disposed = true; 261 | } 262 | } 263 | } 264 | } 265 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox.Core/Receive/SqsPollingQueueReaderOptions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Amazon.SQS.Model; 3 | 4 | namespace DotNetCloud.SqsToolbox.Core.Receive 5 | { 6 | public class SqsPollingQueueReaderOptions 7 | { 8 | private string _queueUrl; 9 | 10 | public string QueueUrl 11 | { 12 | get => _queueUrl; 13 | set 14 | { 15 | if (!Uri.TryCreate(value, UriKind.Absolute, out _)) 16 | { 17 | throw new ArgumentException("The value must be a valid URI", nameof(value)); 18 | } 19 | 20 | _queueUrl = value; 21 | } 22 | } 23 | 24 | public int ChannelCapacity { get; set; } = 100; 25 | 26 | /// 27 | /// The maximum number of messages to request per receive attempt. 28 | /// The value must be between 1 and 10. The default value is 10. 29 | /// 30 | public int MaxMessages { get; set; } = 10; 31 | 32 | public int PollTimeInSeconds { get; set; } = 20; 33 | 34 | public bool UseExponentialBackoff { get; set; } = true; 35 | 36 | public TimeSpan InitialDelay { get; set; } = TimeSpan.FromMinutes(1); 37 | 38 | public TimeSpan MaxDelay { get; set; } = TimeSpan.FromMinutes(5); 39 | 40 | public TimeSpan DelayWhenOverLimit { get; set; } = TimeSpan.FromMinutes(5); 41 | 42 | public ReceiveMessageRequest ReceiveMessageRequest { get; set; } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox.Core/Receive/SqsPollingQueueReaderOptionsExtensions.cs: -------------------------------------------------------------------------------- 1 | namespace DotNetCloud.SqsToolbox.Core.Receive 2 | { 3 | public static class SqsPollingQueueReaderOptionsExtensions 4 | { 5 | public static void CopyFrom(this SqsPollingQueueReaderOptions destination, SqsPollingQueueReaderOptions source) 6 | { 7 | destination.QueueUrl = source.QueueUrl; 8 | destination.ChannelCapacity = source.ChannelCapacity; 9 | destination.MaxMessages = source.MaxMessages; 10 | destination.PollTimeInSeconds = source.PollTimeInSeconds; 11 | destination.InitialDelay = source.InitialDelay; 12 | destination.MaxDelay = source.MaxDelay; 13 | destination.DelayWhenOverLimit = source.DelayWhenOverLimit; 14 | destination.ReceiveMessageRequest = source.ReceiveMessageRequest; 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox.Core/Receive/SqsReceiveDelayCalculator.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using Amazon.SQS.Model; 5 | using DotNetCloud.SqsToolbox.Core.Abstractions; 6 | 7 | namespace DotNetCloud.SqsToolbox.Core.Receive 8 | { 9 | /// 10 | public class SqsReceiveDelayCalculator : ISqsReceiveDelayCalculator 11 | { 12 | private readonly SqsPollingQueueReaderOptions _queueReaderOptions; 13 | private int _emptyResponseCounter; 14 | 15 | public SqsReceiveDelayCalculator(SqsPollingQueueReaderOptions queueReaderOptions) 16 | { 17 | _queueReaderOptions = queueReaderOptions ?? throw new ArgumentNullException(nameof(queueReaderOptions)); 18 | } 19 | 20 | /// 21 | public TimeSpan CalculateSecondsToDelay(IEnumerable messages) 22 | { 23 | _ = messages ?? throw new ArgumentNullException(nameof(messages)); 24 | 25 | if (messages.Any()) 26 | { 27 | _emptyResponseCounter = 0; 28 | 29 | return TimeSpan.Zero; 30 | } 31 | 32 | if (_emptyResponseCounter < 5) 33 | { 34 | _emptyResponseCounter++; 35 | } 36 | 37 | var delaySeconds = _queueReaderOptions.InitialDelay.TotalSeconds; 38 | 39 | if (_queueReaderOptions.UseExponentialBackoff) 40 | { 41 | delaySeconds = Math.Min(Math.Pow(delaySeconds, _emptyResponseCounter), _queueReaderOptions.MaxDelay.TotalSeconds); 42 | } 43 | 44 | return TimeSpan.FromSeconds(delaySeconds); 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox/ConfigurationExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using DotNetCloud.SqsToolbox.Core.Receive; 3 | using Microsoft.Extensions.Configuration; 4 | 5 | namespace DotNetCloud.SqsToolbox 6 | { 7 | internal static class ConfigurationExtensions 8 | { 9 | public static SqsPollingQueueReaderOptions GetPollingQueueReaderOptions(this IConfiguration configuration) 10 | { 11 | _ = configuration ?? throw new ArgumentNullException(nameof(configuration)); 12 | 13 | var section = configuration.GetSection("SQSToolbox"); 14 | 15 | var options = new SqsPollingQueueReaderOptions 16 | { 17 | QueueUrl = section.GetValue("QueueUrl", null) 18 | }; 19 | 20 | return options; 21 | } 22 | 23 | //public static SqsBatchDeletionOptions GetSqsBatchDeleterOptions(this IConfiguration configuration) 24 | //{ 25 | // _ = configuration ?? throw new ArgumentNullException(nameof(configuration)); 26 | 27 | // var section = configuration.GetSection("SQSToolbox"); 28 | 29 | // var options = new SqsBatchDeletionOptions 30 | // { 31 | // QueueUrl = section.GetValue("QueueUrl", null) 32 | // }; 33 | 34 | // return options; 35 | //} 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox/DefaultChannelReaderAccessor.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Channels; 2 | using Amazon.SQS.Model; 3 | 4 | namespace DotNetCloud.SqsToolbox 5 | { 6 | public sealed class DefaultChannelReaderAccessor : IChannelReaderAccessor 7 | { 8 | private readonly ISqsMessageChannelFactory _channelFactory; 9 | 10 | public DefaultChannelReaderAccessor(ISqsMessageChannelFactory channelFactory) 11 | { 12 | _channelFactory = channelFactory; 13 | } 14 | 15 | public ChannelReader GetChannelReader(string logicalQueueName) => _channelFactory.GetOrCreateChannel(logicalQueueName).Reader; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox/DefaultSqsPollingQueueReaderBuilder.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Channels; 3 | using Amazon.SQS.Model; 4 | using DotNetCloud.SqsToolbox.Core.Abstractions; 5 | using DotNetCloud.SqsToolbox.Core.Receive; 6 | using DotNetCloud.SqsToolbox.DependencyInjection; 7 | using DotNetCloud.SqsToolbox.Hosting; 8 | using Microsoft.Extensions.DependencyInjection; 9 | using Microsoft.Extensions.DependencyInjection.Extensions; 10 | using Microsoft.Extensions.Hosting; 11 | 12 | namespace DotNetCloud.SqsToolbox 13 | { 14 | internal class DefaultSqsPollingQueueReaderBuilder : ISqsPollingReaderBuilder 15 | { 16 | public DefaultSqsPollingQueueReaderBuilder(IServiceCollection services, string name) 17 | { 18 | Services = services; 19 | Name = name; 20 | } 21 | 22 | public string Name { get; } 23 | 24 | public IServiceCollection Services { get; } 25 | 26 | public ISqsPollingReaderBuilder WithBackgroundService() 27 | { 28 | Services.AddSingleton(serviceProvider => new SqsPollingBackgroundService(serviceProvider.GetRequiredService(), Name)); 29 | 30 | return this; 31 | } 32 | 33 | public ISqsPollingReaderBuilder WithMessageProcessor() where T : SqsMessageProcessingBackgroundService 34 | { 35 | Services.AddSingleton(serviceProvider => 36 | { 37 | var service = ActivatorUtilities.CreateInstance(serviceProvider); 38 | 39 | service.SetName(Name); 40 | 41 | return service; 42 | }); 43 | 44 | return this; 45 | } 46 | 47 | public ISqsPollingReaderBuilder WithExceptionHandler() where T : IExceptionHandler 48 | { 49 | var type = typeof(T); 50 | 51 | Services.TryAddSingleton(typeof(IExceptionHandler), type); 52 | 53 | Services.Configure(Name, opt => opt.ExceptionHandlerType = type); 54 | 55 | return this; 56 | } 57 | 58 | public ISqsPollingReaderBuilder Configure(Action configure) 59 | { 60 | _ = configure ?? throw new ArgumentNullException(nameof(configure)); 61 | 62 | Services.PostConfigure(Name, opt => configure(opt.Options)); 63 | 64 | return this; 65 | } 66 | 67 | public ISqsPollingReaderBuilder WithChannel(Channel channel) 68 | { 69 | Services.Configure(Name, opt => opt.Channel = channel); 70 | 71 | return this; 72 | } 73 | 74 | #if NETCOREAPP3_1 75 | public ISqsPollingReaderBuilder WithDefaultExceptionHandler() 76 | { 77 | Services.TryAddSingleton(); 78 | 79 | return this; 80 | } 81 | #endif 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox/DependencyInjection/ISqsBatchDeletionBuilder.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace DotNetCloud.SqsToolbox.DependencyInjection 4 | { 5 | //public interface ISqsBatchDeletionBuilder 6 | //{ 7 | // ISqsBatchDeletionBuilder WithBackgroundService(); 8 | // ISqsBatchDeletionBuilder Configure(Action configure); 9 | //} 10 | } 11 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox/DependencyInjection/ISqsPollingReaderBuilder.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Channels; 3 | using Amazon.SQS.Model; 4 | using DotNetCloud.SqsToolbox.Core.Abstractions; 5 | using DotNetCloud.SqsToolbox.Core.Receive; 6 | using DotNetCloud.SqsToolbox.Hosting; 7 | using Microsoft.Extensions.DependencyInjection; 8 | 9 | namespace DotNetCloud.SqsToolbox.DependencyInjection 10 | { 11 | public interface ISqsPollingReaderBuilder 12 | { 13 | /// 14 | /// Gets the name of the client configured by this builder. 15 | /// 16 | string Name { get; } 17 | 18 | /// 19 | /// Gets the application service collection. 20 | /// 21 | IServiceCollection Services { get; } 22 | 23 | ISqsPollingReaderBuilder WithBackgroundService(); 24 | ISqsPollingReaderBuilder WithMessageProcessor() where T : SqsMessageProcessingBackgroundService; 25 | ISqsPollingReaderBuilder WithExceptionHandler() where T : IExceptionHandler; 26 | ISqsPollingReaderBuilder Configure(Action configure); 27 | ISqsPollingReaderBuilder WithChannel(Channel channel); 28 | 29 | #if NETCOREAPP3_1 30 | ISqsPollingReaderBuilder WithDefaultExceptionHandler(); 31 | #endif 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox/DependencyInjection/SqsBatchDeleterServiceCollectionExtensions.cs: -------------------------------------------------------------------------------- 1 | namespace DotNetCloud.SqsToolbox.DependencyInjection 2 | { 3 | //public static class SqsBatchDeleterServiceCollectionExtensions 4 | //{ 5 | // public static ISqsBatchDeletionBuilder AddSqsBatchDeletion(this IServiceCollection services, IConfiguration configuration) 6 | // { 7 | // _ = services ?? throw new ArgumentNullException(nameof(services)); 8 | // _ = configuration ?? throw new ArgumentNullException(nameof(configuration)); 9 | 10 | // AddBatchDeleterCore(services); 11 | 12 | // var options = configuration.GetSqsBatchDeleterOptions(); 13 | 14 | // services.Configure(opt => 15 | // { 16 | // opt.QueueUrl = options.QueueUrl; 17 | // }); 18 | 19 | // services.TryAddSingleton(sp => sp.GetRequiredService>().Value); 20 | 21 | // return new SqsBatchDeletionBuilder(services); 22 | // } 23 | 24 | // /// 25 | // /// Adds the required SQS batch deletion services to the container, using the provided delegate to 26 | // /// configure the . 27 | // /// 28 | // /// The to add the SQS batch deletion services to. 29 | // /// A delegate that is used to configure an . 30 | // /// An instance of used to further configure the SQS batch deletion behaviour. 31 | // public static ISqsBatchDeletionBuilder AddSqsBatchDeletion(this IServiceCollection services, Action configure) 32 | // { 33 | // _ = services ?? throw new ArgumentNullException(nameof(services)); 34 | // _ = configure ?? throw new ArgumentNullException(nameof(configure)); 35 | 36 | // AddBatchDeleterCore(services); 37 | 38 | // services.Configure(configure); 39 | 40 | // services.TryAddSingleton(sp => sp.GetRequiredService>().Value); 41 | 42 | // return new SqsBatchDeletionBuilder(services); 43 | // } 44 | 45 | // private static void AddBatchDeleterCore(IServiceCollection services) 46 | // { 47 | // services.TryAddAWSService(); 48 | 49 | // services.TryAddSingleton(sp => 50 | // { 51 | // var sqs = sp.GetService(); 52 | // var options = sp.GetService>(); 53 | 54 | // return new SqsBatchDeleterBuilder(sqs, options.Value).Build(); 55 | // }); 56 | 57 | // services.TryAddSingleton(sp => sp.GetRequiredService()); 58 | // } 59 | //} 60 | } 61 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox/DependencyInjection/SqsBatchDeletionBuilder.cs: -------------------------------------------------------------------------------- 1 |  2 | 3 | namespace DotNetCloud.SqsToolbox.DependencyInjection 4 | { 5 | //internal sealed class SqsBatchDeletionBuilder : ISqsBatchDeletionBuilder 6 | //{ 7 | // public SqsBatchDeletionBuilder(IServiceCollection services) => Services = services; 8 | 9 | // public IServiceCollection Services { get; } 10 | 11 | // public ISqsBatchDeletionBuilder WithBackgroundService() 12 | // { 13 | // Services.AddHostedService(); 14 | 15 | // return this; 16 | // } 17 | 18 | // public ISqsBatchDeletionBuilder Configure(Action configure) 19 | // { 20 | // Services.PostConfigure(configure); 21 | 22 | // Services.TryAddSingleton(sp => sp.GetRequiredService>()?.Value); 23 | 24 | // return this; 25 | // } 26 | //} 27 | } 28 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox/DependencyInjection/SqsPollingReaderServiceCollectionExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Amazon.SQS; 3 | using DotNetCloud.SqsToolbox.DependencyInjection; 4 | using DotNetCloud.SqsToolbox; 5 | using DotNetCloud.SqsToolbox.Diagnostics; 6 | using DotNetCloud.SqsToolbox.Hosting; 7 | using Microsoft.Extensions.Configuration; 8 | using Microsoft.Extensions.DependencyInjection.Extensions; 9 | 10 | namespace Microsoft.Extensions.DependencyInjection 11 | { 12 | /// 13 | /// Extension methods for . 14 | /// 15 | public static class SqsPollingReaderServiceCollectionExtensions 16 | { 17 | public static ISqsPollingReaderBuilder AddPollingSqs(this IServiceCollection services, IConfigurationSection configurationSection) 18 | { 19 | if (services == null) 20 | { 21 | throw new ArgumentNullException(nameof(services)); 22 | } 23 | 24 | if (configurationSection == null) 25 | { 26 | throw new ArgumentNullException(nameof(configurationSection)); 27 | } 28 | 29 | var queueName = configurationSection["QueueName"]; 30 | var queueUrl = configurationSection["QueueUrl"]; 31 | 32 | if (string.IsNullOrEmpty(queueName) || string.IsNullOrEmpty(queueUrl)) 33 | throw new InvalidOperationException("The configuration is invalid."); 34 | 35 | return services.AddPollingSqs(queueName, queueUrl); 36 | } 37 | 38 | public static ISqsPollingReaderBuilder AddDefaultPollingSqs(this IServiceCollection services, IConfigurationSection configurationSection) where T : SqsMessageProcessingBackgroundService 39 | { 40 | if (services == null) 41 | { 42 | throw new ArgumentNullException(nameof(services)); 43 | } 44 | 45 | if (configurationSection == null) 46 | { 47 | throw new ArgumentNullException(nameof(configurationSection)); 48 | } 49 | 50 | var queueName = configurationSection["QueueName"]; 51 | var queueUrl = configurationSection["QueueUrl"]; 52 | 53 | if (string.IsNullOrEmpty(queueName) || string.IsNullOrEmpty(queueUrl)) 54 | throw new InvalidOperationException("The configuration is invalid."); 55 | 56 | return services.AddDefaultPollingSqs(queueName, queueUrl); 57 | } 58 | 59 | public static ISqsPollingReaderBuilder AddPollingSqs(this IServiceCollection services, string name, string queueUrl) 60 | { 61 | if (services == null) 62 | { 63 | throw new ArgumentNullException(nameof(services)); 64 | } 65 | 66 | if (name == null) 67 | { 68 | throw new ArgumentNullException(nameof(name)); 69 | } 70 | 71 | services.AddOptions(); 72 | 73 | AddPollingSqsCore(services); 74 | 75 | services.TryAddSingleton(); 76 | 77 | services.Configure(name, options => options.Options.QueueUrl = queueUrl); 78 | 79 | return new DefaultSqsPollingQueueReaderBuilder(services, name); 80 | } 81 | 82 | public static ISqsPollingReaderBuilder AddDefaultPollingSqs(this IServiceCollection services, string name, string queueUrl) where T : SqsMessageProcessingBackgroundService 83 | { 84 | if (services == null) 85 | { 86 | throw new ArgumentNullException(nameof(services)); 87 | } 88 | 89 | if (name == null) 90 | { 91 | throw new ArgumentNullException(nameof(name)); 92 | } 93 | 94 | services.AddOptions(); 95 | 96 | AddPollingSqsCore(services); 97 | 98 | services.TryAddSingleton(); 99 | 100 | services.Configure(name, options => options.Options.QueueUrl = queueUrl); 101 | 102 | var builder = new DefaultSqsPollingQueueReaderBuilder(services, name); 103 | 104 | builder.WithBackgroundService(); 105 | builder.WithMessageProcessor(); 106 | 107 | #if NETCOREAPP3_1 108 | builder.WithDefaultExceptionHandler(); 109 | #endif 110 | return builder; 111 | } 112 | 113 | public static IServiceCollection AddSqsToolboxDiagnosticsMonitoring(this IServiceCollection services) where T : DiagnosticsMonitoringService 114 | { 115 | _ = services ?? throw new ArgumentNullException(nameof(services)); 116 | 117 | services.AddHostedService(); 118 | 119 | return services; 120 | } 121 | 122 | private static void AddPollingSqsCore(IServiceCollection services) 123 | { 124 | services.TryAddAWSService(); 125 | 126 | services.TryAddSingleton(); 127 | services.TryAddSingleton(serviceProvider => serviceProvider.GetRequiredService()); 128 | services.TryAddSingleton(serviceProvider => serviceProvider.GetRequiredService()); 129 | } 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox/Diagnostics/DiagnosticsMonitoringService.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Diagnostics; 4 | using System.Reactive.Linq; 5 | using System.Threading; 6 | using System.Threading.Tasks; 7 | using Amazon.SQS.Model; 8 | using DotNetCloud.SqsToolbox.Core.Diagnostics; 9 | using DotNetCloud.SqsToolbox.Core.Receive; 10 | using Microsoft.Extensions.Hosting; 11 | 12 | namespace DotNetCloud.SqsToolbox.Diagnostics 13 | { 14 | public abstract class DiagnosticsMonitoringService : IHostedService 15 | { 16 | private IDisposable _allListenersSubscription; 17 | private readonly ConcurrentBag _subscriptions = new ConcurrentBag(); 18 | 19 | public Task StartAsync(CancellationToken cancellationToken) 20 | { 21 | _allListenersSubscription = DiagnosticListener.AllListeners.Do(source => 22 | { 23 | if (source.Name == SqsPollingQueueReader.DiagnosticListenerName) 24 | { 25 | _subscriptions.Add(source.Do(pair => 26 | { 27 | switch (pair.Key) 28 | { 29 | case DiagnosticEvents.ReceiveMessagesBeginRequest: 30 | { 31 | if (pair.Value is BeginReceiveRequestPayload payload) 32 | { 33 | OnBegin(payload.QueueUrl); 34 | } 35 | 36 | break; 37 | } 38 | case DiagnosticEvents.ReceiveMessagesRequestComplete: 39 | { 40 | if (pair.Value is EndReceiveRequestPayload payload) 41 | { 42 | OnReceived(payload.QueueUrl, payload.MessageCount); 43 | } 44 | 45 | break; 46 | } 47 | //case DiagnosticEvents.DeletionBatchCreated: 48 | //{ 49 | // if (pair.Value is DeletionBatchCreatedPayload payload) 50 | // { 51 | // OnDeleteBatchCreated(payload.MessageCount, payload.MillisecondsTaken); 52 | // } 53 | 54 | // break; 55 | //} 56 | //case DiagnosticEvents.DeleteBatchRequestComplete: 57 | //{ 58 | // if (pair.Value is EndDeletionBatchPayload payload) 59 | // { 60 | // OnBatchDeleted(payload.DeleteMessageBatchResponse, payload.MillisecondsTaken); 61 | // } 62 | 63 | // break; 64 | //} 65 | case DiagnosticEvents.OverLimitException: 66 | { 67 | if (pair.Value is ExceptionPayload payload) 68 | { 69 | OnOverLimit(payload.Exception, payload.Request); 70 | } 71 | 72 | break; 73 | } 74 | case DiagnosticEvents.AmazonSqsException: 75 | { 76 | if (pair.Value is ExceptionPayload payload) 77 | { 78 | OnSqsException(payload.Exception, payload.Request); 79 | } 80 | 81 | break; 82 | } 83 | case DiagnosticEvents.Exception: 84 | { 85 | if (pair.Value is ExceptionPayload payload) 86 | { 87 | OnException(payload.Exception, payload.Request); 88 | } 89 | 90 | break; 91 | } 92 | } 93 | }) 94 | .Subscribe()); 95 | } 96 | 97 | //if (source.Name == SqsBatchDeleter.DiagnosticListenerName) 98 | //{ 99 | // _subscriptions.Add(source.Do(pair => 100 | // { 101 | // switch (pair.Key) 102 | // { 103 | // case DiagnosticEvents.DeletionBatchCreated: 104 | // { 105 | // if (pair.Value is DeletionBatchCreatedPayload payload) 106 | // { 107 | // OnDeleteBatchCreated(payload.MessageCount, payload.MillisecondsTaken); 108 | // } 109 | 110 | // break; 111 | // } 112 | // case DiagnosticEvents.DeleteBatchRequestComplete: 113 | // { 114 | // if (pair.Value is EndDeletionBatchPayload payload) 115 | // { 116 | // OnBatchDeleted(payload.DeleteMessageBatchResponse, payload.MillisecondsTaken); 117 | // } 118 | 119 | // break; 120 | // } 121 | // case DiagnosticEvents.OverLimitException: 122 | // { 123 | // if (pair.Value is ExceptionPayload payload) 124 | // { 125 | // OnOverLimit(payload.Exception, payload.Request); 126 | // } 127 | 128 | // break; 129 | // } 130 | // case DiagnosticEvents.AmazonSqsException: 131 | // { 132 | // if (pair.Value is ExceptionPayload payload) 133 | // { 134 | // OnSqsException(payload.Exception, payload.Request); 135 | // } 136 | 137 | // break; 138 | // } 139 | // case DiagnosticEvents.Exception: 140 | // { 141 | // if (pair.Value is ExceptionPayload payload) 142 | // { 143 | // OnException(payload.Exception, payload.Request); 144 | // } 145 | 146 | // break; 147 | // } 148 | // } 149 | // }) 150 | // .Subscribe()); 151 | //} 152 | 153 | }).Subscribe(); 154 | 155 | return Task.CompletedTask; 156 | } 157 | 158 | public Task StopAsync(CancellationToken cancellationToken) 159 | { 160 | _allListenersSubscription?.Dispose(); 161 | 162 | foreach (var subscription in _subscriptions) 163 | { 164 | subscription.Dispose(); 165 | } 166 | 167 | return Task.CompletedTask; 168 | } 169 | 170 | public virtual void On(string queueUrl) 171 | { 172 | } 173 | 174 | public virtual void OnBegin(string queueUrl) 175 | { 176 | } 177 | 178 | public virtual void OnReceived(string queueUrl, in int messageCount) 179 | { 180 | } 181 | 182 | public virtual void OnOverLimit(Exception ex, ReceiveMessageRequest request) 183 | { 184 | } 185 | 186 | public virtual void OnSqsException(Exception ex, ReceiveMessageRequest request) 187 | { 188 | } 189 | 190 | public virtual void OnException(Exception ex, ReceiveMessageRequest request) 191 | { 192 | } 193 | 194 | public virtual void OnDeleteBatchCreated(in int messageCount, in long millisecondsTaken) 195 | { 196 | } 197 | 198 | public virtual void OnBatchDeleted(DeleteMessageBatchResponse deleteMessageBatchResponse, in long millisecondsTaken) 199 | { 200 | } 201 | } 202 | } 203 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox/DotNetCloud.SqsToolbox.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | .NET Core extensions to support DI registration. 5 | Copyright © 2020, Steve Gordon 6 | netcoreapp2.1;netcoreapp3.1 7 | 1.0.0-alpha.5 8 | 1.0.0.$([System.DateTime]::UtcNow.ToString(mmff)) 9 | 0.0.0.1 10 | $(VersionSuffix) 11 | 1.0.0 Alpha 5 12 | 1.0.0-alpha.5 13 | DotNetCloud.SqsToolbox 14 | sqs;aws;toolbox;tools;extensions;netcore 15 | https://github.com/dotnetcloud/SqsToolbox 16 | false 17 | MIT 18 | git 19 | git://github.com/dotnetcloud/SqsToolbox 20 | Steve Gordon 21 | 8 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox/Hosting/MessageProcessorService.cs: -------------------------------------------------------------------------------- 1 | using System.Threading; 2 | using System.Threading.Channels; 3 | using System.Threading.Tasks; 4 | using Amazon.SQS.Model; 5 | 6 | namespace DotNetCloud.SqsToolbox.Hosting 7 | { 8 | public abstract class MessageProcessorService : SqsMessageProcessingBackgroundService 9 | { 10 | protected MessageProcessorService(IChannelReaderAccessor channelReaderAccessor) : base(channelReaderAccessor) 11 | { 12 | } 13 | 14 | public abstract Task ProcessMessageAsync(Message message, CancellationToken cancellationToken = default); 15 | 16 | public sealed override async Task ProcessFromChannelAsync(ChannelReader channelReader, CancellationToken cancellationToken = default) 17 | { 18 | #if NETCOREAPP3_1 19 | await foreach (var message in channelReader.ReadAllAsync(cancellationToken)) 20 | { 21 | await ProcessMessageAsync(message, cancellationToken).ConfigureAwait(false); 22 | } 23 | #else 24 | while (await channelReader.WaitToReadAsync(cancellationToken).ConfigureAwait(false)) 25 | while (channelReader.TryRead(out var message)) 26 | await ProcessMessageAsync(message, cancellationToken).ConfigureAwait(false); 27 | #endif 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox/Hosting/SqsBatchDeleteBackgroundService.cs: -------------------------------------------------------------------------------- 1 | using System.Threading; 2 | using System.Threading.Tasks; 3 | using Microsoft.Extensions.Hosting; 4 | 5 | namespace DotNetCloud.SqsToolbox.Hosting 6 | { 7 | //public class SqsBatchDeleteBackgroundService : IHostedService 8 | //{ 9 | // private readonly ISqsBatchDeleter _sqsBatchDeleter; 10 | 11 | // public SqsBatchDeleteBackgroundService(ISqsBatchDeleter sqsBatchDeleter) 12 | // { 13 | // _sqsBatchDeleter = sqsBatchDeleter; 14 | // } 15 | 16 | // public Task StartAsync(CancellationToken cancellationToken) 17 | // { 18 | // _sqsBatchDeleter.Start(cancellationToken); 19 | 20 | // return Task.CompletedTask; 21 | // } 22 | 23 | // public async Task StopAsync(CancellationToken cancellationToken) 24 | // { 25 | // await _sqsBatchDeleter.StopAsync(); 26 | // } 27 | //} 28 | } 29 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox/Hosting/SqsMessageProcessingBackgroundService.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading; 3 | using System.Threading.Channels; 4 | using System.Threading.Tasks; 5 | using Amazon.SQS.Model; 6 | using Microsoft.Extensions.Hosting; 7 | 8 | namespace DotNetCloud.SqsToolbox.Hosting 9 | { 10 | /// 11 | /// Base class for implementing long running queue processing. 12 | /// 13 | public abstract class SqsMessageProcessingBackgroundService : BackgroundService 14 | { 15 | private readonly IChannelReaderAccessor _channelReaderAccessor; 16 | private bool _hasStarted; 17 | 18 | protected SqsMessageProcessingBackgroundService(IChannelReaderAccessor channelReaderAccessor) 19 | { 20 | _channelReaderAccessor = channelReaderAccessor ?? throw new ArgumentNullException(nameof(channelReaderAccessor)); 21 | } 22 | 23 | protected string Name { get; private set; } 24 | 25 | /// 26 | /// Sets the logical name of the channel to process. 27 | /// 28 | /// The logical name of the channel. 29 | internal void SetName(string name) 30 | { 31 | if (Name is object) 32 | throw new InvalidOperationException("Name cannot be set twice."); 33 | 34 | if (_hasStarted) 35 | throw new InvalidOperationException("Name cannot be set once the service has been started."); 36 | 37 | Name = name; 38 | } 39 | 40 | /// 41 | protected sealed override async Task ExecuteAsync(CancellationToken stoppingToken) 42 | { 43 | await ProcessFromChannelAsync(_channelReaderAccessor.GetChannelReader(Name), stoppingToken); 44 | } 45 | 46 | public override Task StartAsync(CancellationToken cancellationToken) 47 | { 48 | if (string.IsNullOrEmpty(Name)) return Task.CompletedTask; 49 | 50 | _hasStarted = true; 51 | 52 | return base.StartAsync(cancellationToken); 53 | } 54 | 55 | /// 56 | /// This method is called when the Microsoft.Extensions.Hosting.IHostedService 57 | /// starts. The implementation should return a task that represents a long running task 58 | /// which reads messages from a of . 59 | /// 60 | /// The of from which to receive messages. 61 | /// A triggered when the host is shutting down. 62 | /// A that represents the long running channel reading. 63 | public abstract Task ProcessFromChannelAsync(ChannelReader channelReader, CancellationToken cancellationToken = default); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox/Hosting/SqsPollingBackgroundService.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading; 3 | using System.Threading.Tasks; 4 | using Microsoft.Extensions.Hosting; 5 | 6 | namespace DotNetCloud.SqsToolbox.Hosting 7 | { 8 | public class SqsPollingBackgroundService : IHostedService 9 | { 10 | private readonly ISqsPollingQueueReaderFactory _sqsPollingQueueReader; 11 | private readonly string _name; 12 | 13 | public SqsPollingBackgroundService(ISqsPollingQueueReaderFactory sqsPollingQueueReader, string name) 14 | { 15 | _sqsPollingQueueReader = sqsPollingQueueReader; 16 | _name = name; 17 | } 18 | 19 | public Task StartAsync(CancellationToken cancellationToken) 20 | { 21 | _sqsPollingQueueReader.GetOrCreateReader(_name).Start(cancellationToken); 22 | 23 | return Task.CompletedTask; 24 | } 25 | 26 | public async Task StopAsync(CancellationToken cancellationToken) 27 | { 28 | await _sqsPollingQueueReader.GetOrCreateReader(_name).StopAsync().ConfigureAwait(false); 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox/IChannelReaderAccessor.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Channels; 2 | using Amazon.SQS.Model; 3 | 4 | namespace DotNetCloud.SqsToolbox 5 | { 6 | /// 7 | /// Provides access to a channel reader for polling queue reader. 8 | /// 9 | public interface IChannelReaderAccessor 10 | { 11 | /// 12 | /// Get the channel reader for a logical queue. 13 | /// 14 | /// The logical name of the queue. 15 | /// A of . 16 | ChannelReader GetChannelReader(string logicalQueueName); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox/ISqsMessageChannelFactory.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Channels; 2 | using Amazon.SQS.Model; 3 | 4 | namespace DotNetCloud.SqsToolbox 5 | { 6 | public interface ISqsMessageChannelFactory 7 | { 8 | Channel GetOrCreateChannel(string name); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox/ISqsPollingQueueReaderFactory.cs: -------------------------------------------------------------------------------- 1 | using DotNetCloud.SqsToolbox.Core.Abstractions; 2 | 3 | namespace DotNetCloud.SqsToolbox 4 | { 5 | public interface ISqsPollingQueueReaderFactory 6 | { 7 | ISqsPollingQueueReader GetOrCreateReader(string name); 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox/SqsPollingQueueReaderFactory.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Threading; 4 | using System.Threading.Channels; 5 | using Amazon.SQS; 6 | using Amazon.SQS.Model; 7 | using DotNetCloud.SqsToolbox.Core; 8 | using DotNetCloud.SqsToolbox.Core.Abstractions; 9 | using DotNetCloud.SqsToolbox.Core.Receive; 10 | using Microsoft.Extensions.DependencyInjection; 11 | using Microsoft.Extensions.Logging; 12 | using Microsoft.Extensions.Options; 13 | 14 | namespace DotNetCloud.SqsToolbox 15 | { 16 | internal sealed class SqsPollingQueueReaderFactory : ISqsPollingQueueReaderFactory, ISqsMessageChannelFactory 17 | { 18 | private readonly ILogger _logger; 19 | private readonly IServiceProvider _services; 20 | private readonly IOptionsMonitor _optionsMonitor; 21 | private readonly IExceptionHandler _exceptionHandler; 22 | 23 | internal readonly ConcurrentDictionary> _pollingReaders; 24 | internal readonly ConcurrentDictionary>> _channels; 25 | 26 | public SqsPollingQueueReaderFactory( 27 | IServiceProvider services, 28 | ILoggerFactory loggerFactory, 29 | IOptionsMonitor optionsMonitor, 30 | IExceptionHandler exceptionHandler = null) 31 | { 32 | _ = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); 33 | 34 | _services = services ?? throw new ArgumentNullException(nameof(services)); 35 | _optionsMonitor = optionsMonitor ?? throw new ArgumentNullException(nameof(optionsMonitor)); 36 | _exceptionHandler = exceptionHandler; 37 | 38 | _logger = loggerFactory.CreateLogger(); 39 | 40 | // case-sensitive because named options is. 41 | _pollingReaders = new ConcurrentDictionary>(StringComparer.Ordinal); 42 | _channels = new ConcurrentDictionary>>(StringComparer.Ordinal); 43 | } 44 | 45 | public ISqsPollingQueueReader GetOrCreateReader(string name) 46 | { 47 | _ = name ?? throw new ArgumentNullException(nameof(name)); 48 | 49 | var channel = GetOrCreateChannel(name); 50 | 51 | var options = _optionsMonitor.Get(name); 52 | 53 | var sqs = _services.GetRequiredService(); 54 | var delayCalculator = new SqsReceiveDelayCalculator(options.Options); 55 | 56 | var exceptionHandler = 57 | options.ExceptionHandlerType is object type ? _services.GetService((Type)type) as IExceptionHandler : null; 58 | 59 | exceptionHandler ??= _exceptionHandler ?? DefaultExceptionHandler.Instance; 60 | 61 | var queueReader = new Lazy(() => new SqsPollingQueueReader(options.Options, sqs, delayCalculator, exceptionHandler, channel), LazyThreadSafetyMode.ExecutionAndPublication).Value; 62 | 63 | return queueReader; 64 | } 65 | 66 | public Channel GetOrCreateChannel(string name) 67 | { 68 | _ = name ?? throw new ArgumentNullException(nameof(name)); 69 | 70 | var options = _optionsMonitor.Get(name); 71 | 72 | var channelEntry = new Lazy>(() => options.Channel ?? Channel.CreateBounded(new BoundedChannelOptions(options.Options.ChannelCapacity)), LazyThreadSafetyMode.ExecutionAndPublication); 73 | 74 | var channel = _channels.GetOrAdd(name, channelEntry).Value; 75 | 76 | return channel; 77 | } 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox/SqsPollingQueueReaderFactoryOptions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Channels; 3 | using Amazon.SQS.Model; 4 | using DotNetCloud.SqsToolbox.Core.Receive; 5 | 6 | namespace DotNetCloud.SqsToolbox 7 | { 8 | public class SqsPollingQueueReaderFactoryOptions 9 | { 10 | public SqsPollingQueueReaderOptions Options { get; set; } = new SqsPollingQueueReaderOptions(); 11 | 12 | public Channel Channel { get; set; } 13 | 14 | public Type ExceptionHandlerType { get; set; } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/DotNetCloud.SqsToolbox/StopApplicationExceptionHandler.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Amazon.SQS; 3 | using DotNetCloud.SqsToolbox.Core.Abstractions; 4 | using Microsoft.Extensions.Hosting; 5 | using Microsoft.Extensions.Logging; 6 | 7 | namespace DotNetCloud.SqsToolbox 8 | { 9 | internal class StopApplicationExceptionHandler : IExceptionHandler 10 | { 11 | #if NETCOREAPP3_1 12 | private readonly IHostApplicationLifetime _appLifetime; 13 | #else 14 | private readonly IApplicationLifetime _appLifetime; 15 | #endif 16 | private readonly ILoggerFactory _loggerFactory; 17 | 18 | #if NETCOREAPP3_1 19 | public StopApplicationExceptionHandler(IHostApplicationLifetime hostApplicationLifetime, ILoggerFactory loggerFactory) 20 | { 21 | _appLifetime = hostApplicationLifetime ?? throw new ArgumentNullException(nameof(hostApplicationLifetime)); 22 | _loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); 23 | } 24 | #else 25 | public StopApplicationExceptionHandler(IApplicationLifetime applicationLifetime, ILoggerFactory loggerFactory) 26 | { 27 | _appLifetime = applicationLifetime ?? throw new ArgumentNullException(nameof(applicationLifetime)); 28 | _loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); 29 | } 30 | #endif 31 | 32 | public void OnException(T1 exception, T2 source) where T1 : Exception where T2 : class 33 | { 34 | _ = exception ?? throw new ArgumentNullException(nameof(exception)); 35 | _ = source ?? throw new ArgumentNullException(nameof(source)); 36 | 37 | var logger = _loggerFactory.CreateLogger(); 38 | 39 | if (exception is AmazonSQSException) 40 | { 41 | logger.LogError(exception, "Stopping application. An amazon SQS exception was thrown: {ExceptionMessage}", exception.Message); 42 | } 43 | 44 | _appLifetime.StopApplication(); 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /templates/SqsWorkerService/.template.config/template.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json.schemastore.org/template", 3 | "author": "Steve Gordon", 4 | "classifications": [ "AWS", "Worker" ], 5 | "identity": "DotNetCloud.SqsWorkerService", 6 | "name": ".NET Cloud: SQS Message Processing Worker Service", 7 | "shortName": "sqsworker", 8 | "tags": { 9 | "language": "C#", 10 | "type": "project" 11 | } 12 | } -------------------------------------------------------------------------------- /templates/SqsWorkerService/DiagnosticsMonitorService.cs: -------------------------------------------------------------------------------- 1 | using DotNetCloud.SqsToolbox.Extensions.Diagnostics; 2 | using Microsoft.Extensions.Logging; 3 | 4 | namespace DotNetCloud.SqsWorkerService 5 | { 6 | public class DiagnosticsMonitorService : DiagnosticsMonitoringService 7 | { 8 | private readonly ILogger _logger; 9 | 10 | public DiagnosticsMonitorService(ILogger logger) => _logger = logger; 11 | 12 | public override void OnBegin(string queueUrl) => _logger.LogTrace("Polling for messages from {QueueUrl}.", queueUrl); 13 | 14 | public override void OnReceived(string queueUrl, int messageCount) => _logger.LogInformation("Received {MessageCount} messages from {QueueUrl}.", messageCount, queueUrl); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /templates/SqsWorkerService/DotNetCloud.SqsWorkerService.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp3.1 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /templates/SqsWorkerService/MessageProcessingService.cs: -------------------------------------------------------------------------------- 1 | using System.Threading; 2 | using System.Threading.Channels; 3 | using System.Threading.Tasks; 4 | using Amazon.SQS.Model; 5 | using DotNetCloud.SqsToolbox.Abstractions; 6 | using DotNetCloud.SqsToolbox.Extensions; 7 | using Microsoft.Extensions.Logging; 8 | 9 | namespace DotNetCloud.SqsWorkerService 10 | { 11 | public class MessageProcessingService : SqsMessageProcessingBackgroundService 12 | { 13 | private readonly ILogger _logger; 14 | private readonly ISqsBatchDeleteQueue _sqsBatchDeleteQueue; 15 | 16 | public MessageProcessingService(ILogger logger, ISqsPollingQueueReader sqsPollingQueueReader, ISqsBatchDeleteQueue sqsBatchDeleteQueue) 17 | : base(sqsPollingQueueReader) 18 | { 19 | _logger = logger; 20 | _sqsBatchDeleteQueue = sqsBatchDeleteQueue; 21 | } 22 | 23 | public override async Task ProcessFromChannel(ChannelReader channelReader, CancellationToken cancellationToken) 24 | { 25 | await foreach (var message in channelReader.ReadAllAsync(cancellationToken)) 26 | { 27 | _logger.LogInformation($"Processing {message.MessageId}"); 28 | 29 | // TODO: Message processing 30 | 31 | await _sqsBatchDeleteQueue.AddMessageAsync(message, cancellationToken); 32 | } 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /templates/SqsWorkerService/Program.cs: -------------------------------------------------------------------------------- 1 | using DotNetCloud.SqsToolbox.Extensions.DependencyInjection; 2 | using Microsoft.Extensions.Hosting; 3 | 4 | namespace DotNetCloud.SqsWorkerService 5 | { 6 | public class Program 7 | { 8 | public static void Main(string[] args) 9 | { 10 | CreateHostBuilder(args).Build().Run(); 11 | } 12 | 13 | public static IHostBuilder CreateHostBuilder(string[] args) => 14 | Host.CreateDefaultBuilder(args) 15 | .ConfigureServices((hostContext, services) => 16 | { 17 | var queueUrl = hostContext.Configuration.GetSection("SQS")["ProcessingQueueUrl"]; 18 | 19 | services.AddPollingSqsBackgroundServiceWithProcessor(opt => 20 | { 21 | opt.QueueUrl = queueUrl; 22 | }); 23 | 24 | services.AddSqsToolboxDiagnosticsMonitoring(); 25 | 26 | services.AddSqsBatchDeletion(opt => 27 | { 28 | opt.QueueUrl = queueUrl; 29 | }) 30 | .WithBackgroundService(); 31 | }); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /templates/SqsWorkerService/Properties/launchSettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "profiles": { 3 | "SqsMessageProcessingWorkerService": { 4 | "commandName": "Project", 5 | "environmentVariables": { 6 | "DOTNET_ENVIRONMENT": "Development" 7 | } 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /templates/SqsWorkerService/appsettings.Development.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Information", 5 | "Microsoft": "Warning", 6 | "Microsoft.Hosting.Lifetime": "Information" 7 | } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /templates/SqsWorkerService/appsettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "AWS": { 3 | "Region": "eu-west-2" 4 | }, 5 | 6 | "Logging": { 7 | "LogLevel": { 8 | "Default": "Information", 9 | "Microsoft": "Warning", 10 | "Microsoft.Hosting.Lifetime": "Information" 11 | } 12 | }, 13 | 14 | "SQSToolbox": { 15 | "QueueUrl": "https://sqs.eu-west-2.amazonaws.com/123456789012/test-queue" 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /test/DotNetCloud.SqsToolbox.Core.Tests/DefaultExceptionHandlerTests.cs: -------------------------------------------------------------------------------- 1 | using FluentAssertions; 2 | using Xunit; 3 | 4 | namespace DotNetCloud.SqsToolbox.Core.Tests 5 | { 6 | public class DefaultExceptionHandlerTests 7 | { 8 | [Fact] 9 | public void Instance_ReturnsSameInstanceEachTime() 10 | { 11 | var instance1 = DefaultExceptionHandler.Instance; 12 | var instance2 = DefaultExceptionHandler.Instance; 13 | 14 | instance1.Should().BeSameAs(instance2); 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /test/DotNetCloud.SqsToolbox.Core.Tests/DefaultLogicalQueueNameGeneratorTests.cs: -------------------------------------------------------------------------------- 1 | using FluentAssertions; 2 | using Xunit; 3 | 4 | namespace DotNetCloud.SqsToolbox.Core.Tests 5 | { 6 | public class DefaultLogicalQueueNameGeneratorTests 7 | { 8 | [Fact] 9 | public void GenerateName_ReturnsExpectedName() 10 | { 11 | var sut = new DefaultLogicalQueueNameGenerator(); 12 | 13 | var result = sut.GenerateName("https://sqs.eu-west-2.amazonaws.com/865288682694/TestQueue"); 14 | 15 | result.Should().Be("eu-west-2_TestQueue"); 16 | } 17 | 18 | [Fact] 19 | public void GenerateName_ThrowsArgumentException_WhenUrlIsNotValid() 20 | { 21 | var sut = new DefaultLogicalQueueNameGenerator(); 22 | 23 | var result = sut.GenerateName("https://sqs.eu-west-2.amazonaws.com/865288682694/TestQueue"); 24 | 25 | result.Should().Be("eu-west-2_TestQueue"); 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /test/DotNetCloud.SqsToolbox.Core.Tests/Delete/SqsBatchDeleterTests.cs: -------------------------------------------------------------------------------- 1 |  2 | namespace DotNetCloud.SqsToolbox.Tests.Delete 3 | { 4 | //public class SqsBatchDeleterTests 5 | //{ 6 | // [Fact] 7 | // public async Task Start_ThrowsWhenStartedTwice() 8 | // { 9 | // var options = new SqsBatchDeletionOptions 10 | // { 11 | // QueueUrl = "https://example.com" 12 | // }; 13 | 14 | // var sut = new SqsBatchDeleter(options, Mock.Of(), Mock.Of(), Mock.Of()); 15 | 16 | // sut.Start(); 17 | 18 | // Action act = () => sut.Start(); 19 | 20 | // act.Should().Throw().WithMessage("The batch deleter is already started."); 21 | 22 | // await sut.StopAsync(); 23 | // } 24 | //} 25 | } 26 | -------------------------------------------------------------------------------- /test/DotNetCloud.SqsToolbox.Core.Tests/DotNetCloud.SqsToolbox.Core.Tests.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp3.1 5 | false 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | all 15 | runtime; build; native; contentfiles; analyzers; buildtransitive 16 | 17 | 18 | all 19 | runtime; build; native; contentfiles; analyzers; buildtransitive 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /test/DotNetCloud.SqsToolbox.Core.Tests/SqsPollingDelayerTests.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Amazon.SQS.Model; 3 | using DotNetCloud.SqsToolbox.Core.Receive; 4 | using FluentAssertions; 5 | using Xunit; 6 | 7 | namespace DotNetCloud.SqsToolbox.Core.Tests 8 | { 9 | public class SqsPollingDelayerTests 10 | { 11 | [Theory] 12 | [InlineData(1, 2)] 13 | [InlineData(2, 4)] 14 | [InlineData(3, 8)] 15 | [InlineData(4, 16)] 16 | [InlineData(5, 32)] 17 | public void ReturnsExpectedDelay_WhenUsingExponentialBackOff(int numberOfEmptyPolls, int expected) 18 | { 19 | var sut = new SqsReceiveDelayCalculator(new SqsPollingQueueReaderOptions 20 | { 21 | InitialDelay = TimeSpan.FromSeconds(2) 22 | }); 23 | 24 | TimeSpan result = TimeSpan.FromSeconds(-1); 25 | 26 | for (int i = 0; i < numberOfEmptyPolls; i++) 27 | { 28 | result = sut.CalculateSecondsToDelay(Array.Empty()); 29 | } 30 | 31 | result.TotalSeconds.Should().Be(expected); 32 | } 33 | 34 | [Fact] 35 | public void ReturnsZeroTimeSpan_WhenMoreThanOneMessage() 36 | { 37 | var sut = new SqsReceiveDelayCalculator(new SqsPollingQueueReaderOptions 38 | { 39 | InitialDelay = TimeSpan.FromSeconds(2) 40 | }); 41 | 42 | var result = sut.CalculateSecondsToDelay(new Message[] { new Message() }); 43 | 44 | result.TotalSeconds.Should().Be(0); 45 | } 46 | 47 | [Fact] 48 | public void ReturnsMaxValue_WhenSet() 49 | { 50 | var sut = new SqsReceiveDelayCalculator(new SqsPollingQueueReaderOptions 51 | { 52 | InitialDelay = TimeSpan.FromSeconds(10), 53 | MaxDelay = TimeSpan.FromSeconds(5) 54 | }); 55 | 56 | sut.CalculateSecondsToDelay(Array.Empty()); 57 | 58 | var result = sut.CalculateSecondsToDelay(Array.Empty()); 59 | 60 | result.TotalSeconds.Should().Be(5); 61 | } 62 | 63 | [Fact] 64 | public void ReturnsInitialDelayForAllCalls_WhenNotUsingExponentialBackOff() 65 | { 66 | var sut = new SqsReceiveDelayCalculator(new SqsPollingQueueReaderOptions 67 | { 68 | InitialDelay = TimeSpan.FromSeconds(10), 69 | UseExponentialBackoff = false 70 | }); 71 | 72 | sut.CalculateSecondsToDelay(Array.Empty()); 73 | sut.CalculateSecondsToDelay(Array.Empty()); 74 | 75 | var result = sut.CalculateSecondsToDelay(Array.Empty()); 76 | 77 | result.TotalSeconds.Should().Be(10); 78 | } 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /test/DotNetCloud.SqsToolbox.Tests/DependencyInjection/SqsBatchDeleterServiceCollectionExtensionsTests.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using Amazon.SQS; 3 | using FluentAssertions; 4 | using Microsoft.Extensions.Configuration; 5 | using Microsoft.Extensions.DependencyInjection; 6 | using Microsoft.Extensions.Options; 7 | using Xunit; 8 | 9 | namespace DotNetCloud.SqsToolbox.Extensions.Tests.DependencyInjection 10 | { 11 | //public class SqsBatchDeleterServiceCollectionExtensionsTests 12 | //{ 13 | // [Fact] 14 | // public void AddSqsBatchDeletion_WithConfiguration_AddsRequiredServices() 15 | // { 16 | // var config = new ConfigurationBuilder() 17 | // .AddInMemoryCollection(new[] 18 | // { 19 | // new KeyValuePair("AWS:Region", "eu-west-2"), 20 | // new KeyValuePair("SQSToolbox:QueueUrl", "https://example.com") 21 | // }) 22 | // .Build(); 23 | 24 | // var services = new ServiceCollection(); 25 | 26 | // services.AddSingleton(config); 27 | 28 | // services.AddSqsBatchDeletion(config); 29 | 30 | // var sp = services.BuildServiceProvider(); 31 | 32 | // sp.GetRequiredService(); 33 | 34 | // sp.GetRequiredService>(); 35 | // sp.GetRequiredService(); 36 | 37 | // var deleter = sp.GetRequiredService(); 38 | // var deleterQueue = sp.GetRequiredService(); 39 | 40 | // deleter.Should().BeSameAs(deleterQueue); 41 | // } 42 | //} 43 | } 44 | -------------------------------------------------------------------------------- /test/DotNetCloud.SqsToolbox.Tests/DotNetCloud.SqsToolbox.Tests.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp3.1 5 | false 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | --------------------------------------------------------------------------------