├── .gitignore ├── LICENSE ├── README.md ├── ReleaseNotes.md └── src ├── .vscode ├── launch.json ├── settings.json └── tasks.json ├── LiteQueue.sln ├── LiteQueue ├── IQueue.cs ├── LiteQueue.cs ├── LiteQueue.csproj ├── LiteQueue.nuspec └── QueueEntry.cs ├── LiteQueueTests ├── DTOs │ ├── CustomRecord.cs │ └── DeviceLocation.cs ├── DuplicateException.cs ├── LiteQueueTest_NotTransactional.cs ├── LiteQueueTest_Threaded.cs ├── LiteQueueTest_Transactional.cs ├── LiteQueueTests.csproj └── SampleData.cs └── SampleConsole ├── Program.cs ├── Queue.db └── SampleConsole.csproj /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | ## 4 | ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore 5 | 6 | # User-specific files 7 | *.suo 8 | *.user 9 | *.userosscache 10 | *.sln.docstates 11 | 12 | # User-specific files (MonoDevelop/Xamarin Studio) 13 | *.userprefs 14 | 15 | # Build results 16 | [Dd]ebug/ 17 | [Dd]ebugPublic/ 18 | [Rr]elease/ 19 | [Rr]eleases/ 20 | x64/ 21 | x86/ 22 | bld/ 23 | [Bb]in/ 24 | [Oo]bj/ 25 | [Ll]og/ 26 | 27 | # Visual Studio 2015/2017 cache/options directory 28 | .vs/ 29 | # Uncomment if you have tasks that create the project's static files in wwwroot 30 | #wwwroot/ 31 | 32 | # Visual Studio 2017 auto generated files 33 | Generated\ Files/ 34 | 35 | # MSTest test Results 36 | [Tt]est[Rr]esult*/ 37 | [Bb]uild[Ll]og.* 38 | 39 | # NUNIT 40 | *.VisualState.xml 41 | TestResult.xml 42 | 43 | # Build Results of an ATL Project 44 | [Dd]ebugPS/ 45 | [Rr]eleasePS/ 46 | dlldata.c 47 | 48 | # Benchmark Results 49 | BenchmarkDotNet.Artifacts/ 50 | 51 | # .NET Core 52 | project.lock.json 53 | project.fragment.lock.json 54 | artifacts/ 55 | **/Properties/launchSettings.json 56 | 57 | # StyleCop 58 | StyleCopReport.xml 59 | 60 | # Files built by Visual Studio 61 | *_i.c 62 | *_p.c 63 | *_i.h 64 | *.ilk 65 | *.meta 66 | *.obj 67 | *.iobj 68 | *.pch 69 | *.pdb 70 | *.ipdb 71 | *.pgc 72 | *.pgd 73 | *.rsp 74 | *.sbr 75 | *.tlb 76 | *.tli 77 | *.tlh 78 | *.tmp 79 | *.tmp_proj 80 | *.log 81 | *.vspscc 82 | *.vssscc 83 | .builds 84 | *.pidb 85 | *.svclog 86 | *.scc 87 | 88 | # Chutzpah Test files 89 | _Chutzpah* 90 | 91 | # Visual C++ cache files 92 | ipch/ 93 | *.aps 94 | *.ncb 95 | *.opendb 96 | *.opensdf 97 | *.sdf 98 | *.cachefile 99 | *.VC.db 100 | *.VC.VC.opendb 101 | 102 | # Visual Studio profiler 103 | *.psess 104 | *.vsp 105 | *.vspx 106 | *.sap 107 | 108 | # Visual Studio Trace Files 109 | *.e2e 110 | 111 | # TFS 2012 Local Workspace 112 | $tf/ 113 | 114 | # Guidance Automation Toolkit 115 | *.gpState 116 | 117 | # ReSharper is a .NET coding add-in 118 | _ReSharper*/ 119 | *.[Rr]e[Ss]harper 120 | *.DotSettings.user 121 | 122 | # JustCode is a .NET coding add-in 123 | .JustCode 124 | 125 | # TeamCity is a build add-in 126 | _TeamCity* 127 | 128 | # DotCover is a Code Coverage Tool 129 | *.dotCover 130 | 131 | # AxoCover is a Code Coverage Tool 132 | .axoCover/* 133 | !.axoCover/settings.json 134 | 135 | # Visual Studio code coverage results 136 | *.coverage 137 | *.coveragexml 138 | 139 | # NCrunch 140 | _NCrunch_* 141 | .*crunch*.local.xml 142 | nCrunchTemp_* 143 | 144 | # MightyMoose 145 | *.mm.* 146 | AutoTest.Net/ 147 | 148 | # Web workbench (sass) 149 | .sass-cache/ 150 | 151 | # Installshield output folder 152 | [Ee]xpress/ 153 | 154 | # DocProject is a documentation generator add-in 155 | DocProject/buildhelp/ 156 | DocProject/Help/*.HxT 157 | DocProject/Help/*.HxC 158 | DocProject/Help/*.hhc 159 | DocProject/Help/*.hhk 160 | DocProject/Help/*.hhp 161 | DocProject/Help/Html2 162 | DocProject/Help/html 163 | 164 | # Click-Once directory 165 | publish/ 166 | 167 | # Publish Web Output 168 | *.[Pp]ublish.xml 169 | *.azurePubxml 170 | # Note: Comment the next line if you want to checkin your web deploy settings, 171 | # but database connection strings (with potential passwords) will be unencrypted 172 | *.pubxml 173 | *.publishproj 174 | 175 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 176 | # checkin your Azure Web App publish settings, but sensitive information contained 177 | # in these scripts will be unencrypted 178 | PublishScripts/ 179 | 180 | # NuGet Packages 181 | *.nupkg 182 | # The packages folder can be ignored because of Package Restore 183 | **/[Pp]ackages/* 184 | # except build/, which is used as an MSBuild target. 185 | !**/[Pp]ackages/build/ 186 | # Uncomment if necessary however generally it will be regenerated when needed 187 | #!**/[Pp]ackages/repositories.config 188 | # NuGet v3's project.json files produces more ignorable files 189 | *.nuget.props 190 | *.nuget.targets 191 | 192 | # Microsoft Azure Build Output 193 | csx/ 194 | *.build.csdef 195 | 196 | # Microsoft Azure Emulator 197 | ecf/ 198 | rcf/ 199 | 200 | # Windows Store app package directories and files 201 | AppPackages/ 202 | BundleArtifacts/ 203 | Package.StoreAssociation.xml 204 | _pkginfo.txt 205 | *.appx 206 | 207 | # Visual Studio cache files 208 | # files ending in .cache can be ignored 209 | *.[Cc]ache 210 | # but keep track of directories ending in .cache 211 | !*.[Cc]ache/ 212 | 213 | # Others 214 | ClientBin/ 215 | ~$* 216 | *~ 217 | *.dbmdl 218 | *.dbproj.schemaview 219 | *.jfm 220 | *.pfx 221 | *.publishsettings 222 | orleans.codegen.cs 223 | 224 | # Including strong name files can present a security risk 225 | # (https://github.com/github/gitignore/pull/2483#issue-259490424) 226 | #*.snk 227 | 228 | # Since there are multiple workflows, uncomment next line to ignore bower_components 229 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 230 | #bower_components/ 231 | 232 | # RIA/Silverlight projects 233 | Generated_Code/ 234 | 235 | # Backup & report files from converting an old project file 236 | # to a newer Visual Studio version. Backup files are not needed, 237 | # because we have git ;-) 238 | _UpgradeReport_Files/ 239 | Backup*/ 240 | UpgradeLog*.XML 241 | UpgradeLog*.htm 242 | ServiceFabricBackup/ 243 | *.rptproj.bak 244 | 245 | # SQL Server files 246 | *.mdf 247 | *.ldf 248 | *.ndf 249 | 250 | # Business Intelligence projects 251 | *.rdl.data 252 | *.bim.layout 253 | *.bim_*.settings 254 | *.rptproj.rsuser 255 | 256 | # Microsoft Fakes 257 | FakesAssemblies/ 258 | 259 | # GhostDoc plugin setting file 260 | *.GhostDoc.xml 261 | 262 | # Node.js Tools for Visual Studio 263 | .ntvs_analysis.dat 264 | node_modules/ 265 | 266 | # Visual Studio 6 build log 267 | *.plg 268 | 269 | # Visual Studio 6 workspace options file 270 | *.opt 271 | 272 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) 273 | *.vbw 274 | 275 | # Visual Studio LightSwitch build output 276 | **/*.HTMLClient/GeneratedArtifacts 277 | **/*.DesktopClient/GeneratedArtifacts 278 | **/*.DesktopClient/ModelManifest.xml 279 | **/*.Server/GeneratedArtifacts 280 | **/*.Server/ModelManifest.xml 281 | _Pvt_Extensions 282 | 283 | # Paket dependency manager 284 | .paket/paket.exe 285 | paket-files/ 286 | 287 | # FAKE - F# Make 288 | .fake/ 289 | 290 | # JetBrains Rider 291 | .idea/ 292 | *.sln.iml 293 | 294 | # CodeRush 295 | .cr/ 296 | 297 | # Python Tools for Visual Studio (PTVS) 298 | __pycache__/ 299 | *.pyc 300 | 301 | # Cake - Uncomment if you are using it 302 | # tools/** 303 | # !tools/packages.config 304 | 305 | # Tabs Studio 306 | *.tss 307 | 308 | # Telerik's JustMock configuration file 309 | *.jmconfig 310 | 311 | # BizTalk build output 312 | *.btp.cs 313 | *.btm.cs 314 | *.odx.cs 315 | *.xsd.cs 316 | 317 | # OpenCover UI analysis results 318 | OpenCover/ 319 | 320 | # Azure Stream Analytics local run output 321 | ASALocalRun/ 322 | 323 | # MSBuild Binary and Structured Log 324 | *.binlog 325 | 326 | # NVidia Nsight GPU debugger configuration file 327 | *.nvuser 328 | 329 | # MFractors (Xamarin productivity tool) working folder 330 | .mfractor/ 331 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 NomadeonSoftwareLLC 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # LiteQueue 2 | Lightweight, persisted, thread safe, (optionally) transactional, FIFO queue built on [LiteDB](https://github.com/mbdavid/litedb). 3 | 4 | ### Background 5 | 6 | On unattended or embedded systems, it is often a requirement to reliably deliver messages even when the network is periodically down or the machine is power cycled. When using Windows, [Microsoft Message Queuing (MSMQ)](https://en.wikipedia.org/wiki/Microsoft_Message_Queuing) is an old standby for queuing and can be [called](https://docs.microsoft.com/en-us/dotnet/api/system.messaging.messagequeue?view=netframework-4.7.1) from .NET code. But what if we want to use .NET Core and be OS portable? 7 | 8 | One option is to install something like [ZeroMQ](https://en.wikipedia.org/wiki/ZeroMQ) or [RabbitMQ](https://en.wikipedia.org/wiki/RabbitMQ), but these are heavyweight for a client side queue on a single machine. [Queues.io](http://queues.io/) has a great list with some lighter options, but I did not find any with a nice .NET Standard client library. 9 | 10 | So, it was time to explore creating one. It was not necessary to fully re-invent this wheel. In particular, I did not want to roll my own persistence layer. I came across [LiteDB](http://www.litedb.org/), an embedded NoSQL database specifically for .NET. It behaved quite nicely wrapped within queue logic, which I call LiteQueue. 11 | 12 | ### Nuget 13 | 14 | ```Install-Package LiteQueue``` 15 | 16 | ### LiteQueue 17 | 18 | LiteQueue provides: 19 | 20 | - A persisted FIFO queue. 21 | - Transactions (optionally). More on this below. 22 | - API not too dissimilar from MSMQ for easy porting of legacy code. I use the method name `Dequeue` instead of `Receive` since this is intended to be a local queue. 23 | - Thread safety. Multiple threads can add and remove from the queue. 24 | - Portability via .NET Standard library. 25 | - Storage of both primitives and user defined objects. 26 | - Batch methods. 27 | - Performance is limited by the constraints as LiteDB - recommended for use only on client machines or services with small loads. 28 | - MIT license, same as LiteDB 29 | 30 | ### Transactions 31 | 32 | By default LiteQueue uses transactional logic. In this mode, `Dequeue` will flag an item as checked out but not remove it from the queue. You should call `Commit` (which fully removes the item) or `Abort` (which undoes checkout) after processing the retrieved item. Any other calls to `Dequeue` (on same or different threads) will not see items already checked out. 33 | 34 | To turn transactional logic off, set transactional to false in the constructor. Some methods like `Commit` and `Abort` will throw `InvalidOperationException` if the queue is not transactional. I debated this as it makes it annoying to switch between the two modes, but I want to fail safe if you are trying to use transactional logic when the queue is not in that mode. 35 | 36 | ### Threading 37 | 38 | If accessing the same queue from multiple threads, each thread must reference the same LiteQueue instance to ensure correct locking. If you encounter the following exception, suspect a violation of this rule as the cause: 39 | ``` 40 | SynchronizationLockException: Object synchronization method was called from an unsynchronized block of code. 41 | ``` 42 | This is a limitation of LiteDB after version 5.08. 43 | 44 | ### Message Duplication 45 | Using a queue such as this to send messages to another system, there are two ways in which the receiver could see a duplicate message. The first is when the receiver gets the message, commits it, and sends its ACK but the sender fails to see the ACK (think cellular network). Your code will timeout and logic will trigger a resend. The other possibility is that you receive the final ACK but you get halted (process crash, power cycle) before you can remove the message from your local queue. 46 | 47 | LiteQueue provides some help in the second situation (see `CurrentCheckouts` and `ResetOrphans`), but fundamentally de-duplication will require your receiver to identify and suppress duplicates. You will need to include a unique message identifier (such as GUID) in your message. If you have huge volumes, also consider a timestamp and de-duplication window so your receiver does not need to remember a complete history. 48 | 49 | For further reading: 50 | 51 | * [Two Generals' Problem](https://en.wikipedia.org/wiki/Two_Generals%27_Problem) 52 | * [Duplicate detection in Azure](https://docs.microsoft.com/en-us/azure/service-bus-messaging/duplicate-detection) 53 | * [Duplication detection in Segment](https://segment.com/blog/exactly-once-delivery/) (see also [criticisms](https://news.ycombinator.com/item?id=14664405)) 54 | 55 | ### Code Example 56 | 57 | Here's a quick start C# code snippet using transactional logic. See the unit tests for more usage. 58 | 59 | ```csharp 60 | // LiteQueue depends on LiteDB. You can save other things to same database. 61 | using (var db = new LiteDatabase("Queue.db")) 62 | { 63 | // Creates a "logs" collection in LiteDB. You can also pass a user defined object. 64 | var logs = new LiteQueue(db, "logs"); 65 | 66 | // Recommended on startup to reset anything that was checked out but not committed or aborted. 67 | // Or call CurrentCheckouts to inspect them and abort yourself. See github page for 68 | // notes regarding duplicate messages. 69 | logs.ResetOrphans(); 70 | 71 | // Adds record to queue 72 | logs.Enqueue("Test"); 73 | 74 | // Get next item from queue. Marks it as checked out such that other threads that 75 | // call Checkout will not see it - but does not remove it from the queue. 76 | var record = logs.Dequeue(); 77 | 78 | try 79 | { 80 | // Do something that may fail, i.e. a network call 81 | // record.Payload contains the original string "Test" 82 | 83 | // Removes record from queue 84 | logs.Commit(record); 85 | } 86 | catch 87 | { 88 | // Returns the record to the queue 89 | logs.Abort(record); 90 | } 91 | } 92 | ``` 93 | 94 | ### License 95 | 96 | [MIT](https://github.com/NomadeonSoftwareLLC/LiteQueue/blob/master/LICENSE) 97 | 98 | Copyright (C) by Nomadeon Software LLC 99 | -------------------------------------------------------------------------------- /ReleaseNotes.md: -------------------------------------------------------------------------------- 1 | # Release Notes 2 | 3 | ### 1.3.0 4 | - Added ability to specify the order in which entries are dequeued (FIFO by default). See IQueue::SetOrder and CustomOrder unit tests for usage. 5 | - Added IQueue interface to facilitate mocks/testing 6 | - Upgraded target to net 8.0 7 | - Updated dependencies, particularly LiteDB 5.0.21 8 | - Updated copyright statements -------------------------------------------------------------------------------- /src/.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to find out which attributes exist for C# debugging 3 | // Use hover for the description of the existing attributes 4 | // For further information visit https://github.com/OmniSharp/omnisharp-vscode/blob/master/debugger-launchjson.md 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": ".NET Core Launch (console)", 9 | "type": "coreclr", 10 | "request": "launch", 11 | "preLaunchTask": "build", 12 | // If you have changed target frameworks, make sure to update the program path. 13 | "program": "${workspaceFolder}/SampleConsole/bin/Debug/netcoreapp2.0/SampleConsole.dll", 14 | "args": [], 15 | "cwd": "${workspaceFolder}/SampleConsole", 16 | // For more information about the 'console' field, see https://github.com/OmniSharp/omnisharp-vscode/blob/master/debugger-launchjson.md#console-terminal-window 17 | "console": "internalConsole", 18 | "stopAtEntry": false, 19 | "internalConsoleOptions": "openOnSessionStart" 20 | }, 21 | { 22 | "name": ".NET Core Attach", 23 | "type": "coreclr", 24 | "request": "attach", 25 | "processId": "${command:pickProcess}" 26 | } 27 | ,] 28 | } -------------------------------------------------------------------------------- /src/.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | } -------------------------------------------------------------------------------- /src/.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0.0", 3 | "tasks": [ 4 | { 5 | "label": "build", 6 | "command": "dotnet", 7 | "type": "process", 8 | "args": [ 9 | "build", 10 | "${workspaceFolder}/SampleConsole/SampleConsole.csproj" 11 | ], 12 | "problemMatcher": "$msCompile" 13 | } 14 | ] 15 | } -------------------------------------------------------------------------------- /src/LiteQueue.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio Version 16 4 | VisualStudioVersion = 16.0.30320.27 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LiteQueue", "LiteQueue\LiteQueue.csproj", "{E72D1D9B-0DFB-4696-8255-ABC82EC02682}" 7 | EndProject 8 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SampleConsole", "SampleConsole\SampleConsole.csproj", "{01FEA2BC-6B47-4C5D-AE9A-F7A0BCF4D15C}" 9 | EndProject 10 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LiteQueueTests", "LiteQueueTests\LiteQueueTests.csproj", "{83153733-493B-469C-8484-8B7DDEA9474E}" 11 | EndProject 12 | Global 13 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 14 | Debug|Any CPU = Debug|Any CPU 15 | Release|Any CPU = Release|Any CPU 16 | EndGlobalSection 17 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 18 | {E72D1D9B-0DFB-4696-8255-ABC82EC02682}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 19 | {E72D1D9B-0DFB-4696-8255-ABC82EC02682}.Debug|Any CPU.Build.0 = Debug|Any CPU 20 | {E72D1D9B-0DFB-4696-8255-ABC82EC02682}.Release|Any CPU.ActiveCfg = Release|Any CPU 21 | {E72D1D9B-0DFB-4696-8255-ABC82EC02682}.Release|Any CPU.Build.0 = Release|Any CPU 22 | {01FEA2BC-6B47-4C5D-AE9A-F7A0BCF4D15C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 23 | {01FEA2BC-6B47-4C5D-AE9A-F7A0BCF4D15C}.Debug|Any CPU.Build.0 = Debug|Any CPU 24 | {01FEA2BC-6B47-4C5D-AE9A-F7A0BCF4D15C}.Release|Any CPU.ActiveCfg = Release|Any CPU 25 | {01FEA2BC-6B47-4C5D-AE9A-F7A0BCF4D15C}.Release|Any CPU.Build.0 = Release|Any CPU 26 | {83153733-493B-469C-8484-8B7DDEA9474E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 27 | {83153733-493B-469C-8484-8B7DDEA9474E}.Debug|Any CPU.Build.0 = Debug|Any CPU 28 | {83153733-493B-469C-8484-8B7DDEA9474E}.Release|Any CPU.ActiveCfg = Release|Any CPU 29 | {83153733-493B-469C-8484-8B7DDEA9474E}.Release|Any CPU.Build.0 = Release|Any CPU 30 | EndGlobalSection 31 | GlobalSection(SolutionProperties) = preSolution 32 | HideSolutionNode = FALSE 33 | EndGlobalSection 34 | GlobalSection(ExtensibilityGlobals) = postSolution 35 | SolutionGuid = {E281CD4B-A9E5-4494-B549-FE5B42426F78} 36 | EndGlobalSection 37 | EndGlobal 38 | -------------------------------------------------------------------------------- /src/LiteQueue/IQueue.cs: -------------------------------------------------------------------------------- 1 | /* Copyright 2024 by Nomadeon LLC. Licensed uinder MIT: https://opensource.org/licenses/MIT */ 2 | using System; 3 | using System.Collections.Generic; 4 | 5 | namespace LiteQueue 6 | { 7 | public interface IQueue 8 | { 9 | bool IsTransactional { get; } 10 | 11 | void Abort(IEnumerable> items); 12 | void Abort(QueueEntry item); 13 | void Clear(); 14 | void Commit(IEnumerable> items); 15 | void Commit(QueueEntry item); 16 | int Count(); 17 | List> CurrentCheckouts(); 18 | QueueEntry Dequeue(); 19 | List> Dequeue(int batchSize); 20 | void Enqueue(IEnumerable items); 21 | void Enqueue(T item); 22 | void ResetOrphans(); 23 | void SetOrder(Func, TKey> selector) where TKey : IComparable; 24 | 25 | } 26 | } -------------------------------------------------------------------------------- /src/LiteQueue/LiteQueue.cs: -------------------------------------------------------------------------------- 1 | /* Copyright 2024 by Nomadeon LLC. Licensed uinder MIT: https://opensource.org/licenses/MIT */ 2 | using LiteDB; 3 | using System; 4 | using System.Collections.Generic; 5 | using System.Linq; 6 | 7 | namespace LiteQueue 8 | { 9 | /// 10 | /// Uses LiteDB to provide a persisted, thread safe, (optionally) transactional, FIFO queue. 11 | /// 12 | /// Suitable for use on clients as a lightweight, portable alternative to MSMQ. Not recommended for use 13 | /// on large server side applications due to performance limitations of LiteDB. 14 | /// 15 | public class LiteQueue : IQueue 16 | { 17 | readonly ILiteCollection> _collection; 18 | readonly object _collectionLock = new object(); 19 | 20 | Func, IComparable> _orderFunc = x => x.Id; 21 | bool _transactional = true; 22 | 23 | /// 24 | /// Impacts operation of method. Can only be set once in constructor. 25 | /// 26 | public bool IsTransactional 27 | { 28 | get 29 | { 30 | return _transactional; 31 | } 32 | } 33 | 34 | /// 35 | /// Creates a collection for you in the database 36 | /// 37 | /// The LiteDB database. You are responsible for its lifecycle (using/dispose) 38 | /// Name of the collection to create 39 | /// Whether the queue should use transaction logic, default true 40 | public LiteQueue(LiteDatabase db, string collectionName, bool transactional = true) 41 | { 42 | _collection = db.GetCollection>(collectionName); 43 | _transactional = transactional; 44 | } 45 | 46 | /// 47 | /// Uses the provided database collection 48 | /// 49 | /// A LiteDB collection. 50 | /// Whether the queue should use transaction logic, default true 51 | public LiteQueue(ILiteCollection> collection, bool transactional = true) 52 | { 53 | _collection = collection; 54 | _transactional = transactional; 55 | _collection.EnsureIndex(x => x.Id); 56 | _collection.EnsureIndex(x => x.IsCheckedOut); 57 | } 58 | /// 59 | /// Creates a collection for you in the database, collection's name is 60 | /// 61 | /// The LiteDB database. You are responsible for its lifecycle (using/dispose) 62 | /// Whether the queue should use transaction logic, default true 63 | public LiteQueue(LiteDatabase db, bool transactional = true) 64 | { 65 | _collection = db.GetCollection>(typeof(T).Name); 66 | _transactional = transactional; 67 | } 68 | 69 | /// 70 | /// Adds a single item to queue. See for adding a batch. 71 | /// 72 | /// 73 | public void Enqueue(T item) 74 | { 75 | if (item == null) 76 | { 77 | throw new ArgumentNullException(nameof(item)); 78 | } 79 | 80 | QueueEntry insert = new QueueEntry(item); 81 | 82 | lock (_collectionLock) 83 | { 84 | _collection.Insert(insert); 85 | } 86 | } 87 | 88 | /// 89 | /// Adds a batch of items to the queue. See for adding a single item. 90 | /// 91 | /// 92 | public void Enqueue(IEnumerable items) 93 | { 94 | List> inserts = new List>(); 95 | foreach (var item in items) 96 | { 97 | inserts.Add(new QueueEntry(item)); 98 | } 99 | 100 | lock (_collectionLock) 101 | { 102 | _collection.InsertBulk(inserts); 103 | } 104 | } 105 | 106 | /// 107 | /// Transactional queues: 108 | /// Marks item as checked out but does not remove from queue. You are expected to later call or 109 | /// Non-transactional queues: 110 | /// Removes item from queue with no need to call or 111 | /// 112 | /// An item if found or null 113 | public QueueEntry Dequeue() 114 | { 115 | lock (_collectionLock) 116 | { 117 | var result = Dequeue(1); 118 | if (result.Count == 0) 119 | { 120 | return null; 121 | } 122 | else 123 | { 124 | return result[0]; 125 | } 126 | } 127 | } 128 | 129 | /// 130 | /// Batch equivalent of 131 | /// 132 | /// The maximum number of items to dequeue 133 | /// The items found or an empty collection (never null) 134 | public List> Dequeue(int batchSize) 135 | { 136 | lock (_collectionLock) 137 | { 138 | if (_transactional) 139 | { 140 | // WARN: LiteDB above 5.0.8 requires applying OrderBy or the records are not returned in 141 | // deterministic order (unit tests would sporadically fail). 142 | var items = _collection.Find(x => !x.IsCheckedOut, 0).OrderBy(_orderFunc).Take(batchSize); 143 | 144 | // Capture the result before changing IsCheckedOut, otherwise collection is being changed while iterating 145 | var result = new List>(items); 146 | 147 | foreach (var item in result) 148 | { 149 | item.IsCheckedOut = true; 150 | _collection.Update(item); 151 | } 152 | 153 | return result; 154 | } 155 | else 156 | { 157 | // WARN: LiteDB above 5.0.8 requires applying OrderBy or the records are not returned in 158 | // deterministic order (unit tests would sporadically fail). 159 | var items = _collection.Find(x => true, 0).OrderBy(_orderFunc).Take(batchSize); 160 | var result = new List>(items); 161 | 162 | foreach (var item in items) 163 | { 164 | _collection.Delete(new BsonValue(item.Id)); 165 | } 166 | 167 | return result; 168 | } 169 | } 170 | } 171 | 172 | /// 173 | /// Obtains list of items currently checked out (but not yet commited or aborted) as a result of Dequeue calls on a transactional queue 174 | /// 175 | /// Thrown when queue is not transactional 176 | /// Items found or empty collection (never null) 177 | public List> CurrentCheckouts() 178 | { 179 | if (!_transactional) 180 | { 181 | throw new InvalidOperationException("Cannot call " + nameof(CurrentCheckouts) + " unless queue is transactional"); 182 | } 183 | 184 | lock (_collectionLock) 185 | { 186 | var records = _collection.Find(item => item.IsCheckedOut); 187 | return new List>(records); 188 | } 189 | } 190 | 191 | /// 192 | /// Aborts all currently checked out items. Equivalent of calling followed by 193 | /// 194 | /// Thrown when queue is not transactional 195 | public void ResetOrphans() 196 | { 197 | if (!_transactional) 198 | { 199 | throw new InvalidOperationException("Cannot call " + nameof(ResetOrphans) + " unless queue is transactional"); 200 | } 201 | 202 | lock (_collectionLock) 203 | { 204 | var checkouts = CurrentCheckouts(); 205 | Abort(checkouts); 206 | } 207 | } 208 | 209 | /// 210 | /// Aborts a transaction on a single item. See for batches. 211 | /// 212 | /// An item that was obtained from a call 213 | /// Thrown when queue is not transactional 214 | public void Abort(QueueEntry item) 215 | { 216 | if (!_transactional) 217 | { 218 | throw new InvalidOperationException("Cannot call " + nameof(Abort) + " unless queue is transactional"); 219 | } 220 | else if (item == null) 221 | { 222 | throw new ArgumentNullException(nameof(item)); 223 | } 224 | 225 | lock (_collectionLock) 226 | { 227 | item.IsCheckedOut = false; 228 | _collection.Update(item); 229 | } 230 | } 231 | 232 | /// 233 | /// Aborts a transation on a group of items. See for a single item. 234 | /// 235 | /// Items that were obtained from a call 236 | /// Thrown when queue is not transactional 237 | public void Abort(IEnumerable> items) 238 | { 239 | foreach (var item in items) 240 | { 241 | Abort(item); 242 | } 243 | } 244 | 245 | /// 246 | /// Commits a transaction on a single item. See for batches. 247 | /// 248 | /// An item that was obtained from a call 249 | /// Thrown when queue is not transactional 250 | public void Commit(QueueEntry item) 251 | { 252 | if (item == null) 253 | { 254 | throw new ArgumentNullException(nameof(item)); 255 | } 256 | 257 | if (!_transactional) 258 | { 259 | throw new InvalidOperationException("Cannot call " + nameof(Commit) + " unless queue is transactional"); 260 | } 261 | 262 | lock (_collectionLock) 263 | { 264 | BsonValue id = new BsonValue(item.Id); 265 | _collection.Delete(id); 266 | } 267 | } 268 | 269 | /// 270 | /// Commits a transation on a group of items. See Items that were obtained from a call 273 | /// Thrown when queue is not transactional 274 | public void Commit(IEnumerable> items) 275 | { 276 | foreach (var item in items) 277 | { 278 | Commit(item); 279 | } 280 | } 281 | 282 | /// 283 | /// Number of items in queue, including those that have been checked out. 284 | /// 285 | public int Count() 286 | { 287 | lock (_collectionLock) 288 | { 289 | return _collection.Count(); 290 | } 291 | } 292 | 293 | /// 294 | /// Removes all items from queue, including any that have been checked out. 295 | /// 296 | public void Clear() 297 | { 298 | lock (_collectionLock) 299 | { 300 | _collection.DeleteAll(); 301 | } 302 | } 303 | 304 | public void SetOrder(Func, TKey> selector) where TKey : IComparable 305 | { 306 | _orderFunc = entry => selector(entry); 307 | } 308 | } 309 | } 310 | -------------------------------------------------------------------------------- /src/LiteQueue/LiteQueue.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | net8.0 5 | Nick Bushby 6 | Nomadeon LLC 7 | Lightweight, persisted, thread safe, (optionally) transactional, FIFO queue built on LiteDB. 8 | Nomadeon LLC 9 | http://nomadeon.com/2018/06/07/litequeue/ 10 | https://github.com/NomadeonSoftwareLLC/LiteQueue 11 | Git 12 | LiteDB, MSMQ, Queue 13 | https://github.com/NomadeonSoftwareLLC/LiteQueue/blob/master/LICENSE 14 | 1.3.0 15 | README.md 16 | See ReleaseNotes.md 17 | 18 | 19 | 20 | 21 | True 22 | \ 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /src/LiteQueue/LiteQueue.nuspec: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | LiteQueue 5 | LiteQueue 6 | 1.3.0 7 | Nick Bushby 8 | Nick Bushby 9 | true 10 | https://github.com/NomadeonSoftwareLLC/LiteQueue/blob/master/LICENSE 11 | https://github.com/NomadeonSoftwareLLC/LiteQueue 12 | Lightweight, persisted, thread safe, (optionally) transactional, FIFO queue built on LiteDB. 13 | Lightweight, persisted, thread safe, (optionally) transactional, FIFO queue built on LiteDB. 14 | 2024 by Nomadeon LLC 15 | LiteDB, MSMQ, Queue 16 | See ReleaseNotes.md 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /src/LiteQueue/QueueEntry.cs: -------------------------------------------------------------------------------- 1 | /* Copyright 2024 by Nomadeon LLC. Licensed uinder MIT: https://opensource.org/licenses/MIT */ 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Text; 5 | 6 | namespace LiteQueue 7 | { 8 | public class QueueEntry 9 | { 10 | public long Id { get; set; } 11 | public T Payload { get; set; } 12 | public bool IsCheckedOut { get; set; } 13 | 14 | public QueueEntry() 15 | { 16 | 17 | } 18 | 19 | public QueueEntry(T payload) 20 | { 21 | Payload = payload; 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/LiteQueueTests/DTOs/CustomRecord.cs: -------------------------------------------------------------------------------- 1 | /* Copyright 2024 by Nomadeon LLC. Licensed uinder MIT: https://opensource.org/licenses/MIT */ 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Text; 5 | 6 | namespace LiteQueueTests 7 | { 8 | /// 9 | /// Contrived complex object for testing the T in LiteQueue 10 | /// 11 | public class CustomRecord 12 | { 13 | public DeviceLocation Device { get; set; } 14 | 15 | public double SensorReading { get; set; } 16 | public string LogValue { get; set; } 17 | public DateTime Timestamp { get; set; } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/LiteQueueTests/DTOs/DeviceLocation.cs: -------------------------------------------------------------------------------- 1 | /* Copyright 2024 by Nomadeon LLC. Licensed uinder MIT: https://opensource.org/licenses/MIT */ 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Text; 5 | 6 | namespace LiteQueueTests 7 | { 8 | /// 9 | /// Contrived class to compose in CustomRecord 10 | /// 11 | public class DeviceLocation 12 | { 13 | public double LatitudeDegrees { get; set; } 14 | public double LongitudeDegrees { get; set; } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/LiteQueueTests/DuplicateException.cs: -------------------------------------------------------------------------------- 1 | /* Copyright 2024 by Nomadeon LLC. Licensed uinder MIT: https://opensource.org/licenses/MIT */ 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Text; 5 | 6 | namespace LiteQueueTests 7 | { 8 | class DuplicateException : Exception 9 | { 10 | object _dupe; 11 | 12 | public DuplicateException(object dupe) 13 | { 14 | _dupe = dupe; 15 | } 16 | 17 | public override string ToString() 18 | { 19 | return base.ToString() + " duplicate: " + _dupe; 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/LiteQueueTests/LiteQueueTest_NotTransactional.cs: -------------------------------------------------------------------------------- 1 | /* Copyright 2024 by Nomadeon LLC. Licensed uinder MIT: https://opensource.org/licenses/MIT */ 2 | using LiteDB; 3 | using LiteQueue; 4 | using Microsoft.VisualStudio.TestTools.UnitTesting; 5 | using System; 6 | using System.Collections.Generic; 7 | using System.Linq; 8 | 9 | namespace LiteQueueTests 10 | { 11 | /// 12 | /// Many tests may appear similar to the Transactional tests but there are subtle differences 13 | /// 14 | [TestClass] 15 | public class LiteQueueTest_NotTransactional 16 | { 17 | LiteDatabase _db; 18 | const string _collectionName = "nottransactionaltestcollection"; 19 | 20 | LiteQueue CreateQueue() 21 | { 22 | var logCollection = _db.GetCollection>(_collectionName); 23 | var logs = new LiteQueue(logCollection, false); 24 | return logs; 25 | } 26 | 27 | [TestInitialize] 28 | public void Init() 29 | { 30 | _db = new LiteDatabase("Filename=LiteQueueTest.db;connection=shared"); 31 | _db.DropCollection(_collectionName); 32 | } 33 | 34 | [TestMethod] 35 | public void Ctor_DbCollectionName() 36 | { 37 | var logs = new LiteQueue(_db, _collectionName); 38 | 39 | Assert.AreEqual(0, logs.Count()); 40 | } 41 | 42 | [TestMethod] 43 | public void Ctor_Collection() 44 | { 45 | var logs = CreateQueue(); 46 | 47 | Assert.AreEqual(0, logs.Count()); 48 | } 49 | 50 | [TestMethod] 51 | public void Dequeue() 52 | { 53 | var logs = CreateQueue(); 54 | 55 | const string entry = "NextTest"; 56 | logs.Enqueue(entry); 57 | 58 | var record = logs.Dequeue(); 59 | Assert.IsFalse(record.IsCheckedOut); 60 | Assert.AreEqual(entry, record.Payload); 61 | Assert.AreEqual(0, logs.Count()); 62 | 63 | record = logs.Dequeue(); 64 | Assert.IsNull(record); 65 | } 66 | 67 | [TestMethod] 68 | public void DequeueBatch() 69 | { 70 | var logs = CreateQueue(); 71 | 72 | List batch = new List() { "a", "b", "c" }; 73 | logs.Enqueue(batch); 74 | 75 | var records = logs.Dequeue(1); 76 | Assert.AreEqual(1, records.Count); 77 | Assert.AreEqual("a", records[0].Payload); 78 | Assert.AreEqual(2, logs.Count()); 79 | 80 | records = logs.Dequeue(2); 81 | Assert.AreEqual(2, records.Count); 82 | Assert.AreEqual("b", records[0].Payload); 83 | Assert.AreEqual("c", records[1].Payload); 84 | Assert.AreEqual(0, logs.Count()); 85 | 86 | records = logs.Dequeue(2); 87 | Assert.AreEqual(0, records.Count); 88 | } 89 | 90 | [TestMethod] 91 | public void Fifo() 92 | { 93 | var logs = CreateQueue(); 94 | 95 | const int count = 1000; 96 | 97 | for (int i = 0; i < count; i++) 98 | { 99 | logs.Enqueue(i); 100 | } 101 | 102 | for (int i = 0; i < count; i++) 103 | { 104 | int next = logs.Dequeue().Payload; 105 | Assert.AreEqual(i, next); 106 | } 107 | 108 | Assert.AreEqual(0, logs.Count()); 109 | } 110 | 111 | [TestMethod] 112 | [ExpectedException(typeof(InvalidOperationException))] 113 | public void CurrentCheckouts() 114 | { 115 | var logs = CreateQueue(); 116 | 117 | List batch = new List() { "a", "b", "c" }; 118 | logs.Enqueue(batch); 119 | 120 | var records = logs.Dequeue(1); 121 | var checkouts = logs.CurrentCheckouts(); 122 | } 123 | 124 | [TestMethod] 125 | [ExpectedException(typeof(InvalidOperationException))] 126 | public void ResetOrphans() 127 | { 128 | var logs = CreateQueue(); 129 | 130 | List batch = new List() { "a", "b", "c" }; 131 | logs.Enqueue(batch); 132 | 133 | var records = logs.Dequeue(1); 134 | 135 | logs.ResetOrphans(); 136 | } 137 | 138 | [TestMethod] 139 | [ExpectedException(typeof(InvalidOperationException))] 140 | public void Abort() 141 | { 142 | var logs = CreateQueue(); 143 | 144 | logs.Enqueue("AddTest"); 145 | 146 | var record = logs.Dequeue(); 147 | logs.Abort(record); 148 | } 149 | 150 | [TestMethod] 151 | [ExpectedException(typeof(InvalidOperationException))] 152 | public void AbortBatch() 153 | { 154 | var logs = CreateQueue(); 155 | 156 | List batch = new List() { "a", "b", "c" }; 157 | logs.Enqueue(batch); 158 | 159 | var records = logs.Dequeue(3); 160 | logs.Abort(records); 161 | } 162 | 163 | [TestMethod] 164 | [ExpectedException(typeof(InvalidOperationException))] 165 | public void Commit() 166 | { 167 | var logs = CreateQueue(); 168 | 169 | logs.Enqueue("AddTest"); 170 | 171 | var record = logs.Dequeue(); 172 | logs.Commit(record); 173 | } 174 | 175 | [TestMethod] 176 | [ExpectedException(typeof(InvalidOperationException))] 177 | public void CommitBatch() 178 | { 179 | var logs = CreateQueue(); 180 | 181 | List batch = new List() { "a", "b", "c" }; 182 | logs.Enqueue(batch); 183 | 184 | var records = logs.Dequeue(3); 185 | logs.Commit(records); 186 | } 187 | 188 | [TestMethod] 189 | public void Clear() 190 | { 191 | var logs = CreateQueue(); 192 | 193 | List batch = new List() { "a", "b", "c" }; 194 | logs.Enqueue(batch); 195 | 196 | var records = logs.Dequeue(1); 197 | 198 | logs.Clear(); 199 | Assert.AreEqual(0, logs.Count()); 200 | } 201 | 202 | [TestMethod] 203 | public void ComplexObject() 204 | { 205 | var logs = CreateQueue(); 206 | 207 | var batch = SampleData.GetCustomRecords(); 208 | 209 | logs.Enqueue(batch); 210 | 211 | var records = logs.Dequeue(1); 212 | Assert.AreEqual(1, records.Count); 213 | Assert.AreEqual(2, logs.Count()); 214 | Assert.AreEqual(batch[0].LogValue, records[0].Payload.LogValue); 215 | 216 | records = logs.Dequeue(2); 217 | Assert.AreEqual(2, records.Count); 218 | Assert.AreEqual(0, logs.Count()); 219 | 220 | Assert.AreEqual(batch[1].LogValue, records[0].Payload.LogValue); 221 | Assert.AreEqual(batch[2].LogValue, records[1].Payload.LogValue); 222 | 223 | records = logs.Dequeue(2); 224 | Assert.AreEqual(0, records.Count); 225 | } 226 | 227 | [TestMethod] 228 | public void CustomOrder() 229 | { 230 | var logs = CreateQueue(); 231 | logs.SetOrder((x) => { return x.Payload.Timestamp; }); 232 | 233 | var batch = SampleData.GetCustomRecords(); 234 | var ordered = batch.OrderBy(x => x.Timestamp); 235 | 236 | logs.Enqueue(batch); 237 | 238 | foreach (var record in ordered) 239 | { 240 | var records = logs.Dequeue(1); 241 | Assert.AreEqual(1, records.Count); 242 | Assert.AreEqual(record.LogValue, records[0].Payload.LogValue); 243 | } 244 | } 245 | } 246 | } 247 | -------------------------------------------------------------------------------- /src/LiteQueueTests/LiteQueueTest_Threaded.cs: -------------------------------------------------------------------------------- 1 | /* Copyright 2024 by Nomadeon LLC. Licensed uinder MIT: https://opensource.org/licenses/MIT */ 2 | using LiteDB; 3 | using LiteQueue; 4 | using Microsoft.VisualStudio.TestTools.UnitTesting; 5 | using System; 6 | using System.Collections.Generic; 7 | using System.Threading; 8 | using System.Threading.Tasks; 9 | 10 | namespace LiteQueueTests 11 | { 12 | [TestClass] 13 | public class LiteQueueTest_Threaded 14 | { 15 | LiteDatabase _db; 16 | LiteQueue _queue; 17 | 18 | const string _collectionName = "threadedtestcollection"; 19 | 20 | /// 21 | /// How many records created by each producer thread 22 | /// 23 | const int _recordsToProduce = 100; 24 | 25 | /// 26 | /// Monotomically increasing value shared across producers 27 | /// 28 | int _producerCounter = 0; 29 | 30 | /// 31 | /// Shared by all consumers 32 | /// 33 | HashSet _consumedRecords = new HashSet(); 34 | 35 | /// 36 | /// Consumers keep running until false 37 | /// 38 | bool _keepRunning = true; 39 | 40 | bool _consumerFailed = false; 41 | 42 | [TestInitialize] 43 | public void Init() 44 | { 45 | _db = new LiteDatabase("Filename=LiteQueueTest.db;connection=shared"); 46 | _db.DropCollection(_collectionName); 47 | 48 | _queue = new LiteQueue(_db, _collectionName); 49 | } 50 | 51 | [TestCleanup] 52 | public void Clean() 53 | { 54 | _db.DropCollection(_collectionName); 55 | } 56 | 57 | [TestMethod] 58 | public void Single() 59 | { 60 | Action producer = delegate () { Producer(_queue); }; 61 | Action consumer = delegate () { Consumer(_queue); }; 62 | RunTasks(producer, consumer, producerCount: 1, consumerCount: 1); 63 | } 64 | 65 | [TestMethod] 66 | public void MultipleProducers() 67 | { 68 | Action producer = delegate () { Producer(_queue); }; 69 | Action consumer = delegate () { Consumer(_queue); }; 70 | RunTasks(producer, consumer, producerCount: 10 , consumerCount: 1); 71 | } 72 | 73 | 74 | [TestMethod] 75 | public void MultipleConsumers() 76 | { 77 | Action producer = delegate () { Producer(_queue); }; 78 | Action consumer = delegate () { Consumer(_queue); }; 79 | RunTasks(producer, consumer, producerCount: 1, consumerCount: 10); 80 | } 81 | 82 | [TestMethod] 83 | public void MultipleProducersMultipleConsumers() 84 | { 85 | Action producer = delegate () { Producer(_queue); }; 86 | Action consumer = delegate () { Consumer(_queue); }; 87 | RunTasks(producer, consumer, producerCount: 10, consumerCount: 10); 88 | } 89 | 90 | [TestMethod] 91 | [ExpectedException(typeof(DuplicateException))] 92 | public void Duplicate() 93 | { 94 | Action producer = delegate () { BadProducer(_queue); }; 95 | Action consumer = delegate () { Consumer(_queue); }; 96 | RunTasks(producer, consumer, producerCount: 1, consumerCount: 1); 97 | } 98 | 99 | /// 100 | /// Runs a multi-threaded producer/consumer test 101 | /// 102 | /// # of producer threads to run 103 | /// # of consumer threads to run 104 | /// Function to run for each producer 105 | /// Function to run for each consumer 106 | void RunTasks(Action producer, Action consumer, int producerCount, int consumerCount) 107 | { 108 | List producers = new List(); 109 | for (int i = 0; i < producerCount; i++) 110 | { 111 | Task producerTask = new Task(producer); 112 | producers.Add(producerTask); 113 | producerTask.Start(); 114 | } 115 | 116 | List consumers = new List(); 117 | for (int i = 0; i < consumerCount; i++) 118 | { 119 | Task consumerTask = new Task(consumer); 120 | consumers.Add(consumerTask); 121 | consumerTask.Start(); 122 | } 123 | 124 | Task.WaitAll(producers.ToArray()); 125 | WaitForEmptyQueue(_queue); 126 | 127 | _keepRunning = false; 128 | try 129 | { 130 | Task.WaitAll(consumers.ToArray()); 131 | } 132 | catch (AggregateException ex) 133 | { 134 | throw ex.InnerException; 135 | } 136 | 137 | VerifyAllConsumed(producerCount); 138 | } 139 | 140 | void Producer(LiteQueue queue) 141 | { 142 | for (int i = 0; i < _recordsToProduce; i++) 143 | { 144 | int next = Interlocked.Increment(ref _producerCounter); 145 | 146 | queue.Enqueue(next); 147 | } 148 | } 149 | 150 | void BadProducer(LiteQueue queue) 151 | { 152 | for (int i = 0; i < _recordsToProduce; i++) 153 | { 154 | int next = 1; // Should cause DuplicateException in consumer 155 | 156 | queue.Enqueue(next); 157 | } 158 | } 159 | 160 | void Consumer(LiteQueue queue) 161 | { 162 | try 163 | { 164 | while (_keepRunning) 165 | { 166 | var entry = queue.Dequeue(); 167 | if (entry != null) 168 | { 169 | if (!_consumedRecords.Add(entry.Payload)) 170 | { 171 | throw new DuplicateException(entry.Payload); 172 | } 173 | queue.Commit(entry); 174 | } 175 | else 176 | { 177 | Thread.Sleep(1); 178 | } 179 | } 180 | } 181 | catch 182 | { 183 | _consumerFailed = true; 184 | throw; 185 | } 186 | } 187 | 188 | void WaitForEmptyQueue(LiteQueue queue) 189 | { 190 | while (queue.Count() > 0 && !_consumerFailed) 191 | { 192 | Thread.Sleep(5); 193 | } 194 | } 195 | 196 | void VerifyAllConsumed(int producerThreadCount) 197 | { 198 | int expected = producerThreadCount * _recordsToProduce; 199 | Assert.AreEqual(expected, _consumedRecords.Count); 200 | } 201 | } 202 | } 203 | -------------------------------------------------------------------------------- /src/LiteQueueTests/LiteQueueTest_Transactional.cs: -------------------------------------------------------------------------------- 1 | /* Copyright 2024 by Nomadeon LLC. Licensed uinder MIT: https://opensource.org/licenses/MIT */ 2 | using LiteDB; 3 | using LiteQueue; 4 | using Microsoft.VisualStudio.TestTools.UnitTesting; 5 | using System.Collections.Generic; 6 | using System.Linq; 7 | 8 | namespace LiteQueueTests 9 | { 10 | [TestClass] 11 | public class LiteQueueTest_Transactional 12 | { 13 | LiteDatabase _db; 14 | const string _collectionName = "transactionaltestcollection"; 15 | 16 | LiteQueue CreateQueue() 17 | { 18 | var logCollection = _db.GetCollection>(_collectionName); 19 | var logs = new LiteQueue(logCollection); 20 | return logs; 21 | } 22 | 23 | [TestInitialize] 24 | public void Init() 25 | { 26 | _db = new LiteDatabase("Filename=LiteQueueTest.db;connection=shared"); 27 | _db.DropCollection(_collectionName); 28 | } 29 | 30 | [TestCleanup] 31 | public void Clean() 32 | { 33 | _db.DropCollection(_collectionName); 34 | } 35 | 36 | [TestMethod] 37 | public void Ctor_DbCollectionName() 38 | { 39 | var logs = new LiteQueue(_db, _collectionName); 40 | 41 | Assert.AreEqual(0, logs.Count()); 42 | } 43 | 44 | [TestMethod] 45 | public void Ctor_Collection() 46 | { 47 | var logs = CreateQueue(); 48 | 49 | Assert.AreEqual(0, logs.Count()); 50 | } 51 | 52 | [TestMethod] 53 | public void Enqueue() 54 | { 55 | var logs = CreateQueue(); 56 | 57 | logs.Enqueue("AddTest"); 58 | 59 | Assert.AreEqual(1, logs.Count()); 60 | } 61 | 62 | [TestMethod] 63 | public void EnqueueBatch() 64 | { 65 | var logs = CreateQueue(); 66 | 67 | List batch = new List() { "a", "b", "c" }; 68 | logs.Enqueue(batch); 69 | 70 | Assert.AreEqual(3, logs.Count()); 71 | } 72 | 73 | [TestMethod] 74 | public void Dequeue() 75 | { 76 | var logs = CreateQueue(); 77 | 78 | const string entry = "NextTest"; 79 | logs.Enqueue(entry); 80 | 81 | var record = logs.Dequeue(); 82 | Assert.IsTrue(record.IsCheckedOut); 83 | Assert.AreEqual(entry, record.Payload); 84 | Assert.AreEqual(1, logs.Count()); 85 | 86 | record = logs.Dequeue(); 87 | Assert.IsNull(record); 88 | } 89 | 90 | [TestMethod] 91 | public void DequeueBatch() 92 | { 93 | var logs = CreateQueue(); 94 | 95 | List batch = new List() { "a", "b", "c" }; 96 | logs.Enqueue(batch); 97 | 98 | var records = logs.Dequeue(1); 99 | Assert.AreEqual(1, records.Count); 100 | Assert.AreEqual("a", records[0].Payload); 101 | Assert.AreEqual(3, logs.Count()); 102 | 103 | records = logs.Dequeue(2); 104 | Assert.AreEqual(2, records.Count); 105 | Assert.AreEqual("b", records[0].Payload); 106 | Assert.AreEqual("c", records[1].Payload); 107 | Assert.AreEqual(3, logs.Count()); 108 | 109 | records = logs.Dequeue(2); 110 | Assert.AreEqual(0, records.Count); 111 | } 112 | 113 | [TestMethod] 114 | public void Fifo() 115 | { 116 | var logs = CreateQueue(); 117 | 118 | const int count = 1000; 119 | 120 | for (int i = 0; i < count; i++) 121 | { 122 | logs.Enqueue(i); 123 | } 124 | 125 | for (int i = 0; i < count; i++) 126 | { 127 | var next = logs.Dequeue(); 128 | Assert.AreEqual(i, next.Payload); 129 | logs.Commit(next); 130 | } 131 | 132 | Assert.AreEqual(0, logs.Count()); 133 | } 134 | 135 | [TestMethod] 136 | public void CurrentCheckouts() 137 | { 138 | var logs = CreateQueue(); 139 | 140 | List batch = new List() { "a", "b", "c" }; 141 | logs.Enqueue(batch); 142 | 143 | var records = logs.Dequeue(1); 144 | var checkouts = logs.CurrentCheckouts(); 145 | Assert.AreEqual(1, checkouts.Count); 146 | Assert.IsTrue(checkouts[0].IsCheckedOut); 147 | Assert.AreEqual(batch[0], checkouts[0].Payload); 148 | } 149 | 150 | [TestMethod] 151 | public void ResetOrphans() 152 | { 153 | var logs = CreateQueue(); 154 | 155 | List batch = new List() { "a", "b", "c" }; 156 | logs.Enqueue(batch); 157 | 158 | var records = logs.Dequeue(1); 159 | 160 | Assert.AreEqual(1, logs.CurrentCheckouts().Count); 161 | 162 | logs.ResetOrphans(); 163 | 164 | Assert.AreEqual(0, logs.CurrentCheckouts().Count); 165 | } 166 | 167 | [TestMethod] 168 | public void Abort() 169 | { 170 | var logs = CreateQueue(); 171 | 172 | logs.Enqueue("AddTest"); 173 | 174 | var record = logs.Dequeue(); 175 | logs.Abort(record); 176 | 177 | Assert.AreEqual(1, logs.Count()); 178 | Assert.AreEqual(0, logs.CurrentCheckouts().Count); 179 | } 180 | 181 | [TestMethod] 182 | public void AbortBatch() 183 | { 184 | var logs = CreateQueue(); 185 | 186 | List batch = new List() { "a", "b", "c" }; 187 | logs.Enqueue(batch); 188 | 189 | var records = logs.Dequeue(3); 190 | logs.Abort(records); 191 | 192 | Assert.AreEqual(3, logs.Count()); 193 | Assert.AreEqual(0, logs.CurrentCheckouts().Count); 194 | } 195 | 196 | [TestMethod] 197 | public void Commit() 198 | { 199 | var logs = CreateQueue(); 200 | 201 | logs.Enqueue("AddTest"); 202 | 203 | var record = logs.Dequeue(); 204 | logs.Commit(record); 205 | Assert.AreEqual(0, logs.Count()); 206 | } 207 | 208 | [TestMethod] 209 | public void CommitBatch() 210 | { 211 | var logs = CreateQueue(); 212 | 213 | List batch = new List() { "a", "b", "c" }; 214 | logs.Enqueue(batch); 215 | 216 | var records = logs.Dequeue(3); 217 | logs.Commit(records); 218 | 219 | Assert.AreEqual(0, logs.Count()); 220 | } 221 | 222 | [TestMethod] 223 | public void Clear() 224 | { 225 | var logs = CreateQueue(); 226 | 227 | List batch = new List() { "a", "b", "c" }; 228 | logs.Enqueue(batch); 229 | 230 | var records = logs.Dequeue(1); 231 | 232 | logs.Clear(); 233 | Assert.AreEqual(0, logs.Count()); 234 | } 235 | 236 | [TestMethod] 237 | public void ComplexObject() 238 | { 239 | var logs = CreateQueue(); 240 | 241 | var batch = SampleData.GetCustomRecords(); 242 | 243 | logs.Enqueue(batch); 244 | 245 | var records = logs.Dequeue(1); 246 | Assert.AreEqual(1, records.Count); 247 | Assert.AreEqual(3, logs.Count()); 248 | Assert.AreEqual(batch[0].LogValue, records[0].Payload.LogValue); 249 | Assert.AreEqual(1, logs.CurrentCheckouts().Count); 250 | 251 | logs.Abort(records); 252 | Assert.AreEqual(3, logs.Count()); 253 | Assert.AreEqual(0, logs.CurrentCheckouts().Count); 254 | 255 | records = logs.Dequeue(1); 256 | logs.Commit(records); 257 | Assert.AreEqual(2, logs.Count()); 258 | Assert.AreEqual(0, logs.CurrentCheckouts().Count); 259 | 260 | records = logs.Dequeue(2); 261 | Assert.AreEqual(2, records.Count); 262 | Assert.AreEqual(2, logs.Count()); 263 | Assert.AreEqual(2, logs.CurrentCheckouts().Count); 264 | 265 | Assert.AreEqual(batch[1].LogValue, records[0].Payload.LogValue); 266 | Assert.AreEqual(batch[2].LogValue, records[1].Payload.LogValue); 267 | 268 | logs.Commit(records); 269 | Assert.AreEqual(0, logs.Count()); 270 | Assert.AreEqual(0, logs.CurrentCheckouts().Count); 271 | 272 | records = logs.Dequeue(2); 273 | Assert.AreEqual(0, records.Count); 274 | } 275 | 276 | [TestMethod] 277 | public void CustomOrder() 278 | { 279 | var logs = CreateQueue(); 280 | logs.SetOrder((x) => { return x.Payload.Timestamp; }); 281 | 282 | var batch = SampleData.GetCustomRecords(); 283 | var ordered = batch.OrderBy(x => x.Timestamp); 284 | 285 | logs.Enqueue(batch); 286 | 287 | foreach (var record in ordered) 288 | { 289 | var records = logs.Dequeue(1); 290 | Assert.AreEqual(1, records.Count); 291 | Assert.AreEqual(record.LogValue, records[0].Payload.LogValue); 292 | } 293 | } 294 | } 295 | } 296 | -------------------------------------------------------------------------------- /src/LiteQueueTests/LiteQueueTests.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net8.0 5 | 6 | false 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | all 16 | runtime; build; native; contentfiles; analyzers; buildtransitive 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /src/LiteQueueTests/SampleData.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace LiteQueueTests 8 | { 9 | public static class SampleData 10 | { 11 | public static List GetCustomRecords() 12 | { 13 | var record1 = new CustomRecord() { 14 | Device = new DeviceLocation() { 15 | LatitudeDegrees = 120, 16 | LongitudeDegrees = 30 17 | }, 18 | LogValue = "test", 19 | SensorReading = 2.2, 20 | Timestamp = DateTime.UtcNow.AddHours(-2) // Intentionally out of order 21 | }; 22 | var record2 = new CustomRecord() { 23 | Device = new DeviceLocation() { 24 | LatitudeDegrees = 121, 25 | LongitudeDegrees = 31 26 | }, 27 | LogValue = "test2", 28 | SensorReading = 2.3, 29 | Timestamp = DateTime.UtcNow.AddHours(-3) // Intentionally out of order 30 | }; 31 | var record3 = new CustomRecord() { 32 | Device = new DeviceLocation() { 33 | LatitudeDegrees = 122, 34 | LongitudeDegrees = 32 35 | }, 36 | LogValue = "test3", 37 | SensorReading = 2.4, 38 | Timestamp = DateTime.UtcNow.AddHours(-1) // Intentionally out of order 39 | }; 40 | 41 | var batch = new List() { record1, record2, record3 }; 42 | return batch; 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/SampleConsole/Program.cs: -------------------------------------------------------------------------------- 1 | /* Copyright 2024 by Nomadeon LLC. Licensed uinder MIT: https://opensource.org/licenses/MIT */ 2 | using LiteDB; 3 | using LiteQueue; 4 | using System; 5 | 6 | namespace SampleConsole 7 | { 8 | class Program 9 | { 10 | static void Main(string[] args) 11 | { 12 | // LiteQueue depends on LiteDB. You can save other things to same database. 13 | using (var db = new LiteDatabase("Queue.db")) 14 | { 15 | // Creates a "logs" collection in LiteDB. You can also pass a user defined object. 16 | var logs = new LiteQueue(db, "logs"); 17 | 18 | // Recommended on startup to reset anything that was checked out but not committed or aborted. 19 | // Or call CurrentCheckouts to inspect them and abort yourself. See github page for 20 | // notes regarding duplicate messages. 21 | logs.ResetOrphans(); 22 | 23 | // Adds record to queue 24 | logs.Enqueue("Test"); 25 | 26 | // Get next item from queue. Marks it as checked out such that other threads that 27 | // call Checkout will not see it - but does not remove it from the queue. 28 | var record = logs.Dequeue(); 29 | 30 | try 31 | { 32 | // Do something that may potentially fail, i.e. a network call 33 | // ... 34 | 35 | // Removes record from queue 36 | logs.Commit(record); 37 | } 38 | catch 39 | { 40 | // Returns the record to the queue 41 | logs.Abort(record); 42 | } 43 | } 44 | 45 | Console.WriteLine("Done"); 46 | Console.ReadLine(); 47 | } 48 | } 49 | } -------------------------------------------------------------------------------- /src/SampleConsole/Queue.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NomadeonSoftwareLLC/LiteQueue/b8680461a7f3f42f5bc02b094da4aeeb317bb5bd/src/SampleConsole/Queue.db -------------------------------------------------------------------------------- /src/SampleConsole/SampleConsole.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | net8.0 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | --------------------------------------------------------------------------------