├── .github ├── FUNDING.yml ├── ISSUE_TEMPLATE │ └── bug_report.md └── workflows │ └── swift.yml ├── .spi.yml ├── .gitignore ├── Documentation.docc └── Documentation.md ├── Sources └── MongoQueue │ ├── MongoQueueError.swift │ ├── TaskExecution.swift │ ├── QueuedTask.swift │ ├── ScheduledTask.swift │ ├── RecurringTask.swift │ ├── TaskModel.swift │ ├── KnownType.swift │ └── MongoQueue.swift ├── LICENSE ├── Package.swift ├── Package.resolved ├── README.md └── Tests └── MongoQueueTests └── MongoQueueTests.swift /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: [joannis] 2 | -------------------------------------------------------------------------------- /.spi.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | builder: 3 | configs: 4 | - documentation_targets: [MongoQueue] 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | /.build 3 | /Packages 4 | /*.xcodeproj 5 | xcuserdata/ 6 | DerivedData/ 7 | .swiftpm 8 | -------------------------------------------------------------------------------- /Documentation.docc/Documentation.md: -------------------------------------------------------------------------------- 1 | # ``MongoQueue`` 2 | 3 | A MongoDB-based Job Queue API for Swift applications. 4 | 5 | ## Overview 6 | 7 | This library provides a framework for storing and processing a queue of jobs in MongoDB. It uses [MongoKitten](https://github.com/orlandos-nl/MongoKitten) as a driver, using a `MonogKitten.MongoCollection` of your choosing for storing these jobs. 8 | 9 | ## Topics 10 | 11 | - ``MongoQueue`` 12 | 13 | ### Job Types 14 | 15 | - ``ScheduledTask`` 16 | - ``RecurringTask`` 17 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: Joannis 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **Code to Reproduce** 14 | Example code reproducing the issue 15 | 16 | **Expected behavior** 17 | A clear and concise description of what you expected to happen. 18 | 19 | **Host (please complete the following information):** 20 | - OS: [e.g. macOS Sierra, Ubuntu 24.04] 21 | - MongoDB server [e.g. MongoDB Atlas, or MongoDB 4.4] 22 | - MonogoQueue Version: 23 | 24 | **Additional context** 25 | Add any other context about the problem here. 26 | -------------------------------------------------------------------------------- /.github/workflows/swift.yml: -------------------------------------------------------------------------------- 1 | name: Swift 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | branches: 9 | - main 10 | 11 | jobs: 12 | test-linux: 13 | strategy: 14 | fail-fast: false 15 | matrix: 16 | dbimage: 17 | - mongo 18 | runner: 19 | - swift:5.9-focal 20 | - swift:5.10-focal 21 | - swiftlang/swift:nightly-main-focal 22 | container: ${{ matrix.runner }} 23 | runs-on: ubuntu-latest 24 | services: 25 | mongo-a: 26 | image: ${{ matrix.dbimage }} 27 | mongo-b: 28 | image: ${{ matrix.dbimage }} 29 | steps: 30 | - name: Check out 31 | uses: actions/checkout@v3 32 | - name: Run tests 33 | run: swift test 34 | env: 35 | MONGO_HOSTNAME_A: mongo-a 36 | MONGO_HOSTNAME_B: mongo-b 37 | -------------------------------------------------------------------------------- /Sources/MongoQueue/MongoQueueError.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import MongoKitten 3 | 4 | /// Errors that can occur when using MongoQueue to manage tasks. 5 | public enum MongoQueueError: Error { 6 | public enum TaskExecutionReason { 7 | case failedToClaim 8 | case taskError(Error) 9 | } 10 | 11 | case alreadyStarted 12 | case taskCreationFailed 13 | case taskExecutionFailed(reason: TaskExecutionReason) 14 | case unknownTaskCategory 15 | case reschedulingFailedTaskFailed 16 | case dequeueTaskFailed 17 | case requeueRecurringTaskFailed 18 | } 19 | 20 | public struct QueuedTaskFailure { 21 | /// The context that was used to execute the task. 22 | public let executionContext: Context 23 | 24 | /// The error that occurred when executing the task. 25 | public let error: Error 26 | 27 | /// The number of attempts that have been made to execute the task. 28 | public let attemptsMade: Int 29 | 30 | /// The _id of the task that failed. 31 | public let taskId: ObjectId 32 | } 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Joannis Orlandos 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Package.swift: -------------------------------------------------------------------------------- 1 | // swift-tools-version:5.9 2 | // The swift-tools-version declares the minimum version of Swift required to build this package. 3 | 4 | import PackageDescription 5 | 6 | let package = Package( 7 | name: "MongoQueue", 8 | platforms: [ 9 | .macOS(.v13), 10 | .iOS(.v15), 11 | ], 12 | products: [ 13 | // Products define the executables and libraries a package produces, and make them visible to other packages. 14 | .library( 15 | name: "MongoQueue", 16 | targets: ["MongoQueue"]), 17 | ], 18 | dependencies: [ 19 | // Dependencies declare other packages that this package depends on. 20 | .package(url: "https://github.com/orlandos-nl/MongoKitten.git", from: "7.9.0"), 21 | ], 22 | targets: [ 23 | // Targets are the basic building blocks of a package. A target can define a module or a test suite. 24 | // Targets can depend on other targets in this package, and on products in packages this package depends on. 25 | .target( 26 | name: "MongoQueue", 27 | dependencies: [ 28 | .product(name: "MongoKitten", package: "MongoKitten"), 29 | .product(name: "Meow", package: "MongoKitten") 30 | ] 31 | ), 32 | .testTarget( 33 | name: "MongoQueueTests", 34 | dependencies: ["MongoQueue"]), 35 | ] 36 | ) 37 | -------------------------------------------------------------------------------- /Sources/MongoQueue/TaskExecution.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// The action that should be taken when a task is done executing. 4 | public struct TaskRemovalAction { 5 | enum _Raw { 6 | case dequeue, softDelete 7 | } 8 | 9 | let raw: _Raw 10 | 11 | /// Dequeues the task from the queue, removing it from the queue. 12 | public static func dequeue() -> TaskRemovalAction { 13 | TaskRemovalAction(raw: .dequeue) 14 | } 15 | 16 | /// Soft-deletes the task from the queue, marking it as dequeued, but keeping it in the queue. 17 | public static func softDelete() -> TaskRemovalAction { 18 | TaskRemovalAction(raw: .softDelete) 19 | } 20 | } 21 | 22 | /// The action that should be taken when a task fails to execute. 23 | public struct TaskExecutionFailureAction { 24 | enum _Raw { 25 | case removal(TaskRemovalAction) 26 | case retry(maxAttempts: Int?, action: TaskRemovalAction) 27 | case retryAfter(TimeInterval, maxAttempts: Int?, action: TaskRemovalAction) 28 | } 29 | 30 | let raw: _Raw 31 | 32 | /// Dequeues the task from the queue, removing it from the queue. 33 | public static func dequeue() -> TaskExecutionFailureAction { 34 | TaskExecutionFailureAction(raw: .removal(.dequeue())) 35 | } 36 | 37 | /// Soft-deletes the task from the queue, marking it as dequeued, but keeping it in the queue. 38 | public static func softDelete() -> TaskExecutionFailureAction { 39 | TaskExecutionFailureAction(raw: .removal(.softDelete())) 40 | } 41 | 42 | /// Retries the task, executing it again. 43 | /// - Parameters: 44 | /// - maxAttempts: The maximum number of attempts that should be made to execute the task. If `nil`, the task will be retried indefinitely. 45 | /// - action: The action that should be taken when the task fails to execute after the maximum number of attempts has been reached. 46 | /// - Returns: A `TaskExecutionFailureAction` that will retry the task. 47 | public static func retry( 48 | maxAttempts: Int?, 49 | action: TaskRemovalAction = .dequeue() 50 | ) -> TaskExecutionFailureAction { 51 | TaskExecutionFailureAction(raw: .retry(maxAttempts: maxAttempts, action: action)) 52 | } 53 | 54 | /// Retries the task after a specified interval, executing it again. 55 | /// - Parameters: 56 | /// - interval: The interval after which the task should be retried. 57 | /// - maxAttempts: The maximum number of attempts that should be made to execute the task. If `nil`, the task will be retried indefinitely. 58 | /// - action: The action that should be taken when the task fails to execute after the maximum number of attempts has been reached. 59 | /// - Returns: A `TaskExecutionFailureAction` that will retry the task. 60 | public static func retryAfter( 61 | _ interval: TimeInterval, 62 | maxAttempts: Int?, 63 | action: TaskRemovalAction = .dequeue() 64 | ) -> TaskExecutionFailureAction { 65 | TaskExecutionFailureAction(raw: .retryAfter(interval, maxAttempts: maxAttempts, action: action)) 66 | } 67 | } 68 | 69 | enum TaskExecutionResult { 70 | case noneExecuted 71 | case taskSuccessful 72 | case taskFailure(Error) 73 | } 74 | -------------------------------------------------------------------------------- /Sources/MongoQueue/QueuedTask.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// A QueuedTask is a Codable type that can execute the metadata it carries 4 | /// 5 | /// These tasks can be queued to MongoDB, and an available process built on MongoQueue will pick up the task and run it 6 | /// 7 | /// You cannot implement this protocol directly, but instead need to implement one of the derived protocols. 8 | /// This can be either ``ScheduledTask`` or ``RecurringTask``, but not both at the same time. 9 | public protocol _QueuedTask: Codable { 10 | associatedtype ExecutionContext = Void 11 | 12 | /// The type of task being scheduled, defaults to your Type's name. This chosen value may only be associated with one type at a time. 13 | static var category: String { get } 14 | 15 | /// A group represents custom information that you can query for when disabling or deleting tasks. 16 | /// 17 | /// For example, a user's ID can be associated with a `group`, so that all the tasks that provide information to this user can be cleaned up in bulk when a user is deleted from the system. 18 | var group: String? { get } 19 | 20 | /// The amount of urgency your task has. Tasks with higher priority take precedence over lower priorities. 21 | /// When priorities are equal, the earlier-created task is executed first. 22 | var priority: TaskPriority { get } 23 | 24 | /// If you want only one task of this type to exist, use a static task key 25 | /// If you want to have many tasks, but not duplicate the task, identify this task by the task key 26 | /// If you don't want this task to be uniquely identified, and you want to spawn many of them, use `UUID().uuidString` 27 | var uniqueTaskKey: String { get } 28 | 29 | /// An internal configuration object that MongoQueue uses to pass around internal metadata 30 | /// 31 | /// - Warning: Do not implement or use this yourself, if you need this hook let us know 32 | var configuration: _TaskConfiguration { get } 33 | 34 | /// The expected maximum duration of this task, defaults to 10 minutes 35 | var maxTaskDuration: TimeInterval { get } 36 | 37 | /// If a task is light & quick, you can enable paralellisation. A single worker can execute many parallelised tasks simultaneously. 38 | /// 39 | /// Defaults to `false` 40 | // var allowsParallelisation: Bool { get } 41 | 42 | /// Executes the task using the stored properties in `self`. `ExecutionContext` can be any instance of your choosing, and is used as a means to execute the task. In the case of an newsletter task, this would be the email client. 43 | mutating func execute(withContext context: ExecutionContext) async throws 44 | 45 | /// - Warning: Do not implement this method yourself, if you need this hook let us know 46 | func _onDequeueTask(_ task: TaskModel, withContext context: ExecutionContext, inQueue queue: MongoQueue) async throws -> _DequeueResult 47 | 48 | /// Called when the task failed to execute. Provides an opportunity to decide the fate of this task 49 | /// 50 | /// - Parameters: 51 | /// - totalAttempts: The amount of attempts thus far, including the failed one` 52 | func onExecutionFailure(failureContext: QueuedTaskFailure) async throws -> TaskExecutionFailureAction 53 | } 54 | 55 | /// A publically non-initializable type that prevents users from overriding `onDequeueTask` 56 | public struct _DequeueResult {} 57 | 58 | extension _QueuedTask { 59 | public static var category: String { String(describing: Self.self) } 60 | public var priority: TaskPriority { .normal } 61 | public var group: String? { nil } 62 | public var uniqueTaskKey: String { UUID().uuidString } 63 | public var maxTaskDuration: TimeInterval { 10 * 60 } 64 | // public var allowsParallelisation: Bool { false } 65 | } 66 | -------------------------------------------------------------------------------- /Package.resolved: -------------------------------------------------------------------------------- 1 | { 2 | "pins" : [ 3 | { 4 | "identity" : "bson", 5 | "kind" : "remoteSourceControl", 6 | "location" : "https://github.com/orlandos-nl/BSON.git", 7 | "state" : { 8 | "revision" : "944dfb3b0eb028f477c25ba6a071181de8ab903a", 9 | "version" : "8.0.10" 10 | } 11 | }, 12 | { 13 | "identity" : "dnsclient", 14 | "kind" : "remoteSourceControl", 15 | "location" : "https://github.com/orlandos-nl/DNSClient.git", 16 | "state" : { 17 | "revision" : "770249dcb7259c486f2d68c164091b115ccb765f", 18 | "version" : "2.2.1" 19 | } 20 | }, 21 | { 22 | "identity" : "mongokitten", 23 | "kind" : "remoteSourceControl", 24 | "location" : "https://github.com/orlandos-nl/MongoKitten.git", 25 | "state" : { 26 | "revision" : "929e88ff318a56c8113c692c47f7aed11cc85b0c", 27 | "version" : "7.9.0" 28 | } 29 | }, 30 | { 31 | "identity" : "swift-atomics", 32 | "kind" : "remoteSourceControl", 33 | "location" : "https://github.com/apple/swift-atomics.git", 34 | "state" : { 35 | "revision" : "cd142fd2f64be2100422d658e7411e39489da985", 36 | "version" : "1.2.0" 37 | } 38 | }, 39 | { 40 | "identity" : "swift-collections", 41 | "kind" : "remoteSourceControl", 42 | "location" : "https://github.com/apple/swift-collections.git", 43 | "state" : { 44 | "revision" : "94cf62b3ba8d4bed62680a282d4c25f9c63c2efb", 45 | "version" : "1.1.0" 46 | } 47 | }, 48 | { 49 | "identity" : "swift-distributed-tracing", 50 | "kind" : "remoteSourceControl", 51 | "location" : "https://github.com/apple/swift-distributed-tracing.git", 52 | "state" : { 53 | "revision" : "7fbb8b23b77ee548b3d0686b6faf735c1b3c7cb8", 54 | "version" : "1.1.0" 55 | } 56 | }, 57 | { 58 | "identity" : "swift-log", 59 | "kind" : "remoteSourceControl", 60 | "location" : "https://github.com/apple/swift-log.git", 61 | "state" : { 62 | "revision" : "e97a6fcb1ab07462881ac165fdbb37f067e205d5", 63 | "version" : "1.5.4" 64 | } 65 | }, 66 | { 67 | "identity" : "swift-metrics", 68 | "kind" : "remoteSourceControl", 69 | "location" : "https://github.com/apple/swift-metrics.git", 70 | "state" : { 71 | "revision" : "971ba26378ab69c43737ee7ba967a896cb74c0d1", 72 | "version" : "2.4.1" 73 | } 74 | }, 75 | { 76 | "identity" : "swift-nio", 77 | "kind" : "remoteSourceControl", 78 | "location" : "https://github.com/apple/swift-nio.git", 79 | "state" : { 80 | "revision" : "fc63f0cf4e55a4597407a9fc95b16a2bc44b4982", 81 | "version" : "2.64.0" 82 | } 83 | }, 84 | { 85 | "identity" : "swift-nio-ssl", 86 | "kind" : "remoteSourceControl", 87 | "location" : "https://github.com/apple/swift-nio-ssl.git", 88 | "state" : { 89 | "revision" : "7c381eb6083542b124a6c18fae742f55001dc2b5", 90 | "version" : "2.26.0" 91 | } 92 | }, 93 | { 94 | "identity" : "swift-service-context", 95 | "kind" : "remoteSourceControl", 96 | "location" : "https://github.com/apple/swift-service-context.git", 97 | "state" : { 98 | "revision" : "ce0141c8f123132dbd02fd45fea448018762df1b", 99 | "version" : "1.0.0" 100 | } 101 | }, 102 | { 103 | "identity" : "swift-system", 104 | "kind" : "remoteSourceControl", 105 | "location" : "https://github.com/apple/swift-system.git", 106 | "state" : { 107 | "revision" : "025bcb1165deab2e20d4eaba79967ce73013f496", 108 | "version" : "1.2.1" 109 | } 110 | } 111 | ], 112 | "version" : 2 113 | } 114 | -------------------------------------------------------------------------------- /Sources/MongoQueue/ScheduledTask.swift: -------------------------------------------------------------------------------- 1 | import Logging 2 | import MongoCore 3 | import Foundation 4 | import Meow 5 | 6 | /// A task that is scheduled to be executed at a specific moment in time. 7 | /// This task will be executed once, and then removed from the queue. 8 | /// 9 | /// When conforming to this type, you're also conforming to `Codable`. Using this Codable conformance, all stored properties will be stored in and retrieved from MongoDB. Your task's `execute` function represents your business logic of how tasks are handled, whereas the stored properties of this type represent the input you to execute this work. 10 | /// 11 | /// The context provided into ``execute`` can be any type of your choosing, and is used as a means to execute the task. In the case of an newsletter task, this would be the email client. 12 | /// 13 | /// ```swift 14 | /// struct Reminder: ScheduledTask { 15 | /// typealias ExecutionContext = SMTPClient 16 | /// 17 | /// // Stored properties are encoded to MongoDB 18 | /// // When the task runs, they'll be decodd into a new `Reminder` instance 19 | /// // After which `execute` will be called 20 | /// let username: String 21 | /// 22 | /// // A mandatory property, allowing MongoQueue to set the execution date 23 | /// // In this case, MongoQueue will set the execution date to "now". 24 | /// // This causes the task to be ran as soon as possible. 25 | /// // Because this is computed, the property is not stored in MongoDB. 26 | /// var taskExecutionDate: Date { Date() } 27 | /// 28 | /// // Your business logic for this task 29 | /// func execute(withContext: context: ExecutionContext) async throws { 30 | /// print("I'm running! Hello, \(username)") 31 | /// } 32 | /// 33 | /// // What to do when `execute` throws an error 34 | /// func onExecutionFailure( 35 | /// failureContext: QueuedTaskFailure 36 | /// ) async throws -> TaskExecutionFailureAction { 37 | /// // Removes the task from the queue without re-attempting 38 | /// return .dequeue() 39 | /// } 40 | /// } 41 | /// ``` 42 | public protocol ScheduledTask: _QueuedTask { 43 | /// The date that you want this to be executed (delay) 44 | /// If you want it to be immediate, use `Date()` 45 | var taskExecutionDate: Date { get } 46 | 47 | /// Tasks won't be executed after this moment 48 | var taskExecutionDeadline: Date? { get } 49 | 50 | /// What happens when this task completes successfully 51 | var taskRemovalAction: TaskRemovalAction { get } 52 | } 53 | 54 | extension ScheduledTask { 55 | public var taskExecutionDeadline: Date? { nil } 56 | public var taskRemovalAction: TaskRemovalAction { .dequeue() } 57 | 58 | public func _onDequeueTask(_ task: TaskModel, withContext context: ExecutionContext, inQueue queue: MongoQueue) async throws -> _DequeueResult { 59 | do { 60 | // TODO: We assume this succeeds, but what if it does not? 61 | var concern = WriteConcern() 62 | concern.acknowledgement = .majority 63 | 64 | switch taskRemovalAction.raw { 65 | case .dequeue: 66 | guard try await queue.collection.deleteOne(where: "_id" == task._id, writeConcern: concern).deletes == 1 else { 67 | throw MongoQueueError.dequeueTaskFailed 68 | } 69 | case .softDelete: 70 | var task = task 71 | task.status = .dequeued 72 | task.execution?.lastUpdate = Date() 73 | task.execution?.endState = .success 74 | 75 | let update = try await queue.collection.upsertEncoded(task, where: "_id" == task._id) 76 | 77 | guard update.updatedCount == 1 else { 78 | throw MongoQueueError.dequeueTaskFailed 79 | } 80 | } 81 | } catch { 82 | queue.logger.critical("Failed to delete task \(task._id) of category \"\(Self.category))\" after execution: \(error.localizedDescription)") 83 | } 84 | 85 | return _DequeueResult() 86 | } 87 | 88 | /// The configuration for this task. This is used to identify the task within the queue, for internal use. 89 | public var configuration: _TaskConfiguration { 90 | let scheduled = ScheduledTaskConfiguration( 91 | scheduledDate: taskExecutionDate, 92 | uniqueTaskKey: uniqueTaskKey, 93 | executeBefore: taskExecutionDeadline 94 | ) 95 | return _TaskConfiguration(value: .scheduled(scheduled)) 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /Sources/MongoQueue/RecurringTask.swift: -------------------------------------------------------------------------------- 1 | import Logging 2 | import MongoCore 3 | import Foundation 4 | import Meow 5 | 6 | /// A protocol that describes a task that can be executed on a recurring basis (e.g. every day, every month, etc) 7 | /// 8 | /// When conforming to this type, you're also conforming to `Codable`. Using this Codable conformance, all stored properties will be stored in and retrieved from MongoDB. Your task's `execute` function represents your business logic of how tasks are handled, whereas the stored properties of this type represent the input you to execute this work. 9 | /// 10 | /// The context provided into ``execute`` can be any type of your choosing, and is used as a means to execute the task. In the case of an newsletter task, this would be the email client. 11 | /// 12 | /// ```swift 13 | /// struct DailyReminder: RecurringTask { 14 | /// typealias ExecutionContext = SMTPClient 15 | /// 16 | /// // Stored properties are encoded to MongoDB 17 | /// // When the task runs, they'll be decoded into a new `Reminder` instance 18 | /// // After which `execute` will be called 19 | /// let username: String 20 | /// 21 | /// // A mandatory property, allowing MongoQueue to set the initial execution date 22 | /// // In this case, MongoQueue will set the execution date to "now". 23 | /// // This causes the task to be ran as soon as possible. 24 | /// // Because this is computed, the property is not stored in MongoDB. 25 | /// var initialTaskExecutionDate: Date { Date() } 26 | /// 27 | /// // Your business logic for this task, which can `mutate` self. 28 | /// // This allow it to pass updated info into the next iteration. 29 | /// mutating func execute(withContext: context: ExecutionContext) async throws { 30 | /// print("I'm running! Wake up, \(username)") 31 | /// } 32 | /// 33 | /// // Calculate the next time when this task should be executed again 34 | /// func getNextRecurringTaskDate(_ context: ExecutionContext) async throws -> Date? { 35 | /// // Re-run again in 24 hours 36 | /// return Date().addingTimeInterval(3600 * 24) 37 | /// } 38 | /// 39 | /// // What to do when `execute` throws an error 40 | /// func onExecutionFailure( 41 | /// failureContext: QueuedTaskFailure 42 | /// ) async throws -> TaskExecutionFailureAction { 43 | /// // Removes the task from the queue without re-attempting 44 | /// return .dequeue() 45 | /// } 46 | /// } 47 | /// ``` 48 | public protocol RecurringTask: _QueuedTask { 49 | /// The moment that you want this to be first executed on (delay) 50 | /// If you want it to be immediate, use `Date()` 51 | var initialTaskExecutionDate: Date { get } 52 | 53 | /// Tasks won't be executed after this moment 54 | var taskExecutionDeadline: TimeInterval? { get } 55 | 56 | /// Calculates the next moment that this task should be executed on (e.g. next month, next day, etc). 57 | /// This is called _after_ your `execute` function has successfully completed the work. 58 | /// If you want to stop recurring, return `nil`. 59 | /// - parameter context: The context that was used to execute the task. 60 | func getNextRecurringTaskDate(_ context: ExecutionContext) async throws -> Date? 61 | } 62 | 63 | struct ScheduledInterval: Codable { 64 | private(set) var nextOccurrance: Date 65 | let schedule: Schedule 66 | 67 | enum Schedule: Codable { 68 | case monthly//(..) 69 | case daily//(..) 70 | 71 | func nextMoment(from date: Date = Date()) -> Date { 72 | fatalError() 73 | } 74 | } 75 | 76 | init(schedule: Schedule) { 77 | self.nextOccurrance = schedule.nextMoment() 78 | self.schedule = schedule 79 | } 80 | 81 | mutating func increment() { 82 | nextOccurrance = schedule.nextMoment(from: nextOccurrance) 83 | } 84 | } 85 | 86 | extension RecurringTask { 87 | /// The deadline for this task to be executed on. After this deadline, the task will not be executed, even if it is still in the queue. 88 | public var taskExecutionDeadline: TimeInterval? { nil } 89 | 90 | public func _onDequeueTask(_ task: TaskModel, withContext context: ExecutionContext, inQueue queue: MongoQueue) async throws -> _DequeueResult { 91 | do { 92 | guard case .recurring(let taskConfig) = try task.readConfiguration().value else { 93 | assertionFailure("Invalid internal MongoQueue state") 94 | return _DequeueResult() 95 | } 96 | var concern = WriteConcern() 97 | concern.acknowledgement = .majority 98 | if let nextDate = try await getNextRecurringTaskDate(context) { 99 | var task = task 100 | task.metadata = try BSONEncoder().encode(self) 101 | task.execution = nil 102 | task.status = .scheduled 103 | task.executeAfter = nextDate 104 | task.executeBefore = taskConfig.deadline.map { deadline in 105 | task.executeAfter.addingTimeInterval(deadline) 106 | } 107 | 108 | // TODO: We assume this succeeds, but what if it does not? 109 | // TODO: WriteConcern majority 110 | guard try await queue.collection.upsertEncoded(task, where: "_id" == task._id).updatedCount == 1 else { 111 | throw MongoQueueError.requeueRecurringTaskFailed 112 | } 113 | } else { 114 | // TODO: We assume this succeeds, but what if it does not? 115 | guard try await queue.collection.deleteOne(where: "_id" == task._id, writeConcern: concern).deletes == 1 else { 116 | throw MongoQueueError.dequeueTaskFailed 117 | } 118 | } 119 | } catch { 120 | queue.logger.critical("Failed to delete task \(task._id) of category \"\(Self.category))\" after execution: \(error.localizedDescription)") 121 | } 122 | 123 | return _DequeueResult() 124 | } 125 | 126 | /// The configuration for this task. This is used to identify the task within the queue, for internal use. 127 | public var configuration: _TaskConfiguration { 128 | let recurring = RecurringTaskConfiguration( 129 | scheduledDate: initialTaskExecutionDate, 130 | uniqueTaskKey: uniqueTaskKey, 131 | deadline: taskExecutionDeadline 132 | ) 133 | return _TaskConfiguration(value: .recurring(recurring)) 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MongoQueue 2 | 3 | A [MongoKitten](https://github.com/orlandos-nl/MongoKitten) based JobQueue for MongoDB. 4 | 5 | [Join our Discord](https://discord.gg/H6799jh) for any questions and friendly banter. 6 | 7 | [Read the Docs](https://swiftpackageindex.com/orlandos-nl/MongoQueue/main/documentation/mongoqueue) for more info. 8 | 9 | ### Quick Start 10 | 11 | Connect to MongoDB with MongoKitten regularly: 12 | 13 | ```swift 14 | let db = try await MongoDatabase.connect(to: "mongodb://localhost/my_database") 15 | ``` 16 | 17 | Select a collection for your job queue: 18 | 19 | ```swift 20 | let queue = MongoQueue(collection: db["tasks"]) 21 | ``` 22 | 23 | Define your jobs by conforming to `ScheduledTask` (and implicitly `Codable`): 24 | 25 | ```swift 26 | struct RegistrationEmailTask: ScheduledTask { 27 | // This type has no context 28 | typealias ExecutionContext = Void 29 | 30 | // Computed property, required by ScheduledTask 31 | // This executed the task ASAP 32 | var taskExecutionDate: Date { Date() } 33 | 34 | // Stored properties represent the metadata needed to execute the task 35 | let recipientEmail: String 36 | let userId: ObjectId 37 | let fullName: String 38 | 39 | func execute(withContext context: ExecutionContext) async throws { 40 | // TODO: Send the email 41 | // Throwing an error triggers `onExecutionFailure` 42 | } 43 | 44 | func onExecutionFailure(failureContext: QueuedTaskFailure) async throws -> TaskExecutionFailureAction { 45 | // Only attempt the job once. Failing to send the email cancels the job 46 | return .dequeue() 47 | } 48 | } 49 | ``` 50 | 51 | Register the task to MongoQueue, before it starts. 52 | 53 | ```swift 54 | // Context is `Void`, so we pass in a void here 55 | queue.registerTask(RegistrationEmailTask.self, context: ()) 56 | ``` 57 | 58 | Start the queue in the background - this is helpful in use inside HTTP applications - or when creating separate workers. 59 | 60 | ```swift 61 | try queue.runInBackground() 62 | ``` 63 | 64 | Alternatively, run the queue in the foreground and block until the queue is stopped. Only use this if your queue worker is only running as a worker. I.E., it isn't serving users on the side. 65 | 66 | ```swift 67 | try await queue.run() 68 | ``` 69 | 70 | Queue the task in MongoDB: 71 | 72 | ```swift 73 | let task = RegistrationEmailTask( 74 | recipientEmail: "joannis@orlandos.nl", 75 | userId: ..., 76 | fullName: "Joannis Orlandos" 77 | ) 78 | try await queue.queueTask(task) 79 | ``` 80 | 81 | Tada! Just wait for it to be executed. 82 | 83 | ## Testing 84 | 85 | You can run all currently active jobs in the queue one-by-one using: 86 | 87 | ```swift 88 | try await queue.runUntilEmpty() 89 | ``` 90 | 91 | This will not run any jobs scheduled for the future, and will exit once there are no current jobs available. 92 | 93 | ## Parallelisation 94 | 95 | You can set the parallelisation amount **per job queue instance** using the following code: 96 | 97 | ```swift 98 | queue.setMaxParallelJobs(to: 6) 99 | ``` 100 | 101 | If you have two containers running an instance of MongoQueue, it will therefore be able to run `2 * 6 = 12` jobs simultaneously. 102 | 103 | ## Integrate with Vapor 104 | 105 | To access the `queue` from your Vapor Request, add the following snippet: 106 | 107 | ```swift 108 | import Vapor 109 | import MongoKitten 110 | import MongoQueue 111 | 112 | extension Request { 113 | public var queue: MongoQueue { 114 | return application.queue 115 | } 116 | } 117 | 118 | private struct MongoQueueStorageKey: StorageKey { 119 | typealias Value = MongoQueue 120 | } 121 | 122 | extension Application { 123 | public var queue: MongoQueue { 124 | get { 125 | storage[MongoQueueStorageKey.self]! 126 | } 127 | set { 128 | storage[MongoQueueStorageKey.self] = newValue 129 | } 130 | } 131 | 132 | public func initializeMongoQueue(withCollection collection: MongoCollection) { 133 | self.queue = MongoQueue(collection: collection) 134 | } 135 | } 136 | ``` 137 | 138 | From here, you can add tasks as such: 139 | 140 | ```swift 141 | app.post("tasks") { req in 142 | try await req.queue.queueTask(MyCustomTask()) 143 | return HTTPStatus.created 144 | } 145 | ``` 146 | 147 | # Advanced Use 148 | 149 | Before diving into more (detailed) APIs, here's an overview of how this works: 150 | 151 | When you queue a task, it is used to derive the basic information for queueing the job. Parts of these requirements are in the protocol, but have a default value provided by MongoQueue. 152 | 153 | ## Dequeing Process 154 | 155 | Each task has a **category**, a unique string identifying this task's type in the database. When you register your task with MongoQueue, the category is used to know how to decode & execute the task once it is acquired by a worker. 156 | 157 | MongoQueue regularly checks, on a timer (and if possible with Change Streams for better responsiveness) whether a new task is ready to grab. When it pulls a task from MongoDB, it takes the **highest priority** task that is scheduled for execution at this date. 158 | 159 | The priority is `.normal` by default, but urgency can be increased or decreased in a tasks `var priority: TaskPriority { get }`. 160 | 161 | When the task is taken out of the queue, its `status` is set to `executing`. This means that other jobs can't execute this task right now. While doing so, the task model's `maxTaskDuration` is used as an indication of the expected duration of a task. The expected deadline is set on the model in MongoDB by adding `maxTaskDuration` to the current date. 162 | 163 | If the deadline is reached, other workers can (and will) dequeue the task and put it back into `scheduled`. This assumes the worker has crashed. However, in cases where the task is taking an abnormal amount of time, the worker will update the deadline accordingly. 164 | 165 | Due to this system, it is adviced to set urgent and short-lived tasks to a shorter `maxTaskDuration`. But take network connectivity into consideration, as setting it very low (like 5 seconds) may cause the deadline to be reached before it can be prolonged. 166 | 167 | If the task is dequeued, your task model gets a notification in `func onDequeueTask(withId taskId: ObjectId, withContext context: ExecutionContext, inQueue queue: MongoQueue) async throws`. 168 | 169 | Likewise, on execution failure you get a call on `func onExecutionFailure(failureContext: QueuedTaskFailure) async throws -> TaskExecutionFailureAction` where you can decide whether to requeue, and whether to apply a maximum amount of attempts. 170 | -------------------------------------------------------------------------------- /Sources/MongoQueue/TaskModel.swift: -------------------------------------------------------------------------------- 1 | import Logging 2 | import MongoCore 3 | import Foundation 4 | import Meow 5 | import Foundation 6 | 7 | /// The priority of your task, used to determine the order in which tasks are executed. 8 | public struct TaskPriority { 9 | internal enum _Raw: Int, Codable { 10 | case relaxed = -2, lower = -1, normal = 0, higher = 1, urgent = 2 11 | } 12 | 13 | internal let raw: _Raw 14 | 15 | /// Take your time, it's expected to take a while 16 | public static let relaxed = TaskPriority(raw: .relaxed) 17 | 18 | /// Not as urgent as regular user actions, but please do not take all the time in the world 19 | public static let lower = TaskPriority(raw: .lower) 20 | 21 | /// Regular user actions, this is the default value 22 | public static let normal = TaskPriority(raw: .normal) 23 | 24 | /// This is needed faster than other items 25 | public static let higher = TaskPriority(raw: .higher) 26 | 27 | /// THIS SHOULD NOT WAIT 28 | /// Though, if something is to be executed _immediately_, you probably shouldn't use a job queue 29 | public static let urgent = TaskPriority(raw: .urgent) 30 | } 31 | 32 | 33 | /// The current status of your task 34 | public struct TaskStatus { 35 | internal enum _Raw: String, Codable { 36 | case scheduled 37 | case suspended 38 | case executing 39 | case dequeued 40 | } 41 | 42 | internal let raw: _Raw 43 | 44 | /// The task is scheduled, and is ready for execution 45 | public static let scheduled = TaskStatus(raw: .scheduled) 46 | 47 | /// The task has been suspended until further action 48 | public static let suspended = TaskStatus(raw: .suspended) 49 | 50 | /// The task is currently executing 51 | public static let executing = TaskStatus(raw: .executing) 52 | 53 | /// The task is dequeued / soft deleted 54 | public static let dequeued = TaskStatus(raw: .dequeued) 55 | } 56 | 57 | /// The model that is used to store tasks in the database. 58 | public struct TaskModel: Codable { 59 | struct ExecutingContext: Codable { 60 | enum EndState: String, Codable { 61 | case success, failure 62 | } 63 | 64 | /// Used to represent when the task was first started. Normally it's equal to `executionStartDate` 65 | /// But when a task takes an unexpectedly long amount of time, the two values will be different 66 | let startDate: Date 67 | 68 | /// If `status == .executing`, this marks the start timestamp 69 | /// This allows tasks to be rebooted if the executioner process crashed 70 | /// If the current date exceeds `executionStartDate + maxTaskDuration`, the task likely crashed 71 | /// The executioner of the task **MUST** refresh this date at least every `maxTaskDuration` interval to ensure other executioners don't pick up on the task 72 | var lastUpdate: Date 73 | 74 | var endState: EndState? 75 | 76 | init() { 77 | let now = Date() 78 | self.startDate = now 79 | self.lastUpdate = now 80 | } 81 | 82 | mutating func updateActivity() { 83 | lastUpdate = Date() 84 | } 85 | } 86 | 87 | let _id: ObjectId 88 | 89 | /// Contains `Task.name`, used to identify how to decode the `metadata` 90 | let category: String 91 | let group: String? 92 | 93 | /// If set, only one Task with this `uniqueKey` can be queued or executing for a given `category` 94 | let uniqueKey: String? 95 | 96 | let creationDate: Date 97 | let priority: TaskPriority._Raw 98 | var executeAfter: Date 99 | var executeBefore: Date? 100 | var attempts: Int 101 | var status: TaskStatus._Raw 102 | 103 | /// The Task's stored properties, created by encoding the task using BSONEncoder 104 | var metadata: Document 105 | 106 | /// When this is set in the database, this task is currently being executed 107 | var execution: ExecutingContext? 108 | 109 | /// The maximum time that this task is expected to take. If the task takes longer than this, `execution.lasUpdate` **must** be updated before the time expires. If the times expires, the task's runner is assumed to be killed, and the task will be re-queued for execution. 110 | let maxTaskDuration: TimeInterval 111 | 112 | // let allowsParallelisation: Bool 113 | 114 | private enum ConfigurationType: String, Codable { 115 | case scheduled, recurring 116 | } 117 | 118 | private let configurationType: ConfigurationType 119 | private let configuration: Document 120 | 121 | init(representing task: T) throws { 122 | assert(task.maxTaskDuration >= 30, "maxTaskDuration is set unreasonably low in category \(T.category): \(task.maxTaskDuration)") 123 | 124 | self._id = ObjectId() 125 | self.category = T.category 126 | self.group = task.group 127 | self.priority = task.priority.raw 128 | self.attempts = 0 129 | self.creationDate = Date() 130 | self.status = .scheduled 131 | self.metadata = try BSONEncoder().encode(task) 132 | self.maxTaskDuration = task.maxTaskDuration 133 | 134 | switch task.configuration.value { 135 | case .scheduled(let configuration): 136 | self.configurationType = .scheduled 137 | self.uniqueKey = configuration.uniqueTaskKey 138 | self.executeAfter = configuration.scheduledDate 139 | self.executeBefore = configuration.executeBefore 140 | self.configuration = try BSONEncoder().encode(configuration) 141 | case .recurring(let configuration): 142 | self.configurationType = .recurring 143 | self.uniqueKey = configuration.uniqueTaskKey 144 | self.executeAfter = configuration.scheduledDate 145 | self.executeBefore = configuration.deadline.map { deadline in 146 | configuration.scheduledDate.addingTimeInterval(deadline) 147 | } 148 | self.configuration = try BSONEncoder().encode(configuration) 149 | } 150 | } 151 | 152 | func readConfiguration() throws -> _TaskConfiguration { 153 | switch configurationType { 154 | case .scheduled: 155 | return try _TaskConfiguration( 156 | value: .scheduled( 157 | BSONDecoder().decode(ScheduledTaskConfiguration.self, from: configuration) 158 | ) 159 | ) 160 | case .recurring: 161 | return try _TaskConfiguration( 162 | value: .recurring( 163 | BSONDecoder().decode(RecurringTaskConfiguration.self, from: configuration) 164 | ) 165 | ) 166 | } 167 | } 168 | } 169 | 170 | /// The configuration of a task, used to determine when the task should be executed. This is a wrapper around the actual configuration as to allow for future expansion. 171 | /// 172 | /// - Warning: Do not interact with this type yourself. It exists as a means to discourage/prevent users from creating custom Task types. If you need a different Task type, open an issue instead! 173 | public struct _TaskConfiguration { 174 | internal enum _TaskConfiguration { 175 | case scheduled(ScheduledTaskConfiguration) 176 | case recurring(RecurringTaskConfiguration) 177 | } 178 | 179 | internal var value: _TaskConfiguration 180 | 181 | internal init(value: _TaskConfiguration) { 182 | self.value = value 183 | } 184 | } 185 | 186 | struct RecurringTaskConfiguration: Codable { 187 | let scheduledDate: Date 188 | let uniqueTaskKey: String 189 | let deadline: TimeInterval? 190 | } 191 | 192 | struct ScheduledTaskConfiguration: Codable { 193 | let scheduledDate: Date 194 | let uniqueTaskKey: String? 195 | let executeBefore: Date? 196 | } 197 | -------------------------------------------------------------------------------- /Tests/MongoQueueTests/MongoQueueTests.swift: -------------------------------------------------------------------------------- 1 | import XCTest 2 | import MongoKitten 3 | @testable import MongoQueue 4 | 5 | final class MongoQueueTests: XCTestCase { 6 | static var ranTasks = 0 7 | let settings = try! ConnectionSettings("mongodb://\(ProcessInfo.processInfo.environment["MONGO_HOSTNAME_A"] ?? "localhost")/queues") 8 | 9 | func testExample() async throws { 10 | Self.ranTasks = 0 11 | let db = try await MongoDatabase.connect(to: settings) 12 | try await db.drop() 13 | let queue = MongoQueue(collection: db["tasks"]) 14 | queue.registerTask(_Task.self, context: ()) 15 | try await queue.queueTask(_Task(message: 0)) 16 | try queue.runInBackground() 17 | try await queue.queueTask(_Task(message: 1)) 18 | try await queue.queueTask(_Task(message: 2)) 19 | try await queue.queueTask(_Task(message: 3)) 20 | 21 | // Sleep 10 sec 22 | try await Task.sleep(nanoseconds: 10_000_000_000) 23 | 24 | XCTAssertEqual(Self.ranTasks, 4) 25 | queue.shutdown() 26 | } 27 | 28 | @available(macOS 13.0, *) 29 | func testMaxParallelJobs() async throws { 30 | Self.ranTasks = 0 31 | let db = try await MongoDatabase.connect(to: settings) 32 | try await db.drop() 33 | let queue = MongoQueue(collection: db["tasks"]) 34 | queue.setMaxParallelJobs(to: 6) 35 | queue.registerTask(SlowTask.self, context: ()) 36 | 37 | let start = Date() 38 | 39 | try await queue.queueTask(SlowTask()) 40 | try await queue.queueTask(SlowTask()) 41 | try await queue.queueTask(SlowTask()) 42 | try await queue.queueTask(SlowTask()) 43 | try await queue.queueTask(SlowTask()) 44 | try await queue.queueTask(SlowTask()) 45 | 46 | try await queue.runUntilEmpty() 47 | 48 | XCTAssertLessThanOrEqual(Date().timeIntervalSince(start), 2) 49 | 50 | XCTAssertEqual(Self.ranTasks, 6) 51 | queue.shutdown() 52 | } 53 | 54 | @available(macOS 13.0, *) 55 | func testMaxParallelJobsLow() async throws { 56 | Self.ranTasks = 0 57 | let db = try await MongoDatabase.connect(to: settings) 58 | try await db.drop() 59 | let queue = MongoQueue(collection: db["tasks"]) 60 | queue.setMaxParallelJobs(to: 1) 61 | queue.registerTask(SlowTask.self, context: ()) 62 | 63 | let start = Date() 64 | 65 | try await queue.queueTask(SlowTask()) 66 | try await queue.queueTask(SlowTask()) 67 | try await queue.queueTask(SlowTask()) 68 | try await queue.queueTask(SlowTask()) 69 | try await queue.queueTask(SlowTask()) 70 | try await queue.queueTask(SlowTask()) 71 | 72 | try await queue.runUntilEmpty() 73 | 74 | XCTAssertLessThanOrEqual(Date().timeIntervalSince(start), 7) 75 | 76 | XCTAssertEqual(Self.ranTasks, 6) 77 | queue.shutdown() 78 | } 79 | 80 | func testNoDuplicateQueuedTasksOfSameUniqueKey() async throws { 81 | struct UniqueTask: ScheduledTask { 82 | var taskExecutionDate: Date { Date() } 83 | var uniqueTaskKey: String { "static" } 84 | 85 | func execute(withContext context: Void) async throws {} 86 | 87 | func onExecutionFailure(failureContext: QueuedTaskFailure<()>) async throws -> TaskExecutionFailureAction { 88 | return .dequeue() 89 | } 90 | } 91 | 92 | let db = try await MongoDatabase.connect(to: settings) 93 | try await db.drop() 94 | let queue = MongoQueue(collection: db["tasks"], options: [.enableUniqueKeys]) 95 | queue.registerTask(UniqueTask.self, context: ()) 96 | try await queue.ensureIndexes() 97 | try await queue.queueTask(UniqueTask()) 98 | 99 | do { 100 | try await queue.queueTask(UniqueTask()) 101 | XCTFail("Task should not be able to exist in queue twice") 102 | } catch {} 103 | 104 | try await queue.runUntilEmpty() 105 | try await queue.queueTask(UniqueTask()) 106 | queue.shutdown() 107 | } 108 | 109 | func testDuplicatedOfDifferentTasksCanExist() async throws { 110 | struct UniqueTask: ScheduledTask { 111 | var taskExecutionDate: Date { Date() } 112 | var uniqueTaskKey: String { "static" } 113 | 114 | func execute(withContext context: Void) async throws {} 115 | 116 | func onExecutionFailure(failureContext: QueuedTaskFailure<()>) async throws -> TaskExecutionFailureAction { 117 | return .dequeue() 118 | } 119 | } 120 | 121 | struct UniqueTask2: ScheduledTask { 122 | var taskExecutionDate: Date { Date() } 123 | var uniqueTaskKey: String { "static" } 124 | 125 | func execute(withContext context: Void) async throws {} 126 | 127 | func onExecutionFailure(failureContext: QueuedTaskFailure<()>) async throws -> TaskExecutionFailureAction { 128 | return .dequeue() 129 | } 130 | } 131 | 132 | let db = try await MongoDatabase.connect(to: settings) 133 | try await db.drop() 134 | let queue = MongoQueue(collection: db["tasks"], options: [.enableUniqueKeys]) 135 | queue.registerTask(UniqueTask.self, context: ()) 136 | queue.registerTask(UniqueTask2.self, context: ()) 137 | try await queue.ensureIndexes() 138 | try await queue.queueTask(UniqueTask()) 139 | try await queue.queueTask(UniqueTask2()) 140 | 141 | do { 142 | try await queue.queueTask(UniqueTask()) 143 | XCTFail("Task should not be able to exist in queue twice") 144 | } catch {} 145 | 146 | do { 147 | try await queue.queueTask(UniqueTask2()) 148 | XCTFail("Task should not be able to exist in queue twice") 149 | } catch {} 150 | 151 | try await queue.runUntilEmpty() 152 | try await queue.queueTask(UniqueTask()) 153 | try await queue.queueTask(UniqueTask2()) 154 | queue.shutdown() 155 | } 156 | 157 | func test_recurringTask() async throws { 158 | Self.ranTasks = 0 159 | let db = try await MongoDatabase.connect(to: settings) 160 | try await db.drop() 161 | let queue = MongoQueue(collection: db["tasks"]) 162 | queue.registerTask(RTRecurringTask.self, context: ()) 163 | try await queue.queueTask(RTRecurringTask()) 164 | queue.newTaskPollingFrequency = .milliseconds(100) 165 | try queue.runInBackground() 166 | 167 | // Sleep 30 sec, so each 5-second window is ran, +5 seconds to test if it runs only 5 times 168 | try await Task.sleep(nanoseconds: 5_000_000_000) 169 | 170 | XCTAssertEqual(Self.ranTasks, 5) 171 | queue.shutdown() 172 | } 173 | } 174 | 175 | struct _Task: ScheduledTask { 176 | var taskExecutionDate: Date { 177 | Date() 178 | } 179 | 180 | let message: Int 181 | 182 | func execute(withContext context: Void) async throws { 183 | XCTAssertEqual(MongoQueueTests.ranTasks, message) 184 | MongoQueueTests.ranTasks += 1 185 | } 186 | 187 | func onExecutionFailure(failureContext: QueuedTaskFailure<()>) async throws -> TaskExecutionFailureAction { 188 | return .dequeue() 189 | } 190 | } 191 | 192 | @available(macOS 13.0, *) 193 | struct SlowTask: ScheduledTask { 194 | var taskExecutionDate: Date { 195 | Date() 196 | } 197 | 198 | func execute(withContext context: Void) async throws { 199 | try await Task.sleep(for: .seconds(1)) 200 | MongoQueueTests.ranTasks += 1 201 | } 202 | 203 | func onExecutionFailure(failureContext: QueuedTaskFailure<()>) async throws -> TaskExecutionFailureAction { 204 | return .dequeue() 205 | } 206 | } 207 | 208 | struct RTRecurringTask: RecurringTask { 209 | typealias ExecutionContext = Void 210 | var initialTaskExecutionDate: Date { Date() } 211 | 212 | var uniqueTaskKey: String = "RecurringTask" 213 | 214 | func getNextRecurringTaskDate(_ context: ExecutionContext) async throws -> Date? { 215 | MongoQueueTests.ranTasks >= 5 ? nil : Date().addingTimeInterval(1) 216 | } 217 | 218 | func execute(withContext context: Void) async throws { 219 | MongoQueueTests.ranTasks += 1 220 | } 221 | 222 | func onExecutionFailure(failureContext: QueuedTaskFailure<()>) async throws -> TaskExecutionFailureAction { 223 | return .dequeue() 224 | } 225 | } 226 | -------------------------------------------------------------------------------- /Sources/MongoQueue/KnownType.swift: -------------------------------------------------------------------------------- 1 | import Logging 2 | import MongoCore 3 | import Foundation 4 | import Meow 5 | 6 | internal struct KnownType { 7 | let category: String 8 | let performTask: (inout TaskModel) async throws -> () 9 | 10 | init( 11 | type: T.Type, 12 | queue: MongoQueue, 13 | logger: Logger, 14 | context: T.ExecutionContext 15 | ) { 16 | self.category = type.category 17 | self.performTask = { task in 18 | try await KnownType.performTask( 19 | &task, 20 | queue: queue, 21 | logger: logger, 22 | ofType: type, 23 | context: context 24 | ) 25 | } 26 | } 27 | 28 | private static func performTask( 29 | _ task: inout TaskModel, 30 | queue: MongoQueue, 31 | logger: Logger, 32 | ofType type: T.Type, 33 | context: T.ExecutionContext 34 | ) async throws { 35 | logger.debug("Executing task \(task._id) of category \"\(T.category)\"") 36 | let collection = queue.collection 37 | var metadata: T 38 | 39 | do { 40 | metadata = try BSONDecoder().decode(type, from: task.metadata) 41 | } catch { 42 | logger.error("Task of category \"\(T.category)\" has changed metadata format") 43 | queue.jobsInvalid.increment() 44 | try await collection.deleteOne(where: "_id" == task._id) 45 | queue.jobsRemoved.increment() 46 | throw error 47 | } 48 | 49 | let taskConfig = try task.readConfiguration().value 50 | 51 | switch taskConfig { 52 | case .scheduled(let scheduleConfig): 53 | if let executeBefore = scheduleConfig.executeBefore, executeBefore < Date() { 54 | queue.jobsExpired.increment() 55 | 56 | logger.info("Task of category \"\(T.category)\" expired and will not be executed") 57 | do { 58 | // TODO: We assume this succeeds, but what if it does not? 59 | var concern = WriteConcern() 60 | concern.acknowledgement = .majority 61 | try await collection.deleteOne(where: "_id" == task._id, writeConcern: concern) 62 | queue.jobsRemoved.increment() 63 | } catch { 64 | logger.critical("Failed to delete task \(task._id) of category \"\(T.category))\" after execution: \(error.localizedDescription)") 65 | } 66 | return 67 | } 68 | case .recurring(let recurringConfig): 69 | // No filters exist (yet) that prevent a task from executing 70 | if let deadline = recurringConfig.deadline, recurringConfig.scheduledDate.addingTimeInterval(deadline) < Date() { 71 | queue.jobsExpired.increment() 72 | 73 | logger.info("Task of category \"\(T.category)\" expired and will not be executed") 74 | do { 75 | // TODO: We assume this succeeds, but what if it does not? 76 | var concern = WriteConcern() 77 | concern.acknowledgement = .majority 78 | try await collection.deleteOne(where: "_id" == task._id, writeConcern: concern) 79 | queue.jobsRemoved.increment() 80 | } catch { 81 | logger.critical("Failed to delete task \(task._id) of category \"\(T.category))\" after execution: \(error.localizedDescription)") 82 | } 83 | return 84 | } 85 | } 86 | 87 | do { 88 | task.attempts += 1 89 | let taskId = task._id 90 | assert(task.maxTaskDuration >= 30, "maxTaskDuration is set unreasonably low in category \(task.category): \(task.maxTaskDuration)") 91 | 92 | // We're early on the updates, so that we don't get dequeued 93 | let interval = Swift.max(task.maxTaskDuration - 15, 1) 94 | try await withThrowingTaskGroup(of: T.self) { taskGroup in 95 | taskGroup.addTask { 96 | while !Task.isCancelled { 97 | try await Task.sleep(nanoseconds: UInt64(interval) * 1_000_000_000) 98 | _ = try await collection.findOneAndUpdate( 99 | where: "_id" == taskId , 100 | to: [ 101 | "$set": [ 102 | "execution.lastUpdate": Date() 103 | ] 104 | ] 105 | ).execute() 106 | } 107 | 108 | throw CancellationError() 109 | } 110 | 111 | taskGroup.addTask { [metadata, task] in 112 | var metadata = metadata 113 | queue.jobsRan.increment() 114 | try await metadata.execute(withContext: context) 115 | queue.jobsSucceeded.increment() 116 | logger.debug("Successful execution: task \(task._id) of category \"\(T.category)\"") 117 | _ = try await metadata._onDequeueTask(task, withContext: context, inQueue: queue) 118 | return metadata 119 | } 120 | 121 | guard let _metadata = try await taskGroup.next() else { 122 | throw CancellationError() 123 | } 124 | 125 | metadata = _metadata 126 | taskGroup.cancelAll() 127 | } 128 | } catch { 129 | queue.jobsFailed.increment() 130 | logger.debug("Execution failure for task \(task._id) in category \"\(T.category))\": \(error.localizedDescription)") 131 | let failureContext = QueuedTaskFailure( 132 | executionContext: context, 133 | error: error, 134 | attemptsMade: task.attempts, 135 | taskId: task._id 136 | ) 137 | let onFailure = try await metadata.onExecutionFailure(failureContext: failureContext) 138 | 139 | func applyRemoval(_ removal: TaskRemovalAction) async throws { 140 | switch removal.raw { 141 | case .softDelete: 142 | task.status = .dequeued 143 | task.execution?.lastUpdate = Date() 144 | task.execution?.endState = .failure 145 | 146 | let update = try await queue.collection.upsertEncoded(task, where: "_id" == task._id) 147 | 148 | guard update.updatedCount == 1 else { 149 | logger.error("Failed to soft-delete task \(task._id) of category \"\(T.category)\"") 150 | throw MongoQueueError.dequeueTaskFailed 151 | } 152 | case .dequeue: 153 | guard try await collection.deleteOne(where: "_id" == task._id).deletes == 1 else { 154 | logger.error("Failed to delete task \(task._id) of category \"\(T.category)\"") 155 | throw MongoQueueError.dequeueTaskFailed 156 | } 157 | } 158 | queue.jobsRemoved.increment() 159 | } 160 | 161 | switch onFailure.raw { 162 | case .removal(let removal): 163 | try await applyRemoval(removal) 164 | case .retry(maxAttempts: let maxAttempts, let removal): 165 | if let maxAttempts = maxAttempts, task.attempts >= maxAttempts { 166 | logger.debug("Task Removal: task \(task._id) of category \"\(T.category)\" exceeded \(maxAttempts) attempts") 167 | try await applyRemoval(removal) 168 | } else { 169 | task.status = .scheduled 170 | task.execution = nil 171 | 172 | guard try await collection.upsertEncoded(task, where: "_id" == task._id).updatedCount == 1 else { 173 | throw MongoQueueError.reschedulingFailedTaskFailed 174 | } 175 | queue.jobsRequeued.increment() 176 | } 177 | case .retryAfter(let nextInterval, maxAttempts: let maxAttempts, let removal): 178 | if let maxAttempts = maxAttempts, task.attempts >= maxAttempts { 179 | logger.debug("Task Removal: task \(task._id) of category \"\(T.category)\" exceeded \(maxAttempts) attempts") 180 | try await applyRemoval(removal) 181 | } else { 182 | task.status = .scheduled 183 | task.execution = nil 184 | task.executeAfter = Date().addingTimeInterval(nextInterval) 185 | 186 | guard try await collection.upsertEncoded(task, where: "_id" == task._id).updatedCount == 1 else { 187 | throw MongoQueueError.reschedulingFailedTaskFailed 188 | } 189 | queue.jobsRequeued.increment() 190 | } 191 | } 192 | 193 | // Throw the initial error 194 | throw error 195 | } 196 | } 197 | } 198 | -------------------------------------------------------------------------------- /Sources/MongoQueue/MongoQueue.swift: -------------------------------------------------------------------------------- 1 | import Logging 2 | import MongoCore 3 | import NIOConcurrencyHelpers 4 | import Foundation 5 | import Meow 6 | import Tracing 7 | import Metrics 8 | 9 | /// A MongoQueue is a queue that uses MongoDB as a backend for storing tasks. It is designed to be used in a distributed environment. 10 | /// 11 | /// 1. First, connect to MongoDB and create the MongoQueue. 12 | /// 2. Then, register your tasks _with_ ExecutionContext. 13 | /// 3. Finally, start the job queue. 14 | /// 15 | /// ```swift 16 | /// import MongoKitten 17 | /// 18 | /// let db = try await MongoDatabase.connect(to: "mongodb://localhost/my-db") 19 | /// let queue = MongoQueue(db["job-queue"]) 20 | /// queue.registerTask(Reminder.self, context: executionContext) 21 | /// // Run the queue until it's stopped or a cancellation is received 22 | /// try await queue.run() 23 | /// ``` 24 | /// 25 | /// To insert a new task into the queue: 26 | /// 27 | /// ```swift 28 | /// try await queue.queueTask(Reminder(username: "Joannis")) 29 | /// ``` 30 | public final class MongoQueue: @unchecked Sendable { 31 | public struct Option: Hashable { 32 | internal enum _Option: Hashable { 33 | case uniqueKeysEnabled 34 | } 35 | 36 | internal let raw: _Option 37 | 38 | public static let enableUniqueKeys = Option(raw: .uniqueKeysEnabled) 39 | } 40 | 41 | internal let collection: MongoCollection 42 | internal let logger = Logger(label: "org.openkitten.mongo-queues") 43 | private var knownTypes = [KnownType]() 44 | private let _started = NIOLockedValueBox(false) 45 | private var started: Bool { 46 | get { _started.withLockedValue { $0 } } 47 | set { _started.withLockedValue { $0 = newValue } } 48 | } 49 | private let _serverHasData = NIOLockedValueBox(true) 50 | private var serverHasData: Bool { 51 | get { _serverHasData.withLockedValue { $0 } } 52 | set { _serverHasData.withLockedValue { $0 = newValue } } 53 | } 54 | private let checkServerNotifications = AsyncStream.makeStream(bufferingPolicy: .bufferingNewest(2)) 55 | private var maxParallelJobs = 1 56 | package let jobsRan = Counter(label: "org.orlandos-nl.mongoqueue.jobsRan") 57 | package let jobsSucceeded = Counter(label: "org.orlandos-nl.mongoqueue.jobsSucceeded") 58 | package let jobsInvalid = Counter(label: "org.orlandos-nl.mongoqueue.jobsIgnored") 59 | package let jobsRequeued = Counter(label: "org.orlandos-nl.mongoqueue.jobsRequeued") 60 | package let jobsExpired = Counter(label: "org.orlandos-nl.mongoqueue.jobsExpired") 61 | package let jobsRemoved = Counter(label: "org.orlandos-nl.mongoqueue.jobsRemoved") 62 | package let jobsKilled = Counter(label: "org.orlandos-nl.mongoqueue.jobsKilled") 63 | package let jobsFailed = Counter(label: "org.orlandos-nl.mongoqueue.jobsFailed") 64 | public var newTaskPollingFrequency = NIO.TimeAmount.milliseconds(1000) 65 | public let options: Set