├── .github ├── CODEOWNERS ├── CONTRIBUTING.MD └── workflows │ ├── api-docs.yml │ └── test.yml ├── .gitignore ├── .spi.yml ├── LICENSE ├── Package.swift ├── README.md ├── Sources ├── Queues │ ├── Application+Queues.swift │ ├── AsyncJob.swift │ ├── AsyncJobEventDelegate.swift │ ├── AsyncQueue.swift │ ├── AsyncScheduledJob.swift │ ├── Docs.docc │ │ ├── Resources │ │ │ └── vapor-queues-logo.svg │ │ ├── index.md │ │ └── theme-settings.json │ ├── Exports.swift │ ├── Job.swift │ ├── JobData.swift │ ├── JobIdentifier.swift │ ├── NotificationHook.swift │ ├── Queue.swift │ ├── QueueContext.swift │ ├── QueueName.swift │ ├── QueueWorker.swift │ ├── QueuesCommand.swift │ ├── QueuesConfiguration.swift │ ├── QueuesDriver.swift │ ├── QueuesEventLoopPreference.swift │ ├── RepeatedTask+Cancel.swift │ ├── Request+Queues.swift │ ├── ScheduleBuilder.swift │ └── ScheduledJob.swift └── XCTQueues │ ├── Docs.docc │ ├── Resources │ │ └── vapor-queues-logo.svg │ ├── index.md │ └── theme-settings.json │ └── TestQueueDriver.swift └── Tests └── QueuesTests ├── AsyncQueueTests.swift ├── MetricsTests.swift ├── QueueTests.swift ├── ScheduleBuilderTests.swift ├── Utilities.swift └── Utilities ├── FailingAsyncJob.swift ├── Failure.swift └── MyAsyncJob.swift /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @gwynne @jdmcd 2 | /.github/CONTRIBUTING.md @gwynne @0xTim 3 | /.github/workflows/*.yml @gwynne @0xTim 4 | /.github/workflows/test.yml @gwynne 5 | /.spi.yml @gwynne @0xTim 6 | /.gitignore @gwynne @0xTim 7 | /LICENSE @gwynne @0xTim 8 | /README.md @gwynne @0xTim @jdmcd 9 | -------------------------------------------------------------------------------- /.github/CONTRIBUTING.MD: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | We're excited to see you're interested in contributing to the Queues package! This document covers how you can report any issues you find or contribute with bug fixes and new features! 4 | 5 | ## Reporting Issues 6 | 7 | Go ahead and [open a new issue](https://github.com/vapor/queues/issues/new). The team will be notified and we should get back to you shortly. 8 | 9 | We give you a few sections to fill out to help us hunt down the issue more effectively. Be sure to fill everything out to help us get everything fixed up as fast as possible. 10 | 11 | ## Pull Requests 12 | 13 | We are always willing to have pull requests open for bug fixes or new features. 14 | 15 | Bug fixes are pretty straight forward. Open up a PR and a maintainer will help guide you to make sure everything is in good order. 16 | 17 | New features are little more complex. If you come up with an idea, you should follow these steps: 18 | 19 | - [Open a new issue](https://github.com/vapor/queues/issues/new) or post in the #ideas or #development channel on the [Discord server](http://vapor.team/). This lets the us know what you have in mind so we can discuss it from a higher level down to the implementation details. 20 | 21 | - We'll take a look at your proposed idea and decide if it should be a feature or maybe it isn't needed enough to be added (don't take it personally if we reject it 😄, we still think you're a nice person). If we decide it fits with Queues, you can go ahead and open a PR! 22 | - After you open the PR, make sure it fills out the PR checklist: 23 | - Github Action checks are passing (code compiles and passes tests). 24 | - There are no breaking changes to public API. 25 | - New test cases have been added where appropriate. 26 | - All new code has been commented with doc blocks `///`. 27 | 28 | If it isn't your PR that causes the CI failure, one of the maintainers can helps sort out the issue. 29 | 30 | You can open a PR that adds a breaking change, but it will have to hang around until the next major version to be merged because the repo follows sem-ver. 31 | 32 | - After everything is okayed, one of the maintainers will merge the PR and tag a new release. Congratulations on your contribution! 33 | 34 | # Maintainers 35 | 36 | - [@tanner0101](https://github.com/tanner0101) 37 | - [@jdmcd](https://github.com/jdmcd) 38 | 39 | See the [Vapor maintainers doc](https://github.com/vapor/vapor/blob/main/.github/maintainers.md) for more information. 40 | -------------------------------------------------------------------------------- /.github/workflows/api-docs.yml: -------------------------------------------------------------------------------- 1 | name: deploy-api-docs 2 | on: 3 | push: 4 | branches: 5 | - main 6 | 7 | jobs: 8 | build-and-deploy: 9 | uses: vapor/api-docs/.github/workflows/build-and-deploy-docs-workflow.yml@main 10 | secrets: inherit 11 | with: 12 | package_name: queues 13 | modules: Queues,XCTQueues 14 | pathsToInvalidate: /queues/* /xctqueues/* 15 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: test 2 | concurrency: 3 | group: ${{ github.workflow }}-${{ github.ref }} 4 | cancel-in-progress: true 5 | on: 6 | pull_request: { types: [opened, reopened, synchronize, ready_for_review] } 7 | push: { branches: [ main ] } 8 | 9 | jobs: 10 | unit-tests: 11 | uses: vapor/ci/.github/workflows/run-unit-tests.yml@main 12 | with: 13 | with_tsan: false 14 | with_musl: true 15 | secrets: inherit 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | /.build 3 | /Packages 4 | /*.xcodeproj 5 | DerivedData 6 | Package.resolved 7 | .swiftpm 8 | Tests/LinuxMain.swift 9 | -------------------------------------------------------------------------------- /.spi.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | metadata: 3 | authors: Maintained by the Vapor Core Team with hundreds of contributions from the Vapor Community. 4 | external_links: 5 | documentation: "https://docs.vapor.codes/advanced/queues/" -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2023 Qutheory, LLC 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Package.swift: -------------------------------------------------------------------------------- 1 | // swift-tools-version:5.10 2 | import PackageDescription 3 | 4 | let package = Package( 5 | name: "queues", 6 | platforms: [ 7 | .macOS(.v10_15), 8 | .iOS(.v13), 9 | .watchOS(.v6), 10 | .tvOS(.v13), 11 | ], 12 | products: [ 13 | .library(name: "Queues", targets: ["Queues"]), 14 | .library(name: "XCTQueues", targets: ["XCTQueues"]), 15 | ], 16 | dependencies: [ 17 | .package(url: "https://github.com/vapor/vapor.git", from: "4.104.0"), 18 | .package(url: "https://github.com/apple/swift-nio.git", from: "2.81.0"), 19 | .package(url: "https://github.com/apple/swift-metrics.git", from: "2.5.0"), 20 | ], 21 | targets: [ 22 | .target( 23 | name: "Queues", 24 | dependencies: [ 25 | .product(name: "Vapor", package: "vapor"), 26 | .product(name: "NIOCore", package: "swift-nio"), 27 | .product(name: "Metrics", package: "swift-metrics"), 28 | ], 29 | swiftSettings: swiftSettings 30 | ), 31 | .target( 32 | name: "XCTQueues", 33 | dependencies: [ 34 | .target(name: "Queues"), 35 | ], 36 | swiftSettings: swiftSettings 37 | ), 38 | .testTarget( 39 | name: "QueuesTests", 40 | dependencies: [ 41 | .target(name: "Queues"), 42 | .target(name: "XCTQueues"), 43 | .product(name: "XCTVapor", package: "vapor"), 44 | .product(name: "MetricsTestKit", package: "swift-metrics"), 45 | ], 46 | swiftSettings: swiftSettings 47 | ), 48 | ] 49 | ) 50 | 51 | var swiftSettings: [SwiftSetting] { [ 52 | .enableUpcomingFeature("ForwardTrailingClosures"), 53 | .enableUpcomingFeature("ExistentialAny"), 54 | .enableUpcomingFeature("ConciseMagicFile"), 55 | .enableUpcomingFeature("DisableOutwardActorInference"), 56 | .enableExperimentalFeature("StrictConcurrency=complete"), 57 | ] } 58 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | Queues 3 |
4 |
5 | Documentation 6 | Team Chat 7 | MIT License 8 | Continuous Integration 9 | 10 | Swift 5.10+ 11 |

12 | 13 |
14 | 15 | -------------------------------------------------------------------------------- /Sources/Queues/Application+Queues.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import Logging 3 | import Vapor 4 | import NIO 5 | import NIOConcurrencyHelpers 6 | 7 | extension Application { 8 | /// The application-global ``Queues`` accessor. 9 | public var queues: Queues { 10 | .init(application: self) 11 | } 12 | 13 | /// Contains global configuration for queues and provides methods for registering jobs and retrieving queues. 14 | public struct Queues { 15 | /// A provider for a ``Queues`` driver. 16 | public struct Provider { 17 | let run: @Sendable (Application) -> () 18 | 19 | public init(_ run: @escaping @Sendable (Application) -> ()) { 20 | self.run = run 21 | } 22 | } 23 | 24 | final class Storage: Sendable { 25 | private struct Box: Sendable { 26 | var configuration: QueuesConfiguration 27 | var commands: [QueuesCommand] 28 | var driver: (any QueuesDriver)? 29 | } 30 | private let box: NIOLockedValueBox 31 | 32 | public var configuration: QueuesConfiguration { 33 | get { self.box.withLockedValue { $0.configuration } } 34 | set { self.box.withLockedValue { $0.configuration = newValue } } 35 | } 36 | var commands: [QueuesCommand] { self.box.withLockedValue { $0.commands } } 37 | var driver: (any QueuesDriver)? { 38 | get { self.box.withLockedValue { $0.driver } } 39 | set { self.box.withLockedValue { $0.driver = newValue } } 40 | } 41 | 42 | public init(_ application: Application) { 43 | let command = QueuesCommand(application: application) 44 | 45 | self.box = .init(.init( 46 | configuration: .init(logger: application.logger), 47 | commands: [command], 48 | driver: nil 49 | )) 50 | application.asyncCommands.use(command, as: "queues") 51 | } 52 | 53 | public func add(command: QueuesCommand) { 54 | self.box.withLockedValue { $0.commands.append(command) } 55 | } 56 | } 57 | 58 | struct Key: StorageKey { 59 | typealias Value = Storage 60 | } 61 | 62 | struct Lifecycle: LifecycleHandler { 63 | func shutdown(_ application: Application) { 64 | application.queues.storage.commands.forEach { $0.shutdown() } 65 | application.queues.storage.driver?.shutdown() 66 | } 67 | 68 | func shutdownAsync(_ application: Application) async { 69 | for command in application.queues.storage.commands { 70 | await command.asyncShutdown() 71 | } 72 | await application.queues.storage.driver?.asyncShutdown() 73 | } 74 | } 75 | 76 | /// The ``QueuesConfiguration`` object. 77 | public var configuration: QueuesConfiguration { 78 | get { self.storage.configuration } 79 | nonmutating set { self.storage.configuration = newValue } 80 | } 81 | 82 | /// The selected ``QueuesDriver``. 83 | public var driver: any QueuesDriver { 84 | guard let driver = self.storage.driver else { 85 | fatalError("No Queues driver configured. Configure with app.queues.use(...)") 86 | } 87 | return driver 88 | } 89 | 90 | var storage: Storage { 91 | if self.application.storage[Key.self] == nil { 92 | self.initialize() 93 | } 94 | return self.application.storage[Key.self]! 95 | } 96 | 97 | public let application: Application 98 | 99 | /// Get the default ``Queue``. 100 | public var queue: any Queue { 101 | self.queue(.default) 102 | } 103 | 104 | /// Create or look up an instance of a named ``Queue``. 105 | /// 106 | /// - Parameters: 107 | /// - name: The name of the queue 108 | /// - logger: A logger object 109 | /// - eventLoop: The event loop to run on 110 | public func queue( 111 | _ name: QueueName, 112 | logger: Logger? = nil, 113 | on eventLoop: (any EventLoop)? = nil 114 | ) -> any Queue { 115 | self.driver.makeQueue(with: .init( 116 | queueName: name, 117 | configuration: self.configuration, 118 | application: self.application, 119 | logger: logger ?? self.application.logger, 120 | on: eventLoop ?? self.application.eventLoopGroup.any() 121 | )) 122 | } 123 | 124 | /// Add a new queueable job. 125 | /// 126 | /// This must be called once for each job type that can be queued. 127 | /// 128 | /// - Parameter job: The job to add. 129 | public func add(_ job: some Job) { 130 | self.configuration.add(job) 131 | } 132 | 133 | /// Add a new notification hook. 134 | /// 135 | /// - Parameter hook: The hook to add. 136 | public func add(_ hook: some JobEventDelegate) { 137 | self.configuration.add(hook) 138 | } 139 | 140 | /// Choose which provider to use. 141 | /// 142 | /// - Parameter provider: The provider. 143 | public func use(_ provider: Provider) { 144 | provider.run(self.application) 145 | } 146 | 147 | /// Configure a driver. 148 | /// 149 | /// - Parameter driver: The driver 150 | public func use(custom driver: any QueuesDriver) { 151 | self.storage.driver = driver 152 | } 153 | 154 | /// Schedule a new job. 155 | /// 156 | /// - Parameter job: The job to schedule. 157 | public func schedule(_ job: some ScheduledJob) -> ScheduleBuilder { 158 | self.storage.configuration.schedule(job) 159 | } 160 | 161 | /// Starts an in-process worker to dequeue and run jobs. 162 | /// 163 | /// - Parameter queue: The queue to run the jobs on. Defaults to ``QueueName/default``. 164 | public func startInProcessJobs(on queue: QueueName = .default) throws { 165 | let inProcessJobs = QueuesCommand(application: self.application) 166 | 167 | try inProcessJobs.startJobs(on: queue) 168 | self.storage.add(command: inProcessJobs) 169 | } 170 | 171 | /// Starts an in-process worker to run scheduled jobs. 172 | public func startScheduledJobs() throws { 173 | let scheduledJobs = QueuesCommand(application: self.application) 174 | 175 | try scheduledJobs.startScheduledJobs() 176 | self.storage.add(command: scheduledJobs) 177 | } 178 | 179 | func initialize() { 180 | self.application.lifecycle.use(Lifecycle()) 181 | self.application.storage[Key.self] = .init(self.application) 182 | } 183 | } 184 | } 185 | -------------------------------------------------------------------------------- /Sources/Queues/AsyncJob.swift: -------------------------------------------------------------------------------- 1 | import Vapor 2 | import NIOCore 3 | import Foundation 4 | 5 | /// A task that can be queued for future execution. 6 | public protocol AsyncJob: Job { 7 | associatedtype Payload 8 | 9 | /// Called when it's this Job's turn to be dequeued. 10 | /// 11 | /// - Parameters: 12 | /// - context: The ``QueueContext``. 13 | /// - payload: The typed job payload. 14 | func dequeue( 15 | _ context: QueueContext, 16 | _ payload: Payload 17 | ) async throws 18 | 19 | /// Called when there is an error at any stage of the Job's execution. 20 | /// 21 | /// - Parameters: 22 | /// - context: The ``QueueContext``. 23 | /// - error: The error returned by the job. 24 | /// - payload: The typed job payload. 25 | func error( 26 | _ context: QueueContext, 27 | _ error: any Error, 28 | _ payload: Payload 29 | ) async throws 30 | } 31 | 32 | extension AsyncJob { 33 | /// Default implementation of ``AsyncJob/error(_:_:_:)-8627d``. 34 | public func error(_ context: QueueContext, _ error: any Error, _ payload: Payload) async throws {} 35 | } 36 | 37 | extension AsyncJob { 38 | /// Forward ``Job/dequeue(_:_:)`` to ``AsyncJob/dequeue(_:_:)-9g26t``. 39 | public func dequeue(_ context: QueueContext, _ payload: Payload) -> EventLoopFuture { 40 | context.eventLoop.makeFutureWithTask { 41 | try await self.dequeue(context, payload) 42 | } 43 | } 44 | 45 | /// Forward ``Job/error(_:_:_:)-2brrj`` to ``AsyncJob/error(_:_:_:)-8627d`` 46 | public func error(_ context: QueueContext, _ error: any Error, _ payload: Payload) -> EventLoopFuture { 47 | context.eventLoop.makeFutureWithTask { 48 | try await self.error(context, error, payload) 49 | } 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /Sources/Queues/AsyncJobEventDelegate.swift: -------------------------------------------------------------------------------- 1 | import NIOCore 2 | 3 | /// Represents an object that can receive notifications about job statuses 4 | public protocol AsyncJobEventDelegate: JobEventDelegate { 5 | /// Called when the job is first dispatched 6 | /// - Parameters: 7 | /// - job: The `JobData` associated with the job 8 | func dispatched(job: JobEventData) async throws 9 | 10 | /// Called when the job is dequeued 11 | /// - Parameters: 12 | /// - jobId: The id of the Job 13 | func didDequeue(jobId: String) async throws 14 | 15 | /// Called when the job succeeds 16 | /// - Parameters: 17 | /// - jobId: The id of the Job 18 | func success(jobId: String) async throws 19 | 20 | /// Called when the job returns an error 21 | /// - Parameters: 22 | /// - jobId: The id of the Job 23 | /// - error: The error that caused the job to fail 24 | func error(jobId: String, error: any Error) async throws 25 | } 26 | 27 | extension AsyncJobEventDelegate { 28 | public func dispatched(job: JobEventData) async throws { } 29 | public func didDequeue(jobId: String) async throws { } 30 | public func success(jobId: String) async throws { } 31 | public func error(jobId: String, error: any Error) async throws { } 32 | 33 | public func dispatched(job: JobEventData, eventLoop: any EventLoop) -> EventLoopFuture { 34 | eventLoop.makeFutureWithTask { 35 | try await self.dispatched(job: job) 36 | } 37 | } 38 | 39 | public func didDequeue(jobId: String, eventLoop: any EventLoop) -> EventLoopFuture { 40 | eventLoop.makeFutureWithTask { 41 | try await self.didDequeue(jobId: jobId) 42 | } 43 | } 44 | 45 | public func success(jobId: String, eventLoop: any EventLoop) -> EventLoopFuture { 46 | eventLoop.makeFutureWithTask { 47 | try await self.success(jobId: jobId) 48 | } 49 | } 50 | 51 | public func error(jobId: String, error: any Error, eventLoop: any EventLoop) -> EventLoopFuture { 52 | eventLoop.makeFutureWithTask { 53 | try await self.error(jobId: jobId, error: error) 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /Sources/Queues/AsyncQueue.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import Metrics 3 | import NIOCore 4 | import Vapor 5 | 6 | public protocol AsyncQueue: Queue { 7 | /// The job context 8 | var context: QueueContext { get } 9 | 10 | /// Gets the next job to be run 11 | /// - Parameter id: The ID of the job 12 | func get(_ id: JobIdentifier) async throws -> JobData 13 | 14 | /// Sets a job that should be run in the future 15 | /// - Parameters: 16 | /// - id: The ID of the job 17 | /// - data: Data for the job 18 | func set(_ id: JobIdentifier, to data: JobData) async throws 19 | 20 | /// Removes a job from the queue 21 | /// - Parameter id: The ID of the job 22 | func clear(_ id: JobIdentifier) async throws 23 | 24 | /// Pops the next job in the queue 25 | func pop() async throws -> JobIdentifier? 26 | 27 | /// Pushes the next job into a queue 28 | /// - Parameter id: The ID of the job 29 | func push(_ id: JobIdentifier) async throws 30 | } 31 | 32 | extension AsyncQueue { 33 | public func get(_ id: JobIdentifier) -> EventLoopFuture { 34 | self.context.eventLoop.makeFutureWithTask { try await self.get(id) } 35 | } 36 | 37 | public func set(_ id: JobIdentifier, to data: JobData) -> EventLoopFuture { 38 | self.context.eventLoop.makeFutureWithTask { try await self.set(id, to: data) } 39 | } 40 | 41 | public func clear(_ id: JobIdentifier) -> EventLoopFuture { 42 | self.context.eventLoop.makeFutureWithTask { try await self.clear(id) } 43 | } 44 | 45 | public func pop() -> EventLoopFuture { 46 | self.context.eventLoop.makeFutureWithTask { try await self.pop() } 47 | } 48 | 49 | public func push(_ id: JobIdentifier) -> EventLoopFuture { 50 | self.context.eventLoop.makeFutureWithTask { try await self.push(id) } 51 | } 52 | } 53 | 54 | extension Queue { 55 | /// Dispatch a job into the queue for processing 56 | /// - Parameters: 57 | /// - job: The Job type 58 | /// - payload: The payload data to be dispatched 59 | /// - maxRetryCount: Number of times to retry this job on failure 60 | /// - delayUntil: Delay the processing of this job until a certain date 61 | public func dispatch( 62 | _ job: J.Type, 63 | _ payload: J.Payload, 64 | maxRetryCount: Int = 0, 65 | delayUntil: Date? = nil, 66 | id: JobIdentifier = .init() 67 | ) async throws { 68 | var logger = self.logger 69 | logger[metadataKey: "queue"] = "\(self.queueName.string)" 70 | logger[metadataKey: "job-id"] = "\(id.string)" 71 | logger[metadataKey: "job-name"] = "\(J.name)" 72 | 73 | let storage = try JobData( 74 | payload: J.serializePayload(payload), 75 | maxRetryCount: maxRetryCount, 76 | jobName: J.name, 77 | delayUntil: delayUntil, 78 | queuedAt: .init() 79 | ) 80 | 81 | logger.trace("Storing job data") 82 | try await self.set(id, to: storage).get() 83 | logger.trace("Pusing job to queue") 84 | try await self.push(id).get() 85 | logger.info("Dispatched job") 86 | Counter(label: "dispatched.jobs.counter", dimensions: [ 87 | ("queueName", self.queueName.string), 88 | ("jobName", J.name), 89 | ]).increment() 90 | 91 | await self.sendNotification(of: "dispatch", logger: logger) { 92 | try await $0.dispatched(job: .init(id: id.string, queueName: self.queueName.string, jobData: storage), eventLoop: self.eventLoop).get() 93 | } 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /Sources/Queues/AsyncScheduledJob.swift: -------------------------------------------------------------------------------- 1 | import Vapor 2 | import NIOCore 3 | import Foundation 4 | 5 | /// Describes a job that can be scheduled and repeated 6 | public protocol AsyncScheduledJob: ScheduledJob { 7 | var name: String { get } 8 | 9 | /// The method called when the job is run 10 | /// - Parameter context: A `JobContext` that can be used 11 | func run(context: QueueContext) async throws 12 | } 13 | 14 | extension AsyncScheduledJob { 15 | public var name: String { "\(Self.self)" } 16 | 17 | public func run(context: QueueContext) -> EventLoopFuture { 18 | context.eventLoop.makeFutureWithTask { 19 | try await self.run(context: context) 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /Sources/Queues/Docs.docc/Resources/vapor-queues-logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /Sources/Queues/Docs.docc/index.md: -------------------------------------------------------------------------------- 1 | # ``Queues`` 2 | 3 | Queues is a pure Swift queuing system that allows you to offload task responsibility to a side worker. 4 | 5 | Some of the tasks this package works well for: 6 | 7 | * Sending emails outside of the main request thread 8 | * Performing complex or long-running database operations 9 | * Ensuring job integrity and resilience 10 | * Speeding up response time by delaying non-critical processing 11 | * Scheduling jobs to occur at a specific time -------------------------------------------------------------------------------- /Sources/Queues/Docs.docc/theme-settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "theme": { 3 | "aside": { "border-radius": "16px", "border-style": "double", "border-width": "3px" }, 4 | "border-radius": "0", 5 | "button": { "border-radius": "16px", "border-width": "1px", "border-style": "solid" }, 6 | "code": { "border-radius": "16px", "border-width": "1px", "border-style": "solid" }, 7 | "color": { 8 | "queues": "#e8665a", 9 | "documentation-intro-fill": "radial-gradient(circle at top, var(--color-queues) 30%, #000 100%)", 10 | "documentation-intro-accent": "var(--color-queues)", 11 | "documentation-intro-eyebrow": "white", 12 | "documentation-intro-figure": "white", 13 | "documentation-intro-title": "white", 14 | "logo-base": { "dark": "#fff", "light": "#000" }, 15 | "logo-shape": { "dark": "#000", "light": "#fff" }, 16 | "fill": { "dark": "#000", "light": "#fff" } 17 | }, 18 | "icons": { "technology": "/queues/images/vapor-queues-logo.svg" } 19 | }, 20 | "features": { 21 | "quickNavigation": { "enable": true }, 22 | "i18n": { "enable": true } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /Sources/Queues/Exports.swift: -------------------------------------------------------------------------------- 1 | @_documentation(visibility: internal) @_exported import struct Foundation.Date 2 | @_documentation(visibility: internal) @_exported import struct Logging.Logger 3 | @_documentation(visibility: internal) @_exported import class NIOCore.EventLoopFuture 4 | @_documentation(visibility: internal) @_exported import struct NIOCore.EventLoopPromise 5 | @_documentation(visibility: internal) @_exported import protocol NIOCore.EventLoop 6 | @_documentation(visibility: internal) @_exported import struct NIOCore.TimeAmount 7 | -------------------------------------------------------------------------------- /Sources/Queues/Job.swift: -------------------------------------------------------------------------------- 1 | import NIOCore 2 | import Foundation 3 | import Logging 4 | import Vapor 5 | 6 | /// A task that can be queued for future execution. 7 | public protocol Job: AnyJob { 8 | /// The data associated with a job 9 | associatedtype Payload: Sendable 10 | 11 | /// Called when it's this Job's turn to be dequeued. 12 | /// 13 | /// - Parameters: 14 | /// - context: The ``QueueContext``. 15 | /// - payload: The typed job payload. 16 | func dequeue( 17 | _ context: QueueContext, 18 | _ payload: Payload 19 | ) -> EventLoopFuture 20 | 21 | /// Called when there is an error at any stage of the Job's execution. 22 | /// 23 | /// - Parameters: 24 | /// - context: The ``QueueContext``. 25 | /// - error: The error returned by the job. 26 | /// - payload: The typed job payload. 27 | func error( 28 | _ context: QueueContext, 29 | _ error: any Error, 30 | _ payload: Payload 31 | ) -> EventLoopFuture 32 | 33 | /// Called when there was an error and the job will be retried. 34 | /// 35 | /// - Parameter attempt: Number of job attempts which have failed so far. 36 | /// - Returns: Number of seconds to delay the next retry. Return `0` to place the job back on the queue with no 37 | /// delay added. If this method is not implemented, the default is `0`. Returning `-1` is the same as `0`. 38 | func nextRetryIn(attempt: Int) -> Int 39 | 40 | /// Serialize a typed payload to an array of bytes. When `Payload` is `Codable`, this method will default to 41 | /// encoding to JSON. 42 | /// 43 | /// - Parameter payload: The payload to serialize. 44 | /// - Returns: The array of serialized bytes. 45 | static func serializePayload(_ payload: Payload) throws -> [UInt8] 46 | 47 | /// Deserialize an array of bytes into a typed payload. When `Payload` is `Codable`, this method will default to 48 | /// decoding from JSON. 49 | /// 50 | /// - Parameter bytes: The serialized bytes to decode. 51 | /// - Returns: A decoded payload. 52 | static func parsePayload(_ bytes: [UInt8]) throws -> Payload 53 | } 54 | 55 | extension Job where Payload: Codable { 56 | /// Default implementation for ``Job/serializePayload(_:)-4uro2``. 57 | public static func serializePayload(_ payload: Payload) throws -> [UInt8] { 58 | try .init(JSONEncoder().encode(payload)) 59 | } 60 | 61 | /// Default implementation for ``Job/parsePayload(_:)-9tn3a``. 62 | public static func parsePayload(_ bytes: [UInt8]) throws -> Payload { 63 | try JSONDecoder().decode(Payload.self, from: .init(bytes)) 64 | } 65 | } 66 | 67 | extension Job { 68 | /// Default implementation for ``AnyJob/name``. 69 | public static var name: String { 70 | String(describing: Self.self) 71 | } 72 | 73 | /// Default implementation for ``Job/error(_:_:_:)-jzgw``. 74 | public func error( 75 | _ context: QueueContext, 76 | _ error: any Error, 77 | _ payload: Payload 78 | ) -> EventLoopFuture { 79 | context.eventLoop.makeSucceededVoidFuture() 80 | } 81 | 82 | /// Default implementation for ``Job/nextRetryIn(attempt:)-5gc93``. 83 | public func nextRetryIn(attempt: Int) -> Int { 84 | 0 85 | } 86 | } 87 | 88 | /// A type-erased version of ``Job``. 89 | public protocol AnyJob: Sendable { 90 | /// The name of the job. 91 | static var name: String { get } 92 | 93 | /// Perform ``Job/dequeue(_:_:)`` after deserializing the raw payload bytes. 94 | func _dequeue(_ context: QueueContext, id: String, payload: [UInt8]) -> EventLoopFuture 95 | 96 | /// Perform ``Job/error(_:_:_:)-2brrj`` after deserializing the raw payload bytes. 97 | func _error(_ context: QueueContext, id: String, _ error: any Error, payload: [UInt8]) -> EventLoopFuture 98 | 99 | /// Type-erased accessor for ``Job/nextRetryIn(attempt:)-5gc93``. 100 | func _nextRetryIn(attempt: Int) -> Int 101 | } 102 | 103 | // N.B. These should really not be public. 104 | extension Job { 105 | // See `AnyJob._nextRetryIn(attempt:)`. 106 | public func _nextRetryIn(attempt: Int) -> Int { 107 | self.nextRetryIn(attempt: attempt) 108 | } 109 | 110 | // See `AnyJob._error(_:id:_:payload:)`. 111 | public func _error(_ context: QueueContext, id: String, _ error: any Error, payload: [UInt8]) -> EventLoopFuture { 112 | var context = context 113 | context.logger[metadataKey: "queue"] = "\(context.queueName.string)" 114 | context.logger[metadataKey: "job_id"] = "\(id)" 115 | do { 116 | return try self.error(context, error, Self.parsePayload(payload)) 117 | } catch { 118 | return context.eventLoop.makeFailedFuture(error) 119 | } 120 | } 121 | 122 | // See `AnyJob._dequeue(_:id:payload:)`. 123 | public func _dequeue(_ context: QueueContext, id: String, payload: [UInt8]) -> EventLoopFuture { 124 | var context = context 125 | context.logger[metadataKey: "queue"] = "\(context.queueName.string)" 126 | context.logger[metadataKey: "job_id"] = "\(id)" 127 | do { 128 | return try self.dequeue(context, Self.parsePayload(payload)) 129 | } catch { 130 | return context.eventLoop.makeFailedFuture(error) 131 | } 132 | } 133 | } 134 | 135 | -------------------------------------------------------------------------------- /Sources/Queues/JobData.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// Holds information about the Job that is to be encoded to the persistence store. 4 | public struct JobData: Codable, Sendable { 5 | /// The job data to be encoded. 6 | public let payload: [UInt8] 7 | 8 | /// The maxRetryCount for the `Job`. 9 | public let maxRetryCount: Int 10 | 11 | /// The number of attempts made to run the `Job`. 12 | public let attempts: Int? 13 | 14 | /// A date to execute this job after 15 | public let delayUntil: Date? 16 | 17 | /// The date this job was queued 18 | public let queuedAt: Date 19 | 20 | /// The name of the `Job` 21 | public let jobName: String 22 | 23 | /// Creates a new `JobStorage` holding object 24 | public init( 25 | payload: [UInt8], 26 | maxRetryCount: Int, 27 | jobName: String, 28 | delayUntil: Date?, 29 | queuedAt: Date, 30 | attempts: Int = 0 31 | ) { 32 | assert(maxRetryCount >= 0) 33 | assert(attempts >= 0) 34 | 35 | self.payload = payload 36 | self.maxRetryCount = maxRetryCount 37 | self.jobName = jobName 38 | self.delayUntil = delayUntil 39 | self.queuedAt = queuedAt 40 | self.attempts = attempts 41 | } 42 | } 43 | 44 | // N.B.: These methods are intended for internal use only. 45 | extension JobData { 46 | /// The non-`nil` number of attempts made to run this job (how many times has it failed). 47 | /// This can also be treated as a "retry" count. 48 | /// 49 | /// Value | Meaning 50 | /// -|- 51 | /// 0 | The job has never run, or succeeded on its first attempt 52 | /// 1 | The job has failed once and is queued for its first retry 53 | /// 2... | The job has failed N times and is queued for its Nth retry 54 | var failureCount: Int { 55 | self.attempts ?? 0 56 | } 57 | 58 | /// The number of retries left iff the current (re)try fails. 59 | var remainingAttempts: Int { 60 | Swift.max(0, self.maxRetryCount - self.failureCount) 61 | } 62 | 63 | /// The current attempt number. 64 | /// 65 | /// Value|Meaning 66 | /// -|- 67 | /// 0|Not valid 68 | /// 1|The job has not failed thus far; this the first attempt. 69 | /// 2|The job has failed once; this is the second attempt. 70 | var currentAttempt: Int { 71 | self.failureCount + 1 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /Sources/Queues/JobIdentifier.swift: -------------------------------------------------------------------------------- 1 | import struct Foundation.UUID 2 | 3 | /// An identifier for a job 4 | public struct JobIdentifier: Hashable, Equatable, Sendable { 5 | /// The string value of the ID 6 | public let string: String 7 | 8 | /// Creates a new id from a string 9 | public init(string: String) { 10 | self.string = string 11 | } 12 | 13 | /// Creates a new id with a default UUID value 14 | public init() { 15 | self.init(string: UUID().uuidString) 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /Sources/Queues/NotificationHook.swift: -------------------------------------------------------------------------------- 1 | import NIOCore 2 | import Foundation 3 | 4 | /// Represents an object that can receive notifications about job statuses 5 | public protocol JobEventDelegate: Sendable { 6 | /// Called when the job is first dispatched 7 | /// - Parameters: 8 | /// - job: The `JobData` associated with the job 9 | /// - eventLoop: The eventLoop 10 | func dispatched(job: JobEventData, eventLoop: any EventLoop) -> EventLoopFuture 11 | 12 | /// Called when the job is dequeued 13 | /// - Parameters: 14 | /// - jobId: The id of the Job 15 | /// - eventLoop: The eventLoop 16 | func didDequeue(jobId: String, eventLoop: any EventLoop) -> EventLoopFuture 17 | 18 | 19 | /// Called when the job succeeds 20 | /// - Parameters: 21 | /// - jobId: The id of the Job 22 | /// - eventLoop: The eventLoop 23 | func success(jobId: String, eventLoop: any EventLoop) -> EventLoopFuture 24 | 25 | /// Called when the job returns an error 26 | /// - Parameters: 27 | /// - jobId: The id of the Job 28 | /// - error: The error that caused the job to fail 29 | /// - eventLoop: The eventLoop 30 | func error(jobId: String, error: any Error, eventLoop: any EventLoop) -> EventLoopFuture 31 | } 32 | 33 | extension JobEventDelegate { 34 | public func dispatched(job: JobEventData, eventLoop: any EventLoop) -> EventLoopFuture { 35 | eventLoop.makeSucceededVoidFuture() 36 | } 37 | 38 | public func didDequeue(jobId: String, eventLoop: any EventLoop) -> EventLoopFuture { 39 | eventLoop.makeSucceededVoidFuture() 40 | } 41 | 42 | public func success(jobId: String, eventLoop: any EventLoop) -> EventLoopFuture { 43 | eventLoop.makeSucceededVoidFuture() 44 | } 45 | 46 | public func error(jobId: String, error: any Error, eventLoop: any EventLoop) -> EventLoopFuture { 47 | eventLoop.makeSucceededVoidFuture() 48 | } 49 | } 50 | 51 | /// Data on a job sent via a notification 52 | public struct JobEventData: Sendable { 53 | /// The id of the job, assigned at dispatch 54 | public var id: String 55 | 56 | /// The name of the queue (i.e. `default`) 57 | public var queueName: String 58 | 59 | /// The job data to be encoded. 60 | public var payload: [UInt8] 61 | 62 | /// The maxRetryCount for the `Job`. 63 | public var maxRetryCount: Int 64 | 65 | /// A date to execute this job after 66 | public var delayUntil: Date? 67 | 68 | /// The date this job was queued 69 | public var queuedAt: Date 70 | 71 | /// The name of the `Job` 72 | public var jobName: String 73 | 74 | /// Creates a new `JobStorage` holding object 75 | public init(id: String, queueName: String, jobData: JobData) { 76 | self.id = id 77 | self.queueName = queueName 78 | self.payload = jobData.payload 79 | self.maxRetryCount = jobData.maxRetryCount 80 | self.jobName = jobData.jobName 81 | self.delayUntil = jobData.delayUntil 82 | self.queuedAt = jobData.queuedAt 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /Sources/Queues/Queue.swift: -------------------------------------------------------------------------------- 1 | 2 | import Foundation 3 | import Logging 4 | import Metrics 5 | import NIOCore 6 | 7 | /// A type that can store and retrieve jobs from a persistence layer 8 | public protocol Queue: Sendable { 9 | /// The job context 10 | var context: QueueContext { get } 11 | 12 | /// Gets the next job to be run 13 | /// - Parameter id: The ID of the job 14 | func get(_ id: JobIdentifier) -> EventLoopFuture 15 | 16 | /// Sets a job that should be run in the future 17 | /// - Parameters: 18 | /// - id: The ID of the job 19 | /// - data: Data for the job 20 | func set(_ id: JobIdentifier, to data: JobData) -> EventLoopFuture 21 | 22 | /// Removes a job from the queue 23 | /// - Parameter id: The ID of the job 24 | func clear(_ id: JobIdentifier) -> EventLoopFuture 25 | 26 | /// Pops the next job in the queue 27 | func pop() -> EventLoopFuture 28 | 29 | /// Pushes the next job into a queue 30 | /// - Parameter id: The ID of the job 31 | func push(_ id: JobIdentifier) -> EventLoopFuture 32 | } 33 | 34 | extension Queue { 35 | /// The EventLoop for a job queue 36 | public var eventLoop: any EventLoop { 37 | self.context.eventLoop 38 | } 39 | 40 | /// A logger 41 | public var logger: Logger { 42 | self.context.logger 43 | } 44 | 45 | /// The configuration for the queue 46 | public var configuration: QueuesConfiguration { 47 | self.context.configuration 48 | } 49 | 50 | /// The queue's name 51 | public var queueName: QueueName { 52 | self.context.queueName 53 | } 54 | 55 | /// The key name of the queue 56 | public var key: String { 57 | self.queueName.makeKey(with: self.configuration.persistenceKey) 58 | } 59 | 60 | /// Dispatch a job into the queue for processing 61 | /// - Parameters: 62 | /// - job: The Job type 63 | /// - payload: The payload data to be dispatched 64 | /// - maxRetryCount: Number of times to retry this job on failure 65 | /// - delayUntil: Delay the processing of this job until a certain date 66 | public func dispatch( 67 | _ job: J.Type, 68 | _ payload: J.Payload, 69 | maxRetryCount: Int = 0, 70 | delayUntil: Date? = nil, 71 | id: JobIdentifier = .init() 72 | ) -> EventLoopFuture { 73 | var logger_ = self.logger 74 | logger_[metadataKey: "queue"] = "\(self.queueName.string)" 75 | logger_[metadataKey: "job-id"] = "\(id.string)" 76 | logger_[metadataKey: "job-name"] = "\(J.name)" 77 | let logger = logger_ 78 | 79 | let bytes: [UInt8] 80 | do { 81 | bytes = try J.serializePayload(payload) 82 | } catch { 83 | return self.eventLoop.makeFailedFuture(error) 84 | } 85 | 86 | let storage = JobData( 87 | payload: bytes, 88 | maxRetryCount: maxRetryCount, 89 | jobName: J.name, 90 | delayUntil: delayUntil, 91 | queuedAt: Date() 92 | ) 93 | 94 | logger.trace("Storing job data") 95 | return self.set(id, to: storage).flatMap { 96 | logger.trace("Pusing job to queue") 97 | return self.push(id) 98 | }.flatMapWithEventLoop { _, eventLoop in 99 | Counter(label: "dispatched.jobs.counter", dimensions: [ 100 | ("queueName", self.queueName.string), 101 | ("jobName", J.name), 102 | ]).increment() 103 | self.logger.info("Dispatched queue job") 104 | return self.sendNotification(of: "dispatch", logger: logger) { 105 | $0.dispatched(job: .init(id: id.string, queueName: self.queueName.string, jobData: storage), eventLoop: eventLoop) 106 | } 107 | } 108 | } 109 | } 110 | 111 | extension Queue { 112 | func sendNotification( 113 | of kind: String, logger: Logger, 114 | _ notification: @escaping @Sendable (_ hook: any JobEventDelegate) -> EventLoopFuture 115 | ) -> EventLoopFuture { 116 | logger.trace("Sending notification", metadata: ["kind": "\(kind)"]) 117 | return self.configuration.notificationHooks.map { 118 | notification($0).flatMapErrorWithEventLoop { error, eventLoop in 119 | logger.warning("Failed to send notification", metadata: ["kind": "\(kind)", "error": "\(String(reflecting: error))"]) 120 | return eventLoop.makeSucceededVoidFuture() 121 | } 122 | }.flatten(on: self.eventLoop) 123 | } 124 | 125 | func sendNotification( 126 | of kind: String, logger: Logger, 127 | _ notification: @escaping @Sendable (_ hook: any JobEventDelegate) async throws -> Void 128 | ) async { 129 | logger.trace("Sending notification", metadata: ["kind": "\(kind)"]) 130 | for hook in self.configuration.notificationHooks { 131 | do { 132 | try await notification(hook) 133 | } catch { 134 | logger.warning("Failed to send notification", metadata: ["kind": "\(kind)", "error": "\(String(reflecting: error))"]) 135 | } 136 | } 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /Sources/Queues/QueueContext.swift: -------------------------------------------------------------------------------- 1 | import Logging 2 | import NIOCore 3 | import Vapor 4 | 5 | /// The context for a queue. 6 | public struct QueueContext: Sendable { 7 | /// The name of the queue 8 | public let queueName: QueueName 9 | 10 | /// The configuration object 11 | public let configuration: QueuesConfiguration 12 | 13 | /// The application object 14 | public let application: Application 15 | 16 | /// The logger object 17 | public var logger: Logger 18 | 19 | /// An event loop to run the process on 20 | public let eventLoop: any EventLoop 21 | 22 | /// Creates a new JobContext 23 | /// - Parameters: 24 | /// - queueName: The name of the queue 25 | /// - configuration: The configuration object 26 | /// - application: The application object 27 | /// - logger: The logger object 28 | /// - eventLoop: An event loop to run the process on 29 | public init( 30 | queueName: QueueName, 31 | configuration: QueuesConfiguration, 32 | application: Application, 33 | logger: Logger, 34 | on eventLoop: any EventLoop 35 | ) { 36 | self.queueName = queueName 37 | self.configuration = configuration 38 | self.application = application 39 | self.logger = logger 40 | self.eventLoop = eventLoop 41 | } 42 | 43 | /// Returns the default job `Queue` 44 | public var queue: any Queue { 45 | self.queues(.default) 46 | } 47 | 48 | /// Returns the specific job `Queue` for the given queue name 49 | /// - Parameter queue: The queue name 50 | public func queues(_ queue: QueueName) -> any Queue { 51 | self.application.queues.queue( 52 | queue, 53 | logger: self.logger, 54 | on: self.eventLoop 55 | ) 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /Sources/Queues/QueueName.swift: -------------------------------------------------------------------------------- 1 | /// A specific queue that jobs are run on. 2 | public struct QueueName: Sendable { 3 | /// The default queue that jobs are run on 4 | public static let `default` = Self(string: "default") 5 | 6 | /// The name of the queue 7 | public let string: String 8 | 9 | /// Creates a new ``QueueName`` 10 | /// 11 | /// - Parameter name: The name of the queue 12 | public init(string: String) { 13 | self.string = string 14 | } 15 | 16 | /// Makes the name of the queue 17 | /// 18 | /// - Parameter persistenceKey: The base persistence key 19 | /// - Returns: A string of the queue's fully qualified name 20 | public func makeKey(with persistenceKey: String) -> String { 21 | "\(persistenceKey)[\(self.string)]" 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /Sources/Queues/QueueWorker.swift: -------------------------------------------------------------------------------- 1 | import Dispatch 2 | import Foundation 3 | import Logging 4 | import Metrics 5 | import NIOCore 6 | 7 | extension Queue { 8 | public var worker: QueueWorker { 9 | .init(queue: self) 10 | } 11 | } 12 | 13 | /// The worker that runs ``Job``s. 14 | public struct QueueWorker: Sendable { 15 | let queue: any Queue 16 | 17 | /// Run the queue until there is no more work to be done. 18 | /// This is a thin wrapper for ELF-style callers. 19 | public func run() -> EventLoopFuture { 20 | self.queue.eventLoop.makeFutureWithTask { 21 | try await self.run() 22 | } 23 | } 24 | 25 | /// Run the queue until there is no more work to be done. 26 | /// This is the main async entrypoint for a queue worker. 27 | public func run() async throws { 28 | while try await self.runOneJob() {} 29 | } 30 | 31 | /// Pop a job off the queue and try to run it. If no jobs are available, do 32 | /// nothing. Returns whether a job was run. 33 | private func runOneJob() async throws -> Bool { 34 | var logger = self.queue.logger 35 | logger[metadataKey: "queue"] = "\(self.queue.queueName.string)" 36 | logger.trace("Popping job from queue") 37 | 38 | guard let id = try await self.queue.pop().get() else { 39 | // No job found, go around again. 40 | logger.trace("No pending jobs") 41 | return false 42 | } 43 | 44 | logger[metadataKey: "job-id"] = "\(id.string)" 45 | logger.trace("Found pending job") 46 | 47 | let data = try await self.queue.get(id).get() 48 | logger.trace("Received job data", metadata: ["job-data": "\(data)"]) 49 | logger[metadataKey: "job-name"] = "\(data.jobName)" 50 | 51 | guard let job = self.queue.configuration.jobs[data.jobName] else { 52 | logger.warning("No job with the desired name is registered, discarding") 53 | try await self.queue.clear(id).get() 54 | return false 55 | } 56 | 57 | // If the job has a delay that isn't up yet, requeue it. 58 | guard (data.delayUntil ?? .distantPast) < Date() else { 59 | logger.trace("Job is delayed, requeueing for later execution", metadata: ["delayed-until": "\(data.delayUntil ?? .distantPast)"]) 60 | try await self.queue.push(id).get() 61 | return false 62 | } 63 | 64 | await self.queue.sendNotification(of: "dequeue", logger: logger) { 65 | try await $0.didDequeue(jobId: id.string, eventLoop: self.queue.eventLoop).get() 66 | } 67 | 68 | Meter( 69 | label: "jobs.in.progress.meter", 70 | dimensions: [("queueName", self.queue.queueName.string)] 71 | ).increment() 72 | 73 | try await self.runOneJob(id: id, job: job, jobData: data, logger: logger) 74 | return true 75 | } 76 | 77 | private func runOneJob(id: JobIdentifier, job: any AnyJob, jobData: JobData, logger: Logger) async throws { 78 | let startTime = DispatchTime.now().uptimeNanoseconds 79 | logger.info("Dequeing and running job", metadata: ["attempt": "\(jobData.currentAttempt)", "retries-left": "\(jobData.remainingAttempts)"]) 80 | do { 81 | try await job._dequeue(self.queue.context, id: id.string, payload: jobData.payload).get() 82 | 83 | logger.trace("Job ran successfully", metadata: ["attempts-made": "\(jobData.currentAttempt)"]) 84 | self.updateMetrics(for: jobData.jobName, startTime: startTime, queue: self.queue) 85 | await self.queue.sendNotification(of: "success", logger: logger) { 86 | try await $0.success(jobId: id.string, eventLoop: self.queue.context.eventLoop).get() 87 | } 88 | } catch { 89 | if jobData.remainingAttempts > 0 { 90 | // N.B.: `return` from here so we don't clear the job data. 91 | return try await self.retry(id: id, job: job, jobData: jobData, error: error, logger: logger) 92 | } else { 93 | logger.warning("Job failed, no retries remaining", metadata: ["error": "\(String(reflecting: error))", "attempts-made": "\(jobData.currentAttempt)"]) 94 | self.updateMetrics(for: jobData.jobName, startTime: startTime, queue: self.queue, error: error) 95 | 96 | try await job._error(self.queue.context, id: id.string, error, payload: jobData.payload).get() 97 | await self.queue.sendNotification(of: "failure", logger: logger) { 98 | try await $0.error(jobId: id.string, error: error, eventLoop: self.queue.context.eventLoop).get() 99 | } 100 | } 101 | } 102 | try await self.queue.clear(id).get() 103 | } 104 | 105 | private func retry(id: JobIdentifier, job: any AnyJob, jobData: JobData, error: any Error, logger: Logger) async throws { 106 | let delay = Swift.max(0, job._nextRetryIn(attempt: jobData.currentAttempt)) 107 | let updatedData = JobData( 108 | payload: jobData.payload, 109 | maxRetryCount: jobData.maxRetryCount, 110 | jobName: jobData.jobName, 111 | delayUntil: delay == 0 ? nil : .init(timeIntervalSinceNow: Double(delay)), 112 | queuedAt: .init(), 113 | attempts: jobData.currentAttempt 114 | ) 115 | 116 | logger.warning("Job failed, retrying", metadata: [ 117 | "retry-delay": "\(delay)", "error": "\(String(reflecting: error))", "next-attempt": "\(updatedData.currentAttempt)", "retries-left": "\(updatedData.remainingAttempts)", 118 | ]) 119 | try await self.queue.clear(id).get() 120 | try await self.queue.set(id, to: updatedData).get() 121 | try await self.queue.push(id).get() 122 | } 123 | 124 | private func updateMetrics( 125 | for jobName: String, 126 | startTime: UInt64, 127 | queue: any Queue, 128 | error: (any Error)? = nil 129 | ) { 130 | Timer( 131 | label: "\(jobName).jobDurationTimer", 132 | dimensions: [ 133 | ("success", error == nil ? "true" : "false"), 134 | ("jobName", jobName), 135 | ], 136 | preferredDisplayUnit: .milliseconds 137 | ).recordNanoseconds(DispatchTime.now().uptimeNanoseconds - startTime) 138 | 139 | if error != nil { 140 | Counter( 141 | label: "error.completed.jobs.counter", 142 | dimensions: [("queueName", queue.queueName.string)] 143 | ).increment() 144 | } else { 145 | Counter( 146 | label: "success.completed.jobs.counter", 147 | dimensions: [("queueName", queue.queueName.string)] 148 | ).increment() 149 | } 150 | 151 | Meter( 152 | label: "jobs.in.progress.meter", 153 | dimensions: [("queueName", queue.queueName.string)] 154 | ).decrement() 155 | } 156 | } 157 | -------------------------------------------------------------------------------- /Sources/Queues/QueuesCommand.swift: -------------------------------------------------------------------------------- 1 | import ConsoleKit 2 | @preconcurrency import Dispatch 3 | import Vapor 4 | import NIOConcurrencyHelpers 5 | import NIOCore 6 | import Atomics 7 | 8 | /// The command to start the Queue job 9 | public final class QueuesCommand: AsyncCommand, Sendable { 10 | // See `Command.signature`. 11 | public let signature = Signature() 12 | 13 | // See `Command.Signature`. 14 | public struct Signature: CommandSignature { 15 | public init() {} 16 | 17 | @Option(name: "queue", help: "Specifies a single queue to run") 18 | var queue: String? 19 | 20 | @Flag(name: "scheduled", help: "Runs the scheduled queue jobs") 21 | var scheduled: Bool 22 | } 23 | 24 | // See `Command.help`. 25 | public var help: String { "Starts the Vapor Queues worker" } 26 | 27 | private let application: Application 28 | 29 | private let box: NIOLockedValueBox 30 | 31 | struct Box: Sendable { 32 | var jobTasks: [RepeatedTask] 33 | var scheduledTasks: [String: AnyScheduledJob.Task] 34 | var signalSources: [any DispatchSourceSignal] 35 | var didShutdown: Bool 36 | } 37 | 38 | /// Create a new ``QueuesCommand``. 39 | /// 40 | /// - Parameters: 41 | /// - application: The active Vapor `Application`. 42 | /// - scheduled: This parameter is a historical artifact and has no effect. 43 | public init(application: Application, scheduled: Bool = false) { 44 | self.application = application 45 | self.box = .init(.init(jobTasks: [], scheduledTasks: [:], signalSources: [], didShutdown: false)) 46 | } 47 | 48 | // See `AsyncCommand.run(using:signature:)`. 49 | public func run(using context: CommandContext, signature: QueuesCommand.Signature) async throws { 50 | // shutdown future 51 | let promise = self.application.eventLoopGroup.any().makePromise(of: Void.self) 52 | self.application.running = .start(using: promise) 53 | 54 | // setup signal sources for shutdown 55 | let signalQueue = DispatchQueue(label: "codes.vapor.jobs.command") 56 | func makeSignalSource(_ code: Int32) { 57 | #if canImport(Darwin) 58 | /// https://github.com/swift-server/swift-service-lifecycle/blob/main/Sources/UnixSignals/UnixSignalsSequence.swift#L77-L82 59 | signal(code, SIG_IGN) 60 | #endif 61 | 62 | let source = DispatchSource.makeSignalSource(signal: code, queue: signalQueue) 63 | source.setEventHandler { 64 | print() // clear ^C 65 | promise.succeed(()) 66 | } 67 | source.resume() 68 | self.box.withLockedValue { $0.signalSources.append(source) } 69 | } 70 | makeSignalSource(SIGTERM) 71 | makeSignalSource(SIGINT) 72 | 73 | if signature.scheduled { 74 | self.application.logger.info("Starting scheduled jobs worker") 75 | try self.startScheduledJobs() 76 | } else { 77 | let queue: QueueName = signature.queue.map { .init(string: $0) } ?? .default 78 | 79 | self.application.logger.info("Starting jobs worker", metadata: ["queue": .string(queue.string)]) 80 | try self.startJobs(on: queue) 81 | } 82 | } 83 | 84 | /// Starts an in-process jobs worker for queued tasks 85 | /// 86 | /// - Parameter queueName: The queue to run the jobs on 87 | public func startJobs(on queueName: QueueName) throws { 88 | let workerCount: Int 89 | switch self.application.queues.configuration.workerCount { 90 | case .default: 91 | workerCount = self.application.eventLoopGroup.makeIterator().reduce(0, { n, _ in n + 1 }) 92 | self.application.logger.trace("Using default worker count", metadata: ["workerCount": "\(workerCount)"]) 93 | case .custom(let custom): 94 | workerCount = custom 95 | self.application.logger.trace("Using custom worker count", metadata: ["workerCount": "\(workerCount)"]) 96 | } 97 | 98 | var tasks: [RepeatedTask] = [] 99 | for eventLoop in self.application.eventLoopGroup.makeIterator().prefix(workerCount) { 100 | self.application.logger.trace("Booting worker") 101 | 102 | let worker = self.application.queues.queue(queueName, on: eventLoop).worker 103 | let task = eventLoop.scheduleRepeatedAsyncTask( 104 | initialDelay: .zero, 105 | delay: worker.queue.configuration.refreshInterval 106 | ) { task in 107 | worker.queue.logger.trace("Running refresh task") 108 | return worker.run().map { 109 | worker.queue.logger.trace("Worker ran the task successfully") 110 | }.recover { error in 111 | worker.queue.logger.error("Job run failed", metadata: ["error": "\(String(reflecting: error))"]) 112 | }.map { 113 | if self.box.withLockedValue({ $0.didShutdown }) { 114 | worker.queue.logger.trace("Shutting down, cancelling the task") 115 | task.cancel() 116 | } 117 | } 118 | } 119 | tasks.append(task) 120 | } 121 | 122 | self.box.withLockedValue { $0.jobTasks = tasks } 123 | self.application.logger.trace("Finished adding jobTasks, total count: \(tasks.count)") 124 | } 125 | 126 | /// Starts the scheduled jobs in-process 127 | public func startScheduledJobs() throws { 128 | self.application.logger.trace("Checking for scheduled jobs to begin the worker") 129 | 130 | guard !self.application.queues.configuration.scheduledJobs.isEmpty else { 131 | self.application.logger.warning("No scheduled jobs exist, exiting scheduled jobs worker.") 132 | return 133 | } 134 | 135 | self.application.logger.trace("Beginning the scheduling process") 136 | self.application.queues.configuration.scheduledJobs.forEach { 137 | self.application.logger.trace("Scheduling job", metadata: ["name": "\($0.job.name)"]) 138 | self.schedule($0) 139 | } 140 | } 141 | 142 | private func schedule(_ job: AnyScheduledJob) { 143 | self.box.withLockedValue { box in 144 | if box.didShutdown { 145 | self.application.logger.trace("Application is shutting down, not scheduling job", metadata: ["name": "\(job.job.name)"]) 146 | return 147 | } 148 | 149 | let context = QueueContext( 150 | queueName: QueueName(string: "scheduled"), 151 | configuration: self.application.queues.configuration, 152 | application: self.application, 153 | logger: self.application.logger, 154 | on: self.application.eventLoopGroup.any() 155 | ) 156 | 157 | guard let task = job.schedule(context: context) else { 158 | return 159 | } 160 | 161 | self.application.logger.trace("Job was scheduled successfully", metadata: ["name": "\(job.job.name)"]) 162 | box.scheduledTasks[job.job.name] = task 163 | 164 | task.done.whenComplete { result in 165 | switch result { 166 | case .failure(let error): 167 | context.logger.error("Scheduled job failed", metadata: ["name": "\(job.job.name)", "error": "\(String(reflecting: error))"]) 168 | case .success: break 169 | } 170 | // Explicitly spin the event loop so we don't deadlock on a reentrant call to this method. 171 | context.eventLoop.execute { 172 | self.schedule(job) 173 | } 174 | } 175 | } 176 | } 177 | 178 | /// Shuts down the jobs worker 179 | public func shutdown() { 180 | self.box.withLockedValue { box in 181 | box.didShutdown = true 182 | 183 | // stop running in case shutting down from signal 184 | self.application.running?.stop() 185 | 186 | // clear signal sources 187 | box.signalSources.forEach { $0.cancel() } // clear refs 188 | box.signalSources = [] 189 | 190 | // stop all job queue workers 191 | box.jobTasks.forEach { 192 | $0.syncCancel(on: self.application.eventLoopGroup.any()) 193 | } 194 | // stop all scheduled jobs 195 | box.scheduledTasks.values.forEach { 196 | $0.task.syncCancel(on: self.application.eventLoopGroup.any()) 197 | } 198 | } 199 | } 200 | 201 | public func asyncShutdown() async { 202 | let (jobTasks, scheduledTasks) = self.box.withLockedValue { box in 203 | box.didShutdown = true 204 | 205 | // stop running in case shutting down from signal 206 | self.application.running?.stop() 207 | 208 | // clear signal sources 209 | box.signalSources.forEach { $0.cancel() } // clear refs 210 | box.signalSources = [] 211 | 212 | // Release the lock before we start any suspensions 213 | return (box.jobTasks, box.scheduledTasks) 214 | } 215 | 216 | // stop all job queue workers 217 | for jobTask in jobTasks { 218 | await jobTask.asyncCancel(on: self.application.eventLoopGroup.any()) 219 | } 220 | // stop all scheduled jobs 221 | for scheduledTask in scheduledTasks.values { 222 | await scheduledTask.task.asyncCancel(on: self.application.eventLoopGroup.any()) 223 | } 224 | } 225 | 226 | deinit { 227 | assert(self.box.withLockedValue { $0.didShutdown }, "JobsCommand did not shutdown before deinit") 228 | } 229 | } 230 | -------------------------------------------------------------------------------- /Sources/Queues/QueuesConfiguration.swift: -------------------------------------------------------------------------------- 1 | import ConsoleKitTerminal 2 | import Logging 3 | import NIOCore 4 | import NIOConcurrencyHelpers 5 | 6 | /// Configuration parameters for the Queues module as a whole. 7 | public struct QueuesConfiguration: Sendable { 8 | private struct DataBox: Sendable { 9 | var refreshInterval: TimeAmount = .seconds(1) 10 | var persistenceKey: String = "vapor_queues" 11 | var workerCount: WorkerCount = .default 12 | var userInfo: [AnySendableHashable: any Sendable] = [:] 13 | 14 | var jobs: [String: any AnyJob] = [:] 15 | var scheduledJobs: [AnyScheduledJob] = [] 16 | var notificationHooks: [any JobEventDelegate] = [] 17 | } 18 | 19 | private let dataBox: NIOLockedValueBox = .init(.init()) 20 | 21 | /// The number of seconds to wait before checking for the next job. Defaults to `1` 22 | public var refreshInterval: TimeAmount { 23 | get { self.dataBox.withLockedValue { $0.refreshInterval } } 24 | set { self.dataBox.withLockedValue { $0.refreshInterval = newValue } } 25 | } 26 | 27 | /// The key that stores the data about a job. Defaults to `vapor_queues` 28 | public var persistenceKey: String { 29 | get { self.dataBox.withLockedValue { $0.persistenceKey } } 30 | set { self.dataBox.withLockedValue { $0.persistenceKey = newValue } } 31 | } 32 | 33 | /// Supported options for number of job handling workers. 34 | public enum WorkerCount: ExpressibleByIntegerLiteral, Sendable { 35 | /// One worker per event loop. 36 | case `default` 37 | 38 | /// Specify a custom worker count. 39 | case custom(Int) 40 | 41 | /// See `ExpressibleByIntegerLiteral`. 42 | public init(integerLiteral value: Int) { 43 | self = .custom(value) 44 | } 45 | } 46 | 47 | /// Sets the number of workers used for handling jobs. 48 | public var workerCount: WorkerCount { 49 | get { self.dataBox.withLockedValue { $0.workerCount } } 50 | set { self.dataBox.withLockedValue { $0.workerCount = newValue } } 51 | } 52 | 53 | /// A logger 54 | public let logger: Logger 55 | 56 | // Arbitrary user info to be stored 57 | public var userInfo: [AnySendableHashable: any Sendable] { 58 | get { self.dataBox.withLockedValue { $0.userInfo } } 59 | set { self.dataBox.withLockedValue { $0.userInfo = newValue } } 60 | } 61 | 62 | var jobs: [String: any AnyJob] { 63 | get { self.dataBox.withLockedValue { $0.jobs } } 64 | set { self.dataBox.withLockedValue { $0.jobs = newValue } } 65 | } 66 | 67 | var scheduledJobs: [AnyScheduledJob] { 68 | get { self.dataBox.withLockedValue { $0.scheduledJobs } } 69 | set { self.dataBox.withLockedValue { $0.scheduledJobs = newValue } } 70 | } 71 | 72 | var notificationHooks: [any JobEventDelegate] { 73 | get { self.dataBox.withLockedValue { $0.notificationHooks } } 74 | set { self.dataBox.withLockedValue { $0.notificationHooks = newValue } } 75 | } 76 | 77 | /// Creates an empty ``QueuesConfiguration``. 78 | public init( 79 | refreshInterval: TimeAmount = .seconds(1), 80 | persistenceKey: String = "vapor_queues", 81 | workerCount: WorkerCount = .default, 82 | logger: Logger = .init(label: "codes.vapor.queues") 83 | ) { 84 | self.logger = logger 85 | self.refreshInterval = refreshInterval 86 | self.persistenceKey = persistenceKey 87 | self.workerCount = workerCount 88 | } 89 | 90 | /// Adds a new ``Job`` to the queue configuration. 91 | /// 92 | /// This must be called on all ``Job`` objects before they can be run in a queue. 93 | /// 94 | /// - Parameter job: The ``Job`` to add. 95 | mutating public func add(_ job: J) { 96 | self.logger.trace("Adding job type", metadata: ["name": "\(J.name)"]) 97 | if let existing = self.jobs[J.name] { 98 | self.logger.warning("Job type is already registered", metadata: ["name": "\(J.name)", "existing": "\(existing)"]) 99 | } 100 | self.jobs[J.name] = job 101 | } 102 | 103 | /// Schedules a new job for execution at a later date. 104 | /// 105 | /// config.schedule(Cleanup()) 106 | /// .yearly() 107 | /// .in(.may) 108 | /// .on(23) 109 | /// .at(.noon) 110 | /// 111 | /// - Parameters: 112 | /// - job: The ``ScheduledJob`` to schedule. 113 | /// - builder: A ``ScheduleBuilder`` to use for scheduling. 114 | /// - Returns: The passed-in ``ScheduleBuilder``. 115 | mutating func schedule(_ job: some ScheduledJob, builder: ScheduleBuilder = .init()) -> ScheduleBuilder { 116 | self.logger.trace("Scheduling job", metadata: ["job-name": "\(job.name)"]) 117 | self.scheduledJobs.append(AnyScheduledJob(job: job, scheduler: builder)) 118 | return builder 119 | } 120 | 121 | /// Adds a notification hook that can receive status updates about jobs 122 | /// 123 | /// - Parameter hook: A ``JobEventDelegate`` to register. 124 | mutating public func add(_ hook: some JobEventDelegate) { 125 | self.logger.trace("Adding notification hook") 126 | self.notificationHooks.append(hook) 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /Sources/Queues/QueuesDriver.swift: -------------------------------------------------------------------------------- 1 | /// A new driver for Queues 2 | public protocol QueuesDriver: Sendable { 3 | /// Create or look up a named ``Queue`` instance. 4 | /// 5 | /// - Parameter context: The context for jobs on the queue. Also provides the queue name. 6 | func makeQueue(with context: QueueContext) -> any Queue 7 | 8 | /// Shuts down the driver 9 | func shutdown() 10 | 11 | /// Shut down the driver asynchronously. Helps avoid calling `.wait()` 12 | func asyncShutdown() async 13 | } 14 | 15 | extension QueuesDriver { 16 | public func asyncShutdown() async { 17 | self.shutdown() 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /Sources/Queues/QueuesEventLoopPreference.swift: -------------------------------------------------------------------------------- 1 | import NIOCore 2 | 3 | /// Determines which event loop the jobs worker uses while executing jobs. 4 | public enum QueuesEventLoopPreference { 5 | /// The caller accepts connections and callbacks on any EventLoop. 6 | case indifferent 7 | 8 | /// The caller accepts connections on any event loop, but must be 9 | /// called back (delegated to) on the supplied EventLoop. 10 | /// If possible, the connection should also be on this EventLoop for 11 | /// improved performance. 12 | case delegate(on: any EventLoop) 13 | 14 | /// Returns the delegate EventLoop given an EventLoopGroup. 15 | public func delegate(for eventLoopGroup: any EventLoopGroup) -> any EventLoop { 16 | switch self { 17 | case .indifferent: 18 | return eventLoopGroup.any() 19 | case .delegate(let eventLoop): 20 | return eventLoop 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /Sources/Queues/RepeatedTask+Cancel.swift: -------------------------------------------------------------------------------- 1 | import NIOCore 2 | import Logging 3 | 4 | extension RepeatedTask { 5 | func syncCancel(on eventLoop: any EventLoop) { 6 | do { 7 | let promise = eventLoop.makePromise(of: Void.self) 8 | self.cancel(promise: promise) 9 | try promise.futureResult.wait() 10 | } catch { 11 | Logger(label: "codes.vapor.queues.repeatedtask").debug("Failed cancelling repeated task", metadata: ["error": "\(error)"]) 12 | } 13 | } 14 | 15 | func asyncCancel(on eventLoop: any EventLoop) async { 16 | do { 17 | let promise = eventLoop.makePromise(of: Void.self) 18 | self.cancel(promise: promise) 19 | try await promise.futureResult.get() 20 | } catch { 21 | Logger(label: "codes.vapor.queues.repeatedtask").debug("Failed cancelling repeated task", metadata: ["error": "\(error)"]) 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /Sources/Queues/Request+Queues.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import Vapor 3 | import NIOCore 4 | 5 | extension Request { 6 | /// Get the default ``Queue``. 7 | public var queue: any Queue { 8 | self.queues(.default) 9 | } 10 | 11 | /// Create or look up an instance of a named ``Queue`` and bind it to this request's event loop. 12 | /// 13 | /// - Parameter queue: The queue name 14 | public func queues(_ queue: QueueName, logger: Logger? = nil) -> any Queue { 15 | self.application.queues.queue(queue, logger: logger ?? self.logger, on: self.eventLoop) 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /Sources/Queues/ScheduleBuilder.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// An object that can be used to build a scheduled job 4 | public final class ScheduleBuilder: @unchecked Sendable { 5 | /// Months of the year 6 | public enum Month: Int { 7 | case january = 1 8 | case february 9 | case march 10 | case april 11 | case may 12 | case june 13 | case july 14 | case august 15 | case september 16 | case october 17 | case november 18 | case december 19 | } 20 | 21 | /// Describes a day 22 | public enum Day: ExpressibleByIntegerLiteral { 23 | case first 24 | case last 25 | case exact(Int) 26 | 27 | public init(integerLiteral value: Int) { self = .exact(value) } 28 | } 29 | 30 | /// Describes a day of the week 31 | public enum Weekday: Int { 32 | case sunday = 1 33 | case monday 34 | case tuesday 35 | case wednesday 36 | case thursday 37 | case friday 38 | case saturday 39 | } 40 | 41 | /// Describes a time of day 42 | public struct Time: ExpressibleByStringLiteral, CustomStringConvertible { 43 | /// Returns a `Time` object at midnight (12:00 AM) 44 | public static var midnight: Time { .init(12, 00, .am) } 45 | 46 | /// Returns a `Time` object at noon (12:00 PM) 47 | public static var noon: Time { .init(12, 00, .pm) } 48 | 49 | var hour: Hour24, minute: Minute 50 | 51 | init(_ hour: Hour24, _ minute: Minute) { (self.hour, self.minute) = (hour, minute) } 52 | 53 | init(_ hour: Hour12, _ minute: Minute, _ period: HourPeriod) { 54 | self.init(.init(hour.n % 12 + (period == .am ? 0 : 12)), minute) 55 | } 56 | 57 | public init(stringLiteral value: String) { 58 | let parts = value.split(separator: ":", maxSplits: 1) 59 | 60 | guard let hour = Int(parts[0]) else { fatalError("Could not convert hour to Int") } 61 | switch parts.count { 62 | case 1: 63 | self.init(Hour24(hour), 0) 64 | case 2: 65 | guard let minute = Int(parts[1].prefix(2)) else { fatalError("Could not convert minute to Int") } 66 | switch parts[1].count { 67 | case 2: 68 | self.init(Hour24(hour), Minute(minute)) 69 | case 4: 70 | self.init(Hour12(hour), Minute(minute), HourPeriod(String(parts[1].dropFirst(2)))) 71 | default: 72 | fatalError("Invalid minute format: \(parts[1]), expected 00am/pm") 73 | } 74 | default: 75 | fatalError("Invalid time format: \(value), expected 00:00am/pm") 76 | } 77 | } 78 | 79 | public var description: String { "\(self.hour):\(self.minute)" } 80 | } 81 | 82 | /// Represents an hour numeral that must be in 12 hour format 83 | public struct Hour12: ExpressibleByIntegerLiteral, CustomStringConvertible { 84 | let n: Int 85 | 86 | init(_ n: Int) { precondition((1 ... 12).contains(n), "12-hour clock must be in range 1-12"); self.n = n } 87 | 88 | public init(integerLiteral value: Int) { self.init(value) } 89 | 90 | public var description: String { "\(self.n)" } 91 | } 92 | 93 | /// Represents an hour numeral that must be in 24 hour format 94 | public struct Hour24: ExpressibleByIntegerLiteral, CustomStringConvertible { 95 | let n: Int 96 | 97 | init(_ n: Int) { precondition((0 ..< 24).contains(n), "24-hour clock must be in range 0-23"); self.n = n } 98 | 99 | public init(integerLiteral value: Int) { self.init(value) } 100 | 101 | public var description: String { String("0\(self.n)".suffix(2)) } 102 | } 103 | 104 | /// A period of hours - either `am` or `pm` 105 | public enum HourPeriod: String, ExpressibleByStringLiteral, CustomStringConvertible, Hashable { 106 | case am, pm 107 | 108 | init(_ string: String) { self.init(rawValue: string)! } 109 | 110 | public init(stringLiteral value: String) { self.init(value) } 111 | 112 | public var description: String { self.rawValue } 113 | } 114 | 115 | /// Describes a minute numeral 116 | public struct Minute: ExpressibleByIntegerLiteral, CustomStringConvertible { 117 | let n: Int 118 | 119 | init(_ n: Int) { precondition((0 ..< 60).contains(n), "Minute must be in range 0-59"); self.n = n } 120 | 121 | public init(integerLiteral value: Int) { self.init(value) } 122 | 123 | public var description: String { String("0\(self.n)".suffix(2)) } 124 | } 125 | 126 | /// Describes a second numeral 127 | public struct Second: ExpressibleByIntegerLiteral, CustomStringConvertible { 128 | let n: Int 129 | 130 | init(_ n: Int) { precondition((0 ..< 60).contains(n), "Second must be in range 0-59"); self.n = n } 131 | 132 | public init(integerLiteral value: Int) { self.init(value) } 133 | 134 | public var description: String { String("0\(self.n)".suffix(2)) } 135 | } 136 | 137 | /// An object to build a `Yearly` scheduled job 138 | public struct Yearly { 139 | let builder: ScheduleBuilder 140 | 141 | public func `in`(_ month: Month) -> Monthly { self.builder.month = month; return self.builder.monthly() } 142 | } 143 | 144 | /// An object to build a `Monthly` scheduled job 145 | public struct Monthly { 146 | let builder: ScheduleBuilder 147 | 148 | public func on(_ day: Day) -> Daily { self.builder.day = day; return self.builder.daily() } 149 | } 150 | 151 | /// An object to build a `Weekly` scheduled job 152 | public struct Weekly { 153 | let builder: ScheduleBuilder 154 | 155 | public func on(_ weekday: Weekday) -> Daily { self.builder.weekday = weekday; return self.builder.daily() } 156 | } 157 | 158 | /// An object to build a `Daily` scheduled job 159 | public struct Daily { 160 | let builder: ScheduleBuilder 161 | 162 | public func at(_ time: Time) { self.builder.time = time } 163 | 164 | public func at(_ hour: Hour24, _ minute: Minute) { self.at(.init(hour, minute)) } 165 | 166 | public func at(_ hour: Hour12, _ minute: Minute, _ period: HourPeriod) { self.at(.init(hour, minute, period)) } 167 | } 168 | 169 | /// An object to build a `Hourly` scheduled job 170 | public struct Hourly { 171 | let builder: ScheduleBuilder 172 | 173 | public func at(_ minute: Minute) { self.builder.minute = minute } 174 | } 175 | 176 | /// An object to build a `EveryMinute` scheduled job 177 | public struct Minutely { 178 | let builder: ScheduleBuilder 179 | 180 | public func at(_ second: Second) { self.builder.second = second } 181 | } 182 | 183 | /// Retrieves the next date after the one given. 184 | public func nextDate(current: Date = .init()) -> Date? { 185 | if let date = self.date, date > current { return date } 186 | 187 | var components = DateComponents() 188 | components.nanosecond = self.millisecond.map { $0 * 1_000_000 } 189 | components.second = self.second?.n 190 | components.minute = self.time?.minute.n ?? self.minute?.n 191 | components.hour = self.time?.hour.n 192 | components.weekday = self.weekday?.rawValue 193 | switch self.day { 194 | case .first?: components.day = 1 195 | case .exact(let exact)?: components.day = exact 196 | case .last?: fatalError("Last day of the month is not yet supported.") 197 | default: break 198 | } 199 | components.month = self.month?.rawValue 200 | return calendar.nextDate(after: current, matching: components, matchingPolicy: .strict) 201 | } 202 | 203 | /// The calendar used to compute the next date 204 | var calendar: Calendar 205 | 206 | /// Date to perform task (one-off job) 207 | var date: Date? 208 | var month: Month?, day: Day?, weekday: Weekday? 209 | var time: Time?, minute: Minute?, second: Second?, millisecond: Int? 210 | 211 | public init(calendar: Calendar = .current) { self.calendar = calendar } 212 | 213 | /// Schedules a job using a specific `Calendar` 214 | public func using(_ calendar: Calendar) -> ScheduleBuilder { self.calendar = calendar; return self } 215 | 216 | /// Schedules a job at a specific date 217 | public func at(_ date: Date) { self.date = date } 218 | 219 | /// Creates a yearly scheduled job for further building 220 | @discardableResult public func yearly() -> Yearly { .init(builder: self) } 221 | 222 | /// Creates a monthly scheduled job for further building 223 | @discardableResult public func monthly() -> Monthly { .init(builder: self) } 224 | 225 | /// Creates a weekly scheduled job for further building 226 | @discardableResult public func weekly() -> Weekly { .init(builder: self) } 227 | 228 | /// Creates a daily scheduled job for further building 229 | @discardableResult public func daily() -> Daily { .init(builder: self) } 230 | 231 | /// Creates a hourly scheduled job for further building 232 | @discardableResult public func hourly() -> Hourly { .init(builder: self) } 233 | 234 | /// Creates a minutely scheduled job for further building 235 | @discardableResult public func minutely() -> Minutely { .init(builder: self) } 236 | 237 | /// Runs a job every second 238 | public func everySecond() { self.millisecond = 0 } 239 | } 240 | -------------------------------------------------------------------------------- /Sources/Queues/ScheduledJob.swift: -------------------------------------------------------------------------------- 1 | import NIOCore 2 | import Foundation 3 | import Logging 4 | 5 | /// Describes a job that can be scheduled and repeated 6 | public protocol ScheduledJob: Sendable { 7 | var name: String { get } 8 | 9 | /// The method called when the job is run. 10 | /// 11 | /// - Parameter context: The ``QueueContext``. 12 | func run(context: QueueContext) -> EventLoopFuture 13 | } 14 | 15 | extension ScheduledJob { 16 | public var name: String { "\(Self.self)" } 17 | } 18 | 19 | final class AnyScheduledJob: Sendable { 20 | let job: any ScheduledJob 21 | let scheduler: ScheduleBuilder 22 | 23 | init(job: any ScheduledJob, scheduler: ScheduleBuilder) { 24 | self.job = job 25 | self.scheduler = scheduler 26 | } 27 | 28 | struct Task: Sendable { 29 | let task: RepeatedTask 30 | let done: EventLoopFuture 31 | } 32 | 33 | func schedule(context: QueueContext) -> Task? { 34 | var logger_ = context.logger 35 | logger_[metadataKey: "job-name"] = "\(self.job.name)" 36 | let logger = logger_ 37 | 38 | logger.trace("Beginning the scheduler process") 39 | 40 | guard let date = self.scheduler.nextDate() else { 41 | logger.debug("Scheduler returned no date") 42 | return nil 43 | } 44 | logger.debug("Job scheduled", metadata: ["scheduled-date": "\(date)"]) 45 | 46 | let promise = context.eventLoop.makePromise(of: Void.self) 47 | let task = context.eventLoop.scheduleRepeatedTask( 48 | initialDelay: .microseconds(Int64(date.timeIntervalSinceNow * 1_000_000)), delay: .zero 49 | ) { task in 50 | // always cancel 51 | task.cancel() 52 | logger.trace("Running scheduled job") 53 | self.job.run(context: context).cascade(to: promise) 54 | } 55 | return .init(task: task, done: promise.futureResult) 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /Sources/XCTQueues/Docs.docc/Resources/vapor-queues-logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /Sources/XCTQueues/Docs.docc/index.md: -------------------------------------------------------------------------------- 1 | # ``XCTQueues`` 2 | 3 | XCTQueues provides XCTest extensions to make it easy to write tests that use Vapor's Queues library. -------------------------------------------------------------------------------- /Sources/XCTQueues/Docs.docc/theme-settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "theme": { 3 | "aside": { "border-radius": "16px", "border-style": "double", "border-width": "3px" }, 4 | "border-radius": "0", 5 | "button": { "border-radius": "16px", "border-width": "1px", "border-style": "solid" }, 6 | "code": { "border-radius": "16px", "border-width": "1px", "border-style": "solid" }, 7 | "color": { 8 | "queues": "#e8665a", 9 | "documentation-intro-fill": "radial-gradient(circle at top, var(--color-queues) 30%, #000 100%)", 10 | "documentation-intro-accent": "var(--color-queues)", 11 | "documentation-intro-eyebrow": "white", 12 | "documentation-intro-figure": "white", 13 | "documentation-intro-title": "white", 14 | "logo-base": { "dark": "#fff", "light": "#000" }, 15 | "logo-shape": { "dark": "#000", "light": "#fff" }, 16 | "fill": { "dark": "#000", "light": "#fff" } 17 | }, 18 | "icons": { "technology": "/queues/images/vapor-queues-logo.svg" } 19 | }, 20 | "features": { 21 | "quickNavigation": { "enable": true }, 22 | "i18n": { "enable": true } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /Sources/XCTQueues/TestQueueDriver.swift: -------------------------------------------------------------------------------- 1 | import NIOConcurrencyHelpers 2 | import NIOCore 3 | import Queues 4 | import Vapor 5 | 6 | extension Application.Queues.Provider { 7 | public static var test: Self { 8 | .init { 9 | $0.queues.initializeTestStorage() 10 | return $0.queues.use(custom: TestQueuesDriver()) 11 | } 12 | } 13 | 14 | public static var asyncTest: Self { 15 | .init { 16 | $0.queues.initializeAsyncTestStorage() 17 | return $0.queues.use(custom: AsyncTestQueuesDriver()) 18 | } 19 | } 20 | } 21 | 22 | struct TestQueuesDriver: QueuesDriver { 23 | init() {} 24 | 25 | func makeQueue(with context: QueueContext) -> any Queue { 26 | TestQueue(_context: .init(context)) 27 | } 28 | 29 | func shutdown() { 30 | // nothing 31 | } 32 | } 33 | 34 | struct AsyncTestQueuesDriver: QueuesDriver { 35 | init() {} 36 | func makeQueue(with context: QueueContext) -> any Queue { AsyncTestQueue(_context: .init(context)) } 37 | func shutdown() {} 38 | } 39 | 40 | extension Application.Queues { 41 | public final class TestQueueStorage: Sendable { 42 | private struct Box: Sendable { 43 | var jobs: [JobIdentifier: JobData] = [:] 44 | var queue: [JobIdentifier] = [] 45 | } 46 | 47 | private let box = NIOLockedValueBox(.init()) 48 | 49 | public var jobs: [JobIdentifier: JobData] { 50 | get { self.box.withLockedValue { $0.jobs } } 51 | set { self.box.withLockedValue { $0.jobs = newValue } } 52 | } 53 | 54 | public var queue: [JobIdentifier] { 55 | get { self.box.withLockedValue { $0.queue } } 56 | set { self.box.withLockedValue { $0.queue = newValue } } 57 | } 58 | 59 | /// Returns the payloads of all jobs in the queue having type `J`. 60 | public func all(_: J.Type) -> [J.Payload] { 61 | let filteredJobIds = self.jobs.filter { $1.jobName == J.name }.map { $0.0 } 62 | 63 | return self.queue 64 | .filter { filteredJobIds.contains($0) } 65 | .compactMap { self.jobs[$0] } 66 | .compactMap { try? J.parsePayload($0.payload) } 67 | } 68 | 69 | /// Returns the payload of the first job in the queue having type `J`. 70 | public func first(_: J.Type) -> J.Payload? { 71 | let filteredJobIds = self.jobs.filter { $1.jobName == J.name }.map { $0.0 } 72 | 73 | guard let queueJob = self.queue.first(where: { filteredJobIds.contains($0) }), let jobData = self.jobs[queueJob] else { 74 | return nil 75 | } 76 | return try? J.parsePayload(jobData.payload) 77 | } 78 | 79 | /// Checks whether a job of type `J` was dispatched to queue 80 | public func contains(_ job: J.Type) -> Bool { 81 | self.first(job) != nil 82 | } 83 | } 84 | 85 | struct TestQueueKey: StorageKey, LockKey { 86 | typealias Value = TestQueueStorage 87 | } 88 | 89 | public var test: TestQueueStorage { 90 | self.application.storage[TestQueueKey.self]! 91 | } 92 | 93 | func initializeTestStorage() { 94 | self.application.storage[TestQueueKey.self] = .init() 95 | } 96 | 97 | struct AsyncTestQueueKey: StorageKey, LockKey { 98 | typealias Value = TestQueueStorage 99 | } 100 | 101 | public var asyncTest: TestQueueStorage { 102 | self.application.storage[AsyncTestQueueKey.self]! 103 | } 104 | 105 | func initializeAsyncTestStorage() { 106 | self.application.storage[AsyncTestQueueKey.self] = .init() 107 | } 108 | } 109 | 110 | struct TestQueue: Queue { 111 | let _context: NIOLockedValueBox 112 | var context: QueueContext { self._context.withLockedValue { $0 } } 113 | 114 | func get(_ id: JobIdentifier) -> EventLoopFuture { 115 | self._context.withLockedValue { context in 116 | context.eventLoop.makeSucceededFuture(context.application.queues.test.jobs[id]!) 117 | } 118 | } 119 | 120 | func set(_ id: JobIdentifier, to data: JobData) -> EventLoopFuture { 121 | self._context.withLockedValue { context in 122 | context.application.queues.test.jobs[id] = data 123 | return context.eventLoop.makeSucceededVoidFuture() 124 | } 125 | } 126 | 127 | func clear(_ id: JobIdentifier) -> EventLoopFuture { 128 | self._context.withLockedValue { context in 129 | context.application.queues.test.jobs[id] = nil 130 | return context.eventLoop.makeSucceededVoidFuture() 131 | } 132 | } 133 | 134 | func pop() -> EventLoopFuture { 135 | self._context.withLockedValue { context in 136 | let last = context.application.queues.test.queue.popLast() 137 | return context.eventLoop.makeSucceededFuture(last) 138 | } 139 | } 140 | 141 | func push(_ id: JobIdentifier) -> EventLoopFuture { 142 | self._context.withLockedValue { context in 143 | context.application.queues.test.queue.append(id) 144 | return context.eventLoop.makeSucceededVoidFuture() 145 | } 146 | } 147 | } 148 | 149 | struct AsyncTestQueue: AsyncQueue { 150 | let _context: NIOLockedValueBox 151 | var context: QueueContext { self._context.withLockedValue { $0 } } 152 | 153 | func get(_ id: JobIdentifier) async throws -> JobData { self._context.withLockedValue { $0.application.queues.asyncTest.jobs[id]! } } 154 | func set(_ id: JobIdentifier, to data: JobData) async throws { self._context.withLockedValue { $0.application.queues.asyncTest.jobs[id] = data } } 155 | func clear(_ id: JobIdentifier) async throws { self._context.withLockedValue { $0.application.queues.asyncTest.jobs[id] = nil } } 156 | func pop() async throws -> JobIdentifier? { self._context.withLockedValue { $0.application.queues.asyncTest.queue.popLast() } } 157 | func push(_ id: JobIdentifier) async throws { self._context.withLockedValue { $0.application.queues.asyncTest.queue.append(id) } } 158 | } 159 | -------------------------------------------------------------------------------- /Tests/QueuesTests/AsyncQueueTests.swift: -------------------------------------------------------------------------------- 1 | import Queues 2 | import XCTest 3 | import XCTVapor 4 | 5 | func XCTAssertNoThrowAsync( 6 | _ expression: @autoclosure () async throws -> T, 7 | _ message: @autoclosure () -> String = "", 8 | file: StaticString = #filePath, line: UInt = #line 9 | ) async { 10 | do { 11 | _ = try await expression() 12 | } catch { 13 | XCTAssertNoThrow(try { throw error }(), message(), file: file, line: line) 14 | } 15 | } 16 | 17 | final class AsyncQueueTests: XCTestCase { 18 | var app: Application! 19 | 20 | override class func setUp() { 21 | XCTAssert(isLoggingConfigured) 22 | } 23 | 24 | override func setUp() async throws { 25 | self.app = try await Application.make(.testing) 26 | } 27 | 28 | override func tearDown() async throws { 29 | try await self.app.asyncShutdown() 30 | } 31 | 32 | func testAsyncJobWithSyncQueue() async throws { 33 | self.app.queues.use(.test) 34 | 35 | let promise = self.app.eventLoopGroup.any().makePromise(of: Void.self) 36 | self.app.queues.add(MyAsyncJob(promise: promise)) 37 | 38 | self.app.get("foo") { req in 39 | try await req.queue.dispatch(MyAsyncJob.self, .init(foo: "bar")) 40 | try await req.queue.dispatch(MyAsyncJob.self, .init(foo: "baz")) 41 | try await req.queue.dispatch(MyAsyncJob.self, .init(foo: "quux")) 42 | return "done" 43 | } 44 | 45 | try await self.app.testable().test(.GET, "foo") { res async in 46 | XCTAssertEqual(res.status, .ok) 47 | XCTAssertEqual(res.body.string, "done") 48 | } 49 | 50 | XCTAssertEqual(self.app.queues.test.queue.count, 3) 51 | XCTAssertEqual(self.app.queues.test.jobs.count, 3) 52 | let job = self.app.queues.test.first(MyAsyncJob.self) 53 | XCTAssert(self.app.queues.test.contains(MyAsyncJob.self)) 54 | XCTAssertNotNil(job) 55 | XCTAssertEqual(job!.foo, "bar") 56 | 57 | try await self.app.queues.queue.worker.run().get() 58 | XCTAssertEqual(self.app.queues.test.queue.count, 0) 59 | XCTAssertEqual(self.app.queues.test.jobs.count, 0) 60 | 61 | await XCTAssertNoThrowAsync(try await promise.futureResult.get()) 62 | } 63 | 64 | func testAsyncJobWithAsyncQueue() async throws { 65 | self.app.queues.use(.asyncTest) 66 | 67 | let promise = self.app.eventLoopGroup.any().makePromise(of: Void.self) 68 | self.app.queues.add(MyAsyncJob(promise: promise)) 69 | 70 | self.app.get("foo") { req in 71 | try await req.queue.dispatch(MyAsyncJob.self, .init(foo: "bar")) 72 | try await req.queue.dispatch(MyAsyncJob.self, .init(foo: "baz")) 73 | try await req.queue.dispatch(MyAsyncJob.self, .init(foo: "quux")) 74 | return "done" 75 | } 76 | 77 | try await self.app.testable().test(.GET, "foo") { res async in 78 | XCTAssertEqual(res.status, .ok) 79 | XCTAssertEqual(res.body.string, "done") 80 | } 81 | 82 | XCTAssertEqual(self.app.queues.asyncTest.queue.count, 3) 83 | XCTAssertEqual(self.app.queues.asyncTest.jobs.count, 3) 84 | let job = self.app.queues.asyncTest.first(MyAsyncJob.self) 85 | XCTAssert(self.app.queues.asyncTest.contains(MyAsyncJob.self)) 86 | XCTAssertNotNil(job) 87 | XCTAssertEqual(job!.foo, "bar") 88 | 89 | try await self.app.queues.queue.worker.run().get() 90 | XCTAssertEqual(self.app.queues.asyncTest.queue.count, 0) 91 | XCTAssertEqual(self.app.queues.asyncTest.jobs.count, 0) 92 | 93 | await XCTAssertNoThrowAsync(try await promise.futureResult.get()) 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /Tests/QueuesTests/MetricsTests.swift: -------------------------------------------------------------------------------- 1 | @testable import CoreMetrics 2 | import Metrics 3 | import MetricsTestKit 4 | import NIOConcurrencyHelpers 5 | import Queues 6 | @testable import Vapor 7 | import XCTQueues 8 | import XCTVapor 9 | 10 | final class MetricsTests: XCTestCase { 11 | var app: Application! 12 | var metrics: TestMetrics! 13 | 14 | override func setUp() async throws { 15 | self.metrics = TestMetrics() 16 | MetricsSystem.bootstrapInternal(self.metrics) 17 | 18 | self.app = try await Application.make(.testing) 19 | self.app.queues.use(.test) 20 | } 21 | 22 | override func tearDown() async throws { 23 | try await self.app.asyncShutdown() 24 | } 25 | 26 | func testJobDurationTimer() async throws { 27 | let promise = self.app.eventLoopGroup.next().makePromise(of: Void.self) 28 | self.app.queues.add(MyAsyncJob(promise: promise)) 29 | 30 | self.app.get("foo") { req async throws in 31 | try await req.queue.dispatch(MyAsyncJob.self, .init(foo: "bar"), id: JobIdentifier(string: "some-id")) 32 | return "done" 33 | } 34 | 35 | try await self.app.testable().test(.GET, "foo") { res async in 36 | XCTAssertEqual(res.status, .ok) 37 | XCTAssertEqual(res.body.string, "done") 38 | } 39 | 40 | try await self.app.queues.queue.worker.run() 41 | 42 | let timer = try XCTUnwrap(self.metrics.timers.first(where: { $0.label == "MyAsyncJob.jobDurationTimer" })) 43 | let successDimension = try XCTUnwrap(timer.dimensions.first(where: { $0.0 == "success" })) 44 | let idDimension = try XCTUnwrap(timer.dimensions.first(where: { $0.0 == "jobName" })) 45 | XCTAssertEqual(successDimension.1, "true") 46 | XCTAssertEqual(idDimension.1, "MyAsyncJob") 47 | 48 | try XCTAssertNoThrow(promise.futureResult.wait()) 49 | } 50 | 51 | func testSuccessfullyCompletedJobsCounter() async throws { 52 | let promise = self.app.eventLoopGroup.next().makePromise(of: Void.self) 53 | self.app.queues.add(MyAsyncJob(promise: promise)) 54 | 55 | self.app.get("foo") { req async throws in 56 | try await req.queue.dispatch(MyAsyncJob.self, .init(foo: "bar")) 57 | return "done" 58 | } 59 | 60 | try await self.app.testable().test(.GET, "foo") { res async in 61 | XCTAssertEqual(res.status, .ok) 62 | XCTAssertEqual(res.body.string, "done") 63 | } 64 | 65 | try await self.app.queues.queue.worker.run() 66 | let counter = try XCTUnwrap(self.metrics.counters.first(where: { $0.label == "success.completed.jobs.counter" })) 67 | let queueNameDimension = try XCTUnwrap(counter.dimensions.first(where: { $0.0 == "queueName" })) 68 | XCTAssertEqual(queueNameDimension.1, self.app.queues.queue.queueName.string) 69 | XCTAssertEqual(counter.lastValue, 1) 70 | } 71 | 72 | func testErroringJobsCounter() async throws { 73 | let promise = self.app.eventLoopGroup.next().makePromise(of: Void.self) 74 | self.app.queues.add(FailingAsyncJob(promise: promise)) 75 | 76 | self.app.get("foo") { req async throws in 77 | try await req.queue.dispatch(FailingAsyncJob.self, .init(foo: "bar")) 78 | return "done" 79 | } 80 | 81 | try await self.app.testable().test(.GET, "foo") { res async in 82 | XCTAssertEqual(res.status, .ok) 83 | XCTAssertEqual(res.body.string, "done") 84 | } 85 | 86 | try await self.app.queues.queue.worker.run() 87 | let counter = try XCTUnwrap(self.metrics.counters.first(where: { $0.label == "error.completed.jobs.counter" })) 88 | let queueNameDimension = try XCTUnwrap(counter.dimensions.first(where: { $0.0 == "queueName" })) 89 | XCTAssertEqual(queueNameDimension.1, self.app.queues.queue.queueName.string) 90 | XCTAssertEqual(counter.lastValue, 1) 91 | } 92 | 93 | func testDispatchedJobsCounter() async throws { 94 | let promise = self.app.eventLoopGroup.next().makePromise(of: Void.self) 95 | self.app.queues.add(MyAsyncJob(promise: promise)) 96 | 97 | self.app.get("foo") { req async throws in 98 | try await req.queue.dispatch(MyAsyncJob.self, .init(foo: "bar"), id: JobIdentifier(string: "first")) 99 | try await req.queue.dispatch(MyAsyncJob.self, .init(foo: "rab"), id: JobIdentifier(string: "second")) 100 | return "done" 101 | } 102 | 103 | try await self.app.testable().test(.GET, "foo") { res async in 104 | XCTAssertEqual(res.status, .ok) 105 | XCTAssertEqual(res.body.string, "done") 106 | } 107 | 108 | try await self.app.queues.queue.worker.run() 109 | 110 | let counter = try XCTUnwrap(self.metrics.counters.first(where: { $0.label == "dispatched.jobs.counter" })) 111 | let queueNameDimension = try XCTUnwrap(counter.dimensions.first(where: { $0.0 == "queueName" })) 112 | let jobNameDimension = try XCTUnwrap(counter.dimensions.first(where: { $0.0 == "jobName" })) 113 | XCTAssertEqual(queueNameDimension.1, self.app.queues.queue.queueName.string) 114 | XCTAssertEqual(jobNameDimension.1, MyAsyncJob.name) 115 | XCTAssertEqual(counter.totalValue, 2) 116 | } 117 | 118 | func testInProgressJobsGauge() async throws { 119 | let promise = self.app.eventLoopGroup.next().makePromise(of: Void.self) 120 | self.app.queues.add(MyAsyncJob(promise: promise)) 121 | 122 | self.app.get("foo") { req async throws in 123 | try await req.queue.dispatch(MyAsyncJob.self, .init(foo: "bar")) 124 | return "done" 125 | } 126 | 127 | try await self.app.testable().test(.GET, "foo") { res async in 128 | XCTAssertEqual(res.status, .ok) 129 | XCTAssertEqual(res.body.string, "done") 130 | } 131 | 132 | try await self.app.queues.queue.worker.run() 133 | 134 | let meter = try XCTUnwrap(self.metrics.meters.first(where: { $0.label == "jobs.in.progress.meter" })) 135 | let queueNameDimension = try XCTUnwrap(meter.dimensions.first(where: { $0.0 == "queueName" })) 136 | XCTAssertEqual(queueNameDimension.1, self.app.queues.queue.queueName.string) 137 | XCTAssertEqual(meter.values, [1, 0]) 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /Tests/QueuesTests/QueueTests.swift: -------------------------------------------------------------------------------- 1 | import Atomics 2 | import NIOConcurrencyHelpers 3 | import Queues 4 | import XCTest 5 | import XCTQueues 6 | import XCTVapor 7 | 8 | func XCTAssertEqualAsync( 9 | _ expression1: @autoclosure () async throws -> T, 10 | _ expression2: @autoclosure () async throws -> T, 11 | _ message: @autoclosure () -> String = "", 12 | file: StaticString = #filePath, line: UInt = #line 13 | ) async where T: Equatable { 14 | do { 15 | let expr1 = try await expression1(), expr2 = try await expression2() 16 | return XCTAssertEqual(expr1, expr2, message(), file: file, line: line) 17 | } catch { 18 | return XCTAssertEqual(try { () -> Bool in throw error }(), false, message(), file: file, line: line) 19 | } 20 | } 21 | 22 | func XCTAssertTrueAsync( 23 | _ predicate: @autoclosure () async throws -> Bool, 24 | _ message: @autoclosure () -> String = "", 25 | file: StaticString = #filePath, line: UInt = #line 26 | ) async { 27 | do { 28 | let result = try await predicate() 29 | XCTAssertTrue(result, message(), file: file, line: line) 30 | } catch { 31 | return XCTAssertTrue(try { throw error }(), message(), file: file, line: line) 32 | } 33 | } 34 | 35 | func XCTAssertFalseAsync( 36 | _ predicate: @autoclosure () async throws -> Bool, 37 | _ message: @autoclosure () -> String = "", 38 | file: StaticString = #filePath, line: UInt = #line 39 | ) async { 40 | do { 41 | let result = try await predicate() 42 | XCTAssertFalse(result, message(), file: file, line: line) 43 | } catch { 44 | return XCTAssertFalse(try { throw error }(), message(), file: file, line: line) 45 | } 46 | } 47 | 48 | final class QueueTests: XCTestCase { 49 | var app: Application! 50 | 51 | override class func setUp() { 52 | XCTAssert(isLoggingConfigured) 53 | } 54 | 55 | override func setUp() async throws { 56 | self.app = try await Application.make(.testing) 57 | self.app.queues.use(.test) 58 | } 59 | 60 | override func tearDown() async throws { 61 | try await self.app.asyncShutdown() 62 | self.app = nil 63 | } 64 | 65 | func testVaporIntegrationWithInProcessJob() async throws { 66 | let jobSignal1 = self.app.eventLoopGroup.any().makePromise(of: String.self) 67 | self.app.queues.add(Foo1(promise: jobSignal1)) 68 | let jobSignal2 = self.app.eventLoopGroup.any().makePromise(of: String.self) 69 | self.app.queues.add(Foo2(promise: jobSignal2)) 70 | try self.app.queues.startInProcessJobs(on: .default) 71 | 72 | self.app.get("bar1") { req in 73 | try await req.queue.dispatch(Foo1.self, .init(foo: "Bar payload")).get() 74 | return "job bar dispatched" 75 | } 76 | 77 | self.app.get("bar2") { req in 78 | try await req.queue.dispatch(Foo2.self, .init(foo: "Bar payload")) 79 | return "job bar dispatched" 80 | } 81 | 82 | try await self.app.testable().test(.GET, "bar1") { res async in 83 | XCTAssertEqual(res.status, .ok) 84 | XCTAssertEqual(res.body.string, "job bar dispatched") 85 | }.test(.GET, "bar2") { res async in 86 | XCTAssertEqual(res.status, .ok) 87 | XCTAssertEqual(res.body.string, "job bar dispatched") 88 | } 89 | 90 | await XCTAssertEqualAsync(try await jobSignal1.futureResult.get(), "Bar payload") 91 | await XCTAssertEqualAsync(try await jobSignal2.futureResult.get(), "Bar payload") 92 | } 93 | 94 | func testVaporIntegration() async throws { 95 | let promise = self.app.eventLoopGroup.any().makePromise(of: String.self) 96 | self.app.queues.add(Foo1(promise: promise)) 97 | 98 | self.app.get("foo") { req in 99 | try await req.queue.dispatch(Foo1.self, .init(foo: "bar")) 100 | return "done" 101 | } 102 | 103 | try await self.app.testable().test(.GET, "foo") { res async in 104 | XCTAssertEqual(res.status, .ok) 105 | XCTAssertEqual(res.body.string, "done") 106 | } 107 | 108 | XCTAssertEqual(self.app.queues.test.queue.count, 1) 109 | XCTAssertEqual(self.app.queues.test.jobs.count, 1) 110 | let job = self.app.queues.test.first(Foo1.self) 111 | XCTAssert(self.app.queues.test.contains(Foo1.self)) 112 | XCTAssertNotNil(job) 113 | XCTAssertEqual(job!.foo, "bar") 114 | 115 | try await self.app.queues.queue.worker.run() 116 | XCTAssertEqual(self.app.queues.test.queue.count, 0) 117 | XCTAssertEqual(self.app.queues.test.jobs.count, 0) 118 | 119 | await XCTAssertEqualAsync(try await promise.futureResult.get(), "bar") 120 | } 121 | 122 | func testRunUntilEmpty() async throws { 123 | let promise1 = self.app.eventLoopGroup.any().makePromise(of: String.self) 124 | self.app.queues.add(Foo1(promise: promise1)) 125 | let promise2 = self.app.eventLoopGroup.any().makePromise(of: String.self) 126 | self.app.queues.add(Foo2(promise: promise2)) 127 | 128 | self.app.get("foo") { req in 129 | try await req.queue.dispatch(Foo1.self, .init(foo: "bar")) 130 | try await req.queue.dispatch(Foo1.self, .init(foo: "quux")) 131 | try await req.queue.dispatch(Foo2.self, .init(foo: "baz")) 132 | return "done" 133 | } 134 | 135 | try await self.app.testable().test(.GET, "foo") { res async in 136 | XCTAssertEqual(res.status, .ok) 137 | XCTAssertEqual(res.body.string, "done") 138 | } 139 | 140 | XCTAssertEqual(self.app.queues.test.queue.count, 3) 141 | XCTAssertEqual(self.app.queues.test.jobs.count, 3) 142 | try await self.app.queues.queue.worker.run() 143 | XCTAssertEqual(self.app.queues.test.queue.count, 0) 144 | XCTAssertEqual(self.app.queues.test.jobs.count, 0) 145 | 146 | await XCTAssertEqualAsync(try await promise1.futureResult.get(), "quux") 147 | await XCTAssertEqualAsync(try await promise2.futureResult.get(), "baz") 148 | } 149 | 150 | func testSettingCustomId() async throws { 151 | let promise = self.app.eventLoopGroup.any().makePromise(of: String.self) 152 | self.app.queues.add(Foo1(promise: promise)) 153 | 154 | self.app.get("foo") { req in 155 | try await req.queue.dispatch(Foo1.self, .init(foo: "bar"), id: JobIdentifier(string: "my-custom-id")) 156 | return "done" 157 | } 158 | 159 | try await self.app.testable().test(.GET, "foo") { res async in 160 | XCTAssertEqual(res.status, .ok) 161 | XCTAssertEqual(res.body.string, "done") 162 | } 163 | 164 | XCTAssertEqual(self.app.queues.test.queue.count, 1) 165 | XCTAssertEqual(self.app.queues.test.jobs.count, 1) 166 | XCTAssert(self.app.queues.test.jobs.keys.map(\.string).contains("my-custom-id")) 167 | 168 | try await self.app.queues.queue.worker.run() 169 | XCTAssertEqual(self.app.queues.test.queue.count, 0) 170 | XCTAssertEqual(self.app.queues.test.jobs.count, 0) 171 | 172 | await XCTAssertEqualAsync(try await promise.futureResult.get(), "bar") 173 | } 174 | 175 | func testScheduleBuilderAPI() async throws { 176 | // yearly 177 | self.app.queues.schedule(Cleanup()).yearly().in(.may).on(23).at(.noon) 178 | 179 | // monthly 180 | self.app.queues.schedule(Cleanup()).monthly().on(15).at(.midnight) 181 | 182 | // weekly 183 | self.app.queues.schedule(Cleanup()).weekly().on(.monday).at("3:13am") 184 | 185 | // daily 186 | self.app.queues.schedule(Cleanup()).daily().at("5:23pm") 187 | 188 | // daily 2 189 | self.app.queues.schedule(Cleanup()).daily().at(5, 23, .pm) 190 | 191 | // daily 3 192 | self.app.queues.schedule(Cleanup()).daily().at(17, 23) 193 | 194 | // hourly 195 | self.app.queues.schedule(Cleanup()).hourly().at(30) 196 | } 197 | 198 | func testRepeatingScheduledJob() async throws { 199 | let scheduledJob = TestingScheduledJob() 200 | XCTAssertEqual(scheduledJob.count.load(ordering: .relaxed), 0) 201 | self.app.queues.schedule(scheduledJob).everySecond() 202 | try self.app.queues.startScheduledJobs() 203 | 204 | let promise = self.app.eventLoopGroup.any().makePromise(of: Void.self) 205 | self.app.eventLoopGroup.any().scheduleTask(in: .seconds(5)) { 206 | XCTAssert(scheduledJob.count.load(ordering: .relaxed) > 4) 207 | promise.succeed() 208 | } 209 | 210 | try await promise.futureResult.get() 211 | } 212 | 213 | func testAsyncRepeatingScheduledJob() async throws { 214 | let scheduledJob = AsyncTestingScheduledJob() 215 | XCTAssertEqual(scheduledJob.count.load(ordering: .relaxed), 0) 216 | self.app.queues.schedule(scheduledJob).everySecond() 217 | try self.app.queues.startScheduledJobs() 218 | 219 | let promise = self.app.eventLoopGroup.any().makePromise(of: Void.self) 220 | self.app.eventLoopGroup.any().scheduleTask(in: .seconds(5)) { 221 | XCTAssert(scheduledJob.count.load(ordering: .relaxed) > 4) 222 | promise.succeed() 223 | } 224 | 225 | try await promise.futureResult.get() 226 | } 227 | 228 | func testFailingScheduledJob() async throws { 229 | self.app.queues.schedule(FailingScheduledJob()).everySecond() 230 | try self.app.queues.startScheduledJobs() 231 | 232 | let promise = self.app.eventLoopGroup.any().makePromise(of: Void.self) 233 | self.app.eventLoopGroup.any().scheduleTask(in: .seconds(1)) { 234 | promise.succeed() 235 | } 236 | try await promise.futureResult.get() 237 | } 238 | 239 | func testAsyncFailingScheduledJob() async throws { 240 | self.app.queues.schedule(AsyncFailingScheduledJob()).everySecond() 241 | try self.app.queues.startScheduledJobs() 242 | 243 | let promise = self.app.eventLoopGroup.any().makePromise(of: Void.self) 244 | self.app.eventLoopGroup.any().scheduleTask(in: .seconds(1)) { 245 | promise.succeed() 246 | } 247 | try await promise.futureResult.get() 248 | } 249 | 250 | func testCustomWorkerCount() async throws { 251 | // Setup custom ELG with 4 threads 252 | let eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: 4) 253 | 254 | do { 255 | let count = self.app.eventLoopGroup.any().makePromise(of: Int.self) 256 | self.app.queues.use(custom: WorkerCountDriver(count: count)) 257 | // Limit worker count to less than 4 threads 258 | self.app.queues.configuration.workerCount = 2 259 | 260 | try self.app.queues.startInProcessJobs(on: .default) 261 | await XCTAssertEqualAsync(try await count.futureResult.get(), 2) 262 | } catch { 263 | try? await eventLoopGroup.shutdownGracefully() 264 | throw error 265 | } 266 | try await eventLoopGroup.shutdownGracefully() 267 | } 268 | 269 | func testSuccessHooks() async throws { 270 | let promise = self.app.eventLoopGroup.any().makePromise(of: String.self) 271 | let successHook = SuccessHook() 272 | let errorHook = ErrorHook() 273 | let dispatchHook = DispatchHook() 274 | let dequeuedHook = DequeuedHook() 275 | self.app.queues.add(Foo1(promise: promise)) 276 | self.app.queues.add(successHook) 277 | self.app.queues.add(errorHook) 278 | self.app.queues.add(dispatchHook) 279 | self.app.queues.add(dequeuedHook) 280 | 281 | self.app.get("foo") { req in 282 | try await req.queue.dispatch(Foo1.self, .init(foo: "bar")) 283 | return "done" 284 | } 285 | 286 | XCTAssertFalse(dispatchHook.successHit) 287 | try await self.app.testable().test(.GET, "foo") { res async in 288 | XCTAssertEqual(res.status, .ok) 289 | XCTAssertEqual(res.body.string, "done") 290 | XCTAssertTrue(dispatchHook.successHit) 291 | } 292 | 293 | XCTAssertFalse(successHook.successHit) 294 | XCTAssertEqual(errorHook.errorCount, 0) 295 | XCTAssertEqual(self.app.queues.test.queue.count, 1) 296 | XCTAssertEqual(self.app.queues.test.jobs.count, 1) 297 | let job = self.app.queues.test.first(Foo1.self) 298 | XCTAssert(self.app.queues.test.contains(Foo1.self)) 299 | XCTAssertNotNil(job) 300 | XCTAssertEqual(job!.foo, "bar") 301 | XCTAssertFalse(dequeuedHook.successHit) 302 | 303 | try await self.app.queues.queue.worker.run() 304 | XCTAssertTrue(successHook.successHit) 305 | XCTAssertEqual(errorHook.errorCount, 0) 306 | XCTAssertEqual(self.app.queues.test.queue.count, 0) 307 | XCTAssertEqual(self.app.queues.test.jobs.count, 0) 308 | XCTAssertTrue(dequeuedHook.successHit) 309 | 310 | await XCTAssertEqualAsync(try await promise.futureResult.get(), "bar") 311 | } 312 | 313 | func testAsyncSuccessHooks() async throws { 314 | let promise = self.app.eventLoopGroup.any().makePromise(of: String.self) 315 | let successHook = AsyncSuccessHook() 316 | let errorHook = AsyncErrorHook() 317 | let dispatchHook = AsyncDispatchHook() 318 | let dequeuedHook = AsyncDequeuedHook() 319 | self.app.queues.add(Foo1(promise: promise)) 320 | self.app.queues.add(successHook) 321 | self.app.queues.add(errorHook) 322 | self.app.queues.add(dispatchHook) 323 | self.app.queues.add(dequeuedHook) 324 | 325 | self.app.get("foo") { req in 326 | try await req.queue.dispatch(Foo1.self, .init(foo: "bar")) 327 | return "done" 328 | } 329 | 330 | await XCTAssertFalseAsync(await dispatchHook.successHit) 331 | try await self.app.testable().test(.GET, "foo") { res async in 332 | XCTAssertEqual(res.status, .ok) 333 | XCTAssertEqual(res.body.string, "done") 334 | await XCTAssertTrueAsync(await dispatchHook.successHit) 335 | } 336 | 337 | await XCTAssertFalseAsync(await successHook.successHit) 338 | await XCTAssertEqualAsync(await errorHook.errorCount, 0) 339 | XCTAssertEqual(self.app.queues.test.queue.count, 1) 340 | XCTAssertEqual(self.app.queues.test.jobs.count, 1) 341 | let job = self.app.queues.test.first(Foo1.self) 342 | XCTAssert(self.app.queues.test.contains(Foo1.self)) 343 | XCTAssertNotNil(job) 344 | XCTAssertEqual(job!.foo, "bar") 345 | await XCTAssertFalseAsync(await dequeuedHook.successHit) 346 | 347 | try await self.app.queues.queue.worker.run() 348 | await XCTAssertTrueAsync(await successHook.successHit) 349 | await XCTAssertEqualAsync(await errorHook.errorCount, 0) 350 | XCTAssertEqual(self.app.queues.test.queue.count, 0) 351 | XCTAssertEqual(self.app.queues.test.jobs.count, 0) 352 | await XCTAssertTrueAsync(await dequeuedHook.successHit) 353 | 354 | await XCTAssertEqualAsync(try await promise.futureResult.get(), "bar") 355 | } 356 | 357 | func testFailureHooks() async throws { 358 | self.app.queues.use(.test) 359 | self.app.queues.add(Bar()) 360 | let successHook = SuccessHook() 361 | let errorHook = ErrorHook() 362 | self.app.queues.add(successHook) 363 | self.app.queues.add(errorHook) 364 | 365 | self.app.get("foo") { req in 366 | try await req.queue.dispatch(Bar.self, .init(foo: "bar"), maxRetryCount: 3) 367 | return "done" 368 | } 369 | 370 | try await self.app.testable().test(.GET, "foo") { res async in 371 | XCTAssertEqual(res.status, .ok) 372 | XCTAssertEqual(res.body.string, "done") 373 | } 374 | 375 | XCTAssertFalse(successHook.successHit) 376 | XCTAssertEqual(errorHook.errorCount, 0) 377 | XCTAssertEqual(self.app.queues.test.queue.count, 1) 378 | XCTAssertEqual(self.app.queues.test.jobs.count, 1) 379 | let job = self.app.queues.test.first(Bar.self) 380 | XCTAssert(self.app.queues.test.contains(Bar.self)) 381 | XCTAssertNotNil(job) 382 | 383 | try await self.app.queues.queue.worker.run() 384 | try await self.app.queues.queue.worker.run() 385 | try await self.app.queues.queue.worker.run() 386 | try await self.app.queues.queue.worker.run() 387 | XCTAssertFalse(successHook.successHit) 388 | XCTAssertEqual(errorHook.errorCount, 1) 389 | XCTAssertEqual(self.app.queues.test.queue.count, 0) 390 | XCTAssertEqual(self.app.queues.test.jobs.count, 0) 391 | } 392 | 393 | func testAsyncFailureHooks() async throws { 394 | self.app.queues.use(.test) 395 | self.app.queues.add(Bar()) 396 | let successHook = AsyncSuccessHook() 397 | let errorHook = AsyncErrorHook() 398 | self.app.queues.add(successHook) 399 | self.app.queues.add(errorHook) 400 | 401 | self.app.get("foo") { req in 402 | try await req.queue.dispatch(Bar.self, .init(foo: "bar"), maxRetryCount: 3) 403 | return "done" 404 | } 405 | 406 | try await self.app.testable().test(.GET, "foo") { res async in 407 | XCTAssertEqual(res.status, .ok) 408 | XCTAssertEqual(res.body.string, "done") 409 | } 410 | 411 | await XCTAssertFalseAsync(await successHook.successHit) 412 | await XCTAssertEqualAsync(await errorHook.errorCount, 0) 413 | XCTAssertEqual(self.app.queues.test.queue.count, 1) 414 | XCTAssertEqual(self.app.queues.test.jobs.count, 1) 415 | let job = self.app.queues.test.first(Bar.self) 416 | XCTAssert(self.app.queues.test.contains(Bar.self)) 417 | XCTAssertNotNil(job) 418 | 419 | try await self.app.queues.queue.worker.run() 420 | try await self.app.queues.queue.worker.run() 421 | try await self.app.queues.queue.worker.run() 422 | try await self.app.queues.queue.worker.run() 423 | await XCTAssertFalseAsync(await successHook.successHit) 424 | await XCTAssertEqualAsync(await errorHook.errorCount, 1) 425 | XCTAssertEqual(self.app.queues.test.queue.count, 0) 426 | XCTAssertEqual(self.app.queues.test.jobs.count, 0) 427 | } 428 | 429 | func testFailureHooksWithDelay() async throws { 430 | self.app.queues.add(Baz()) 431 | let successHook = SuccessHook() 432 | let errorHook = ErrorHook() 433 | self.app.queues.add(successHook) 434 | self.app.queues.add(errorHook) 435 | 436 | self.app.get("foo") { req in 437 | try await req.queue.dispatch(Baz.self, .init(foo: "baz"), maxRetryCount: 1) 438 | return "done" 439 | } 440 | 441 | try await self.app.testable().test(.GET, "foo") { res async in 442 | XCTAssertEqual(res.status, .ok) 443 | XCTAssertEqual(res.body.string, "done") 444 | } 445 | 446 | XCTAssertFalse(successHook.successHit) 447 | XCTAssertEqual(errorHook.errorCount, 0) 448 | XCTAssertEqual(self.app.queues.test.queue.count, 1) 449 | XCTAssertEqual(self.app.queues.test.jobs.count, 1) 450 | var job = self.app.queues.test.first(Baz.self) 451 | XCTAssert(self.app.queues.test.contains(Baz.self)) 452 | XCTAssertNotNil(job) 453 | 454 | try await self.app.queues.queue.worker.run() 455 | XCTAssertFalse(successHook.successHit) 456 | XCTAssertEqual(errorHook.errorCount, 0) 457 | XCTAssertEqual(self.app.queues.test.queue.count, 1) 458 | XCTAssertEqual(self.app.queues.test.jobs.count, 1) 459 | job = self.app.queues.test.first(Baz.self) 460 | XCTAssert(self.app.queues.test.contains(Baz.self)) 461 | XCTAssertNotNil(job) 462 | 463 | sleep(1) 464 | 465 | try await self.app.queues.queue.worker.run() 466 | XCTAssertFalse(successHook.successHit) 467 | XCTAssertEqual(errorHook.errorCount, 1) 468 | XCTAssertEqual(self.app.queues.test.queue.count, 0) 469 | XCTAssertEqual(self.app.queues.test.jobs.count, 0) 470 | } 471 | 472 | func testAsyncFailureHooksWithDelay() async throws { 473 | self.app.queues.add(Baz()) 474 | let successHook = AsyncSuccessHook() 475 | let errorHook = AsyncErrorHook() 476 | self.app.queues.add(successHook) 477 | self.app.queues.add(errorHook) 478 | 479 | self.app.get("foo") { req in 480 | try await req.queue.dispatch(Baz.self, .init(foo: "baz"), maxRetryCount: 1) 481 | return "done" 482 | } 483 | 484 | try await self.app.testable().test(.GET, "foo") { res async in 485 | XCTAssertEqual(res.status, .ok) 486 | XCTAssertEqual(res.body.string, "done") 487 | } 488 | 489 | await XCTAssertFalseAsync(await successHook.successHit) 490 | await XCTAssertEqualAsync(await errorHook.errorCount, 0) 491 | XCTAssertEqual(self.app.queues.test.queue.count, 1) 492 | XCTAssertEqual(self.app.queues.test.jobs.count, 1) 493 | var job = self.app.queues.test.first(Baz.self) 494 | XCTAssert(self.app.queues.test.contains(Baz.self)) 495 | XCTAssertNotNil(job) 496 | 497 | try await self.app.queues.queue.worker.run() 498 | await XCTAssertFalseAsync(await successHook.successHit) 499 | await XCTAssertEqualAsync(await errorHook.errorCount, 0) 500 | XCTAssertEqual(self.app.queues.test.queue.count, 1) 501 | XCTAssertEqual(self.app.queues.test.jobs.count, 1) 502 | job = self.app.queues.test.first(Baz.self) 503 | XCTAssert(self.app.queues.test.contains(Baz.self)) 504 | XCTAssertNotNil(job) 505 | 506 | sleep(1) 507 | 508 | try await self.app.queues.queue.worker.run() 509 | await XCTAssertFalseAsync(await successHook.successHit) 510 | await XCTAssertEqualAsync(await errorHook.errorCount, 1) 511 | XCTAssertEqual(self.app.queues.test.queue.count, 0) 512 | XCTAssertEqual(self.app.queues.test.jobs.count, 0) 513 | } 514 | 515 | func testStuffThatIsntActuallyUsedAnywhere() { 516 | XCTAssertEqual(self.app.queues.queue(.default).key, "vapor_queues[default]") 517 | XCTAssertNotNil(QueuesEventLoopPreference.indifferent.delegate(for: self.app.eventLoopGroup)) 518 | XCTAssertNotNil(QueuesEventLoopPreference.delegate(on: self.app.eventLoopGroup.any()).delegate(for: self.app.eventLoopGroup)) 519 | } 520 | } 521 | 522 | final class DispatchHook: JobEventDelegate, @unchecked Sendable { 523 | var successHit = false 524 | 525 | func dispatched(job _: JobEventData, eventLoop: any EventLoop) -> EventLoopFuture { 526 | self.successHit = true 527 | return eventLoop.makeSucceededVoidFuture() 528 | } 529 | } 530 | 531 | final class SuccessHook: JobEventDelegate, @unchecked Sendable { 532 | var successHit = false 533 | 534 | func success(jobId _: String, eventLoop: any EventLoop) -> EventLoopFuture { 535 | self.successHit = true 536 | return eventLoop.makeSucceededVoidFuture() 537 | } 538 | } 539 | 540 | final class ErrorHook: JobEventDelegate, @unchecked Sendable { 541 | var errorCount = 0 542 | 543 | func error(jobId _: String, error _: any Error, eventLoop: any EventLoop) -> EventLoopFuture { 544 | self.errorCount += 1 545 | return eventLoop.makeSucceededVoidFuture() 546 | } 547 | } 548 | 549 | final class DequeuedHook: JobEventDelegate, @unchecked Sendable { 550 | var successHit = false 551 | 552 | func didDequeue(jobId _: String, eventLoop: any EventLoop) -> EventLoopFuture { 553 | self.successHit = true 554 | return eventLoop.makeSucceededVoidFuture() 555 | } 556 | } 557 | 558 | actor AsyncDispatchHook: AsyncJobEventDelegate { 559 | var successHit = false 560 | func dispatched(job _: JobEventData) async throws { self.successHit = true } 561 | } 562 | 563 | actor AsyncSuccessHook: AsyncJobEventDelegate { 564 | var successHit = false 565 | func success(jobId _: String) async throws { self.successHit = true } 566 | } 567 | 568 | actor AsyncErrorHook: AsyncJobEventDelegate { 569 | var errorCount = 0 570 | func error(jobId _: String, error _: any Error) async throws { self.errorCount += 1 } 571 | } 572 | 573 | actor AsyncDequeuedHook: AsyncJobEventDelegate { 574 | var successHit = false 575 | func didDequeue(jobId _: String) async throws { self.successHit = true } 576 | } 577 | 578 | final class WorkerCountDriver: QueuesDriver, @unchecked Sendable { 579 | let count: EventLoopPromise 580 | let lock: NIOLock 581 | var recordedEventLoops: Set 582 | 583 | init(count: EventLoopPromise) { 584 | self.count = count 585 | self.lock = .init() 586 | self.recordedEventLoops = [] 587 | } 588 | 589 | func makeQueue(with context: QueueContext) -> any Queue { 590 | WorkerCountQueue(driver: self, context: context) 591 | } 592 | 593 | func record(eventLoop: any EventLoop) { 594 | self.lock.lock() 595 | defer { self.lock.unlock() } 596 | let previousCount = self.recordedEventLoops.count 597 | self.recordedEventLoops.insert(.init(eventLoop)) 598 | if self.recordedEventLoops.count == previousCount { 599 | // we've detected all unique event loops now 600 | self.count.succeed(previousCount) 601 | } 602 | } 603 | 604 | func shutdown() { 605 | // nothing 606 | } 607 | 608 | private struct WorkerCountQueue: Queue { 609 | let driver: WorkerCountDriver 610 | var context: QueueContext 611 | 612 | func get(_: JobIdentifier) -> EventLoopFuture { fatalError() } 613 | func set(_: JobIdentifier, to _: JobData) -> EventLoopFuture { fatalError() } 614 | func clear(_: JobIdentifier) -> EventLoopFuture { fatalError() } 615 | func pop() -> EventLoopFuture { 616 | self.driver.record(eventLoop: self.context.eventLoop) 617 | return self.context.eventLoop.makeSucceededFuture(nil) 618 | } 619 | 620 | func push(_: JobIdentifier) -> EventLoopFuture { fatalError() } 621 | } 622 | } 623 | 624 | struct FailingScheduledJob: ScheduledJob { 625 | func run(context: QueueContext) -> EventLoopFuture { context.eventLoop.makeFailedFuture(Failure()) } 626 | } 627 | 628 | struct AsyncFailingScheduledJob: AsyncScheduledJob { 629 | func run(context _: QueueContext) async throws { throw Failure() } 630 | } 631 | 632 | struct TestingScheduledJob: ScheduledJob { 633 | var count = ManagedAtomic(0) 634 | 635 | func run(context: QueueContext) -> EventLoopFuture { 636 | self.count.wrappingIncrement(ordering: .relaxed) 637 | return context.eventLoop.makeSucceededVoidFuture() 638 | } 639 | } 640 | 641 | struct AsyncTestingScheduledJob: AsyncScheduledJob { 642 | var count = ManagedAtomic(0) 643 | func run(context _: QueueContext) async throws { self.count.wrappingIncrement(ordering: .relaxed) } 644 | } 645 | 646 | struct Foo1: Job { 647 | let promise: EventLoopPromise 648 | 649 | struct Data: Codable { 650 | var foo: String 651 | } 652 | 653 | func dequeue(_ context: QueueContext, _ data: Data) -> EventLoopFuture { 654 | self.promise.succeed(data.foo) 655 | return context.eventLoop.makeSucceededVoidFuture() 656 | } 657 | 658 | func error(_ context: QueueContext, _ error: any Error, _: Data) -> EventLoopFuture { 659 | self.promise.fail(error) 660 | return context.eventLoop.makeSucceededVoidFuture() 661 | } 662 | } 663 | 664 | struct Foo2: Job { 665 | let promise: EventLoopPromise 666 | 667 | struct Data: Codable { 668 | var foo: String 669 | } 670 | 671 | func dequeue(_ context: QueueContext, _ data: Data) -> EventLoopFuture { 672 | self.promise.succeed(data.foo) 673 | return context.eventLoop.makeSucceededVoidFuture() 674 | } 675 | 676 | func error(_ context: QueueContext, _ error: any Error, _: Data) -> EventLoopFuture { 677 | self.promise.fail(error) 678 | return context.eventLoop.makeSucceededVoidFuture() 679 | } 680 | } 681 | 682 | struct Bar: Job { 683 | struct Data: Codable { 684 | var foo: String 685 | } 686 | 687 | func dequeue(_ context: QueueContext, _: Data) -> EventLoopFuture { 688 | context.eventLoop.makeFailedFuture(Abort(.badRequest)) 689 | } 690 | 691 | func error(_ context: QueueContext, _: any Error, _: Data) -> EventLoopFuture { 692 | context.eventLoop.makeSucceededVoidFuture() 693 | } 694 | } 695 | 696 | struct Baz: Job { 697 | struct Data: Codable { 698 | var foo: String 699 | } 700 | 701 | func dequeue(_ context: QueueContext, _: Data) -> EventLoopFuture { 702 | context.eventLoop.makeFailedFuture(Abort(.badRequest)) 703 | } 704 | 705 | func error(_ context: QueueContext, _: any Error, _: Data) -> EventLoopFuture { 706 | context.eventLoop.makeSucceededVoidFuture() 707 | } 708 | 709 | func nextRetryIn(attempt: Int) -> Int { 710 | attempt 711 | } 712 | } 713 | -------------------------------------------------------------------------------- /Tests/QueuesTests/ScheduleBuilderTests.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import Queues 3 | import XCTest 4 | 5 | final class ScheduleBuilderTests: XCTestCase { 6 | override class func setUp() { 7 | XCTAssert(isLoggingConfigured) 8 | } 9 | 10 | func testHourlyBuilder() throws { 11 | let builder = ScheduleBuilder() 12 | builder.hourly().at(30) 13 | // same time 14 | XCTAssertEqual( 15 | builder.nextDate(current: Date(hour: 5, minute: 30)), 16 | // plus one hour 17 | Date(hour: 6, minute: 30) 18 | ) 19 | // just before 20 | XCTAssertEqual( 21 | builder.nextDate(current: Date(hour: 5, minute: 29)), 22 | // plus one minute 23 | Date(hour: 5, minute: 30) 24 | ) 25 | // just after 26 | XCTAssertEqual( 27 | builder.nextDate(current: Date(hour: 5, minute: 31)), 28 | // plus one hour 29 | Date(hour: 6, minute: 30) 30 | ) 31 | } 32 | 33 | func testDailyBuilder() throws { 34 | let builder = ScheduleBuilder() 35 | builder.daily().at("5:23am") 36 | // same time 37 | XCTAssertEqual( 38 | builder.nextDate(current: Date(day: 1, hour: 5, minute: 23)), 39 | // plus one day 40 | Date(day: 2, hour: 5, minute: 23) 41 | ) 42 | // just before 43 | XCTAssertEqual( 44 | builder.nextDate(current: Date(day: 1, hour: 5, minute: 22)), 45 | // plus one minute 46 | Date(day: 1, hour: 5, minute: 23) 47 | ) 48 | // just after 49 | XCTAssertEqual( 50 | builder.nextDate(current: Date(day: 1, hour: 5, minute: 24)), 51 | // plus one day 52 | Date(day: 2, hour: 5, minute: 23) 53 | ) 54 | } 55 | 56 | func testWeeklyBuilder() throws { 57 | let builder = ScheduleBuilder() 58 | builder.weekly().on(.monday).at(.noon) 59 | // sunday before 60 | XCTAssertEqual( 61 | builder.nextDate(current: Date(year: 2019, month: 1, day: 6, hour: 5, minute: 23)), 62 | // next day at noon 63 | Date(year: 2019, month: 1, day: 7, hour: 12, minute: 00) 64 | ) 65 | // monday at 1pm 66 | XCTAssertEqual( 67 | builder.nextDate(current: Date(year: 2019, month: 1, day: 7, hour: 13, minute: 00)), 68 | // next monday at noon 69 | Date(year: 2019, month: 1, day: 14, hour: 12, minute: 00) 70 | ) 71 | // monday at 11:30am 72 | XCTAssertEqual( 73 | builder.nextDate(current: Date(year: 2019, month: 1, day: 7, hour: 11, minute: 30)), 74 | // same day at noon 75 | Date(year: 2019, month: 1, day: 7, hour: 12, minute: 00) 76 | ) 77 | } 78 | 79 | func testMonthlyBuilderFirstDay() throws { 80 | let builder = ScheduleBuilder() 81 | builder.monthly().on(.first).at(.noon) 82 | // middle of jan 83 | XCTAssertEqual( 84 | builder.nextDate(current: Date(year: 2019, month: 1, day: 15, hour: 5, minute: 23)), 85 | // first of feb 86 | Date(year: 2019, month: 2, day: 1, hour: 12, minute: 00) 87 | ) 88 | // just before 89 | XCTAssertEqual( 90 | builder.nextDate(current: Date(year: 2019, month: 2, day: 1, hour: 11, minute: 30)), 91 | // first of feb 92 | Date(year: 2019, month: 2, day: 1, hour: 12, minute: 00) 93 | ) 94 | // just after 95 | XCTAssertEqual( 96 | builder.nextDate(current: Date(year: 2019, month: 2, day: 1, hour: 12, minute: 30)), 97 | // first of feb 98 | Date(year: 2019, month: 3, day: 1, hour: 12, minute: 00) 99 | ) 100 | } 101 | 102 | func testMonthlyBuilder15th() throws { 103 | let builder = ScheduleBuilder() 104 | builder.monthly().on(15).at(.noon) 105 | // just before 106 | XCTAssertEqual( 107 | builder.nextDate(current: Date(year: 2019, month: 2, day: 15, hour: 11, minute: 30)), 108 | // first of feb 109 | Date(year: 2019, month: 2, day: 15, hour: 12, minute: 00) 110 | ) 111 | // just after 112 | XCTAssertEqual( 113 | builder.nextDate(current: Date(year: 2019, month: 2, day: 15, hour: 12, minute: 30)), 114 | // first of feb 115 | Date(year: 2019, month: 3, day: 15, hour: 12, minute: 00) 116 | ) 117 | } 118 | 119 | func testYearlyBuilder() throws { 120 | let builder = ScheduleBuilder() 121 | builder.yearly().in(.may).on(23).at("2:58pm") 122 | // early in the year 123 | XCTAssertEqual( 124 | builder.nextDate(current: Date(year: 2019, month: 1, day: 15, hour: 5, minute: 23)), 125 | // 2019 126 | Date(year: 2019, month: 5, day: 23, hour: 14, minute: 58) 127 | ) 128 | // just before 129 | XCTAssertEqual( 130 | builder.nextDate(current: Date(year: 2019, month: 5, day: 23, hour: 14, minute: 57)), 131 | // one minute later 132 | Date(year: 2019, month: 5, day: 23, hour: 14, minute: 58) 133 | ) 134 | // just after 135 | XCTAssertEqual( 136 | builder.nextDate(current: Date(year: 2019, month: 5, day: 23, hour: 14, minute: 59)), 137 | // one year later 138 | Date(year: 2020, month: 5, day: 23, hour: 14, minute: 58) 139 | ) 140 | } 141 | 142 | func testCustomCalendarBuilder() throws { 143 | let est = Calendar.calendar(timezone: "EST") 144 | let mst = Calendar.calendar(timezone: "MST") 145 | 146 | // Create a date at 8:00pm EST 147 | let estDate = Date(calendar: est, hour: 20, minute: 00) 148 | 149 | // Schedule it for 7:00pm MST 150 | let builder = ScheduleBuilder(calendar: mst) 151 | builder.daily().at("7:00pm") 152 | 153 | XCTAssertEqual( 154 | builder.nextDate(current: estDate), 155 | // one hour later 156 | Date(calendar: est, hour: 21, minute: 00) 157 | ) 158 | } 159 | 160 | } 161 | 162 | final class Cleanup: ScheduledJob { 163 | func run(context: QueueContext) -> EventLoopFuture { 164 | context.eventLoop.makeSucceededVoidFuture() 165 | } 166 | } 167 | 168 | extension Date { 169 | init( 170 | calendar: Calendar = .current, 171 | year: Int = 2020, 172 | month: Int = 1, 173 | day: Int = 1, 174 | hour: Int = 0, 175 | minute: Int = 0, 176 | second: Int = 0 177 | ) { 178 | self = DateComponents( 179 | calendar: calendar, 180 | year: year, month: month, day: day, hour: hour, minute: minute, second: second 181 | ).date! 182 | } 183 | } 184 | 185 | extension Calendar { 186 | fileprivate static func calendar(timezone identifier: String) -> Calendar { 187 | var calendar = Calendar.current 188 | calendar.timeZone = TimeZone(identifier: identifier)! 189 | return calendar 190 | } 191 | } 192 | -------------------------------------------------------------------------------- /Tests/QueuesTests/Utilities.swift: -------------------------------------------------------------------------------- 1 | import Logging 2 | import class Foundation.ProcessInfo 3 | 4 | func env(_ name: String) -> String? { 5 | return ProcessInfo.processInfo.environment[name] 6 | } 7 | 8 | let isLoggingConfigured: Bool = { 9 | LoggingSystem.bootstrap { label in 10 | var handler = StreamLogHandler.standardOutput(label: label) 11 | handler.logLevel = env("LOG_LEVEL").flatMap { .init(rawValue: $0) } ?? .info 12 | return handler 13 | } 14 | return true 15 | }() 16 | -------------------------------------------------------------------------------- /Tests/QueuesTests/Utilities/FailingAsyncJob.swift: -------------------------------------------------------------------------------- 1 | import Queues 2 | 3 | struct FailingAsyncJob: AsyncJob { 4 | let promise: EventLoopPromise 5 | 6 | struct Payload: Codable { 7 | var foo: String 8 | } 9 | 10 | func dequeue(_: QueueContext, _: Payload) async throws { 11 | let failure = Failure() 12 | self.promise.fail(failure) 13 | throw failure 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /Tests/QueuesTests/Utilities/Failure.swift: -------------------------------------------------------------------------------- 1 | struct Failure: Error {} 2 | -------------------------------------------------------------------------------- /Tests/QueuesTests/Utilities/MyAsyncJob.swift: -------------------------------------------------------------------------------- 1 | import Queues 2 | 3 | struct MyAsyncJob: AsyncJob { 4 | let promise: EventLoopPromise 5 | 6 | struct Payload: Codable { 7 | var foo: String 8 | } 9 | 10 | func dequeue(_: QueueContext, _: Payload) async throws { 11 | self.promise.succeed() 12 | } 13 | } 14 | --------------------------------------------------------------------------------