├── Tests ├── LinuxMain.swift └── QueuesFluentDriverTests │ └── QueuesFluentDriverTests.swift ├── .gitignore ├── Sources └── QueuesFluentDriver │ ├── PopQueries │ ├── PopQueryProtocol.swift │ ├── PostgresPopQuery.swift │ ├── SqlitePopQuery.swift │ └── MySQLPopQuery.swift │ ├── QueuesFluentError.swift │ ├── SQLExpressionExtensions.swift │ ├── Queues.Provider+fluent.swift │ ├── JobDataModel.swift │ ├── JobModel.swift │ ├── FluentQueuesDriver.swift │ ├── JobModelMigrate.swift │ └── FluentQueue.swift ├── .github └── workflows │ └── test.yml ├── LICENSE ├── Package.swift └── README.md /Tests/LinuxMain.swift: -------------------------------------------------------------------------------- 1 | fatalError("Please use swift test --enable-test-discovery to run the tests instead") 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | /.build 3 | /Packages 4 | /*.xcodeproj 5 | xcuserdata/ 6 | .swiftpm 7 | Package.resolved 8 | -------------------------------------------------------------------------------- /Sources/QueuesFluentDriver/PopQueries/PopQueryProtocol.swift: -------------------------------------------------------------------------------- 1 | import SQLKit 2 | import Fluent 3 | 4 | protocol PopQueryProtocol { 5 | func pop(db: Database, select: SQLExpression) -> EventLoopFuture 6 | } 7 | -------------------------------------------------------------------------------- /Tests/QueuesFluentDriverTests/QueuesFluentDriverTests.swift: -------------------------------------------------------------------------------- 1 | import XCTest 2 | @testable import QueuesFluentDriver 3 | 4 | final class QueuesFluentDriverTests: XCTestCase { 5 | func testExample() { 6 | // This is an example of a functional test case. 7 | // Use XCTAssert and related functions to verify your tests produce the correct 8 | // results. 9 | //XCTAssertEqual(QueuesFluentDriver().text, "Hello, World!") 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /Sources/QueuesFluentDriver/QueuesFluentError.swift: -------------------------------------------------------------------------------- 1 | import Queues 2 | 3 | enum QueuesFluentError: Error { 4 | /// Couldn't find a job with this Id 5 | case missingJob(_ id: JobIdentifier) 6 | /// The JobIdentifier is not a valid UUID 7 | case invalidIdentifier 8 | /// Error encoding the json Payload to JSON 9 | case jobDataEncodingError(_ message: String? = nil) 10 | case jobDataDecodingError(_ message: String? = nil) 11 | /// The given DatabaseID doesn't match any existing database configured in the Vapor app. 12 | case databaseNotFound 13 | } 14 | -------------------------------------------------------------------------------- /Sources/QueuesFluentDriver/SQLExpressionExtensions.swift: -------------------------------------------------------------------------------- 1 | import Fluent 2 | import SQLKit 3 | 4 | enum SQLSkipLocked: SQLExpression { 5 | case forUpdateSkipLocked 6 | case forShareSkipLocked 7 | 8 | public func serialize(to serializer: inout SQLSerializer) { 9 | switch self { 10 | case .forUpdateSkipLocked: 11 | serializer.write("FOR UPDATE SKIP LOCKED") 12 | case .forShareSkipLocked: 13 | // This is the "lightest" locking that is supported by both Postgres and Mysql 14 | serializer.write("FOR SHARE SKIP LOCKED") 15 | } 16 | } 17 | } 18 | 19 | -------------------------------------------------------------------------------- /Sources/QueuesFluentDriver/Queues.Provider+fluent.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import Vapor 3 | import Fluent 4 | import Queues 5 | 6 | extension Application.Queues.Provider { 7 | /// `databaseId`: a Fluent `DatabaseID` configured in your application. 8 | /// `useSoftDeletes`: if set to `true`, flag completed jobs using Fluent's default SoftDelete feature instead of actually deleting them. 9 | public static func fluent(_ databaseId: DatabaseID? = nil, useSoftDeletes: Bool = false) -> Self { 10 | .init { 11 | $0.queues.use(custom: 12 | FluentQueuesDriver(on: databaseId, useSoftDeletes: useSoftDeletes, on: $0.eventLoopGroup) 13 | ) 14 | } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: test 2 | on: 3 | pull_request: 4 | push: 5 | branches: 6 | - master 7 | jobs: 8 | linux: 9 | runs-on: ubuntu-latest 10 | container: swift:5.2-bionic 11 | steps: 12 | - name: Check out code 13 | uses: actions/checkout@v2 14 | - name: Run tests with Thread Sanitizer 15 | run: swift test --enable-test-discovery --sanitize=thread 16 | macOS: 17 | runs-on: macos-latest 18 | steps: 19 | - name: Select latest available Xcode 20 | uses: maxim-lobanov/setup-xcode@1.0 21 | with: 22 | xcode-version: latest 23 | - name: Check out code 24 | uses: actions/checkout@v2 25 | - name: Run tests with Thread Sanitizer 26 | run: swift test --enable-test-discovery --sanitize=thread -------------------------------------------------------------------------------- /Sources/QueuesFluentDriver/PopQueries/PostgresPopQuery.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import Foundation 3 | import SQLKit 4 | import Fluent 5 | 6 | final class PostgresPop: PopQueryProtocol { 7 | func pop(db: Database, select: SQLExpression) -> EventLoopFuture { 8 | let db = db as! SQLDatabase 9 | let subQueryGroup = SQLGroupExpression.init(select) 10 | let query = db 11 | .update (JobModel.schema) 12 | .set (SQLColumn("\(FieldKey.state)"), to: SQLBind(QueuesFluentJobState.processing)) 13 | .set (SQLColumn("\(FieldKey.updatedAt)"), to: SQLBind(Date())) 14 | .where ( 15 | SQLBinaryExpression(left: SQLColumn("\(FieldKey.id)"), op: SQLBinaryOperator.equal , right: subQueryGroup) 16 | ) 17 | .returning(SQLColumn("\(FieldKey.id)")) 18 | .query 19 | 20 | var id: String? 21 | return db.execute(sql: query) { (row) -> Void in 22 | id = try? row.decode(column: "\(FieldKey.id)", as: String.self) 23 | }.map { 24 | return id 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Matthieu Barthélemy 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Sources/QueuesFluentDriver/PopQueries/SqlitePopQuery.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import SQLKit 3 | import Fluent 4 | 5 | final class SqlitePop: PopQueryProtocol { 6 | func pop(db: Database, select: SQLExpression) -> EventLoopFuture { 7 | db.transaction { transaction in 8 | let database = transaction as! SQLDatabase 9 | var id: String? 10 | 11 | return database.execute(sql: select) { (row) -> Void in 12 | id = try? row.decode(column: "\(FieldKey.id)", as: String.self) 13 | } 14 | .flatMap { 15 | guard let id = id else { 16 | return database.eventLoop.makeSucceededFuture(nil) 17 | } 18 | let updateQuery = database 19 | .update(JobModel.schema) 20 | .set(SQLColumn("\(FieldKey.state)"), to: SQLBind(QueuesFluentJobState.processing)) 21 | .set(SQLColumn("\(FieldKey.updatedAt)"), to: SQLBind(Date())) 22 | .where(SQLColumn("\(FieldKey.id)"), .equal, SQLBind(id)) 23 | .where(SQLColumn("\(FieldKey.state)"), .equal, SQLBind(QueuesFluentJobState.pending)) 24 | .query 25 | return database.execute(sql: updateQuery) { (row) in } 26 | .map { id } 27 | } 28 | 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /Package.swift: -------------------------------------------------------------------------------- 1 | // swift-tools-version:5.4 2 | // The swift-tools-version declares the minimum version of Swift required to build this package. 3 | 4 | import PackageDescription 5 | 6 | let package = Package( 7 | name: "QueuesFluentDriver", 8 | platforms: [ 9 | .macOS(.v10_15) 10 | ], 11 | products: [ 12 | .library( 13 | name: "QueuesFluentDriver", 14 | targets: ["QueuesFluentDriver"]), 15 | ], 16 | dependencies: [ 17 | .package(url: "https://github.com/vapor/vapor.git", from: "4.0.0"), 18 | .package(url: "https://github.com/vapor/fluent.git", from: "4.0.0"), 19 | .package(url: "https://github.com/vapor/sql-kit.git", from: "3.6.0"), 20 | .package(url: "https://github.com/vapor/queues.git", from: "1.11.1"), 21 | ], 22 | targets: [ 23 | .target( 24 | name: "QueuesFluentDriver", 25 | dependencies: [ 26 | .product(name: "Vapor", package: "vapor"), 27 | .product(name: "Fluent", package: "fluent"), 28 | .product(name: "SQLKit", package: "sql-kit"), 29 | .product(name: "Queues", package: "queues") 30 | ], 31 | path: "Sources" 32 | ), 33 | .testTarget( 34 | name: "QueuesFluentDriverTests", 35 | dependencies: ["QueuesFluentDriver"] 36 | ), 37 | ] 38 | ) 39 | -------------------------------------------------------------------------------- /Sources/QueuesFluentDriver/JobDataModel.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import Fluent 3 | import Queues 4 | 5 | extension FieldKey { 6 | static var payload: Self { "payload" } 7 | static var maxRetryCount: Self { "max_retries" } 8 | static var attempts: Self { "attempt" } 9 | static var delayUntil: Self { "delay_until" } 10 | static var queuedAt: Self { "queued_at" } 11 | static var jobName: Self { "job_name" } 12 | } 13 | 14 | /// Handles storage of a `JobData` into the database 15 | final class JobDataModel: Fields { 16 | required init() {} 17 | 18 | /// The job data to be encoded. 19 | @Field(key: .payload) 20 | var payload: [UInt8] 21 | 22 | /// The maxRetryCount for the `Job`. 23 | @Field(key: .maxRetryCount) 24 | var maxRetryCount: Int 25 | 26 | /// The number of attempts made to run the `Job`. 27 | @Field(key: .attempts) 28 | var attempts: Int? 29 | 30 | /// A date to execute this job after 31 | @OptionalField(key: .delayUntil) 32 | var delayUntil: Date? 33 | 34 | /// The date this job was queued 35 | @Field(key: .queuedAt) 36 | var queuedAt: Date 37 | 38 | /// The name of the `Job` 39 | @Field(key: .jobName) 40 | var jobName: String 41 | 42 | init(jobData: JobData) { 43 | self.payload = jobData.payload 44 | self.maxRetryCount = jobData.maxRetryCount 45 | self.attempts = jobData.attempts 46 | self.delayUntil = jobData.delayUntil 47 | self.jobName = jobData.jobName 48 | self.queuedAt = jobData.queuedAt 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /Sources/QueuesFluentDriver/PopQueries/MySQLPopQuery.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import SQLKit 3 | import Fluent 4 | 5 | final class MySQLPop: PopQueryProtocol { 6 | // MySQL is a bit challenging since it doesn't support updating a table that is 7 | // used in a subquery. 8 | // So we first select, then update, with the whole process wrapped in a transaction. 9 | func pop(db: Database, select: SQLExpression) -> EventLoopFuture { 10 | db.transaction { transaction in 11 | let database = transaction as! SQLDatabase 12 | var id: String? 13 | 14 | return database.execute(sql: select) { (row) -> Void in 15 | id = try? row.decode(column: "\(FieldKey.id)", as: String.self) 16 | } 17 | .flatMap { 18 | guard let id = id else { 19 | return database.eventLoop.makeSucceededFuture(nil) 20 | } 21 | let updateQuery = database 22 | .update(JobModel.schema) 23 | .set(SQLColumn("\(FieldKey.state)"), to: SQLBind(QueuesFluentJobState.processing)) 24 | .set(SQLColumn("\(FieldKey.updatedAt)"), to: SQLBind(Date())) 25 | .where(SQLColumn("\(FieldKey.id)"), .equal, SQLBind(id)) 26 | .where(SQLColumn("\(FieldKey.state)"), .equal, SQLBind(QueuesFluentJobState.pending)) 27 | .query 28 | return database.execute(sql: updateQuery) { (row) in } 29 | .map { id } 30 | } 31 | 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /Sources/QueuesFluentDriver/JobModel.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import Fluent 3 | import Queues 4 | 5 | public enum QueuesFluentJobState: String, Codable, CaseIterable { 6 | /// Ready to be picked up for execution 7 | case pending 8 | case processing 9 | /// Executed, regardless if it was successful or not 10 | case completed 11 | } 12 | 13 | extension FieldKey { 14 | static var queue: Self { "queue" } 15 | static var data: Self { "data" } 16 | static var state: Self { "state" } 17 | 18 | static var runAt: Self { "run_at" } 19 | static var updatedAt: Self { "updated_at" } 20 | static var deletedAt: Self { "deleted_at" } 21 | } 22 | 23 | class JobModel: Model { 24 | public required init() {} 25 | 26 | public static var schema = "_jobs_meta" 27 | 28 | /// The unique Job ID 29 | @ID(custom: .id, generatedBy: .user) 30 | var id: String? 31 | 32 | /// The Job key 33 | @Field(key: .queue) 34 | var queue: String 35 | 36 | /// The current state of the Job 37 | @Field(key: .state) 38 | var state: QueuesFluentJobState 39 | 40 | /// Earliest date the job can run 41 | @OptionalField(key: .runAt) 42 | var runAtOrAfter: Date? 43 | 44 | @Timestamp(key: .updatedAt, on: .update) 45 | var updatedAt: Date? 46 | 47 | @Timestamp(key: .deletedAt, on: .delete) 48 | var deletedAt: Date? 49 | 50 | @Group(key: .data) 51 | var data: JobDataModel 52 | 53 | init(id: JobIdentifier, queue: String, jobData: JobDataModel) { 54 | self.id = id.string 55 | self.queue = queue 56 | self.state = .pending 57 | self.data = jobData 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /Sources/QueuesFluentDriver/FluentQueuesDriver.swift: -------------------------------------------------------------------------------- 1 | import Fluent 2 | import SQLKit 3 | import Queues 4 | 5 | public enum QueuesFluentDbType: String { 6 | case postgresql 7 | case mysql 8 | case sqlite 9 | } 10 | 11 | public struct FluentQueuesDriver { 12 | let databaseId: DatabaseID? 13 | let useSoftDeletes: Bool 14 | let eventLoopGroup: EventLoopGroup 15 | 16 | init(on databaseId: DatabaseID? = nil, useSoftDeletes: Bool, on eventLoopGroup: EventLoopGroup) { 17 | self.databaseId = databaseId 18 | self.useSoftDeletes = useSoftDeletes 19 | self.eventLoopGroup = eventLoopGroup 20 | } 21 | } 22 | 23 | extension FluentQueuesDriver: QueuesDriver { 24 | public func makeQueue(with context: QueueContext) -> Queue { 25 | let db = context 26 | .application 27 | .databases 28 | .database(databaseId, logger: context.logger, on: context.eventLoop) 29 | 30 | // How do we report that something goes wrong here? Since makeQueue cannot throw. 31 | let dialect = (db as? SQLDatabase)?.dialect.name 32 | if db == nil || dialect == nil { 33 | context.logger.error( 34 | "\(Self.self): Database misconfigured or unsupported." 35 | ) 36 | } 37 | 38 | let dbType = QueuesFluentDbType(rawValue: dialect!) 39 | if dbType == nil { 40 | context.logger.error("\(Self.self): Unsupported Database type '\(dialect!)'") 41 | } 42 | 43 | return FluentQueue( 44 | context: context, 45 | db: db!, 46 | dbType: dbType!, 47 | useSoftDeletes: self.useSoftDeletes 48 | ) 49 | } 50 | 51 | public func shutdown() { 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /Sources/QueuesFluentDriver/JobModelMigrate.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import Fluent 3 | import SQLKit 4 | 5 | public struct JobMetadataMigrate: Migration { 6 | public init() {} 7 | 8 | public init(schema: String) { 9 | JobModel.schema = schema 10 | } 11 | 12 | public func prepare(on database: Database) -> EventLoopFuture { 13 | return database.schema(JobModel.schema) 14 | .field(.id, .string, .identifier(auto: false)) 15 | .field(FieldKey.queue, .string, .required) 16 | .field(FieldKey.state, .string, .required) 17 | .field(FieldKey.runAt, .datetime) 18 | .field(FieldKey.updatedAt, .datetime) 19 | .field(FieldKey.deletedAt, .datetime) 20 | // "group"/nested model JobDataModel 21 | .field(.init(stringLiteral: "\(FieldKey.data)_\(FieldKey.payload)"), .array(of: .uint8), .required) 22 | .field(.init(stringLiteral: "\(FieldKey.data)_\(FieldKey.maxRetryCount)"), .int, .required) 23 | .field(.init(stringLiteral: "\(FieldKey.data)_\(FieldKey.attempts)"), .int) 24 | .field(.init(stringLiteral: "\(FieldKey.data)_\(FieldKey.delayUntil)"), .datetime) 25 | .field(.init(stringLiteral: "\(FieldKey.data)_\(FieldKey.queuedAt)"), .datetime, .required) 26 | .field(.init(stringLiteral: "\(FieldKey.data)_\(FieldKey.jobName)"), .string, .required) 27 | .create() 28 | .flatMap { 29 | // Mysql could lock the entire table if there's no index on the fields of the WHERE clause used in `FluentQueue.pop()`. 30 | // Order of the fields in the composite index and order of the fields in the WHERE clauses should match. 31 | // Or I got totally confused reading their doc, which is also a possibility. 32 | // Postgres seems to not be so sensitive and should be happy with the following indices. 33 | let sqlDb = database as! SQLDatabase 34 | let stateIndex = sqlDb.create(index: "i_\(JobModel.schema)_\(FieldKey.state)_\(FieldKey.queue)") 35 | .on(JobModel.schema) 36 | .column("\(FieldKey.state)") 37 | .column("\(FieldKey.queue)") 38 | .column("\(FieldKey.runAt)") 39 | .run() 40 | return stateIndex.map { index in 41 | return 42 | } 43 | } 44 | } 45 | 46 | public func revert(on database: Database) -> EventLoopFuture { 47 | return database.schema(JobModel.schema).delete() 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # QueuesFluentDriver 2 | 3 | This Vapor Queues driver stores the Queues jobs metadata into a relational database. It is an alternative to the default Redis driver. 4 | 5 | 6 | ## Compatibility 7 | 8 | This package makes use of some relatively recent, non standard SQL extensions added to some major database engines to support this exact use case: queuing systems, where there must be a guarantee that a task or job won't be picked by multiple workers. 9 | 10 | This package should be compatible with: 11 | 12 | - Postgres >= 11 13 | - Mysql >= 8.0.1 14 | - MariaDB >= 10.3 15 | 16 | > Sqlite will only work if you have a custom, very low number of Queues workers (1-2), which makes it useless except for testing purposes 17 | 18 |   19 | 20 | ## Usage 21 | 22 | 23 | 24 | Add it to the `Package.swift` of your Vapor4 project: 25 | 26 | ```swift 27 | 28 | // swift-tools-version:5.4 29 | import PackageDescription 30 | 31 | let package = Package( 32 | name: "app", 33 | platforms: [ 34 | .macOS(.v10_15) 35 | ], 36 | ... 37 | dependencies: [ 38 | ... 39 | .package(url: "https://github.com/m-barthelemy/vapor-queues-fluent-driver.git", from: "3.0.0-beta1"), 40 | ... 41 | ], 42 | targets: [ 43 | .target(name: "App", dependencies: [ 44 | ... 45 | .product(name: "QueuesFluentDriver", package: "vapor-queues-fluent-driver"), 46 | ... 47 | ]), 48 | ... 49 | ] 50 | ) 51 | 52 | ``` 53 | 54 |   55 | 56 | This package needs a table, named `_jobs_meta` by default, to store the Vapor Queues jobs. Make sure to add this to your migrations: 57 | ```swift 58 | // Ensure the table for storing jobs is created 59 | app.migrations.add(JobMetadataMigrate()) 60 | ``` 61 | 62 |   63 | 64 | Finally, load the `QueuesFluentDriver` driver: 65 | ```swift 66 | app.queues.use(.fluent()) 67 | ``` 68 | 69 | ⚠️ Make sure you call `app.databases.use(...)` **before** calling `app.queues.use(.fluent())`! 70 | 71 |   72 | 73 | ## Options 74 | 75 | ### Using a custom Database 76 | You can optionally create a dedicated Database, set to `isdefault: false` and with a custom `DatabaseID` and use it for your Queues. 77 | In that case you would initialize the Queues configuration like this: 78 | 79 | ```swift 80 | let queuesDb = DatabaseID(string: "my_queues_db") 81 | app.databases.use(.postgres(configuration: dbConfig), as: queuesDb, isDefault: false) 82 | app.queues.use(.fluent(queuesDb)) 83 | ``` 84 | 85 | ### Customizing the jobs table name 86 | You can customize the name of the table used by this driver during the migration : 87 | ```swift 88 | app.migrations.add(JobMetadataMigrate(schema: "my_jobs")) 89 | ``` 90 | 91 | ### Soft Deletes 92 | By default, completed jobs are deleted from the two database tables used by this driver. 93 | If you want to keep them, you can use Fluent's "soft delete" feature, which just sets the `deleted_at` field to a non-null value and excludes rows from queries by default: 94 | 95 | ```swift 96 | app.queues.use(.fluent(useSoftDeletes: true)) 97 | ``` 98 | 99 | When enabling this option, it is probably a good idea to cleanup the completed jobs from time to time. 100 | 101 |   102 | 103 | 104 | ## Caveats 105 | 106 | 107 | ### Polling interval and number of workers 108 | By default, the Vapor Queues package creates 2 workers per CPU core, and each worker would poll the database for jobs to be run every second. 109 | On a 4 cores system, this means 8 workers querying the database every second by default. 110 | 111 | You can change the jobs polling interval by calling: 112 | 113 | ```swift 114 | app.queues.configuration.refreshInterval = .seconds(custom_value) 115 | ``` 116 | 117 | With Queues >=1.4.0, you can also configure the number of workers that will be started by setting `app.queues.configuration.workerCount` 118 | 119 | -------------------------------------------------------------------------------- /Sources/QueuesFluentDriver/FluentQueue.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import Queues 3 | import Fluent 4 | import SQLKit 5 | 6 | public struct FluentQueue { 7 | public let context: QueueContext 8 | let db: Database 9 | let dbType: QueuesFluentDbType 10 | let useSoftDeletes: Bool 11 | } 12 | 13 | extension FluentQueue: Queue { 14 | public func get(_ id: JobIdentifier) -> EventLoopFuture { 15 | return db.query(JobModel.self) 16 | .filter(\.$id == id.string) 17 | .first() 18 | .unwrap(or: QueuesFluentError.missingJob(id)) 19 | .flatMapThrowing { job in 20 | return JobData( 21 | payload: Array(job.data.payload), 22 | maxRetryCount: job.data.maxRetryCount, 23 | jobName: job.data.jobName, 24 | delayUntil: job.data.delayUntil, 25 | queuedAt: job.data.queuedAt, 26 | attempts: job.data.attempts ?? 0 27 | ) 28 | } 29 | } 30 | 31 | public func set(_ id: JobIdentifier, to jobStorage: JobData) -> EventLoopFuture { 32 | let jobModel = JobModel(id: id, queue: queueName.string, jobData: JobDataModel(jobData: jobStorage)) 33 | // If the job must run at a later time, ensure it won't be picked earlier since 34 | // we sort pending jobs by date when querying 35 | jobModel.runAtOrAfter = jobStorage.delayUntil ?? Date() 36 | 37 | return jobModel.save(on: db).map { metadata in 38 | return 39 | } 40 | } 41 | 42 | public func clear(_ id: JobIdentifier) -> EventLoopFuture { 43 | // This does the equivalent of a Fluent Softdelete but sets the `state` to `completed` 44 | return db.query(JobModel.self) 45 | .filter(\.$id == id.string) 46 | .filter(\.$state != .completed) 47 | .first() 48 | .unwrap(or: QueuesFluentError.missingJob(id)) 49 | .flatMap { job in 50 | if self.useSoftDeletes { 51 | job.state = .completed 52 | job.deletedAt = Date() 53 | return job.update(on: self.db) 54 | } else { 55 | return job.delete(force: true, on: self.db) 56 | } 57 | } 58 | } 59 | 60 | public func push(_ id: JobIdentifier) -> EventLoopFuture { 61 | guard let sqlDb = db as? SQLDatabase else { 62 | return self.context.eventLoop.makeFailedFuture(QueuesFluentError.databaseNotFound) 63 | } 64 | return sqlDb 65 | .update(JobModel.schema) 66 | .set(SQLColumn("\(FieldKey.state)"), to: SQLBind(QueuesFluentJobState.pending)) 67 | .where(SQLColumn("\(FieldKey.id)"), .equal, SQLBind(id.string)) 68 | .run() 69 | } 70 | 71 | /// Currently selects the oldest job pending execution 72 | public func pop() -> EventLoopFuture { 73 | guard let sqlDb = db as? SQLDatabase else { 74 | return self.context.eventLoop.makeFailedFuture(QueuesFluentError.databaseNotFound) 75 | } 76 | 77 | var selectQuery = sqlDb 78 | .select() 79 | .column("\(FieldKey.id)") 80 | .from(JobModel.schema) 81 | .where(SQLColumn("\(FieldKey.state)"), .equal, SQLBind(QueuesFluentJobState.pending)) 82 | .where(SQLColumn("\(FieldKey.queue)"), .equal, SQLBind(self.queueName.string)) 83 | .where(SQLColumn("\(FieldKey.runAt)"), .lessThanOrEqual, SQLBind(Date())) 84 | .orderBy("\(FieldKey.runAt)") 85 | .limit(1) 86 | if self.dbType != .sqlite { 87 | selectQuery = selectQuery.lockingClause(SQLSkipLocked.forUpdateSkipLocked) 88 | } 89 | 90 | var popProvider: PopQueryProtocol! 91 | switch (self.dbType) { 92 | case .postgresql: 93 | popProvider = PostgresPop() 94 | case .mysql: 95 | popProvider = MySQLPop() 96 | case .sqlite: 97 | popProvider = SqlitePop() 98 | } 99 | return popProvider.pop(db: db, select: selectQuery.query).optionalMap { id in 100 | return JobIdentifier(string: id) 101 | } 102 | } 103 | } 104 | --------------------------------------------------------------------------------