├── .gitignore ├── .travis.yml ├── CHANGELOG.md ├── LICENSE ├── README.md ├── project.clj ├── src └── mysql_queue │ ├── core.clj │ ├── queries.clj │ └── utils.clj └── test └── mysql_queue ├── core_test.clj └── utils_test.clj /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /classes 3 | /checkouts 4 | pom.xml 5 | pom.xml.asc 6 | *.jar 7 | *.class 8 | /.lein-* 9 | /.nrepl-port 10 | .hgignore 11 | .hg/ 12 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: trusty 2 | language: clojure 3 | services: 4 | - mysql 5 | jdk: 6 | - openjdk7 7 | - oraclejdk8 8 | - oraclejdk9 9 | - oraclejdk11 10 | script: "lein test-all" 11 | before_script: "echo 'CREATE DATABASE IF NOT EXISTS clj_mysql_queue;' | mysql -uroot" 12 | 13 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | All notable changes to this project are documented in this file. This change log follows the conventions of [keepachangelog.com](http://keepachangelog.com/). 3 | 4 | ## [0.2.0] - 2016-09-10 5 | 6 | ### Fixed 7 | - Fix `queue/initialize!` that fails to create MySQL tables due to index conflict. 8 | - Internal: Fix possible test failures in non-UTC timezones. 9 | - Internal: Fix graceful shutdown test. 10 | 11 | ### Changed 12 | - Use JVM time when querying for pending jobs instead of relying on UTC. 13 | - Verify the library works properly on Java 8. 14 | 15 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC 2 | LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM 3 | CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. 4 | 5 | 1. DEFINITIONS 6 | 7 | "Contribution" means: 8 | 9 | a) in the case of the initial Contributor, the initial code and 10 | documentation distributed under this Agreement, and 11 | 12 | b) in the case of each subsequent Contributor: 13 | 14 | i) changes to the Program, and 15 | 16 | ii) additions to the Program; 17 | 18 | where such changes and/or additions to the Program originate from and are 19 | distributed by that particular Contributor. A Contribution 'originates' from 20 | a Contributor if it was added to the Program by such Contributor itself or 21 | anyone acting on such Contributor's behalf. Contributions do not include 22 | additions to the Program which: (i) are separate modules of software 23 | distributed in conjunction with the Program under their own license 24 | agreement, and (ii) are not derivative works of the Program. 25 | 26 | "Contributor" means any person or entity that distributes the Program. 27 | 28 | "Licensed Patents" mean patent claims licensable by a Contributor which are 29 | necessarily infringed by the use or sale of its Contribution alone or when 30 | combined with the Program. 31 | 32 | "Program" means the Contributions distributed in accordance with this 33 | Agreement. 34 | 35 | "Recipient" means anyone who receives the Program under this Agreement, 36 | including all Contributors. 37 | 38 | 2. GRANT OF RIGHTS 39 | 40 | a) Subject to the terms of this Agreement, each Contributor hereby grants 41 | Recipient a non-exclusive, worldwide, royalty-free copyright license to 42 | reproduce, prepare derivative works of, publicly display, publicly perform, 43 | distribute and sublicense the Contribution of such Contributor, if any, and 44 | such derivative works, in source code and object code form. 45 | 46 | b) Subject to the terms of this Agreement, each Contributor hereby grants 47 | Recipient a non-exclusive, worldwide, royalty-free patent license under 48 | Licensed Patents to make, use, sell, offer to sell, import and otherwise 49 | transfer the Contribution of such Contributor, if any, in source code and 50 | object code form. This patent license shall apply to the combination of the 51 | Contribution and the Program if, at the time the Contribution is added by the 52 | Contributor, such addition of the Contribution causes such combination to be 53 | covered by the Licensed Patents. The patent license shall not apply to any 54 | other combinations which include the Contribution. No hardware per se is 55 | licensed hereunder. 56 | 57 | c) Recipient understands that although each Contributor grants the licenses 58 | to its Contributions set forth herein, no assurances are provided by any 59 | Contributor that the Program does not infringe the patent or other 60 | intellectual property rights of any other entity. Each Contributor disclaims 61 | any liability to Recipient for claims brought by any other entity based on 62 | infringement of intellectual property rights or otherwise. As a condition to 63 | exercising the rights and licenses granted hereunder, each Recipient hereby 64 | assumes sole responsibility to secure any other intellectual property rights 65 | needed, if any. For example, if a third party patent license is required to 66 | allow Recipient to distribute the Program, it is Recipient's responsibility 67 | to acquire that license before distributing the Program. 68 | 69 | d) Each Contributor represents that to its knowledge it has sufficient 70 | copyright rights in its Contribution, if any, to grant the copyright license 71 | set forth in this Agreement. 72 | 73 | 3. REQUIREMENTS 74 | 75 | A Contributor may choose to distribute the Program in object code form under 76 | its own license agreement, provided that: 77 | 78 | a) it complies with the terms and conditions of this Agreement; and 79 | 80 | b) its license agreement: 81 | 82 | i) effectively disclaims on behalf of all Contributors all warranties and 83 | conditions, express and implied, including warranties or conditions of title 84 | and non-infringement, and implied warranties or conditions of merchantability 85 | and fitness for a particular purpose; 86 | 87 | ii) effectively excludes on behalf of all Contributors all liability for 88 | damages, including direct, indirect, special, incidental and consequential 89 | damages, such as lost profits; 90 | 91 | iii) states that any provisions which differ from this Agreement are offered 92 | by that Contributor alone and not by any other party; and 93 | 94 | iv) states that source code for the Program is available from such 95 | Contributor, and informs licensees how to obtain it in a reasonable manner on 96 | or through a medium customarily used for software exchange. 97 | 98 | When the Program is made available in source code form: 99 | 100 | a) it must be made available under this Agreement; and 101 | 102 | b) a copy of this Agreement must be included with each copy of the Program. 103 | 104 | Contributors may not remove or alter any copyright notices contained within 105 | the Program. 106 | 107 | Each Contributor must identify itself as the originator of its Contribution, 108 | if any, in a manner that reasonably allows subsequent Recipients to identify 109 | the originator of the Contribution. 110 | 111 | 4. COMMERCIAL DISTRIBUTION 112 | 113 | Commercial distributors of software may accept certain responsibilities with 114 | respect to end users, business partners and the like. While this license is 115 | intended to facilitate the commercial use of the Program, the Contributor who 116 | includes the Program in a commercial product offering should do so in a 117 | manner which does not create potential liability for other Contributors. 118 | Therefore, if a Contributor includes the Program in a commercial product 119 | offering, such Contributor ("Commercial Contributor") hereby agrees to defend 120 | and indemnify every other Contributor ("Indemnified Contributor") against any 121 | losses, damages and costs (collectively "Losses") arising from claims, 122 | lawsuits and other legal actions brought by a third party against the 123 | Indemnified Contributor to the extent caused by the acts or omissions of such 124 | Commercial Contributor in connection with its distribution of the Program in 125 | a commercial product offering. The obligations in this section do not apply 126 | to any claims or Losses relating to any actual or alleged intellectual 127 | property infringement. In order to qualify, an Indemnified Contributor must: 128 | a) promptly notify the Commercial Contributor in writing of such claim, and 129 | b) allow the Commercial Contributor tocontrol, and cooperate with the 130 | Commercial Contributor in, the defense and any related settlement 131 | negotiations. The Indemnified Contributor may participate in any such claim 132 | at its own expense. 133 | 134 | For example, a Contributor might include the Program in a commercial product 135 | offering, Product X. That Contributor is then a Commercial Contributor. If 136 | that Commercial Contributor then makes performance claims, or offers 137 | warranties related to Product X, those performance claims and warranties are 138 | such Commercial Contributor's responsibility alone. Under this section, the 139 | Commercial Contributor would have to defend claims against the other 140 | Contributors related to those performance claims and warranties, and if a 141 | court requires any other Contributor to pay any damages as a result, the 142 | Commercial Contributor must pay those damages. 143 | 144 | 5. NO WARRANTY 145 | 146 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON 147 | AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 148 | EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR 149 | CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A 150 | PARTICULAR PURPOSE. Each Recipient is solely responsible for determining the 151 | appropriateness of using and distributing the Program and assumes all risks 152 | associated with its exercise of rights under this Agreement , including but 153 | not limited to the risks and costs of program errors, compliance with 154 | applicable laws, damage to or loss of data, programs or equipment, and 155 | unavailability or interruption of operations. 156 | 157 | 6. DISCLAIMER OF LIABILITY 158 | 159 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY 160 | CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, 161 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION 162 | LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 163 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 164 | ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE 165 | EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY 166 | OF SUCH DAMAGES. 167 | 168 | 7. GENERAL 169 | 170 | If any provision of this Agreement is invalid or unenforceable under 171 | applicable law, it shall not affect the validity or enforceability of the 172 | remainder of the terms of this Agreement, and without further action by the 173 | parties hereto, such provision shall be reformed to the minimum extent 174 | necessary to make such provision valid and enforceable. 175 | 176 | If Recipient institutes patent litigation against any entity (including a 177 | cross-claim or counterclaim in a lawsuit) alleging that the Program itself 178 | (excluding combinations of the Program with other software or hardware) 179 | infringes such Recipient's patent(s), then such Recipient's rights granted 180 | under Section 2(b) shall terminate as of the date such litigation is filed. 181 | 182 | All Recipient's rights under this Agreement shall terminate if it fails to 183 | comply with any of the material terms or conditions of this Agreement and 184 | does not cure such failure in a reasonable period of time after becoming 185 | aware of such noncompliance. If all Recipient's rights under this Agreement 186 | terminate, Recipient agrees to cease use and distribution of the Program as 187 | soon as reasonably practicable. However, Recipient's obligations under this 188 | Agreement and any licenses granted by Recipient relating to the Program shall 189 | continue and survive. 190 | 191 | Everyone is permitted to copy and distribute copies of this Agreement, but in 192 | order to avoid inconsistency the Agreement is copyrighted and may only be 193 | modified in the following manner. The Agreement Steward reserves the right to 194 | publish new versions (including revisions) of this Agreement from time to 195 | time. No one other than the Agreement Steward has the right to modify this 196 | Agreement. The Eclipse Foundation is the initial Agreement Steward. The 197 | Eclipse Foundation may assign the responsibility to serve as the Agreement 198 | Steward to a suitable separate entity. Each new version of the Agreement will 199 | be given a distinguishing version number. The Program (including 200 | Contributions) may always be distributed subject to the version of the 201 | Agreement under which it was received. In addition, after a new version of 202 | the Agreement is published, Contributor may elect to distribute the Program 203 | (including its Contributions) under the new version. Except as expressly 204 | stated in Sections 2(a) and 2(b) above, Recipient receives no rights or 205 | licenses to the intellectual property of any Contributor under this 206 | Agreement, whether expressly, by implication, estoppel or otherwise. All 207 | rights in the Program not expressly granted under this Agreement are 208 | reserved. 209 | 210 | This Agreement is governed by the laws of the State of New York and the 211 | intellectual property laws of the United States of America. No party to this 212 | Agreement will bring a legal action under this Agreement more than one year 213 | after the cause of action arose. Each party waives its rights to a jury trial 214 | in any resulting litigation. 215 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # mysql-queue 2 | 3 | [![Build Status](https://travis-ci.org/wildbit/mysql-queue.svg?branch=master)](https://travis-ci.org/wildbit/mysql-queue) 4 | 5 | A Clojure library that implements a MySQL-backed durable queue with support for scheduled jobs. 6 | 7 | ## Install 8 | 9 | Get it from Clojars: 10 | 11 | [![Clojars Project](https://img.shields.io/clojars/v/mysql-queue.svg)](https://clojars.org/mysql-queue) 12 | 13 | ## Why 14 | 15 | Database-backed queues are known to be fraught with various problems. 16 | It'd be unwise to deny this: compared to specialized queue and messaging systems they're slow and inefficient. 17 | If one's looking at processing hundreds of jobs per second, it is certainly advisable to consider a specialized solution. 18 | For smaller projects though, using a relational database as a queue offers considerable benefits: 19 | 20 | * **No additional infrastructure requirements.** Most web applications already have a database. 21 | * **Centralized queue.** Compared to embeddable solutions, you can easily add more workers running on different servers. 22 | * **Scheduled jobs.** Specialized queue and messaging systems tend to not support scheduled messages, because they're simply not built for storing data. On the contrary, any in-database queue is inherently great at storing and filtering data. 23 | 24 | The latter point is also important if what you're after is just a scheduler to push jobs to your internal queue. 25 | For example, you may decide to use a database table to store scheduled jobs that will later be executed via Zach Tellman's [durable-queue](https://github.com/Factual/durable-queue). 26 | Nevertheless, relegating your database-backed queue to a position of a scheduler doesn't make it less of a queue. 27 | It still has to grapple with all of the concomitant issues, such as: 28 | 29 | * **State management.** It is far from trivial to represent jobs' states using semantics of traditional relational database operations. 30 | * **Concurrency.** Unless there is only a single worker process running at any time, you're going to need some sort of locking functionality that will ensure that only one worker gets a particular job. 31 | * **Recovery.** Sometimes a worker fails without acknowledging successful job execution or the entire process crashes. Unless you're comfortable with dropping those messages, a recovery mechanism is required. 32 | * **Graceful shutdown.** Even with "real" queues and messaging systems this important part is often overlooked and has to be implemented by the client application. 33 | * **Logging and error handling.** Quick home-grown solutions tend to cut on logging, and often complicate debugging when something going wrong. 34 | 35 | The `mysql-queue` library aims at providing a sane implementation of a MySQL-backed queue while addressing all of the issues above. 36 | It is especially useful for compact monolith web applications requiring both scheduled and real-time message processing (at a smaller scale). 37 | In real world, this library is running at the core of the free DMARC monitoring tool by Postmark responsible for sending weekly digests to the subscribers, as well as executing a handful of other repeating tasks. 38 | 39 | ## Usage 40 | 41 | All public functions are located in the `mysql-queue.core` namespace: 42 | 43 | ``` clojure 44 | (require '[mysql-queue.core :as queue]) 45 | 46 | (def db-conn {:subprotocol "mysql" 47 | :subname "//localhost:3306/myapp" 48 | :user "user" 49 | :password "password"}) 50 | ``` 51 | 52 | This lib uses two MySQL tables: `jobs` and `scheduled_jobs`. The names are not configurable. 53 | You can create these tables when your application starts by calling `initialize!` function. 54 | 55 | ``` clojure 56 | ; The following call is idempotent, meaning it will not attempt to create the tables if they already exist 57 | (queue/initialize! db-conn) 58 | ``` 59 | 60 | Schedule jobs via `schedule-job` function: 61 | 62 | ``` clojure 63 | (queue/schedule-job db-conn :my-job :begin {:echo "Hello, world!"} (java.util.Date.)) 64 | ``` 65 | 66 | Jobs are functions of two arguments: `status` (Clojure keyword) and `args` (anything serializable with EDN). 67 | A job function is expected to return a vector of two elements: new status and (potentially) updated args. 68 | 69 | Examples: `[:done {}]`, `[:step2 {:step1-result 42}]`. 70 | 71 | If a job returns something that isn't a vector of 2 elements, the return value considered to be `[:done nil]`. 72 | The job is considered completed if the returned status is `:done`. 73 | Two synonym statuses for `:done` are available to provide more context in logs: `:canceled` and `:failed`. 74 | If the returned status isn't `:done`, it will be persisted and the job function will be executed again with returned arguments. 75 | See Advanced section for more details on this behavior. 76 | 77 | ``` clojure 78 | ; Job function example 79 | (defn my-job 80 | [status {:keys [echo] :as args}] 81 | (println echo) 82 | [:done args]) 83 | ``` 84 | 85 | Symbolic job names passed to `schedule-job` are associated with job functions via a user-defined bindings map: 86 | 87 | ``` clojure 88 | (def jobs {:my-job my-job}) 89 | ``` 90 | 91 | Use `worker` function to create a worker that will periodically check the database for new jobs and execute 92 | associated job functions on a thread pool. See docstring for advanced configuration options such as concurrency 93 | settings and user-defined logging functions. 94 | 95 | ``` clojure 96 | (def worker (queue/worker db-conn jobs)) 97 | ``` 98 | 99 | To gracefully stop a worker, use `stop` function. 100 | It will immediately stop publisher and recovery threads, and will then wait for the worker threads to process all loaded jobs for up to a specified number of seconds before unblocking the calling thread. 101 | 102 | ``` clojure 103 | (queue/stop worker 5) 104 | ``` 105 | 106 | This covers the basics. Proceed to Advanced section for more details. 107 | 108 | ## Advanced 109 | 110 | ### Failproof job functions 111 | 112 | Job functions used by `mysql-queue` are simple state machines. If your job is a combination of side effects (such as write to a database, notify an external service, then send an email), you may need to be able to pick up the job exactly where it left off in case any one of these multiple steps fails. 113 | 114 | Take a look at the following job function: 115 | 116 | ``` clojure 117 | (defn multi-step-job 118 | [status {data :data}] 119 | (->> data write-to-db (send-email :new-item) save-message-id)) 120 | ``` 121 | 122 | This (valid) job function is quite terse, but still easy to understand if you're familiar with Clojure idioms. 123 | Unfortunately, it couples three dependent side effects together into a single operation. 124 | Assume there is a connection error to your mail transfer agent when sending that email. 125 | It means that a given piece of data was recorded to the database, but the email wasn't yet sent. 126 | If we retry this job, the data will be written to the database again, resulting in a duplicate. 127 | Even worse, if `save-message-id` throws a database connection error, retrying this job will mean sending another notification. 128 | To mitigate this problem, we could structure our job in a bit different way: 129 | 130 | ``` clojure 131 | (defmulti multi-step-job (fn [status _] status)) 132 | 133 | (defmethod multi-step-job :begin 134 | [_ {data :data}] 135 | (let [id (write-to-db data)] 136 | [:send-email {:id id}])) 137 | 138 | (defmethod multi-step-job :send-email 139 | [_ {id :id}] 140 | (let [message-id (send-email :new-item id)] 141 | [:save-message-id {:message-id id}])) 142 | 143 | (defmethod multi-step-job :save-message-id 144 | [_ {:keys [message-id]}] 145 | (save-message-id message-id) 146 | [:done {}]) 147 | ``` 148 | 149 | Here we used multimethods to represent all stateful transitions in our job. 150 | It doesn't look nearly as concise as before, but the new representation reveals how many actions that threading macro was hiding. 151 | Since the library persists any results returned by the job function, even if your VM crashes in the middle of the job, the recovery will later be able to pick up from the last successfully acknowledged step. 152 | 153 | **Warning:** Be careful about returning large chunks of data because all job arguments have to be serialized and stored in the database. 154 | 155 | ## License 156 | 157 | Copyright © 2016 Wildbit 158 | 159 | Distributed under the Eclipse Public License either version 1.0 or (at 160 | your option) any later version. 161 | 162 | -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject mysql-queue "1.1.0" 2 | :description "A durable queue with scheduled job support that is backed by MySQL." 3 | :url "https://github.com/wildbit/mysql-queue" 4 | :license {:name "Eclipse Public License" 5 | :url "http://www.eclipse.org/legal/epl-v10.html"} 6 | :dependencies [[org.clojure/clojure "1.9.0"] 7 | [org.clojure/core.async "0.3.465"] 8 | [org.clojure/java.jdbc "0.7.5"] 9 | [mysql/mysql-connector-java "5.1.28"]] 10 | :scm {:name "git" :url "https://github.com/wildbit/mysql-queue"} 11 | :profiles {:1.7 {:dependencies [[org.clojure/clojure "1.7.0"]]} 12 | :1.8 {:dependencies [[org.clojure/clojure "1.8.0"]]} 13 | :1.9 {:dependencies [[org.clojure/clojure "1.9.0"]]}} 14 | :aliases {"test-all" ["with-profile" "+1.7:+1.8:+1.9" "test"]}) 15 | -------------------------------------------------------------------------------- /src/mysql_queue/core.clj: -------------------------------------------------------------------------------- 1 | (ns mysql-queue.core 2 | "A MySQL-backed durable queue implementation with scheduled jobs support." 3 | (:require [mysql-queue.queries :as queries] 4 | [mysql-queue.utils :refer [while-let fn-options with-error-handler profile-block meter ns->ms numeric-stats]] 5 | [clojure.string :as string] 6 | [clojure.set :as clj-set] 7 | [clojure.edn :as edn] 8 | [clojure.core.async :as async :refer [chan >!! >! > consumer-threads 45 | (concat [scheduler-thread recovery-thread]) 46 | async/merge 47 | (async/into [])) 48 | [v ch] (alts!! [(timeout (* 1000 timeout-secs)) consumer-shutdowns])] 49 | (= ch consumer-shutdowns))))) 50 | 51 | (defprotocol ToDb (*->db [entity])) 52 | 53 | (defprotocol Persistent 54 | (persist [entity conn]) 55 | (cleanup [entity conn])) 56 | 57 | (defprotocol Executable 58 | (finished? [job]) 59 | (execute [job db-conn log-fn err-fn timeout])) 60 | 61 | (defprotocol Fertile 62 | (beget [parent] [parent status] [parent status parameters])) 63 | 64 | (defrecord Job [user-fn id scheduled-job-id parent-id name status parameters attempt] 65 | Object 66 | (toString [this] 67 | (job-summary-string this)) 68 | ToDb 69 | (*->db [this] 70 | [scheduled-job-id 71 | parent-id 72 | (clojure.core/name name) 73 | (clojure.core/name status) 74 | (pr-str parameters) 75 | attempt 76 | (Date.)]) 77 | Persistent 78 | (persist [this conn] 79 | (if id 80 | this 81 | (try 82 | (let [{id :generated_key} (apply queries/insert-jobdb this))] 83 | (assoc this :id id)) 84 | (catch MySQLIntegrityConstraintViolationException e nil)))) 85 | (cleanup [this conn] 86 | (when scheduled-job-id 87 | (queries/delete-scheduled-job-by-id! conn scheduled-job-id) 88 | nil)) 89 | Fertile 90 | (beget [this] (beget this status parameters)) 91 | (beget [this status] (beget this status parameters)) 92 | (beget [this status parameters] 93 | (assoc this 94 | :id nil 95 | :parent-id id 96 | :status status 97 | :parameters parameters 98 | :attempt (if (= status (:status this)) (inc attempt) 1)))) 99 | 100 | (defn- job 101 | "Creates a new Job (or fn-constructor) record from a JDBC map representing a row of jobs table." 102 | ([db-row fn-bindings] 103 | (job db-row fn-bindings ->Job)) 104 | ([db-row fn-bindings fn-constructor] 105 | (fn-constructor (fn-bindings (-> db-row :name keyword)) 106 | (:id db-row) 107 | (:scheduled_job_id db-row) 108 | (:parent_id db-row) 109 | (-> db-row :name keyword) 110 | (-> db-row :status keyword) 111 | (-> db-row :parameters (String. "UTF-8") edn/read-string) 112 | (:attempt db-row)))) 113 | 114 | (defrecord StuckJob [user-fn id scheduled-job-id parent-id name status parameters attempt] 115 | Object 116 | (toString [this] 117 | (job-summary-string this)) 118 | Fertile 119 | (beget [this] 120 | (if (< attempt max-retries) 121 | (->Job user-fn nil scheduled-job-id id name status parameters (inc attempt)) 122 | (->Job user-fn nil scheduled-job-id id name :failed parameters attempt))) 123 | (beget [this _] (beget this)) 124 | (beget [this _ _] (beget this))) 125 | 126 | (defn- stuck-job 127 | [db-row fn-bindings] 128 | (job db-row fn-bindings ->StuckJob)) 129 | 130 | (defrecord ScheduledJob [user-fn id name status parameters due-at] 131 | Object 132 | (toString [this] 133 | (job-summary-string this)) 134 | ToDb 135 | (*->db [this] 136 | [(clojure.core/name name) 137 | (clojure.core/name status) 138 | (pr-str parameters) 139 | due-at]) 140 | Persistent 141 | (persist [this conn] 142 | (if id 143 | this 144 | (let [{id :generated_key} (apply queries/insert-scheduled-jobdb this))] 145 | (assoc this :id id)))) 146 | (cleanup [this conn] 147 | (queries/delete-scheduled-job-by-id! conn id) 148 | nil) 149 | Fertile 150 | (beget [this] (beget this status parameters)) 151 | (beget [this status] (beget this status parameters)) 152 | (beget [this status parameters] 153 | (->Job user-fn nil id 0 name status parameters 1))) 154 | 155 | (defn- scheduled-job 156 | [db-row fn-bindings] 157 | (->ScheduledJob (fn-bindings (-> db-row :name keyword)) 158 | (:id db-row) 159 | (-> db-row :name keyword) 160 | (-> db-row :status keyword) 161 | (-> db-row :parameters (String. "UTF-8") edn/read-string) 162 | (:scheduled_for db-row))) 163 | 164 | (defn- job-result-or-nil 165 | "Returns its value if it's a valid job result or nil. 166 | Job result is a vector of two elements where the first element is a keyword." 167 | [result] 168 | (when (and (vector? result) (= 2 (count result)) (keyword? (first result))) 169 | result)) 170 | 171 | (extend-protocol Executable 172 | Job 173 | (finished? [job] 174 | (ultimate-job-states (:status job))) 175 | (execute [{:as job job-fn :user-fn :keys [status parameters attempt]} db-conn log-fn err-fn timeout] 176 | (profile-block [m] 177 | (if (finished? job) 178 | (cleanup job db-conn) 179 | (try 180 | (log-fn :info job "Executing job " job) 181 | (let [[status params] (-> (meter m :user (with-timeout timeout (job-fn status parameters))) 182 | job-result-or-nil (or [:done nil]))] 183 | (-> job (beget status params) (persist db-conn))) 184 | (catch Exception e 185 | (err-fn e) 186 | (if (< attempt max-retries) 187 | (-> job beget (persist db-conn)) 188 | (-> job (beget :failed) (persist db-conn)))))))) 189 | StuckJob 190 | (finished? [job] false) 191 | (execute [job db-conn log-fn err-fn timeout] 192 | (profile-block [_] 193 | (log-fn :info job "Recovering job " job) 194 | (-> job beget (persist db-conn)))) 195 | ScheduledJob 196 | (finished? [job] 197 | (throw (UnsupportedOperationException. "finished? is not implemented for ScheduledJob."))) 198 | (execute [job db-conn log-fn err-fn timeout] 199 | (profile-block [_] 200 | (log-fn :info job "Executing job " job) 201 | (-> job beget (persist db-conn))))) 202 | 203 | (defn- bound-job-names 204 | "Takes a map of job bindings and returns a sequence of names as strings." 205 | [fn-bindings] 206 | (map name (keys fn-bindings))) 207 | 208 | (defn- get-scheduled-jobs 209 | "Searches for ready scheduled jobs and attempts to insert root jobs for each of those. 210 | Returns the number of jobs added, or false if channel was closed." 211 | [db-conn n fn-bindings sieve] 212 | (->> (queries/select-n-ready-scheduled-jobs db-conn (bound-job-names fn-bindings) sieve n) 213 | (map #(scheduled-job % fn-bindings)))) 214 | 215 | (defn- get-stuck-jobs 216 | "Searches DB for long-running jobs and enqueues follow-up jobs for them. 217 | Should not be run too often or the channel will be clogged 218 | with stuck IDs. Also keep in mind the number of concurrently running nodes." 219 | [db-conn n fn-bindings threshold sieve] 220 | (->> (queries/select-n-stuck-jobs db-conn 221 | (map name ultimate-job-states) 222 | (bound-job-names fn-bindings) 223 | sieve 224 | threshold 225 | n) 226 | (map #(stuck-job % fn-bindings)))) 227 | 228 | (defn- batch-publish 229 | "Puts gitven jobs onto the channel chan and returns the number of successfully 230 | published jobs or false if the channel is closed." 231 | [publish-chan jobs] 232 | (let [total (->> jobs (take-while #(>!! publish-chan %)) count)] 233 | (if (or (not (async-proto/closed? publish-chan)) (pos? total)) 234 | total 235 | false))) 236 | 237 | (defn- default-status 238 | "Build the default status map for a given number of consumer threads." 239 | [{:keys [num-consumer-threads 240 | recovery-threshold-mins 241 | min-scheduler-sleep-interval 242 | max-scheduler-sleep-interval 243 | min-recovery-sleep-interval 244 | max-recovery-sleep-interval]}] 245 | {:running true 246 | :consumers (mapv #(hash-map :n (inc %) 247 | :started-at (Date.) 248 | :jobs-executed 0N 249 | :recent-jobs []) 250 | (range num-consumer-threads)) 251 | :recovery {:started-at (Date.) 252 | :min-interval min-recovery-sleep-interval 253 | :max-interval max-recovery-sleep-interval 254 | :iterations 0N 255 | :jobs-published 0N} 256 | :scheduler {:started-at (Date.) 257 | :min-interval min-scheduler-sleep-interval 258 | :max-interval max-scheduler-sleep-interval 259 | :recovery-threshold-mins recovery-threshold-mins 260 | :iterations 0N 261 | :jobs-published 0N}}) 262 | 263 | (defn- consumer-thread 264 | "Consumer loop. Automatically quits if the listen-chan is closed. Runs in a go-thread." 265 | [id listen-chan status-chan db-conn log-fn err-fn job-timeout] 266 | (go 267 | (while-let [job (! status-chan {:id id :state :running-job :job current-job}) 276 | (let [[next-job dmetrics] (! status-chan {:id id :state :finished-job :job last-job :metrics metrics}) 280 | (log-fn :info last-job "Completed job " last-job " in " (ns->ms (:full metrics)) "ms") 281 | nil))) 282 | (catch Exception e 283 | (>! status-chan {:id id :state :error :job job}) 284 | (log-fn :error job "Unexpected error " e " in consumer loop when running job " job) 285 | (err-fn e)))) 286 | (>! status-chan {:id id :state :quit}) 287 | (log-fn :debug "Consumer Thread" "Consumer is stopping...") 288 | :done)) 289 | 290 | (defn- publisher-thread 291 | "Publisher loop. Automatically quits if the publish-chan is closed. Runs in a go-thread." 292 | [status-chan min-sleep-secs max-sleep-secs source-fn log-fn] 293 | (go-loop [last-exec (System/currentTimeMillis)] 294 | (if-let [published (! status-chan {:state :running :jobs-published published}) 297 | (if (zero? published) 298 | (! status-chan {:state :quit :jobs-published 0}) 305 | (log-fn :debug nil "Publisher is stopping...") 306 | :done)))) 307 | 308 | (defn- status-threads 309 | "Status loop. Listens for status updates from consumer and publisher threads. 310 | Runs multiple go-threads." 311 | [status num-kept-jobs] 312 | (let [consumer-chan (chan) 313 | scheduler-chan (chan) 314 | recovery-chan (chan) 315 | consumer-fn (fn [{:keys [recent-jobs] :as status} consumer-state job metrics] 316 | (let [current-time (Date.) 317 | finished? (= consumer-state :finished-job) 318 | overflow (inc (- (count recent-jobs) num-kept-jobs)) 319 | overflow? (pos? overflow) 320 | job (assoc job :metrics metrics :processed-at current-time)] 321 | (cond-> status 322 | true (assoc :last-update-at current-time 323 | :last-job job 324 | :state consumer-state) 325 | (and finished? overflow?) (update :recent-jobs subvec overflow) 326 | finished? (update :jobs-executed inc) 327 | finished? (update :recent-jobs conj job)))) 328 | consumer-thread (go 329 | (while-let [{:keys [id job state metrics]} ( status 333 | (assoc :last-update-at (Date.) 334 | :state publisher-state 335 | :jobs-published-last-run jobs-published) 336 | (update :jobs-published + jobs-published) 337 | (update :iterations inc))) 338 | scheduler-thread (go 339 | (while-let [{:keys [state jobs-published]} (ids 349 | "Returns a sieve seq that can be used to filter SQL queries for 350 | certain job types. Includes a 0 id to simplify the case when the 351 | sieve is empty." 352 | [sieve sieved-type] 353 | (->> sieve 354 | (clj-set/select #(instance? sieved-type %)) 355 | (map :id) 356 | (concat [0]))) 357 | 358 | (defn- deduplicate 359 | "Takes a channel and returns a new input channel and n-outs output channels. 360 | The returned pipeline is deduplicated via an in-memory sieve of currently 361 | processed elements." 362 | [ch n-outs] 363 | (let [sieve (atom #{}) 364 | in-ch (chan) 365 | out-chs (vec (repeatedly n-outs chan))] 366 | (go-loop [] 367 | (if-let [v (! ch v)) 372 | (recur)) 373 | (close! ch))) 374 | (go-loop [occupations {}] 375 | (if-let [v (> e .getStackTrace (string/join "\n"))) 410 | (err-fn e) 411 | 0)) 412 | 413 | (defn initialize! 414 | "Create required MySQL tables if they don't exist. Returns true." 415 | [db-conn] 416 | (queries/create-scheduled-jobs! db-conn) 417 | (queries/create-jobs! db-conn) 418 | true) 419 | 420 | (defn schedule-job 421 | "Creates a scheduled job with provided name, status, args and due time. 422 | Returns the unique numeric id of a created job." 423 | [db-conn name status params due-at] 424 | {:pre [(keyword? name) 425 | (keyword? status)]} 426 | (-> (->ScheduledJob nil nil name status params due-at) 427 | (persist db-conn) 428 | :id)) 429 | 430 | (defn status 431 | "Returns a map describing the current status of the worker." 432 | [{:keys [db-conn fn-bindings] 433 | {recovery-threshold :recovery-threshold-mins} :options 434 | :as worker}] 435 | (let [serialize-job (fn [job] 436 | (and job 437 | (select-keys job [:id :name :status :metrics :processed-at]))) 438 | sieve @(:sieve worker) 439 | {:keys [consumers] :as raw-status} @(:status worker) 440 | recent-jobs (mapcat :recent-jobs (:consumers raw-status)) 441 | scheduled-status (queries/select-scheduled-jobs-status db-conn (bound-job-names fn-bindings)) 442 | in-progress-status (queries/select-jobs-status 443 | db-conn 444 | (bound-job-names fn-bindings) 445 | ultimate-job-states 446 | recovery-threshold) 447 | consumers (map #(-> % 448 | (dissoc :recent-jobs) 449 | (update :last-job serialize-job)) 450 | consumers)] 451 | (-> raw-status 452 | (assoc :db-queue {:scheduled-jobs scheduled-status :jobs in-progress-status}) 453 | (assoc :prefetched-jobs (map serialize-job sieve)) 454 | (assoc :recent-jobs-stats 455 | {:job-types (frequencies (map :name recent-jobs)) 456 | :performance {:user (numeric-stats (keep #(get-in % [:metrics :user]) recent-jobs)) 457 | :full (numeric-stats (keep #(get-in % [:metrics :full]) recent-jobs))}}) 458 | (assoc :recent-jobs (map serialize-job recent-jobs)) 459 | (assoc :consumers consumers)))) 460 | 461 | (defn worker 462 | "Creates a new worker. Takes a database connection db-conn, 463 | a map of fn-bindings binding job names to job functions, and a number 464 | of optional keyword arguments: 465 | 466 | * buffer-size - maximum number of jobs allowed into internal queue. Determines 467 | when the publisher will block. Default 10. 468 | * job-timeout-mins - the number of minutes after which the job times out. Default 20. 469 | * prefetch - the number of jobs a publisher fetches from the database at once. 470 | Default 10. 471 | * num-stats-jobs - the number of jobs to keep in memory for statistical purpose. 472 | Per consumer thread. Default 50. 473 | * num-consumer-threads - the number of concurrent threads that run jobs at the 474 | same time. 475 | * min-scheduler-sleep-interval - the minimum time in seconds the scheduler will sleep 476 | before querying the database for due jobs. Default 0 seconds. 477 | * max-scheduler-sleep-interval - the maximum time in seconds the scheduler will sleep 478 | before querying the database for due jobs. Default 10 seconds. 479 | * min-recovery-sleep-interval - the minimum time in seconds the recovery thread will 480 | sleep before querying the database for stuck jobs. Default 0 seconds. 481 | * max-recovery-sleep-interval - the maximum time in seconds the recovery thread will 482 | sleep before qerying the database for stuck jobs. Default 10 seconds. 483 | * recovery-threshold-mins - the number of minutes after which a job is considered 484 | stuck and will be picked up by the recovery thread. Default 20. 485 | * log-fn - user-provided logging function of 3 arguments: level (keyword), job (record), message (msg). 486 | * err-fn - user-provided error function of one argument: error (Exception)." 487 | [db-conn 488 | fn-bindings 489 | &{:keys [buffer-size 490 | prefetch 491 | job-timeout-mins 492 | num-stats-jobs 493 | num-consumer-threads 494 | min-scheduler-sleep-interval 495 | max-scheduler-sleep-interval 496 | min-recovery-sleep-interval 497 | max-recovery-sleep-interval 498 | recovery-threshold-mins 499 | log-fn 500 | err-fn] 501 | :or {buffer-size 10 502 | job-timeout-mins 20 503 | prefetch 10 504 | num-stats-jobs 50 505 | num-consumer-threads 2 506 | min-scheduler-sleep-interval 0 507 | max-scheduler-sleep-interval 10 508 | min-recovery-sleep-interval 0 509 | max-recovery-sleep-interval 10 510 | recovery-threshold-mins 20 511 | log-fn (constantly nil) 512 | err-fn (constantly nil)} 513 | :as options}] 514 | {:pre [(every? keyword? (keys fn-bindings)) 515 | (every? fn? (vals fn-bindings)) 516 | (every? (fn-options #'worker) (keys options)) 517 | (fn? log-fn) 518 | (fn? err-fn)]} 519 | (let [log-fn (quiet-log-fn log-fn) 520 | err-fn (quiet-err-fn err-fn) 521 | options {:buffer-size buffer-size 522 | :prefetch prefetch 523 | :num-stats-jobs num-stats-jobs 524 | :num-consumer-threads num-consumer-threads 525 | :min-scheduler-sleep-interval min-scheduler-sleep-interval 526 | :max-scheduler-sleep-interval max-scheduler-sleep-interval 527 | :min-recovery-sleep-interval min-recovery-sleep-interval 528 | :max-recovery-sleep-interval max-recovery-sleep-interval 529 | :recovery-threshold-mins recovery-threshold-mins 530 | :log-fn log-fn 531 | :err-fn err-fn} 532 | status (atom (default-status options)) 533 | {:as status-threads 534 | {consumer-status-channel :channel} :consumer 535 | {recovery-status-channel :channel} :recovery 536 | {scheduler-status-channel :channel} :scheduler} (status-threads status num-stats-jobs) 537 | queue-chan (chan buffer-size) 538 | [in-ch out-chs sieve] (deduplicate queue-chan num-consumer-threads) 539 | consumer-threads (->> out-chs 540 | (map-indexed 541 | #(consumer-thread %1 %2 consumer-status-channel db-conn log-fn err-fn (* 60 1000 job-timeout-mins))) 542 | (into []) 543 | doall) 544 | handler (partial publisher-error-handler log-fn err-fn) 545 | scheduler-thread (publisher-thread scheduler-status-channel 546 | min-scheduler-sleep-interval 547 | max-scheduler-sleep-interval 548 | (with-error-handler [(handler "scheduler thread")] 549 | (batch-publish in-ch 550 | (get-scheduled-jobs db-conn 551 | prefetch 552 | fn-bindings 553 | (sieve->ids @sieve ScheduledJob)))) 554 | log-fn) 555 | recovery-thread (publisher-thread recovery-status-channel 556 | min-recovery-sleep-interval 557 | max-recovery-sleep-interval 558 | (with-error-handler [(handler "recovery thread")] 559 | (batch-publish in-ch 560 | (get-stuck-jobs db-conn 561 | prefetch 562 | fn-bindings 563 | recovery-threshold-mins 564 | (sieve->ids @sieve StuckJob)))) 565 | log-fn)] 566 | (log-fn :info nil "Starting a new worker...") 567 | (->Worker db-conn 568 | fn-bindings 569 | in-ch 570 | status 571 | sieve 572 | status-threads 573 | consumer-threads 574 | scheduler-thread 575 | recovery-thread 576 | options))) 577 | 578 | -------------------------------------------------------------------------------- /src/mysql_queue/queries.clj: -------------------------------------------------------------------------------- 1 | (ns mysql-queue.queries 2 | (:require [clojure.java.jdbc :as sql] 3 | [clojure.string :as string])) 4 | 5 | (defn create-scheduled-jobs! 6 | [db] 7 | (sql/execute! db 8 | ["CREATE TABLE IF NOT EXISTS `scheduled_jobs` ( 9 | `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT, 10 | `name` varchar(255) COLLATE utf8_bin NOT NULL, 11 | `status` varchar(255) COLLATE utf8_bin NOT NULL DEFAULT 'running', 12 | `parameters` blob NOT NULL, 13 | `scheduled_for` datetime NOT NULL, 14 | `created_at` datetime NOT NULL, 15 | PRIMARY KEY (`id`), 16 | KEY `scheduled_jobs_by_scheduled_for` (`scheduled_for`), 17 | KEY `scheduled_jobs_by_name` (`name`) 18 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;"])) 19 | 20 | (defn create-jobs! 21 | [db] 22 | (sql/execute! db 23 | ["CREATE TABLE IF NOT EXISTS `jobs` ( 24 | `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT, 25 | `scheduled_job_id` bigint(20) unsigned DEFAULT NULL, 26 | `parent_id` bigint(20) unsigned DEFAULT NULL, 27 | `name` varchar(255) COLLATE utf8_bin NOT NULL, 28 | `status` varchar(255) COLLATE utf8_bin NOT NULL DEFAULT 'running', 29 | `parameters` mediumblob NOT NULL, 30 | `attempt` int(10) unsigned NOT NULL DEFAULT '1', 31 | `created_at` datetime NOT NULL, 32 | PRIMARY KEY (`id`), 33 | UNIQUE KEY `jobs_by_scheduled_job_id_and_parent_id` (`scheduled_job_id`,`parent_id`), 34 | KEY `jobs_by_name` (`name`), 35 | KEY `jobs_by_status` (`status`), 36 | KEY `jobs_by_created_at` (`created_at`), 37 | CONSTRAINT `jobs_ibfk_1` FOREIGN KEY (`scheduled_job_id`) REFERENCES `scheduled_jobs` (`id`) ON DELETE CASCADE 38 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;"])) 39 | 40 | (defn in-query-stubs 41 | [xs] 42 | (string/join "," (repeat (count xs) "?"))) 43 | 44 | (defn insert-scheduled-job> f-var 17 | meta 18 | :arglists 19 | (map last) 20 | (filter map?) 21 | (mapcat :keys) 22 | (map keyword) 23 | (into #{}))) 24 | 25 | (defmacro with-error-handler 26 | "Returns a function of no arguments that calls fn with an exception object if one is thrown." 27 | [[f :as bindings] & body] 28 | {:pre [(= 1 (count bindings))]} 29 | `(fn [] 30 | (try 31 | ~@body 32 | (catch Exception e# 33 | (~f e#))))) 34 | 35 | (defn numeric-stats 36 | "Returns a map containing the minimum (:min), the maximum (:max), 37 | the median (:median), the mean (:mean), and the 90 percentile (:90p)." 38 | [s] 39 | (when (seq s) 40 | (let [sorted (sort s) 41 | length (count s)] 42 | {:min (first sorted) 43 | :max (last sorted) 44 | :mean (float (/ (reduce + s) length)) 45 | :median (nth sorted (dec (/ length 2))) 46 | :90p (nth sorted (dec (/ (* 9 length) 10)))}))) 47 | 48 | (defn ns->ms 49 | "Converts a value in nanoseconds to milliseconds." 50 | [t] 51 | (Math/round (/ (double t) 1000000))) 52 | 53 | (defmacro profile 54 | "Profiles a block of code. Returns a vector with original return value and 55 | elapsed time in ns." 56 | [& body] 57 | `(let [start# (System/nanoTime) 58 | ret# (do ~@body) 59 | elapsed# (- (System/nanoTime) start#)] 60 | [ret# elapsed#])) 61 | 62 | (defmacro meter 63 | "Profiles a block of code. Assocs result to a named key in a provided atom. 64 | Returns the original return value." 65 | [metrics name & body] 66 | `(let [[ret# elapsed#] (profile ~@body)] 67 | (swap! ~metrics assoc ~name elapsed#) 68 | ret#)) 69 | 70 | (defmacro profile-block 71 | "Profiles a block of code with multiple named hot spots. Requires a binding for 72 | optionally used atom. Pass this atom to `meter` to register hot spots. 73 | Returns a map of hotspot => execution time (in ns). 74 | The entire block is wrapped in :full hotspot by default." 75 | [[metrics :as bindings] & body] 76 | {:pre [(= 1 (count bindings))]} 77 | `(let [metrics# (atom {}) 78 | ~metrics metrics# 79 | ret# (meter metrics# :full ~@body)] 80 | [ret# (deref metrics#)])) 81 | 82 | -------------------------------------------------------------------------------- /test/mysql_queue/core_test.clj: -------------------------------------------------------------------------------- 1 | (ns mysql-queue.core-test 2 | (:require [clojure.test :refer :all] 3 | [clojure.java.jdbc :as sql] 4 | [clojure.set :as clj-set] 5 | [clojure.string :as string] 6 | [mysql-queue.core :refer :all] 7 | [mysql-queue.queries :as queries])) 8 | 9 | (Thread/setDefaultUncaughtExceptionHandler 10 | (reify Thread$UncaughtExceptionHandler 11 | (uncaughtException [_ thread throwable] 12 | (println "WARNING!!! Uncaught exception in core async:") 13 | (println throwable)))) 14 | 15 | (def db-conn {:subprotocol "mysql" 16 | :subname "//localhost:3306/clj_mysql_queue?useSSL=false" 17 | :user "root" 18 | :password ""}) 19 | 20 | (defn delete-scheduled-jobs-by-name! 21 | [db-conn job-name] 22 | (sql/delete! db-conn :scheduled_jobs ["name = ?" job-name])) 23 | 24 | (defn count-jobs 25 | [db-conn] 26 | (sql/query db-conn ["SELECT COUNT(*) AS c FROM jobs"] {:result-set-fn (comp :c first)})) 27 | 28 | (defn queue-size 29 | [db-conn] 30 | (sql/query db-conn ["SELECT COUNT(*) AS c FROM scheduled_jobs"] {:result-set-fn (comp :c first)})) 31 | 32 | (defn setup-db 33 | [f] 34 | (initialize! db-conn) 35 | (f)) 36 | 37 | (defn clean-up 38 | [f] 39 | (delete-scheduled-jobs-by-name! db-conn "test-foo") 40 | (delete-scheduled-jobs-by-name! db-conn "slow-job") 41 | (delete-scheduled-jobs-by-name! db-conn "quick-job") 42 | (f)) 43 | 44 | (use-fixtures :once setup-db) 45 | (use-fixtures :each clean-up) 46 | 47 | (defmacro with-worker 48 | [[bound-name expr :as args] & body] 49 | {:pre (= 2 (count args))} 50 | `(let [~bound-name ~expr] 51 | (try 52 | ~@body 53 | (finally 54 | (stop ~bound-name 2))))) 55 | 56 | (defn check-in-atom [expected-set success-promise] 57 | (doto (atom []) 58 | (add-watch :test (fn [_ _ _ v] 59 | (when (= (set v) expected-set) 60 | (deliver success-promise true)))))) 61 | 62 | (deftest job-processing-test 63 | (let [num-jobs 100 64 | expected-set (->> num-jobs range (map inc) (into #{})) 65 | success? (promise) 66 | exception (promise) 67 | check-ins (check-in-atom expected-set success?) 68 | jobs {:test-foo (fn [status {id :id :as args}] 69 | (Thread/sleep 10) 70 | (swap! check-ins conj id) 71 | [:done args])} 72 | _ (dotimes [n num-jobs] 73 | (schedule-job db-conn :test-foo :begin {:id (inc n)} (java.util.Date.)))] 74 | (with-worker [wrk (worker db-conn 75 | jobs 76 | :num-consumer-threads 1 77 | :err-fn #(deliver exception %) 78 | :max-scheduler-sleep-interval 0.5 79 | :max-recovery-sleep-interval 0.5)] 80 | (is (deref success? 15000 false) 81 | (str "Failed to process " num-jobs " test jobs in 15 seconds.\n" 82 | "Missing job IDs: " (clj-set/difference expected-set @check-ins) "\n" 83 | "Exception?: " (deref exception 0 "nope"))) 84 | (is (= num-jobs (count @check-ins)) 85 | "The number of executed jobs doesn't match the number of jobs queued.")))) 86 | 87 | (deftest parallel-job-processing-test 88 | (let [num-jobs 100 89 | expected-set (->> num-jobs range (map inc) (into #{})) 90 | success? (promise) 91 | exception (promise) 92 | check-ins (check-in-atom expected-set success?) 93 | jobs {:test-foo (fn [status {id :id :as args}] 94 | (Thread/sleep 10) 95 | (swap! check-ins conj id) 96 | [:done args])} 97 | _ (dotimes [n num-jobs] 98 | (schedule-job db-conn :test-foo :begin {:id (inc n)} (java.util.Date.)))] 99 | (with-worker [wrk (worker db-conn 100 | jobs 101 | :prefetch 4 102 | :num-consumer-threads 4 103 | :err-fn #(deliver exception %) 104 | :max-scheduler-sleep-interval 0.5 105 | :max-recovery-sleep-interval 0.5)] 106 | (is (deref success? 15000 false) 107 | (str "Failed to process " num-jobs " test jobs in 15 seconds.\n" 108 | "Missing job IDs: " (clj-set/difference expected-set @check-ins) "\n" 109 | "Exception?: " (deref exception 0 "nope"))) 110 | (is (= num-jobs (count @check-ins)) 111 | "The number of executed jobs doesn't match the number of jobs queued.")))) 112 | 113 | (deftest unbalanced-parallel-job-processing-test 114 | (let [num-slow-jobs 1 115 | num-quick-jobs 5 116 | expected-slow-set (->> num-slow-jobs range (map inc) (into #{})) 117 | expected-quick-set (->> num-quick-jobs range (map inc) (into #{})) 118 | quick-success? (promise) 119 | slow-success? (promise) 120 | exception (promise) 121 | slow-check-ins (check-in-atom expected-slow-set slow-success?) 122 | quick-check-ins (check-in-atom expected-quick-set quick-success?) 123 | jobs {:quick-job (fn [status {id :id :as args}] 124 | (swap! quick-check-ins conj id) 125 | [:done args]) 126 | :slow-job (fn [status {id :id :as args}] 127 | (when (deref quick-success? 2000 false) 128 | (swap! slow-check-ins conj id)) 129 | [:done args])} 130 | _ (dotimes [n num-slow-jobs] 131 | (schedule-job db-conn :slow-job :begin {:id (inc n)} (java.util.Date.))) 132 | _ (dotimes [n num-quick-jobs] 133 | (schedule-job db-conn :quick-job :begin {:id (inc n)} (java.util.Date.)))] 134 | (with-worker [wrk (worker db-conn 135 | jobs 136 | :prefetch 3 137 | :num-consumer-threads 2 138 | :err-fn #(deliver exception %) 139 | :max-scheduler-sleep-interval 0.1)] 140 | (is (deref slow-success? 2000 false) 141 | (str "Failed to process 1 slow job and " num-quick-jobs 142 | " quick jobs in 2 seconds.\n" 143 | "Missing slow job IDs: " (clj-set/difference expected-slow-set 144 | @slow-check-ins) "\n" 145 | "Missing quick job IDs: " (clj-set/difference expected-quick-set 146 | @quick-check-ins) "\n" 147 | "Exception?: " (deref exception 0 "nope"))) 148 | (is (= num-slow-jobs (count @slow-check-ins)) 149 | "The number of executed slow jobs doesn't match the number of jobs queued.") 150 | (is (= num-quick-jobs (count @quick-check-ins)) 151 | "The number of executed quick jobs doesn't match the number of jobs queued.")))) 152 | 153 | (deftest distributed-job-processing-test 154 | (let [num-jobs 100 155 | expected-set (->> num-jobs range (map inc) (into #{})) 156 | success? (promise) 157 | exception (promise) 158 | check-ins (check-in-atom expected-set success?) 159 | jobs {:test-foo (fn [status {id :id :as args}] 160 | (Thread/sleep 10) 161 | (swap! check-ins conj id) 162 | [:done args])} 163 | _ (dotimes [n num-jobs] 164 | (schedule-job db-conn :test-foo :begin {:id (inc n)} (java.util.Date.)))] 165 | (with-worker [wrk-1 (worker db-conn 166 | jobs 167 | :prefetch 4 168 | :num-consumer-threads 2 169 | :err-fn #(deliver exception %) 170 | :max-scheduler-sleep-interval 0.5 171 | :max-recovery-sleep-interval 0.5)] 172 | (with-worker [wrk-2 (worker db-conn 173 | jobs 174 | :prefetch 4 175 | :num-consumer-threads 2 176 | :err-fn #(deliver exception %) 177 | :max-scheduler-sleep-interval 0.5 178 | :max-recovery-sleep-interval 0.5)] 179 | (is (deref success? 15000 false) 180 | (str "Failed to process " num-jobs " test jobs in 15 seconds.\n" 181 | "Missing job IDs: " (clj-set/difference expected-set @check-ins) "\n" 182 | "Exception?: " (deref exception 0 "nope"))) 183 | (is (= num-jobs (count @check-ins)) 184 | "The number of executed jobs doesn't match the number of jobs queued."))))) 185 | 186 | (deftest stuck-job-processing-test 187 | (let [num-jobs 100 188 | expected-set (->> num-jobs range (map inc) (into #{})) 189 | success? (promise) 190 | exception (promise) 191 | check-ins (check-in-atom expected-set success?) 192 | jobs {:test-foo (fn [status {id :id :as args}] 193 | (Thread/sleep 10) 194 | (swap! check-ins conj id) 195 | [:done args])} 196 | _ (dotimes [n num-jobs] 197 | (let [scheduled-id (schedule-job db-conn :test-foo :begin {:id (inc n)} (java.util.Date.))] 198 | (queries/insert-job> num-jobs range (map inc) (into #{})) 249 | success? (promise) 250 | exception (promise) 251 | lock (promise) 252 | check-ins (check-in-atom expected-set success?) 253 | jobs {:test-foo (fn [status {id :id :as args}] 254 | @lock 255 | (Thread/sleep 1500) 256 | (swap! check-ins conj id) 257 | [:done args])} 258 | _ (dotimes [n num-jobs] 259 | (schedule-job db-conn :test-foo :begin {:id (inc n)} (java.util.Date.)))] 260 | (with-worker [wrk (worker db-conn 261 | jobs 262 | :num-consumer-threads 2 263 | :err-fn #(deliver exception %) 264 | :recovery-threshold-mins 0 265 | :max-scheduler-sleep-interval 0.5 266 | :max-recovery-sleep-interval 0.5)] 267 | (deliver lock :unlocked) 268 | (Thread/sleep 1000)) 269 | (is (deref success? 10 false) 270 | (str "Failed to finish " num-jobs " test jobs taking 1500ms with 2s quit timeout.\n" 271 | "Missing job IDs: " (clj-set/difference expected-set @check-ins) "\n" 272 | "Exception?: " (deref exception 0 "nope"))) 273 | (is (= num-jobs (count @check-ins)) 274 | "The number of executed jobs doesn't match the number of jobs queued."))) 275 | 276 | (deftest status-test 277 | (let [num-jobs 100 278 | expected-set (->> num-jobs range (map inc) (into #{})) 279 | success? (promise) 280 | exception (promise) 281 | check-ins (check-in-atom expected-set success?) 282 | jobs {:test-foo (fn [status {id :id :as args}] 283 | (Thread/sleep 20) 284 | (swap! check-ins conj id) 285 | [:done args])}] 286 | (with-worker [wrk (worker db-conn 287 | jobs 288 | :num-consumer-threads 1 289 | :err-fn #(deliver exception %) 290 | :max-scheduler-sleep-interval 0.5 291 | :max-recovery-sleep-interval 2)] 292 | ; Initial status 293 | (let [{{:keys [scheduled-jobs jobs]} :db-queue 294 | :keys [consumers prefetched-jobs]} 295 | (status wrk)] 296 | (is (= 1 (count consumers))) 297 | (is (zero? (:overdue scheduled-jobs))) 298 | (is (zero? (:total scheduled-jobs))) 299 | (is (zero? (:stuck jobs))) 300 | (is (zero? (:total jobs))) 301 | (is (empty? prefetched-jobs))) 302 | 303 | ; Publishing jobs 304 | (dotimes [n num-jobs] 305 | (schedule-job db-conn :test-foo :begin {:id (inc n)} (java.util.Date.))) 306 | 307 | ; Publishing one stuck job 308 | (let [scheduled-id (schedule-job db-conn :test-foo :begin {:id 1} (java.util.Date. 0))] 309 | (queries/insert-jobms-test 11 | (is (= 0 (ns->ms 0))) 12 | (is (= 1 (ns->ms 1000000)))) 13 | 14 | (deftest profile-test 15 | (let [p (promise) 16 | [ret t] (profile (deref p 50 42))] 17 | (is (= 42 ret)) 18 | (is (<= 50000000 t)))) 19 | 20 | (deftest profile-block-test 21 | (let [p (promise) 22 | [ret m] (profile-block [m] 23 | (meter m :subop (deref p 50 nil)) 24 | (Thread/sleep 50) 25 | 42)] 26 | (is (= 42 ret)) 27 | (is (<= 50000000 (:subop m) 60000000)) 28 | (is (<= 100000000 (:full m))))) 29 | 30 | (deftest numeric-stats-test 31 | (let [stats (numeric-stats (shuffle (range 1 101)))] 32 | (is (= 1 (:min stats))) 33 | (is (= 100 (:max stats))) 34 | (is (= 50 (:median stats))) 35 | (is (= 50.5 (:mean stats))) 36 | (is (= 90 (:90p stats))))) 37 | 38 | --------------------------------------------------------------------------------