├── .github └── workflows │ ├── release.yaml │ └── tests.yml ├── .gitignore ├── .gitmodules ├── LICENSE ├── Makefile ├── README.md ├── deps.edn ├── dev └── com │ └── moclojer │ └── tools │ └── build.clj ├── docker └── docker-compose.yml ├── docs └── README.md ├── resources ├── command-allowlist.edn └── logback.xml ├── src └── com │ └── moclojer │ ├── internal │ └── reflection.clj │ ├── rq.clj │ └── rq │ ├── adapters.clj │ ├── pubsub.clj │ └── queue.clj └── test └── com └── moclojer ├── rq ├── adapters_test.clj ├── pubsub_test.clj └── queue_test.clj ├── rq_test.clj └── test_helpers.clj /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*' 7 | workflow_dispatch: 8 | 9 | jobs: 10 | clojars: 11 | runs-on: ubuntu-latest 12 | environment: env 13 | defaults: 14 | run: 15 | shell: bash 16 | steps: 17 | - uses: actions/checkout@v4 18 | - uses: "DeLaGuardo/setup-clojure@12.5" 19 | with: 20 | cli: "1.11.1.1139" 21 | - name: deploy 22 | env: 23 | CLOJARS_USERNAME: ${{ secrets.CLOJARS_USERNAME }} 24 | CLOJARS_PASSWORD: ${{ secrets.CLOJARS_PASSWORD }} 25 | run: | 26 | make jedis 27 | clojure -M:dev --report stderr -m com.moclojer.tools.build && \ 28 | clojure -X:deploy-clojars 29 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Run Tests 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | - 'refs/tags/*' 8 | pull_request: 9 | branches: 10 | - main 11 | 12 | jobs: 13 | test: 14 | runs-on: ubuntu-latest 15 | services: 16 | redis: 17 | image: "redis:7" 18 | ports: 19 | - 6379:6379 20 | options: --rm 21 | 22 | steps: 23 | - name: Checkout code 24 | uses: actions/checkout@v3 25 | 26 | - name: Set up Java 27 | uses: actions/setup-java@v1 28 | with: 29 | distribution: 'adopt' 30 | java-version: '11' 31 | 32 | - name: Clone Submodules 33 | run: make jedis 34 | 35 | - name: Install clojure cli 36 | uses: DeLaGuardo/setup-clojure@master 37 | with: 38 | cli: 1.11.1.1113 39 | 40 | - name: Cache Maven packages 41 | uses: actions/cache@v3 42 | with: 43 | path: ~/.m2 44 | key: ${{ runner.os }}-m2-${{ hashFiles('**/deps.edn') }} 45 | restore-keys: ${{ runner.os }}-m2 46 | 47 | - name: Run tests 48 | run: clojure -M:test 49 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | pom.xml 2 | pom.xml.asc 3 | *.pom.asc 4 | *.jar 5 | *.class 6 | /lib/ 7 | /classes/ 8 | /target/ 9 | /checkouts/ 10 | .lein-deps-sum 11 | .lein-repl-history 12 | .lein-plugins/ 13 | .lein-failures 14 | .nrepl-port 15 | .cpcache/ 16 | **/.lsp/.cache 17 | /.clj-kondo/.cache 18 | /.calva/output-window 19 | output.calva-repl 20 | # emacs 21 | *~ 22 | /.clj-kondo 23 | .envrc 24 | redis 25 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "vendor/jedis"] 2 | path = vendor/jedis 3 | url = https://github.com/moclojer/jedis.git 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 moclojer 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | jedis: 2 | git submodule update --init --recursive --remote 3 | cd vendor/jedis && make mvn-package-no-tests 4 | 5 | all: jedis 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # clj-rq 2 | 3 | RQ (Redis Queue) is a simple Clojure package for queueing jobs and processing them in the background with workers. It is backed by Redis and it is designed to have a low barrier to entry, inspired by [python-rq](https://python-rq.org). 4 | 5 | > "simple is better than complex" - [The Zen of Python](https://peps.python.org/pep-0020/) 6 | 7 | > clj-rq arose from the need to simplify the use of the redis queue feature used in the [SaaS version of moclojer](https://app.moclojer.com), [here's](https://www.reddit.com/r/Clojure/comments/1d1567t/redis_queue_in_clojure/) a more detailed explanation of the motivation *(a thread that generated controversy)* 8 | 9 | ## installation 10 | 11 | We distribute the library via [Clojars](https://clojars.org/com.moclojer/rq). 12 | 13 | 14 | [![Clojars Project](https://img.shields.io/clojars/v/com.moclojer/rq.svg)](https://clojars.org/com.moclojer/rq) 15 | 16 | ```edn 17 | com.moclojer/rq {:mvn/version "0.x.x"} 18 | ``` 19 | 20 | ```clojure 21 | [com.moclojer/rq "0.x.x"] 22 | ``` 23 | 24 | > see the versions distributed on clojars 25 | 26 | ## building from source 27 | 28 | We build Jedis ourselves to enable building queue functions directly using reflection. This approach ensures full compatibility with our library's features. 29 | 30 | ### prerequisites 31 | 32 | - Make sure you have Java JDK (version X.X or higher) installed 33 | - Ensure you have Make installed on your system 34 | 35 | ### build steps 36 | 37 | 1. Clone the repository: `git clone [repository URL]` 38 | 2. Navigate to the project directory: `cd clj-rq` 39 | 3. Run the build command: `make jedis` 40 | 41 | After running `make jedis`, the library will be built and ready to be linked with your project. Linking in this context means that the built Jedis library will be properly referenced and used by clj-rq when you include it in your project. 42 | 43 | ## how clj-rq works under the hood 44 | 45 | The `clj-rq` library leverages the `->wrap-method` macro to dynamically generate queue functions by wrapping methods from the Jedis library. This approach ensures that the library is always up-to-date with the latest changes in Jedis, providing enhanced security and compatibility. 46 | 47 | The `->wrap-method` macro is defined in `src/com/moclojer/internal/reflection.clj` and is used in `src/com/moclojer/rq/queue.clj` to generate the queue functions. By using reflection, the library can dynamically adapt to changes in the Jedis API, ensuring that the generated functions are always in sync with the underlying Jedis methods. 48 | 49 | This dynamic generation process is a key differentiator of the `clj-rq` library, making it more secure and future-proof compared to other libraries that rely on static function definitions. 50 | 51 | ## functions 52 | 53 | This section outlines the key functions available in the clj-rq library, covering both queue and pub/sub operations. For detailed descriptions and examples of each function, please refer to the specific subsections below. 54 | 55 | ### queue 56 | 57 | The `clj-rq` library provides a set of queue functions that are dynamically generated by wrapping methods from the Jedis library. These functions are defined in `src/com/moclojer/rq/queue.clj` and include: 58 | 59 | - `push!`: Adds elements to the queue. 60 | - `pop!`: Removes and returns elements from the queue. 61 | - `bpop!`: Blocks until an element is available to pop from the queue. 62 | - `index`: Retrieves an element at a specific index in the queue. 63 | - `range`: Retrieves a range of elements from the queue. 64 | - `set!`: Sets the value of an element at a specific index in the queue. 65 | - `len`: Returns the length of the queue. 66 | - `rem!`: Removes elements from the queue. 67 | - `insert!`: Inserts an element into the queue at a specific position. 68 | - `trim!`: Trims the queue to a specified range. 69 | 70 | #### common options 71 | 72 | All these functions share common options, such as specifying the queue name and handling encoding/decoding of messages. The options are passed as arguments to the functions and allow for flexible configuration. 73 | 74 | #### examples 75 | 76 | - **push!**: This function adds an element to the queue. It supports options for specifying the direction (left or right) and encoding the message before pushing it to the queue. 77 | 78 | > [!WARNING] 79 | > The element or elements to be pushed into a queue has to be passed inside a sequentiable (a vector for example). 80 | 81 | ```clojure 82 | (rq-queue/push! client "my-queue" ["message"] {:direction :left}) 83 | ``` 84 | 85 | - **pop!**: This function removes and returns an element from the queue. It supports options for specifying the direction (left or right) and decoding the message after popping it from the queue. 86 | 87 | ```clojure 88 | (rq-queue/pop! client "my-queue" {:direction :right}) 89 | ``` 90 | 91 | - **bpop!**: This function blocks until an element is available to pop from the queue. It is useful in scenarios where you need to wait for new messages to arrive. 92 | 93 | ```clojure 94 | (rq-queue/bpop! client "my-queue" {:timeout 5}) 95 | ``` 96 | 97 | - **index**: This function retrieves an element at a specific index in the queue. It supports options for decoding the retrieved message. 98 | 99 | ```clojure 100 | (rq-queue/index client "my-queue" 0) 101 | ``` 102 | 103 | - **range**: This function retrieves a range of elements from the queue. It supports options for decoding the retrieved messages. 104 | 105 | ```clojure 106 | (rq-queue/range client "my-queue" 0 -1) 107 | ``` 108 | 109 | - **set!**: This function sets the value of an element at a specific index in the queue. It supports options for encoding the message before setting it. 110 | 111 | ```clojure 112 | (rq-queue/set! client "my-queue" 0 "new-message") 113 | ``` 114 | 115 | - **len**: This function returns the length of the queue. It is useful for monitoring the size of the queue. 116 | 117 | ```clojure 118 | (rq-queue/len client "my-queue") 119 | ``` 120 | 121 | - **rem!**: This function removes elements from the queue based on a specified pattern. It supports options for specifying the number of elements to remove. 122 | 123 | ```clojure 124 | (rq-queue/rem! client "my-queue" "message" {:count 2}) 125 | ``` 126 | 127 | - **insert!**: This function inserts an element into the queue at a specific position. It supports options for encoding the message before inserting it. 128 | 129 | ```clojure 130 | (rq-queue/insert! client "my-queue" "pivot-message" "new-message" {:position :before}) 131 | ``` 132 | 133 | - **trim!**: This function trims the queue to a specified range. It is useful for maintaining the size of the queue within certain limits. 134 | 135 | ```clojure 136 | (rq-queue/trim! client "my-queue" 0 10) 137 | ``` 138 | 139 | ### pubsub 140 | 141 | The `clj-rq` library provides a set of pub/sub functions that facilitate message publishing and subscription in a Redis-backed system. These functions are defined in `src/com/moclojer/rq/pubsub.clj` and include: 142 | 143 | - `publish!`: Publishes a message to a specified channel. 144 | - `group-handlers-by-channel`: Groups message handlers by their associated channels. 145 | - `create-listener`: Creates a listener that processes messages from subscribed channels. 146 | - `unarquive-channel!`: Unarchives a channel, making it active again. 147 | - `pack-workers-channels`: Packs worker channels into a format suitable for processing. 148 | - `subscribe!`: Subscribes to one or more channels and processes incoming messages. 149 | 150 | #### examples 151 | 152 | - **publish!**: This function publishes a message to a specified channel. It is used to send messages to subscribers listening on that channel. 153 | 154 | ```clojure 155 | (rq-pubsub/publish! client "my-channel" "Hello, World!") 156 | ``` 157 | 158 | - **subscribe!**: This function subscribes to one or more channels and processes incoming messages using the provided handlers. 159 | 160 | ```clojure 161 | (rq-pubsub/subscribe! client ["my-channel"] handlers) 162 | ``` 163 | 164 | - **unarquive-channel!**: This function unarchives a channel, making it active again. It is useful for reactivating channels that were previously archived. 165 | 166 | ```clojure 167 | (rq-pubsub/unarquive-channel! client "my-channel") 168 | ``` 169 | 170 | ## complete example 171 | 172 | ```clojure 173 | (ns rq.example 174 | (:require [com.moclojer.rq :as rq] 175 | [com.moclojer.rq.queue :as queue] 176 | [com.moclojer.rq.pubsub :as pubsub])) 177 | 178 | (def *redis-pool* (rq/create-client "redis://localhost:6379/0")) 179 | 180 | ;; queue 181 | (queue/push! *redis-pool* "my-queue" 182 | ;; has to be an array of the elements to push 183 | [{:now (java.time.LocalDateTime/now) 184 | :foo "bar"}]) 185 | 186 | (println :size (queue/len *redis-pool* "my-queue")) 187 | (prn :popped (queue/pop! *redis-pool* "my-queue")) 188 | 189 | ;; pub/sub 190 | (def my-workers 191 | [{:channel "my-channel" 192 | :handler (fn [msg] 193 | (prn :msg :my-channel msg))} 194 | {:channel "my-other-channel" 195 | :handler (fn [{:keys [my data hello]}] 196 | (my-function my data hello))}]) 197 | 198 | (pubsub/subscribe! *redis-pool* my-workers) 199 | (pubsub/publish! *redis-pool* "my-channel" "hello world") 200 | (pubsub/publish! *redis-pool* "my-other-channel" 201 | {:my "moclojer team" 202 | :data "app.moclojer.com" 203 | :hello "maybe you'll like this website"}) 204 | 205 | (rq/close-client *redis-pool*) 206 | ``` 207 | 208 | The workflow in the given example can be represented as follows: 209 | 210 | ```mermaid 211 | sequenceDiagram 212 | participant User 213 | participant Client 214 | participant Queue 215 | participant PubSub 216 | participant Logger 217 | 218 | User->>Client: create-client URL 219 | Client-->>Logger: log client creation 220 | Client-->>User: return client 221 | 222 | User->>Queue: push! message 223 | Queue-->>Logger: log push message 224 | Queue-->>Queue: push message to queue 225 | 226 | User->>PubSub: publish! channel, message 227 | PubSub-->>Logger: log publish message 228 | PubSub-->>PubSub: publish message to channel 229 | 230 | User->>Queue: pop! queue-name 231 | Queue-->>Logger: log pop operation 232 | Queue-->>User: return popped message 233 | 234 | User->>Client: close-client client 235 | Client-->>Logger: log closing client 236 | Client-->>User: confirm client closure 237 | ``` 238 | 239 | --- 240 | 241 | Made with 💜 by [moclojer](https://moclojer.com). 242 | -------------------------------------------------------------------------------- /deps.edn: -------------------------------------------------------------------------------- 1 | {:paths ["src" "resources"] 2 | :deps 3 | {redis.clients/jedis {#_#_:mvn/version "5.1.2" 4 | :local/root "vendor/jedis/target/jedis-5.2.0-SNAPSHOT.jar"} 5 | org.clojure/tools.logging {:mvn/version "1.3.0"} 6 | ch.qos.logback/logback-classic {:mvn/version "1.5.6"} 7 | camel-snake-kebab/camel-snake-kebab {:mvn/version "0.4.3"} 8 | org.clojure/data.json {:mvn/version "2.5.0"}} 9 | 10 | :aliases 11 | {;; clj -A:dev -m com.moclojer.rq 12 | :dev 13 | {:extra-paths ["dev"] 14 | :extra-deps {io.github.clojure/tools.build {:git/tag "v0.9.6" 15 | :git/sha "8e78bcc" 16 | :exclusions [org.slf4j/slf4j-nop]}}} 17 | 18 | ;; Run all tests 19 | ;; clj -M:test 20 | ;; Run specific test 21 | ;; clj -M:test -n com.moclojer.rq.excel-test 22 | :test 23 | {:extra-paths ["test"] 24 | :extra-deps {io.github.cognitect-labs/test-runner {:git/tag "v0.5.1" 25 | :git/sha "dfb30dd"} 26 | clj-kondo/clj-kondo {:mvn/version "2023.10.20"} 27 | cljfmt/cljfmt {:mvn/version "0.9.2"}} 28 | :main-opts ["-m" "cognitect.test-runner"] 29 | :exec-fn cognitect.test-runner.api/test} 30 | 31 | ;; clj -M:nrepl 32 | :nrepl 33 | {:extra-deps {cider/cider-nrepl {:mvn/version "0.30.0"}} 34 | :main-opts ["-m" "nrepl.cmdline" "--middleware" "[cider.nrepl/cider-middleware]"]} 35 | 36 | ;; Lint the source 37 | ;; clj -M:lint 38 | :lint 39 | {:replace-deps {clj-kondo/clj-kondo {:mvn/version "2023.10.20"}} 40 | :main-opts ["-m" "clj-kondo.main" "--lint" "src"]} 41 | 42 | ;; clj -M:dev --report stderr -m com.moclojer.tools.build 43 | ;; env CLOJARS_USERNAME=username CLOJARS_PASSWORD=clojars-token 44 | ;; clj -X:deploy-clojars 45 | :deploy-clojars 46 | {:extra-deps {slipset/deps-deploy {:mvn/version "RELEASE"}} 47 | :exec-fn deps-deploy.deps-deploy/deploy 48 | :exec-args {:installer :remote 49 | :sign-releases? false 50 | :pom-file "target/classes/META-INF/maven/com.moclojer/rq/pom.xml" 51 | :artifact "target/com.moclojer.rq.jar"}}}} 52 | 53 | 54 | 55 | -------------------------------------------------------------------------------- /dev/com/moclojer/tools/build.clj: -------------------------------------------------------------------------------- 1 | (ns com.moclojer.tools.build 2 | (:refer-clojure :exclude [test]) 3 | (:require 4 | [clojure.string :as string] 5 | [clojure.tools.build.api :as b] 6 | [com.moclojer.rq :as rq])) 7 | 8 | (def class-dir "target/classes") 9 | (def jar-file "target/com.moclojer.rq.jar") 10 | 11 | (set! *warn-on-reflection* true) 12 | 13 | (defmacro with-err-str 14 | [& body] 15 | `(let [s# (new java.io.StringWriter)] 16 | (binding [*err* s#] 17 | ~@body 18 | (str s#)))) 19 | 20 | (def pom-template 21 | [[:description "RQ (Redis Queue) is a simple Clojure package for queueing jobs and processing them in the background with workers. It is backed by Redis and it is designed to have a low barrier to entry"] 22 | [:url "https://github.com/moclojer/clj-rq"] 23 | [:licenses 24 | [:license 25 | [:name "MIT License"] 26 | [:url "https://opensource.org/licenses/MIT"]]] 27 | [:scm 28 | [:url "https://github.com/moclojer/clj-rq"] 29 | [:connection "scm:git:https://github.com/moclojer/clj-rq.git"] 30 | [:developerConnection "scm:git:ssh:git@github.com:moclojer/clj-rq.git"] 31 | [:tag (str "v" rq/version)]]]) 32 | 33 | (def options 34 | (let [basis (b/create-basis {:project "deps.edn"})] 35 | {:class-dir class-dir 36 | :lib 'com.moclojer/rq 37 | :main 'com.moclojer.rq 38 | :version rq/version 39 | :basis basis 40 | :ns-compile '[com.moclojer.rq com.moclojer.rq.queue] 41 | :uber-file jar-file 42 | :jar-file jar-file 43 | :target "target" 44 | :src-dirs (:paths basis) 45 | :pom-data pom-template 46 | :exclude ["docs/*" "test/*" "target/*"]})) 47 | 48 | (defn -main 49 | [& _] 50 | (let [basis (b/create-basis {:project "deps.edn"})] 51 | (println "Clearing target directory") 52 | (b/delete {:path "target"}) 53 | 54 | (println "Writing pom") 55 | (->> (b/write-pom options) 56 | with-err-str 57 | string/split-lines 58 | ;; Avoid confusing future me/you: suppress "Skipping coordinate" messages for our jars, we don't care, we are creating an uberjar 59 | (remove #(re-matches #"^Skipping coordinate: \{:local/root .*target/(lib1|lib2|graal-build-time).jar.*" %)) 60 | (run! println)) 61 | (b/copy-dir {:src-dirs (:paths basis) 62 | :target-dir class-dir}) 63 | 64 | (println "Compile sources to classes") 65 | (b/compile-clj options) 66 | 67 | (println "Packaging classes into jar") 68 | (b/jar options))) 69 | -------------------------------------------------------------------------------- /docker/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.7" 2 | 3 | services: 4 | redis: 5 | image: redis:latest 6 | ports: 7 | - "6379:6379" 8 | volumes: 9 | - ./redis:/redis 10 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # Code structure 2 | 3 | We have now, [rq.clj](../src/rq.clj) responsible for stablishing the connection to the redis server. If you do not have a redis running on your machine, you can either download redis and run it locally or use our [dockerfile or docker-compose.yml](../docker/docker-compose.yml) to run it. 4 | Also, our [rq folder](../src/rq/) contains all the code we need. You can see how we handle redis on clj or you can test them by yourself. 5 | 6 | 7 | ## Running some tests 8 | 9 | To run the tests we've set, you can use one of the following commands: 10 | 11 | ### Using Clojure CLI 12 | 13 | This command will run all the test cases on our deps.edn file. 14 | 15 | ```sh 16 | 17 | clj -M:tests 18 | 19 | ``` 20 | 21 | ### Using Leiningen 22 | if you want to use lein, you will have to set the environmet before. But this will be the command to run the tests. 23 | 24 | ```sh 25 | 26 | lein test com.moclojer.rq.queue-test 27 | 28 | ``` 29 | 30 | This commands will run all the test cases defined in the our tests namespaces and provide feedback on their status. 31 | By running these tests, you can verify the correctness and reliability of the queue operations, such as seeing some important info output on debug mode. -------------------------------------------------------------------------------- /resources/command-allowlist.edn: -------------------------------------------------------------------------------- 1 | #{"lpush" "rpush" "lpop" "rpop" "brpop" 2 | "blpop" "lrange" "lindex" "lset" "lrem" 3 | "llen" "linsert" "ltrim" "rpoplpush" 4 | "brpoplpush" "lmove"} 5 | -------------------------------------------------------------------------------- /resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | %d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /src/com/moclojer/internal/reflection.clj: -------------------------------------------------------------------------------- 1 | (ns com.moclojer.internal.reflection 2 | (:require 3 | [camel-snake-kebab.core :as csk] 4 | [clojure.string :as str] 5 | [com.moclojer.rq.adapters :as adapters])) 6 | 7 | (defn unpack-parameter 8 | [parameter] 9 | {:type (.. parameter getType getName) 10 | :name (csk/->kebab-case (.getName parameter))}) 11 | 12 | (defn unpack-method 13 | [method] 14 | {:name (csk/->kebab-case (.getName method)) 15 | :parameters (map unpack-parameter (.getParameters method))}) 16 | 17 | (defn underload-methods 18 | "Given a list of overloaded `methods`, returns each one's parameter 19 | list that matches given its `paramlist`." 20 | [paramlist methods] 21 | (reduce 22 | (fn [underloaded-methods {:keys [name parameters]}] 23 | (let [allowed-params (get paramlist name) 24 | param-names (map :name parameters)] 25 | (if (and (= (count parameters) (count allowed-params)) 26 | (every? #(some #{%} param-names) allowed-params)) 27 | (assoc underloaded-methods name parameters) 28 | underloaded-methods))) 29 | {} methods)) 30 | 31 | (defn get-klazz-methods 32 | [klazz allowmap] 33 | (let [allowlist (set (keys allowmap)) 34 | paramlist (reduce-kv 35 | (fn [acc name method] 36 | (assoc acc name (second method))) 37 | {} allowmap)] 38 | (->> (.getMethods klazz) 39 | (map unpack-method) 40 | (filter #(contains? allowlist (:name %))) 41 | (underload-methods paramlist)))) 42 | 43 | (defmacro ->wrap-method 44 | "Wraps given jedis `method` and its respective `parameters` into a 45 | common function for this library, which includes, besides the wrapped 46 | function itself, options like key pattern and encoding/decoding." 47 | [method parameters allowmap] 48 | (let [wrapped-method (clojure.string/replace method #"[`0-9]" "") 49 | param-syms (map #(-> % :name symbol) parameters) 50 | [_ _ enc dec] (get allowmap method ["" nil :none :none])] 51 | `(fn 52 | ~(-> (into ['client] param-syms) 53 | (conj '& 'options)) 54 | 55 | (let [~{:keys ['pattern 'encoding 'decoding] 56 | :or {'pattern :rq 57 | 'encoding enc 58 | 'decoding dec}} ~'options 59 | 60 | ~'result ~(->> (reduce 61 | (fn [acc par] 62 | (->> (cond 63 | (= par 'key) 64 | `(com.moclojer.rq.adapters/pack-pattern 65 | ~'pattern ~par) 66 | 67 | (some #{'value 'string 68 | 'args 'pivot} [par]) 69 | `(com.moclojer.rq.adapters/encode 70 | ~'encoding ~par) 71 | 72 | :else par) 73 | (conj acc))) 74 | [] 75 | param-syms) 76 | (into [(symbol (str "." wrapped-method)) '@client]) 77 | (seq))] 78 | (try 79 | (com.moclojer.rq.adapters/decode ~'decoding ~'result) 80 | (catch ~'Exception ~'e 81 | (.printStackTrace ~'e) 82 | ~'result)))))) 83 | 84 | (comment 85 | (get-klazz-methods 86 | redis.clients.jedis.JedisPooled 87 | {"rpop" ["hello" ["key" "count"] :edn-array :none]}) 88 | 89 | (require '[clojure.pprint :refer [pprint]]) 90 | (let [allowmap {"linsert" ["Inserts a message into a queue in reference to given pivot" 91 | ["key" "where" "pivot" "value"] :edn :none]} 92 | [method parameters] (first 93 | (get-klazz-methods 94 | redis.clients.jedis.JedisPooled 95 | allowmap))] 96 | (pprint 97 | (macroexpand-1 `(->wrap-method ~method ~parameters ~allowmap)))) 98 | 99 | ;; 100 | ) 101 | -------------------------------------------------------------------------------- /src/com/moclojer/rq.clj: -------------------------------------------------------------------------------- 1 | (ns com.moclojer.rq 2 | (:import 3 | [redis.clients.jedis JedisPooled])) 4 | 5 | (def version "0.2.2") 6 | 7 | ;; redis connection pool to be thread safe 8 | (def 9 | ^{:private true :dynamic true} 10 | *redis-pool* (ref nil)) 11 | 12 | (defn create-client 13 | "Connect to redis client. If `ref?` is true, will save the created instance 14 | in the global var `*redis-pool*. Just returns the created instance otherwise." 15 | ([url] 16 | (create-client url false)) 17 | ([url ref?] 18 | (let [pool (JedisPooled. url)] 19 | (if (and ref? (not @*redis-pool*)) 20 | (dosync 21 | (ref-set *redis-pool* pool) 22 | *redis-pool*) 23 | (atom pool))))) 24 | 25 | (defn close-client 26 | "Disconnect and close redis client. 27 | If no specific client is passed, the global client stored is closed;" 28 | ([] (close-client *redis-pool*)) 29 | ([client] (.close @client))) 30 | -------------------------------------------------------------------------------- /src/com/moclojer/rq/adapters.clj: -------------------------------------------------------------------------------- 1 | (ns com.moclojer.rq.adapters 2 | (:require 3 | [clojure.data.json :as json] 4 | [clojure.edn :as edn] 5 | [clojure.string :as str] 6 | [clojure.tools.logging :as log]) 7 | (:import 8 | [redis.clients.jedis.args ListPosition])) 9 | 10 | (def patterns 11 | {:none "" 12 | :rq "rq:" 13 | :pubsub "rq:pubsub:" 14 | :pending "rq:pubsub:pending:"}) 15 | 16 | (defn- pattern->str 17 | "Adapts given pattern keyword to a known internal pattern. Raises 18 | an exception if invalid." 19 | [pattern] 20 | (or (get patterns pattern) 21 | (throw (ex-info (str "No pattern named " pattern) 22 | {:cause :illegal-argument 23 | :value pattern 24 | :expected (keys patterns)})))) 25 | 26 | (defn pack-pattern 27 | [pattern queue-name] 28 | (log/debug :packing pattern queue-name) 29 | (str (pattern->str pattern) queue-name)) 30 | 31 | (defn unpack-pattern 32 | [pattern queue-name] 33 | (log/debug :unpacking pattern queue-name) 34 | (let [prefix (pattern->str pattern)] 35 | (if (str/starts-with? queue-name prefix) 36 | (subs queue-name (count prefix)) 37 | (do 38 | (log/warn :invalid-prefix 39 | :queue-name queue-name 40 | :expected-prefix prefix) 41 | queue-name)))) 42 | 43 | (def encoding-fns 44 | {:none identity 45 | :edn pr-str 46 | :json json/write-str 47 | :array #(into-array (map pr-str %)) 48 | :edn-array #(into-array (map pr-str %)) 49 | :json-array #(into-array (map json/write-str %))}) 50 | 51 | (defn- keyword-enc->fn 52 | [enc] 53 | (or (get encoding-fns enc) 54 | (throw (ex-info (str "No encoding " (name enc)) 55 | {:cause :illegal-argument 56 | :value enc 57 | :expected (set (keys encoding-fns))})))) 58 | 59 | (defn encode 60 | [enc message] 61 | (log/debug :encoding enc message) 62 | ((cond 63 | (keyword? enc) (keyword-enc->fn enc) 64 | (fn? enc) enc 65 | :else (throw (ex-info 66 | (str "`encoding` must be either keyword or function") 67 | {:cause :illegal-argument 68 | :value enc 69 | :expected #{keyword? fn?}}))) 70 | message)) 71 | 72 | (def decoding-fns 73 | (let [json-dec-fn #(json/read-str % :key-fn keyword) 74 | array? #(or (seq? %) 75 | (some-> % class .isArray) 76 | (instance? java.util.ArrayList %))] 77 | {:none identity 78 | :edn edn/read-string 79 | :json json-dec-fn 80 | :array #(if (array? %) 81 | (vec %) 82 | [%]) 83 | :edn-array #(if (array? %) 84 | (vec (map edn/read-string %)) 85 | [(edn/read-string %)]) 86 | :json-array #(if (array? %) 87 | (vec (map json-dec-fn %)) 88 | [(json-dec-fn %)])})) 89 | 90 | (defn- keyword-dec->fn 91 | [dec] 92 | (or (get decoding-fns dec) 93 | (throw (ex-info (str "No decoding " (name dec)) 94 | {:cause :illegal-argument 95 | :value dec 96 | :expected (set (keys decoding-fns))})))) 97 | 98 | (defn decode 99 | [dec message] 100 | (log/debug :decoding dec message) 101 | ((cond 102 | (keyword? dec) (keyword-dec->fn dec) 103 | (fn? dec) dec 104 | :else (throw (ex-info 105 | (str "`decoding` must be either keyword or function") 106 | {:cause :illegal-argument 107 | :value dec 108 | :expected #{keyword? fn?}}))) 109 | message)) 110 | 111 | (defn ->list-position 112 | [pos] 113 | (or (get {:before ListPosition/BEFORE 114 | :after ListPosition/AFTER} 115 | pos) 116 | (throw (ex-info (str "No list position named " pos) 117 | {:cause :illegal-argument 118 | :value pos 119 | :expected #{:before :after}})))) 120 | -------------------------------------------------------------------------------- /src/com/moclojer/rq/pubsub.clj: -------------------------------------------------------------------------------- 1 | (ns com.moclojer.rq.pubsub 2 | (:require 3 | [clojure.edn :as edn] 4 | [clojure.tools.logging :as log] 5 | [com.moclojer.rq.adapters :as adapters] 6 | [com.moclojer.rq.queue :as queue]) 7 | (:import 8 | [redis.clients.jedis JedisPubSub] 9 | [redis.clients.jedis.exceptions JedisConnectionException])) 10 | 11 | (defn publish! 12 | "Publish a message to a channel. When `consumer-min` isn't met, 13 | archives the message. Returns whether or not `consumer-min` was met." 14 | [client channel message & options] 15 | (let [{:keys [consumer-min] 16 | :or {consumer-min 1} 17 | :as opts} options 18 | consumer-count (.publish 19 | @client 20 | (adapters/pack-pattern :pubsub channel) 21 | (pr-str message)) 22 | consumer-met? (>= consumer-count consumer-min) 23 | debug-args {:channel channel 24 | :message message 25 | :options opts 26 | :consumer-count consumer-count 27 | :consumer-met? consumer-met?}] 28 | (if consumer-met? 29 | (log/debug "published message to channel" debug-args) 30 | (do 31 | (log/warn "published message, but didn't meet min consumers. archiving..." 32 | debug-args) 33 | (queue/push! client channel [message] {:pattern :pending}))) 34 | 35 | consumer-met?)) 36 | 37 | (defn group-handlers-by-channel 38 | [workers] 39 | (reduce 40 | (fn [acc {:keys [channel handler]}] 41 | (assoc acc channel handler)) 42 | {} workers)) 43 | 44 | (defn create-listener 45 | "Create a listener for the pubsub. It will be entry point for any 46 | published data, being responsible for routing the right consumer. 47 | However, that's on the enduser." 48 | [workers] 49 | (let [handlers-by-channel (group-handlers-by-channel workers)] 50 | (proxy [JedisPubSub] [] 51 | (onMessage [channel message] 52 | (log/info "consuming channel message" 53 | {:channel channel 54 | :message message}) 55 | (if-let [handler-fn (get handlers-by-channel channel)] 56 | (handler-fn (if (string? message) 57 | (edn/read-string message) 58 | message)) 59 | (log/error "no worker handler found for channel" 60 | {:channel channel 61 | :channels (keys handlers-by-channel) 62 | :message message})))))) 63 | 64 | (defn unarquive-channel! 65 | "Unarquives every pending message from given `channel`, calling 66 | `on-msg-fn` on each of them." 67 | [client channel on-msg-fn] 68 | (loop [message-count 0] 69 | (if-let [?message (first 70 | (queue/pop! client channel 1 71 | {:pattern :pending}))] 72 | (do 73 | (try 74 | (on-msg-fn ?message) 75 | (catch Exception e 76 | (.printStackTrace e) 77 | (log/error "failed to unarchive channel message" 78 | {:channel channel 79 | :message ?message 80 | :on-msg-fn on-msg-fn 81 | :exception e 82 | :ex-message (.getMessage e)}))) 83 | (recur (inc message-count))) 84 | (log/info "unarchived channel" 85 | {:channel channel 86 | :message-count message-count})))) 87 | 88 | (defn pack-workers-channels 89 | [workers] 90 | (map 91 | #(update % :channel (partial adapters/pack-pattern :pubsub)) 92 | workers)) 93 | 94 | (defn subscribe! 95 | "Subscribe given `workers` to their respective channels. 96 | 97 | The list of `workers` should look something like this: 98 | 99 | `[{:channel \"my-channel\" 100 | :handler (fn [msg] (do-something-with-my-msg))} 101 | {:channel \"my-other-channel\" 102 | :handler (fn [msg] (do-something-else msg))}]` 103 | 104 | Options: 105 | 106 | - reconnect-sleep: Time to sleep before reconnecting, right after an 107 | abrupt or unexpected disconnection. 108 | - blocking?: Whether or not to block the current thread." 109 | [client workers & options] 110 | (let [packed-workers (pack-workers-channels workers) 111 | packed-channels (vec (map :channel packed-workers)) 112 | listener (create-listener packed-workers) 113 | {:keys [reconnect-sleep blocking?] 114 | :or {reconnect-sleep 2500 115 | blocking? false} 116 | :as opts} options] 117 | 118 | (doseq [channel (map :channel workers)] 119 | (unarquive-channel! 120 | client channel 121 | #(.onMessage 122 | listener 123 | (adapters/pack-pattern :pubsub channel) 124 | %))) 125 | 126 | (let [sub-fn #(try 127 | (.subscribe @client listener 128 | (into-array packed-channels)) 129 | 130 | (log/debug "subscribed workers to channels" 131 | {:channels packed-channels 132 | :options opts}) 133 | 134 | (catch JedisConnectionException e 135 | 136 | (log/warn "subscriber connection got killed. trying to reconnect..." 137 | {:channels packed-channels 138 | :exception e 139 | :ex-message (.getMessage e)}) 140 | 141 | (Thread/sleep reconnect-sleep) 142 | (apply subscribe! [client workers options])))] 143 | (if blocking? 144 | (sub-fn) 145 | (future (sub-fn)))) 146 | 147 | listener)) 148 | -------------------------------------------------------------------------------- /src/com/moclojer/rq/queue.clj: -------------------------------------------------------------------------------- 1 | (ns com.moclojer.rq.queue 2 | (:refer-clojure :exclude [pop! range]) 3 | (:require 4 | [clojure.string :as str] 5 | [com.moclojer.internal.reflection :as reflection] 6 | [com.moclojer.rq.adapters :as adapters])) 7 | 8 | ;; The allowlisted redis commands followed by their respective 9 | ;; documentation, param names and default encoding/decoding formats. 10 | ;; `lpush` for example encodes a given `value` through the `:edn-array`, 11 | ;; and decodes the result through the `:none` format (`identity`). 12 | 13 | (def allowmap 14 | {"lpush" ["Pushes a message into a queue" 15 | ["key" "string"] :edn-array :none] 16 | "rpush" ["Pushes a message into a queue" 17 | ["key" "string"] :edn-array :none] 18 | "lpop" ["Left-Pops a message from a queue" 19 | ["key" "count"] :none :edn-array] 20 | "rpop" ["Right-Pops a message from a queue" 21 | ["key" "count"] :none :edn-array] 22 | "brpop" ["Right-Pops a message from a queue (blocking)" 23 | ["timeout" "key"] :none :edn-array] 24 | "blpop" ["Left-Pops a message from a queue (blocking)" 25 | ["timeout" "key"] :none :edn-array] 26 | "lindex" ["Get the element from a queue at given index" 27 | ["key", "index"] :none :edn-array] 28 | "lrange" ["Get the elements from a queue" 29 | ["key" "start" "stop"] :none :edn-array] 30 | "lset" ["Sets the element from a queue at given index" 31 | ["key" "index" "value"] :edn :none] 32 | "lrem" ["Removes matching count of given message from a queue" 33 | ["key" "count" "value"] :edn :none] 34 | "llen" ["Gets the length of a queue" 35 | ["key"] :none :none] 36 | "linsert" ["Inserts a message into a queue in reference to given pivot" 37 | ["key" "where" "pivot" "value"] :edn :none] 38 | "ltrim" ["Trim a queue between the given limit values" 39 | ["key" "start" "stop"] 40 | :none :none]}) 41 | 42 | (doseq [[method parameters] (reflection/get-klazz-methods 43 | redis.clients.jedis.JedisPooled 44 | allowmap)] 45 | (let [method' (str/replace method #"[`0-9]" "") 46 | _base-doc (str "Wraps redis.clients.jedis.JedisPooled." method')] 47 | (intern 48 | *ns* (symbol method') 49 | (eval `(reflection/->wrap-method ~method ~parameters ~allowmap))))) 50 | 51 | ;; --- directional --- 52 | 53 | (defn push! 54 | [client queue-name values & [options]] 55 | (let [{:keys [direction] 56 | :or {direction :l}} options 57 | push-fn (if (= direction :l) lpush rpush)] 58 | (apply push-fn [client queue-name values options]))) 59 | 60 | (defn pop! 61 | [client queue-name count & [options]] 62 | (let [{:keys [direction timeout] 63 | :or {direction :r}} options 64 | pop-fn (if (= direction :r) 65 | (if timeout brpop rpop) 66 | (if timeout blpop lpop)) 67 | num (or timeout count)] 68 | (apply pop-fn (flatten [client 69 | (if timeout 70 | [num queue-name] 71 | [queue-name num]) 72 | options])))) 73 | 74 | (defn bpop! 75 | [client queue-name timeout & [options]] 76 | (apply pop! [client queue-name count 77 | (assoc options :timeout timeout)])) 78 | 79 | (defn index 80 | [client queue-name index & [options]] 81 | (first (apply lindex [client queue-name index options]))) 82 | 83 | (defn range 84 | [client queue-name start stop & [options]] 85 | (apply lrange [client queue-name start stop options])) 86 | 87 | (defn set! 88 | [client queue-name index value & [options]] 89 | (apply lset [client queue-name index value options])) 90 | 91 | (defn len 92 | [client queue-name & [options]] 93 | (apply llen [client queue-name options])) 94 | 95 | (defn rem! 96 | [client queue-name count value & [options]] 97 | (apply lrem [client queue-name count value options])) 98 | 99 | (defn insert! 100 | [client queue-name where pivot value & [options]] 101 | (apply linsert [client queue-name 102 | (adapters/->list-position where) 103 | pivot value options])) 104 | 105 | (defn trim! 106 | [client queue-name start stop & [options]] 107 | (apply ltrim [client queue-name start stop options])) 108 | -------------------------------------------------------------------------------- /test/com/moclojer/rq/adapters_test.clj: -------------------------------------------------------------------------------- 1 | (ns com.moclojer.rq.adapters-test 2 | (:require 3 | [clojure.string :as str] 4 | [clojure.test :as t] 5 | [com.moclojer.rq.adapters :as adapters])) 6 | 7 | (t/deftest pattern->str-test 8 | (t/are [expected pattern queue-name] (= expected 9 | (adapters/pack-pattern 10 | pattern queue-name)) 11 | "my-queue" :none "my-queue" 12 | "rq:my-queue" :rq "my-queue" 13 | "rq:pubsub:my-queue" :pubsub "my-queue" 14 | "rq:pubsub:pending:my-queue" :pending "my-queue") 15 | 16 | (t/are [expected pattern queue-name] (= expected 17 | (adapters/unpack-pattern 18 | pattern queue-name)) 19 | "my-queue" :none "my-queue" 20 | "my-queue" :rq "rq:my-queue" 21 | "my-queue" :pubsub "rq:pubsub:my-queue" 22 | "my-queue" :pending "rq:pubsub:pending:my-queue")) 23 | 24 | (t/deftest encode-test 25 | (t/testing "keyword encoders" 26 | [(t/is (= "hello world" (adapters/encode :none "hello world"))) 27 | (t/is (= "{:hello? true}" (adapters/encode :edn {:hello? true}))) 28 | (t/is (= "{\"hello?\":true}" (adapters/encode :json {:hello? true}))) 29 | (t/is (= ["3" "true"] (vec (adapters/encode :array [3 true])))) 30 | (t/is (= ["{\"hello?\":true}"] (vec (adapters/encode 31 | :json-array 32 | [{:hello? true}]))))]) 33 | (t/testing "function encoder" 34 | (t/is (= "HELLO WORLD" (adapters/encode str/upper-case "hello world"))))) 35 | 36 | (t/deftest decode-test 37 | (t/are [expected decoding value] (= expected 38 | (adapters/decode decoding value)) 39 | "hello world" :none "hello world" 40 | {:hello? true} :edn "{:hello? true}" 41 | {:hello? true} :json "{\"hello?\":true}" 42 | ["3" "true"] :array (into-array ["3" "true"]) 43 | [3 true] :edn-array (into-array ["3" "true"]) 44 | [{:hello? true}] :json-array (into-array ["{\"hello?\":true}"]))) 45 | -------------------------------------------------------------------------------- /test/com/moclojer/rq/pubsub_test.clj: -------------------------------------------------------------------------------- 1 | (ns com.moclojer.rq.pubsub-test 2 | (:require 3 | [clojure.test :as t] 4 | [com.moclojer.rq :as rq] 5 | [com.moclojer.rq.pubsub :as rq-pubsub] 6 | [com.moclojer.test-helpers :as helpers])) 7 | 8 | (defn build-workers 9 | [qtt state] 10 | (let [channels (repeatedly qtt #(str (random-uuid))) 11 | messages (repeatedly qtt helpers/gen-message) 12 | chans-msgs (zipmap channels messages)] 13 | {:chans-msgs chans-msgs 14 | :msgs messages 15 | :workers (map (fn [[chan msg]] 16 | {:channel chan 17 | :handler (fn [_] 18 | (swap! state conj msg))}) 19 | chans-msgs)})) 20 | 21 | (t/deftest pubsub-test 22 | (let [client (rq/create-client "redis://localhost:6379")] 23 | 24 | (t/testing "archiving/unarchiving" 25 | (let [channel (str (random-uuid)) 26 | message (helpers/gen-message) 27 | state (atom nil)] 28 | (rq-pubsub/publish! client channel message) 29 | (Thread/sleep 500) 30 | (rq-pubsub/unarquive-channel! client channel 31 | (fn [msg] (reset! state msg))) 32 | (t/is (= message @state)))) 33 | 34 | (t/testing "unarchiving after subscribing" 35 | (let [channel (str (random-uuid)) 36 | message (helpers/gen-message) 37 | state (atom nil)] 38 | (rq-pubsub/publish! client channel message) 39 | (rq-pubsub/publish! client channel message) 40 | (rq-pubsub/subscribe! client [{:channel channel 41 | :handler (fn [msg] 42 | (swap! state conj msg))}]) 43 | (Thread/sleep 1000) 44 | (t/is (= (repeatedly 2 (constantly message)) @state)))) 45 | 46 | (t/testing "multi pub/sub" 47 | (let [state (atom []) 48 | {:keys [chans-msgs msgs workers]} (build-workers 5 state)] 49 | (rq-pubsub/subscribe! client workers) 50 | (Thread/sleep 500) 51 | (doseq [[chan msg] chans-msgs] 52 | (rq-pubsub/publish! client chan msg)) 53 | (Thread/sleep 1000) 54 | (t/is (= msgs @state)))) 55 | 56 | (rq/close-client client))) 57 | -------------------------------------------------------------------------------- /test/com/moclojer/rq/queue_test.clj: -------------------------------------------------------------------------------- 1 | (ns com.moclojer.rq.queue-test 2 | (:require 3 | [clojure.test :as t] 4 | [com.moclojer.rq :as rq] 5 | [com.moclojer.rq.queue :as rq-queue] 6 | [com.moclojer.test-helpers :as helpers])) 7 | 8 | (t/deftest queue-test 9 | (let [client (rq/create-client "redis://localhost:6379") 10 | queue-name (str (random-uuid)) 11 | message (helpers/gen-message) 12 | message2 (helpers/gen-message)] 13 | 14 | [(t/testing "simple" 15 | (rq-queue/push! client queue-name [message message2]) 16 | (t/is (= 2 (rq-queue/len client queue-name))) 17 | (t/is (= [message message2] 18 | (rq-queue/pop! client queue-name 2)))) 19 | 20 | (t/testing "direction" 21 | ;; pushing from the right, then reverse popping from the left 22 | (rq-queue/push! client queue-name [message message2] 23 | {:direction :r}) 24 | (t/is (= [message message2] 25 | (rq-queue/pop! client queue-name 2 26 | {:direction :l})))) 27 | 28 | (t/testing "pattern" 29 | (rq-queue/push! client queue-name [message] 30 | {:pattern :pending}) 31 | (t/is (= [message] 32 | (rq-queue/pop! client queue-name 1 33 | {:pattern :pending})))) 34 | 35 | (t/testing "blocking" 36 | (rq-queue/push! client queue-name [message]) 37 | (t/is (= message 38 | (second (rq-queue/bpop! client queue-name 1))))) 39 | 40 | (t/testing "index" 41 | (rq-queue/push! client queue-name [message]) 42 | (t/is (= message (rq-queue/index client queue-name 0))) 43 | (rq-queue/pop! client queue-name 1)) 44 | 45 | (t/testing "range" 46 | (rq-queue/push! client queue-name [message message2]) 47 | (t/is (= [message2 message] 48 | (rq-queue/range client queue-name 0 -1))) 49 | (rq-queue/pop! client queue-name 2)) 50 | 51 | (t/testing "set!" 52 | (rq-queue/push! client queue-name [message message2]) 53 | (rq-queue/set! client queue-name 0 message2) 54 | (rq-queue/set! client queue-name 1 message) 55 | (t/is (= [message message2] (rq-queue/pop! client queue-name 2)))) 56 | 57 | (t/testing "rem!" 58 | (rq-queue/push! client queue-name [message message message]) 59 | (rq-queue/rem! client queue-name 3 message) 60 | (t/is (= 0 (rq-queue/len client queue-name)))) 61 | 62 | (t/testing "insert!" 63 | (rq-queue/push! client queue-name [message]) 64 | (rq-queue/insert! client queue-name :before message message2) 65 | (t/is (= [message message2] (rq-queue/pop! client queue-name 2)))) 66 | 67 | (t/testing "trim!" 68 | (let [base-message {:test "hello", :my/test2 "123", :foobar ["321"]} 69 | message (assoc base-message :uuid (random-uuid)) 70 | another-message (assoc base-message :uuid (random-uuid))] 71 | (rq-queue/push! client queue-name [another-message message]) 72 | [(t/is (= "OK" (rq-queue/trim! client queue-name 1 -1))) 73 | (t/is (= [(dissoc another-message :uuid)] 74 | (map #(dissoc % :uuid) 75 | (rq-queue/range client queue-name 0 -1))))]) 76 | (rq-queue/pop! client queue-name 2))] 77 | 78 | (rq/close-client client))) 79 | 80 | (comment 81 | (def my-client (rq/create-client "redis://localhost:6379")) 82 | 83 | (rq-queue/push! my-client "my-queue2" [{:hello true}]) 84 | 85 | (rq-queue/insert! my-client "my-queue2" :before {:hello true} {:bye false}) 86 | 87 | (rq-queue/range my-client "my-queue2" 0 -1) 88 | 89 | (rq-queue/len my-client "my-queue2") 90 | 91 | (rq-queue/pop! my-client "my-queue2" 2) 92 | 93 | (rq/close-client my-client) 94 | 95 | (rq/close-client my-client) 96 | ;; 97 | ) 98 | -------------------------------------------------------------------------------- /test/com/moclojer/rq_test.clj: -------------------------------------------------------------------------------- 1 | (ns com.moclojer.rq-test 2 | (:require 3 | [clojure.test :as t] 4 | [com.moclojer.rq :as rq])) 5 | 6 | ;; WARNING: redis needs to be runing. 7 | (t/deftest create-client-test 8 | (t/testing "redis-client being created" 9 | (let [client (rq/create-client "redis://localhost:6379")] 10 | (t/is (.. @client getPool getResource)) 11 | (rq/close-client client)))) 12 | 13 | 14 | -------------------------------------------------------------------------------- /test/com/moclojer/test_helpers.clj: -------------------------------------------------------------------------------- 1 | (ns com.moclojer.test-helpers) 2 | 3 | (defn gen-message 4 | "Generates a fuzzy message" 5 | [] 6 | {(random-uuid) 1 7 | (keyword (str (random-uuid))) true 8 | :test 'hello 9 | :my/test2 "123" 10 | :foobar ["321"]}) 11 | --------------------------------------------------------------------------------