├── wiki ├── .gitignore ├── README.md ├── Home.md ├── 4-Community-resources.md ├── 3-Message-queue.md ├── 1-Getting-started.md ├── 0-Breaking-changes.md └── 2-Further-usage.md ├── doc └── cljdoc.edn ├── FUNDING.yml ├── mq-architecture.monopic ├── check-resp3.sh ├── test └── taoensso │ ├── graal_tests.clj │ ├── carmine │ └── tests │ │ ├── config.clj │ │ ├── locks.clj │ │ ├── tundra.clj │ │ └── message_queue.clj │ └── carmine_v4 │ └── tests │ └── main.clj ├── .gitignore ├── src └── taoensso │ ├── carmine_v4 │ ├── classes.clj │ ├── utils.clj │ ├── cluster.clj │ ├── opts.clj │ └── resp │ │ └── write.clj │ ├── carmine │ ├── lua │ │ ├── cas-get.lua │ │ ├── tundra │ │ │ └── extend-exists.lua │ │ ├── cas-hget.lua │ │ ├── cas-set.lua │ │ ├── cas-hset.lua │ │ ├── hmsetnx.lua │ │ └── mq │ │ │ ├── msg-status.lua │ │ │ ├── enqueue.lua │ │ │ └── dequeue.lua │ ├── tundra │ │ ├── carmine.clj │ │ ├── disk.clj │ │ ├── s3.clj │ │ └── faraday.clj │ ├── ring.clj │ ├── locks.clj │ ├── connections.clj │ └── commands.clj │ └── carmine_v4.clj ├── bb.edn ├── SECURITY.md ├── .github └── workflows │ ├── graal-tests.yml │ └── main-tests.yml ├── bb └── graal_tests.clj ├── deps ├── project.clj ├── carmine-v4.org ├── README.md ├── CHANGELOG.md └── LICENSE.txt /wiki/.gitignore: -------------------------------------------------------------------------------- 1 | README.md 2 | -------------------------------------------------------------------------------- /doc/cljdoc.edn: -------------------------------------------------------------------------------- 1 | {:cljdoc/docstring-format :plaintext} 2 | 3 | -------------------------------------------------------------------------------- /FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: ptaoussanis 2 | custom: "https://www.taoensso.com/clojure" 3 | -------------------------------------------------------------------------------- /mq-architecture.monopic: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/taoensso/carmine/HEAD/mq-architecture.monopic -------------------------------------------------------------------------------- /check-resp3.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo -e "*2\r\n\$5\r\nHELLO\r\n\$1\r\n3\r\n*1\r\n\$4\r\nPING\r\n" | nc localhost 6379 4 | -------------------------------------------------------------------------------- /wiki/README.md: -------------------------------------------------------------------------------- 1 | # Attention! 2 | 3 | This wiki is designed for viewing from [here](../../../wiki)! 4 | 5 | Viewing from GitHub's file browser will result in **broken links**. -------------------------------------------------------------------------------- /test/taoensso/graal_tests.clj: -------------------------------------------------------------------------------- 1 | (ns taoensso.graal-tests 2 | (:require [taoensso.carmine :as car]) 3 | (:gen-class)) 4 | 5 | (defn -main [& args] (println "Namespace loaded successfully")) 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | pom.xml* 2 | .lein* 3 | .nrepl-port 4 | *.jar 5 | *.class 6 | .env 7 | .DS_Store 8 | /lib/ 9 | /classes/ 10 | /target/ 11 | /checkouts/ 12 | /logs/ 13 | /docs/ 14 | .idea/ 15 | *.iml 16 | /wiki/.git 17 | -------------------------------------------------------------------------------- /src/taoensso/carmine_v4/classes.clj: -------------------------------------------------------------------------------- 1 | (ns ^:no-doc taoensso.carmine-v4.classes 2 | "Private ns, implementation detail. 3 | Classes, interfaces, etc. isolated from other code to prevent 4 | identity issues during REPL work.") 5 | 6 | (definterface ReplyError) 7 | -------------------------------------------------------------------------------- /bb.edn: -------------------------------------------------------------------------------- 1 | {:paths ["bb"] 2 | :tasks 3 | {:requires ([graal-tests]) 4 | graal-tests 5 | {:doc "Run Graal native-image tests" 6 | :task 7 | (do 8 | (graal-tests/uberjar) 9 | (graal-tests/native-image) 10 | (graal-tests/run-tests))}}} 11 | -------------------------------------------------------------------------------- /src/taoensso/carmine/lua/cas-get.lua: -------------------------------------------------------------------------------- 1 | local val = redis.call('get', _:k); 2 | local ex = redis.call('exists', _:k); 3 | local len = redis.call('strlen', _:k); 4 | local sha = ""; 5 | if len > 40 then -- Longer than SHA hash 6 | sha = redis.sha1hex(val); 7 | end 8 | return {val, ex, sha}; 9 | -------------------------------------------------------------------------------- /wiki/Home.md: -------------------------------------------------------------------------------- 1 | See the **menu to the right** for content 👉 2 | 3 | # Contributions welcome 4 | 5 | **PRs very welcome** to help improve this documentation! 6 | See the [wiki](../tree/master/wiki) folder in the main repo for the relevant files. 7 | 8 | \- [Peter Taoussanis](https://www.taoensso.com) -------------------------------------------------------------------------------- /src/taoensso/carmine/lua/tundra/extend-exists.lua: -------------------------------------------------------------------------------- 1 | local result = {}; 2 | local ttl_ms = tonumber(ARGV[1]); 3 | for i,k in pairs(KEYS) do 4 | if ttl_ms > 0 and redis.call('pttl', k) > 0 then 5 | result[i] = redis.call('pexpire', k, ttl_ms); 6 | else 7 | result[i] = redis.call('exists', k); 8 | end 9 | end 10 | return result; 11 | -------------------------------------------------------------------------------- /src/taoensso/carmine/lua/cas-hget.lua: -------------------------------------------------------------------------------- 1 | local val = redis.call('hget', _:k, _:field); 2 | local ex = redis.call('hexists', _:k, _:field); 3 | -- local len = redis.call('hstrlen', _:k, _:field); -- Needs Redis 3.2+ 4 | local len = 0; 5 | if val then 6 | len = string.len(val); 7 | end 8 | local sha = ""; 9 | if len > 40 then -- Longer than SHA hash 10 | sha = redis.sha1hex(val); 11 | end 12 | return {val, ex, sha}; 13 | -------------------------------------------------------------------------------- /test/taoensso/carmine/tests/config.clj: -------------------------------------------------------------------------------- 1 | (ns taoensso.carmine.tests.config) 2 | 3 | (def conn-opts 4 | "Connection opts passed to `wcar`. 5 | Example for Redis cloud: 6 | {:spec 7 | {:host \"redis-12345.c49334.us-east-1-mz.ec2.cloud.rlrcp.com\" 8 | :port 12345 9 | :username \"default\" 10 | :password \"YOUR_PASSWORD\" 11 | :timeout-ms 4000 12 | :db 0}}" 13 | {}) 14 | -------------------------------------------------------------------------------- /src/taoensso/carmine/tundra/carmine.clj: -------------------------------------------------------------------------------- 1 | (ns taoensso.carmine.tundra.carmine 2 | "Secondary Redis server DataStore implementation for Tundra." 3 | {:author "Peter Taoussanis"} 4 | (:require [taoensso.encore :as enc] 5 | [taoensso.carmine.tundra :as tundra]) 6 | (:import [taoensso.carmine.tundra IDataStore])) 7 | 8 | ;; TODO 9 | 10 | ;; (defrecord CarmineDataStore [conn-opts] 11 | ;; IDataStore 12 | ;; (put-key [dstore k v]) 13 | ;; (fetch-keys [dstore ks])) 14 | -------------------------------------------------------------------------------- /src/taoensso/carmine/lua/cas-set.lua: -------------------------------------------------------------------------------- 1 | local curr_val = redis.call('get', _:k); 2 | 3 | local do_swap = false; 4 | if (_:old-?sha ~= '') then 5 | local curr_sha = redis.sha1hex(curr_val); 6 | if (curr_sha == _:old-?sha) then do_swap = true; end 7 | else 8 | if (curr_val == _:old-?val) then do_swap = true; end 9 | end 10 | 11 | if do_swap then 12 | if (_:delete == '1') then 13 | redis.call('del', _:k); 14 | else 15 | redis.call('set', _:k, _:new-val); 16 | end 17 | return 1; 18 | else 19 | return 0; 20 | end 21 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security policy 2 | 3 | ## Advisories 4 | 5 | All security advisories will be posted [on GitHub](https://github.com/taoensso/carmine/security/advisories). 6 | 7 | ## Reporting a vulnerability 8 | 9 | Please report possible security vulnerabilities [via GitHub](https://github.com/taoensso/carmine/security/advisories), or by emailing me at `my first name at taoensso.com`. You may encrypt emails with [my public PGP/GPG key](https://www.taoensso.com/pgp). 10 | 11 | Thank you! 12 | 13 | \- [Peter Taoussanis](https://www.taoensso.com) 14 | -------------------------------------------------------------------------------- /src/taoensso/carmine/lua/cas-hset.lua: -------------------------------------------------------------------------------- 1 | local curr_val = redis.call('hget', _:k, _:field); 2 | 3 | local do_swap = false; 4 | if (_:old-?sha ~= '') then 5 | local curr_sha = redis.sha1hex(curr_val); 6 | if (curr_sha == _:old-?sha) then do_swap = true; end 7 | else 8 | if (curr_val == _:old-?val) then do_swap = true; end 9 | end 10 | 11 | if do_swap then 12 | if (_:delete == '1') then 13 | redis.call('hdel', _:k, _:field); 14 | else 15 | redis.call('hset', _:k, _:field, _:new-val); 16 | end 17 | return 1; 18 | else 19 | return 0; 20 | end 21 | -------------------------------------------------------------------------------- /src/taoensso/carmine/lua/hmsetnx.lua: -------------------------------------------------------------------------------- 1 | local hkey = KEYS[1]; 2 | 3 | if redis.call('exists', hkey) == 0 then 4 | redis.call('hmset', hkey, unpack(ARGV)); 5 | return 1; 6 | else 7 | local proceed = true; 8 | for i,x in ipairs(ARGV) do 9 | if (i % 2 ~= 0) then -- odd i => `x` is field 10 | if redis.call('hexists', hkey, x) == 1 then 11 | proceed = false; 12 | break; 13 | end 14 | end 15 | end 16 | 17 | if proceed then 18 | redis.call('hmset', hkey, unpack(ARGV)); 19 | return 1; 20 | else 21 | return 0; 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /wiki/4-Community-resources.md: -------------------------------------------------------------------------------- 1 | If you spot issues with any linked resources, please **contact the relevant authors** to let them know! 2 | 3 | Contributor | Link | Description 4 | :-- | :-- | :-- 5 | [@lantiga](https://github.com/lantiga) | [redlock-clj](https://github.com/lantiga/redlock-clj) | Distributed locks for uncoordinated Redis clusters 6 | [@oliyh](https://github.com/oliyh) | [carmine-streams](https://github.com/oliyh/carmine-streams) | Utility functions for working with [Redis streams](https://redis.io/topics/streams-intro) 7 | [@danielsz](https://github.com/danielsz) | [system](https://github.com/danielsz/system/tree/master/src/system/components) | Example `system` components for [Redis Pub/Sub](https://redis.io/docs/interact/pubsub/), etc. 8 | _ | _ | Your link here? [PRs](../wiki#contributions-welcome) welcome! -------------------------------------------------------------------------------- /.github/workflows/graal-tests.yml: -------------------------------------------------------------------------------- 1 | name: Graal tests 2 | on: [push, pull_request] 3 | 4 | jobs: 5 | test: 6 | strategy: 7 | matrix: 8 | java: ['17'] 9 | os: [ubuntu-latest, macOS-latest, windows-latest] 10 | 11 | runs-on: ${{ matrix.os }} 12 | steps: 13 | - uses: actions/checkout@v4 14 | - uses: graalvm/setup-graalvm@v1 15 | with: 16 | version: 'latest' 17 | java-version: ${{ matrix.java }} 18 | components: 'native-image' 19 | github-token: ${{ secrets.GITHUB_TOKEN }} 20 | 21 | - uses: DeLaGuardo/setup-clojure@12.5 22 | with: 23 | lein: latest 24 | bb: latest 25 | 26 | - uses: actions/cache@v4 27 | with: 28 | path: ~/.m2/repository 29 | key: deps-${{ hashFiles('deps.edn') }} 30 | restore-keys: deps- 31 | 32 | - run: bb graal-tests 33 | -------------------------------------------------------------------------------- /.github/workflows/main-tests.yml: -------------------------------------------------------------------------------- 1 | name: Main tests 2 | on: [push, pull_request] 3 | 4 | jobs: 5 | tests: 6 | strategy: 7 | matrix: 8 | java: ['17', '18', '19'] 9 | os: [ubuntu-latest] 10 | 11 | runs-on: ${{ matrix.os }} 12 | steps: 13 | - uses: actions/checkout@v4 14 | - uses: actions/setup-java@v4 15 | with: 16 | distribution: 'corretto' 17 | java-version: ${{ matrix.java }} 18 | 19 | - uses: DeLaGuardo/setup-clojure@12.5 20 | with: 21 | lein: latest 22 | 23 | - uses: supercharge/redis-github-action@1.7.0 24 | with: 25 | redis-version: 7 26 | 27 | - uses: actions/cache@v4 28 | id: cache-deps 29 | with: 30 | path: ~/.m2/repository 31 | key: deps-${{ hashFiles('project.clj') }} 32 | restore-keys: deps- 33 | 34 | - run: lein test-all 35 | -------------------------------------------------------------------------------- /src/taoensso/carmine/lua/mq/msg-status.lua: -------------------------------------------------------------------------------- 1 | -- Careful! Logic here is subtle, see mq-architecture.svg for assistance. 2 | local mid = _:mid; 3 | local now = tonumber(_:now); 4 | 5 | local status = nil; -- base status e/o nil, done, queued, locked 6 | local is_bo = false; -- backoff flag for: done, queued 7 | local is_rq = false; -- requeue flag for: done, locked 8 | -- 8x cases: nil, done(bo/rq), queued(bo), locked(rq) 9 | -- Describe with: {status, $bo, $rq} with $ prefixes e/o: _, +, -, * 10 | 11 | if (redis.call('hexists', _:qk-messages, mid) == 1) then 12 | local exp_lock = tonumber(redis.call('hget', _:qk-locks, mid)) or 0; 13 | local exp_bo = tonumber(redis.call('hget', _:qk-backoffs, mid)) or 0; 14 | 15 | is_bo = (now < exp_bo); 16 | is_rq = (redis.call('sismember', _:qk-requeue, mid) == 1) or -- Deprecated 17 | (redis.call('hexists', _:qk-messages-rq, mid) == 1); 18 | 19 | if (redis.call('sismember', _:qk-done, mid) == 1) then status = 'done'; 20 | elseif (now < exp_lock) then status = 'locked'; 21 | else status = 'queued'; end 22 | else 23 | status = 'nx'; 24 | end 25 | 26 | return {status, is_bo, is_rq}; 27 | -------------------------------------------------------------------------------- /bb/graal_tests.clj: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bb 2 | 3 | (ns graal-tests 4 | (:require 5 | [clojure.string :as str] 6 | [babashka.fs :as fs] 7 | [babashka.process :refer [shell]])) 8 | 9 | (defn uberjar [] 10 | (let [command "lein with-profiles +graal-tests uberjar" 11 | command 12 | (if (fs/windows?) 13 | (if (fs/which "lein") 14 | command 15 | ;; Assume PowerShell powershell module 16 | (str "powershell.exe -command " (pr-str command))) 17 | command)] 18 | 19 | (shell command))) 20 | 21 | (defn executable [dir name] 22 | (-> (fs/glob dir (if (fs/windows?) (str name ".{exe,bat,cmd}") name)) 23 | first 24 | fs/canonicalize 25 | str)) 26 | 27 | (defn native-image [] 28 | (let [graalvm-home (System/getenv "GRAALVM_HOME") 29 | bin-dir (str (fs/file graalvm-home "bin"))] 30 | (shell (executable bin-dir "gu") "install" "native-image") 31 | (shell (executable bin-dir "native-image") 32 | "--features=clj_easy.graal_build_time.InitClojureClasses" 33 | "--no-fallback" "-jar" "target/graal-tests.jar" "graal_tests"))) 34 | 35 | (defn run-tests [] 36 | (let [{:keys [out]} (shell {:out :string} (executable "." "graal_tests"))] 37 | (assert (str/includes? out "loaded") out) 38 | (println "Native image works!"))) 39 | -------------------------------------------------------------------------------- /src/taoensso/carmine/ring.clj: -------------------------------------------------------------------------------- 1 | (ns taoensso.carmine.ring 2 | "Carmine-backed Ring session store." 3 | {:author "Peter Taoussanis"} 4 | (:require 5 | [ring.middleware.session] 6 | [taoensso.encore :as enc] 7 | [taoensso.carmine :as car :refer [wcar]])) 8 | 9 | (defrecord CarmineSessionStore [conn-opts prefix ttl-secs extend-on-read?] 10 | ring.middleware.session.store/SessionStore 11 | (read-session [_ k] 12 | (last 13 | (wcar conn-opts :as-pipeline 14 | (when (and extend-on-read? ttl-secs) 15 | (car/expire k ttl-secs)) 16 | (car/get k)))) 17 | 18 | (delete-session [_ k] (wcar conn-opts (car/del k)) nil) 19 | (write-session [_ k data] 20 | (let [k (or k (str prefix ":" (enc/uuid-str)))] 21 | (wcar conn-opts 22 | (if ttl-secs 23 | (car/setex k ttl-secs data) 24 | (car/set k data))) 25 | k))) 26 | 27 | (defn carmine-store 28 | "Creates and returns a Carmine-backed Ring `SessionStore`. 29 | 30 | Options include: 31 | `:expiration-secs` - How long session data should persist after last 32 | write. nil => persist forever. 33 | `:extend-on-read?` - If true, expiration will also be extended by 34 | session reads." 35 | [conn-opts & [{:keys [key-prefix expiration-secs extend-on-read?] 36 | :or {key-prefix "carmine:session" 37 | expiration-secs (enc/secs :days 30) 38 | extend-on-read? false}}]] 39 | (->CarmineSessionStore conn-opts key-prefix expiration-secs extend-on-read?)) 40 | 41 | (enc/deprecated 42 | (defn ^:no-doc ^:deprecated make-carmine-store ; 1.x backwards compatiblity 43 | "Prefer `carmine-store`." 44 | [& [s1 s2 & sn :as args]] 45 | (if (instance? taoensso.carmine.connections.ConnectionPool s1) 46 | (carmine-store {:pool s1 :spec s2} (apply hash-map sn)) 47 | (apply carmine-store args)))) 48 | -------------------------------------------------------------------------------- /deps: -------------------------------------------------------------------------------- 1 | [clj-aws-s3 "0.3.10" :scope "test"] 2 | [clj-time "0.6.0" :scope "test"] 3 | [com.amazonaws/aws-java-sdk "1.7.5" :scope "test"] 4 | [com.fasterxml.jackson.core/jackson-annotations "2.1.1" :scope "test"] 5 | [com.fasterxml.jackson.core/jackson-core "2.1.1" :scope "test"] 6 | [com.fasterxml.jackson.core/jackson-databind "2.1.1" :scope "test"] 7 | [org.apache.httpcomponents/httpclient "4.2" :scope "test"] 8 | [org.apache.httpcomponents/httpcore "4.2" :scope "test"] 9 | [com.taoensso/encore "3.62.1"] 10 | [com.taoensso/truss "1.10.1"] 11 | [org.clojure/tools.reader "1.3.6"] 12 | [com.taoensso/faraday "1.12.0" :scope "test"] 13 | [com.amazonaws/aws-java-sdk-dynamodb "1.12.410" :scope "test" :exclusions [[joda-time] [commons-logging]]] 14 | [com.amazonaws/aws-java-sdk-core "1.12.410" :scope "test"] 15 | [com.fasterxml.jackson.dataformat/jackson-dataformat-cbor "2.12.6" :scope "test"] 16 | [software.amazon.ion/ion-java "1.0.2" :scope "test"] 17 | [com.amazonaws/aws-java-sdk-s3 "1.12.410" :scope "test"] 18 | [com.amazonaws/aws-java-sdk-kms "1.12.410" :scope "test"] 19 | [com.amazonaws/jmespath-java "1.12.410" :scope "test"] 20 | [commons-logging "1.2" :scope "test"] 21 | [joda-time "2.12.2" :scope "test"] 22 | [com.taoensso/nippy "3.2.0"] 23 | [org.iq80.snappy/snappy "0.4"] 24 | [org.lz4/lz4-java "1.8.0"] 25 | [org.tukaani/xz "1.9"] 26 | [com.taoensso/timbre "6.2.1"] 27 | [io.aviso/pretty "1.1.1"] 28 | [com.taoensso/tufte "2.5.0"] 29 | [commons-codec "1.16.0"] 30 | [nrepl "1.0.0" :exclusions [[org.clojure/clojure]]] 31 | [org.apache.commons/commons-pool2 "2.11.1"] 32 | [org.clojure/clojure "1.11.1" :scope "test"] 33 | [org.clojure/core.specs.alpha "0.2.62" :scope "test"] 34 | [org.clojure/spec.alpha "0.3.218" :scope "test"] 35 | [org.clojure/data.json "2.4.0" :scope "test"] 36 | [org.clojure/test.check "1.1.1" :scope "test"] 37 | [org.nrepl/incomplete "0.1.0" :exclusions [[org.clojure/clojure]]] 38 | [ring/ring-core "1.10.0" :scope "test"] 39 | [commons-fileupload "1.5" :scope "test"] 40 | [commons-io "2.11.0" :scope "test"] 41 | [crypto-equality "1.0.1" :scope "test"] 42 | [crypto-random "1.2.1" :scope "test"] 43 | [ring/ring-codec "1.2.0" :scope "test"] 44 | -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject com.taoensso/carmine "3.5.0" 2 | :author "Peter Taoussanis " 3 | :description "Redis client + message queue for Clojure" 4 | :url "https://github.com/taoensso/carmine" 5 | 6 | :license 7 | {:name "Eclipse Public License - v 1.0" 8 | :url "https://www.eclipse.org/legal/epl-v10.html"} 9 | 10 | :test-paths ["test" #_"src"] 11 | 12 | :dependencies 13 | [[com.taoensso/encore "3.158.0"] 14 | [com.taoensso/nippy "3.6.0"] 15 | [com.taoensso/trove "1.1.0"] 16 | [org.apache.commons/commons-pool2 "2.12.1"] 17 | [commons-codec/commons-codec "1.20.0"]] 18 | 19 | :profiles 20 | {;; :default [:base :system :user :provided :dev] 21 | :provided {:dependencies [[org.clojure/clojure "1.12.3"]]} 22 | :c1.12 {:dependencies [[org.clojure/clojure "1.12.3"]]} 23 | :c1.11 {:dependencies [[org.clojure/clojure "1.11.4"]]} 24 | :c1.10 {:dependencies [[org.clojure/clojure "1.10.3"]]} 25 | 26 | :graal-tests 27 | {:source-paths ["test"] 28 | :main taoensso.graal-tests 29 | :aot [taoensso.graal-tests] 30 | :uberjar-name "graal-tests.jar" 31 | :dependencies 32 | [[org.clojure/clojure "1.11.1"] 33 | [com.github.clj-easy/graal-build-time "1.0.5"]]} 34 | 35 | :dev 36 | {;; :jvm-opts ["-server" "-Dtaoensso.elide-deprecated=true"] 37 | :global-vars 38 | {*warn-on-reflection* true 39 | *assert* true 40 | *unchecked-math* false #_:warn-on-boxed} 41 | 42 | :dependencies 43 | [[org.clojure/test.check "1.1.1"] 44 | [org.clojure/data.json "2.5.1"] 45 | [com.taoensso/faraday "1.12.3"] 46 | [clj-aws-s3 "0.3.10"] 47 | [ring/ring-core "1.15.3"]] 48 | 49 | :plugins 50 | [[lein-pprint "1.3.2"] 51 | [lein-ancient "0.7.0"]]}} 52 | 53 | :test-selectors 54 | {:v3 (fn [{:keys [ns]} & _] (.startsWith (str ns) "taoensso.carmine.")) 55 | :v4 (fn [{:keys [ns]} & _] (.startsWith (str ns) "taoensso.carmine-v4."))} 56 | 57 | :aliases 58 | {"start-dev" ["with-profile" "+dev" "repl" ":headless"] 59 | ;; "build-once" ["do" ["clean"] ["cljsbuild" "once"]] 60 | "deploy-lib" ["do" #_["build-once"] ["deploy" "clojars"] ["install"]] 61 | 62 | "test-v3" ["with-profile" "+c1.12:+c1.11:+c1.10" "test" ":v3"] 63 | "test-v4" ["with-profile" "+c1.12:+c1.11:+c1.10" "test" ":v4"] 64 | "test-clj" ["with-profile" "+c1.12:+c1.11:+c1.10" "test"] 65 | ;; "test-cljs" ["with-profile" "+c1.12" "cljsbuild" "test"] 66 | "test-all" ["do" ["clean"] ["test-clj"] #_["test-cljs"]]}) 67 | -------------------------------------------------------------------------------- /src/taoensso/carmine/tundra/disk.clj: -------------------------------------------------------------------------------- 1 | (ns taoensso.carmine.tundra.disk 2 | "Simple disk-based DataStore implementation for Tundra." 3 | {:author "Peter Taoussanis"} 4 | (:require 5 | [taoensso.encore :as enc] 6 | [taoensso.truss :as truss] 7 | [taoensso.carmine.tundra :as tundra]) 8 | 9 | (:import 10 | [taoensso.carmine.tundra IDataStore] 11 | [java.nio.file CopyOption Files LinkOption OpenOption Path Paths 12 | StandardCopyOption StandardOpenOption NoSuchFileException])) 13 | 14 | ;;;; Private utils 15 | 16 | (defn- uuid [] (java.util.UUID/randomUUID)) 17 | (defn- path* [path] (Paths/get "" (into-array String [path]))) 18 | (defn- mkdirs [path] (.mkdirs ^java.io.File (.toFile ^Path (path* path)))) 19 | (defn- mv [path-source path-dest] 20 | (Files/move (path* path-source) (path* path-dest) 21 | (into-array CopyOption [StandardCopyOption/ATOMIC_MOVE 22 | StandardCopyOption/REPLACE_EXISTING]))) 23 | 24 | (defn- read-ba [path] (Files/readAllBytes (path* path))) 25 | (defn- write-ba [path ba] 26 | (Files/write ^Path (path* path) ^bytes ba 27 | ^"[Ljava.nio.file.OpenOption;" 28 | (into-array OpenOption [StandardOpenOption/CREATE 29 | StandardOpenOption/TRUNCATE_EXISTING 30 | StandardOpenOption/WRITE 31 | StandardOpenOption/SYNC]))) 32 | 33 | ;;;; 34 | 35 | (defrecord DiskDataStore [path] 36 | IDataStore 37 | (fetch-keys [this ks] 38 | (let [fetch1 (fn [k] (tundra/catcht (read-ba (format "%s/%s" (path* path) k))))] 39 | (mapv fetch1 ks))) 40 | 41 | (put-key [this k v] 42 | (assert (enc/bytes? v)) 43 | (let [result 44 | (try (let [path-full-temp (format "%s/tmp-%s" (path* path) (uuid)) 45 | path-full (format "%s/%s" (path* path) k)] 46 | (write-ba path-full-temp v) 47 | (mv path-full-temp path-full)) 48 | (catch Exception e e))] 49 | (cond 50 | (instance? Path result) true 51 | (instance? NoSuchFileException result) 52 | (if (mkdirs path) (recur k v) result) 53 | 54 | (instance? Exception result) result 55 | :else (truss/ex-info (str "Unexpected result: " result) {:result result}))))) 56 | 57 | (defn disk-datastore 58 | "Alpha - subject to change. 59 | Requires JVM 1.7+. 60 | Supported Freezer io types: byte[]s." 61 | [path] {:pre [(string? path)]} (->DiskDataStore path)) 62 | 63 | (comment 64 | (def dstore (disk-datastore "./tundra")) 65 | (def hardkey (tundra/>urlsafe-str "foo:bar /♡\\:baz ")) 66 | (tundra/put-key dstore hardkey (.getBytes "hello world")) 67 | (String. (first (tundra/fetch-keys dstore [hardkey]))) 68 | (time (dotimes [_ 10000] 69 | (tundra/put-key dstore hardkey (.getBytes "hello world")) 70 | (tundra/fetch-keys dstore [hardkey])))) 71 | -------------------------------------------------------------------------------- /carmine-v4.org: -------------------------------------------------------------------------------- 1 | #+TITLE: Title 2 | #+STARTUP: indent overview hidestars 3 | #+TAGS: { Cost: c1(1) c2(2) c3(3) c4(4) c5(5) } 4 | #+TAGS: nb(n) urgent(u) 5 | 6 | * Next 7 | ** Re-eval choice to switch away from KeyedObjectPool 8 | ** Review arch needs for Cluster, esp. re: conns 9 | 10 | ** Add SSB stats to pooled manager (borrow time, etc.)? 11 | ** Add SSB stats to Sentinel? 12 | ** Common & core util to parse-?marked-ba -> [ ] 13 | ** Some way to implement a parser over >1 replies? 14 | E.g. fetch two sets, and parser to merge -> single reply 15 | 16 | * Later 17 | ** New Pub/Sub API? (note RESP2 vs RESP3 differences) 18 | Pub/Sub + Sentinel integration 19 | psubscribe* to Sentinel server 20 | check for `switch-master` channel name 21 | "switch-master" 22 | 23 | ** Implement Cluster (enough user demand?) 24 | ** Use Telemere (shell API?) 25 | 26 | * Polish 27 | ** Print methods, toString content, etc. 28 | ** Check all errors: eids, messages, data, cbids 29 | ** Check all dynamic bindings and sys-vals, ensure accessible 30 | ** Document `*default-conn-opts*`, incl. cbs 31 | ** Document `*default-sentinel-opts*`, incl. cbs 32 | ** Complete (esp. high-level / integration) tests 33 | ** Review all config, docstring, privacy, etc. 34 | ** Grep for TODOs 35 | 36 | * Refactor commands 37 | ** Add modules support 38 | ** Support custom (e.g. newer) commands.json or edn 39 | ** Refactor helpers API, etc. 40 | ** Modern Tundra? 41 | ** Further MQ improvements? 42 | 43 | * Release 44 | ** v4 upgrade/migration plan 45 | ** v4 wiki with changes, migration, new features, examples, etc. 46 | ** First public alpha 47 | 48 | * CHANGELOG 49 | ** [new] Full RESP3 support, incl. streaming, etc. 50 | *** Enabled by default, requires Redis >= v6 (2020-04-30). 51 | ** [new] Full Redis Sentinel support - incl. auto failover and read replicas. 52 | ** [mod] Hugely improved connections API, incl. improved: 53 | *** Flexibility 54 | *** Docs 55 | *** Usability (e.g. opts validation, hard shutdowns, closing managed conns, etc.). 56 | *** Transparency (deref stats, cbs, timings for profiling, etc.). 57 | **** Derefs: Conns, ConnManagers, SentinelSpecs. 58 | *** Protocols for extension by advanced users. 59 | *** Full integration with Sentinel, incl.: 60 | **** Auto invalidation of pool conns on master changes. 61 | **** Auto verification of addresses on pool borrows. 62 | *** [new] Common conn utils are now aliased in core Carmine ns for convenience. 63 | *** [new] Improved pool efficiency, incl. smarter sub-pool keying. 64 | *** [mod] Improved parsing API, incl.: 65 | **** General simplifications. 66 | **** Aggregate parsers, with xform support. 67 | *** [new] *auto-serialize?*, *auto-deserialize?* 68 | *** [new] Greatly improved `skip-replies` performance 69 | *** [mod] Simplified parsers API 70 | *** [new] Improvements to docs, error messages, debug data, etc. 71 | *** [new] New Wiki with further documentation and examples. 72 | -------------------------------------------------------------------------------- /src/taoensso/carmine/tundra/s3.clj: -------------------------------------------------------------------------------- 1 | (ns taoensso.carmine.tundra.s3 2 | "AWS S3 (clj-aws-s3) DataStore implementation for Tundra." 3 | {:author "Peter Taoussanis"} 4 | (:require 5 | [clojure.string :as str] 6 | [aws.sdk.s3 :as s3] 7 | [taoensso.encore :as enc] 8 | [taoensso.truss :as truss] 9 | [taoensso.carmine.tundra :as tundra]) 10 | 11 | (:import 12 | [java.io ByteArrayInputStream DataInputStream] 13 | [taoensso.carmine.tundra IDataStore] 14 | [com.amazonaws.services.s3.model AmazonS3Exception PutObjectResult])) 15 | 16 | (defn- base64-md5 [^bytes x] 17 | (-> x (org.apache.commons.codec.digest.DigestUtils/md5) 18 | (org.apache.commons.codec.binary.Base64/encodeBase64String))) 19 | 20 | (defrecord S3DataStore [creds bucket] 21 | IDataStore 22 | (put-key [this k v] 23 | (truss/have? enc/bytes? v) 24 | (let [reply (try (s3/put-object creds bucket k (ByteArrayInputStream. v) 25 | {:content-length (count v) 26 | ;; Nb! Prevents overwriting with corrupted data: 27 | :content-md5 (base64-md5 v)}) 28 | (catch Exception e e))] 29 | (cond 30 | (instance? PutObjectResult reply) true 31 | (and (instance? AmazonS3Exception reply) 32 | (= (.getMessage ^AmazonS3Exception reply) 33 | "The specified bucket does not exist")) 34 | (let [bucket (first (str/split bucket #"/"))] 35 | (s3/create-bucket creds bucket) 36 | (recur k v)) 37 | (instance? Exception reply) reply 38 | :else (truss/ex-info (str "Unexpected reply: " reply) {:reply reply})))) 39 | 40 | (fetch-keys [this ks] 41 | (let [fetch1 42 | (fn [k] 43 | (tundra/catcht 44 | (let [obj (s3/get-object creds bucket k)] 45 | (with-open 46 | [^com.amazonaws.services.s3.model.S3ObjectInputStream cnt 47 | (:content obj)] 48 | (let [ba (byte-array (truss/have enc/pos-int? 49 | (-> obj :metadata :content-length)))] 50 | (.readFully (DataInputStream. cnt) ba) 51 | ba)))))] 52 | (->> (mapv #(future (fetch1 %)) ks) 53 | (mapv deref))))) 54 | 55 | (defn s3-datastore 56 | "Alpha - subject to change. 57 | Returns a Faraday DataStore using given AWS S3 credentials and bucket. 58 | 59 | (s3-datastore {:access-key \"\" :secret-key \"S3DataStore creds bucket)) 65 | 66 | (comment 67 | (def dstore (s3-datastore creds "ensso-store/folder")) 68 | (def hardkey (tundra/>urlsafe-str "00temp-test-foo:bar /♡\\:baz ")) 69 | (tundra/put-key dstore hardkey (.getBytes "hello world")) 70 | (String. (first (tundra/fetch-keys dstore [hardkey])))) 71 | -------------------------------------------------------------------------------- /wiki/3-Message-queue.md: -------------------------------------------------------------------------------- 1 | Carmine includes a simple **distributed message queue** originally inspired by a [post](http://oldblog.antirez.com/post/250) by Redis's original author Salvatore Sanfilippo. 2 | 3 | # API 4 | 5 | See linked docstrings below for features and usage: 6 | 7 | | Name | Description | 8 | | :------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------ | 9 | | [`worker`](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine.message-queue#worker) | Returns a worker for named queue. Deref worker for detailed status info! | 10 | | [`enqueue`](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine.message-queue#enqueue) | Enqueues given message for processing by active worker/s. | 11 | | [`set-min-log-level!`](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine.message-queue#set-min-log-level!) | Sets minimum log level for message queue logs. | 12 | 13 | # Example 14 | 15 | 16 | ```clojure 17 | (def my-conn-opts {:pool {} :spec {}}) 18 | 19 | (def my-worker 20 | (car-mq/worker my-conn-opts "my-queue" 21 | {:handler 22 | (fn [{:keys [message attempt]}] 23 | (try 24 | (println "Received" message) 25 | {:status :success} 26 | (catch Throwable _ 27 | (println "Handler error!") 28 | {:status :retry})))})) 29 | 30 | (wcar* (car-mq/enqueue "my-queue" "my message!")) 31 | 32 | ;; Deref your worker to get detailed status info 33 | @my-worker => 34 | {:qname "my-queue" 35 | :opts 36 | :conn-opts 37 | :running? true 38 | :nthreads {:worker 1, :handler 1} 39 | :stats 40 | {:queue-size {:last 1332, :max 1352, :p90 1323, ...} 41 | :queueing-time-ms {:last 203, :max 4774, :p90 300, ...} 42 | :handling-time-ms {:last 11, :max 879, :p90 43, ...} 43 | :counts 44 | {:handler/success 5892 45 | :handler/retry 808 46 | :handler/error 2 47 | :handler/backoff 2034 48 | :sleep/end-of-circle 350}}} 49 | ``` 50 | 51 | # Semantics 52 | 53 | The following semantics are provided: 54 | 55 | - Messages are **persistent** (durable as per Redis config). 56 | - Messages are **handled once and only once**. 57 | - Messages are **handled in loose order** (exact order may be affected by the number of concurrent handler threads, and retry/backoff features, etc.). 58 | - Messages are **fault-tolerant** (preserved until acknowledged as handled). 59 | - Messages support optional per-message **de-duplication**, preventing the same message from being simultaneously queued more than once within a configurable per-message backoff period. 60 | - Messages are serialized with [Nippy](https://www.taoensso.com/nippy) and stored as [byte strings](https://redis.io/docs/latest/develop/data-types/strings) in Redis hashes, so each serialized message has a **maximum size of 512MiB**. You'll normally want to use *much* smaller messages though (typically small maps or UUIDs/pointers to larger data stores when necessary). -------------------------------------------------------------------------------- /src/taoensso/carmine/locks.clj: -------------------------------------------------------------------------------- 1 | (ns taoensso.carmine.locks 2 | "Alpha - subject to change. 3 | Distributed lock implementation for Carmine. 4 | Based on work by Ronen Narkis and Josiah Carlson. 5 | 6 | Redis keys: 7 | * carmine:lock: -> ttl str, lock owner's UUID. 8 | 9 | Ref. for implementation details." 10 | (:require 11 | [taoensso.truss :as truss] 12 | [taoensso.carmine :as car :refer [wcar]])) 13 | 14 | (def ^:private lkey (partial car/key :carmine :lock)) 15 | 16 | (defn acquire-lock 17 | "Attempts to acquire a distributed lock, returning an owner UUID iff successful." 18 | [conn-opts lock-name timeout-ms wait-ms] 19 | (let [max-udt (+ wait-ms (System/currentTimeMillis)) 20 | uuid (str (java.util.UUID/randomUUID))] 21 | (wcar conn-opts ; Hold one connection for all attempts 22 | (loop [] 23 | (when (> max-udt (System/currentTimeMillis)) 24 | (if (-> (car/set (lkey lock-name) uuid "nx" "px" timeout-ms) 25 | (car/with-replies) 26 | (= "OK")) 27 | (car/return uuid) 28 | (do (Thread/sleep 1) (recur)))))))) 29 | 30 | (comment (acquire-lock {} "my-lock" 2000 500)) 31 | 32 | (defn release-lock 33 | "Attempts to release a distributed lock, returning true iff successful." 34 | [conn-opts lock-name owner-uuid] 35 | (wcar conn-opts 36 | (car/parse-bool 37 | (car/lua 38 | "if redis.call('get', _:lkey) == _:uuid then 39 | redis.call('del', _:lkey); 40 | return 1; 41 | else 42 | return 0; 43 | end" 44 | {:lkey (lkey lock-name)} 45 | {:uuid owner-uuid})))) 46 | 47 | (comment 48 | (when-let [uuid (acquire-lock {} "my-lock" 2000 500)] 49 | [(Thread/sleep 100) 50 | (release-lock {} "my-lock" uuid) 51 | (release-lock {} "my-lock" uuid)])) 52 | 53 | (defn have-lock? [conn-opts lock-name owner-uuid] 54 | (= (wcar conn-opts (car/get (lkey lock-name))) owner-uuid)) 55 | 56 | (comment 57 | (when-let [uuid (acquire-lock {} "my-lock" 2000 500)] 58 | [(Thread/sleep 100) 59 | (have-lock? {} "my-lock" uuid) 60 | (Thread/sleep 2000) 61 | (have-lock? {} "my-lock" uuid)])) 62 | 63 | (defmacro with-lock 64 | "Attempts to acquire a distributed lock, executing body and then releasing 65 | lock when successful. Returns {:result } on successful release, 66 | or nil if the lock could not be acquired. If the lock is successfully acquired 67 | but expires before being released, throws an exception." 68 | [conn-opts lock-name timeout-ms wait-ms & body] 69 | `(let [conn-opts# ~conn-opts 70 | lock-name# ~lock-name] 71 | (when-let [uuid# (acquire-lock conn-opts# lock-name# ~timeout-ms ~wait-ms)] 72 | (try 73 | {:result (do ~@body)} ; Wrapped to distinguish nil body result 74 | (catch Throwable t# (throw t#)) 75 | (finally 76 | (when-not (release-lock conn-opts# ~lock-name uuid#) 77 | (truss/ex-info! (str "Lock expired before it was released: " ~lock-name) 78 | {:lock-name ~lock-name}))))))) 79 | 80 | (comment 81 | (with-lock {} "my-lock" 2000 500 (Thread/sleep 1000) "m") ; {:result "m"} 82 | (with-lock {} "my-lock" 2000 500 (Thread/sleep 1000) (/ 1 0)) ; ex 83 | (with-lock {} "my-lock" 2000 500 (Thread/sleep 2500) "m") ; ex 84 | (do (future (with-lock {} "my-lock" 2000 500 (Thread/sleep 1000) (println "m"))) 85 | (future (with-lock {} "my-lock" 2000 500 (Thread/sleep 1000) (println "m"))) 86 | (future (with-lock {} "my-lock" 2000 2000 (Thread/sleep 1000) (println "m"))))) 87 | 88 | (defn- release-all-locks! [conn-opts] 89 | (when-let [lkeys (seq (wcar conn-opts (car/keys (lkey :*))))] 90 | (wcar conn-opts (apply car/del lkeys)))) 91 | 92 | (comment (release-all-locks! {})) 93 | -------------------------------------------------------------------------------- /src/taoensso/carmine/tundra/faraday.clj: -------------------------------------------------------------------------------- 1 | (ns taoensso.carmine.tundra.faraday 2 | "Faraday (DynamoDB) DataStore implementation for Tundra. 3 | 4 | Use AWS Data Pipeline to setup scheduled backups of DynamoDB table(s) to S3 5 | (there is a template pipeline for exactly this purpose)." 6 | {:author "Peter Taoussanis"} 7 | (:require 8 | [taoensso.encore :as enc] 9 | [taoensso.truss :as truss] 10 | [taoensso.faraday :as far] 11 | [taoensso.carmine.tundra :as tundra]) 12 | 13 | (:import [taoensso.carmine.tundra IDataStore])) 14 | 15 | (def default-table :faraday.tundra.datastore.default.prod) 16 | (defn ensure-table ; Slow, so we won't do automatically 17 | "Creates an appropriate Faraday Tundra table iff it doesn't already exist. 18 | Options: 19 | :name - Default is :faraday.tundra.datastore.default.prod. You may 20 | want additional tables for different apps/environments. 21 | :throughput - Default is {:read 1 :write 1}." 22 | [client-opts & [{:keys [name throughput block?] 23 | :or {name default-table}}]] 24 | (far/ensure-table client-opts name [:key-ns :s] 25 | {:range-keydef [:redis-key :s] 26 | :throughput throughput 27 | :block? block?})) 28 | 29 | (defrecord FaradayDataStore [client-opts opts] 30 | IDataStore 31 | (put-key [this k v] 32 | (assert (enc/bytes? v)) 33 | (far/put-item client-opts (:table opts) 34 | {:key-ns (:key-ns opts) 35 | :redis-key k 36 | :frozen-val v}) 37 | true) 38 | 39 | (fetch-keys [this ks] 40 | (assert (<= (count ks) 100)) ; API limit 41 | (let [{:keys [table key-ns]} opts 42 | vals-map ; { } 43 | (try 44 | (->> (far/batch-get-item client-opts 45 | {table {:prim-kvs {:key-ns key-ns 46 | :redis-key ks} 47 | :attrs [:redis-key :frozen-val]}}) 48 | (table) ; [{:frozen-val _ :redis-key _} ...] 49 | (far/items-by-attrs :redis-key) 50 | (enc/map-vals :frozen-val) 51 | (reduce merge {})) 52 | (catch Throwable t (zipmap ks (repeat t))))] 53 | (mapv #(get vals-map % (truss/ex-info "Missing value" {})) ks)))) 54 | 55 | (defn faraday-datastore 56 | "Alpha - subject to change. 57 | Returns a Faraday DataStore with options: 58 | :table - Tundra table name. Useful for different apps/environments with 59 | individually provisioned throughput. 60 | :key-ns - Optional key namespacing mechanism useful for different 61 | apps/environments under a single table (i.e. shared provisioned 62 | throughput). 63 | Supported Freezer io types: byte[]s." 64 | [client-opts & [{:keys [table key-ns] 65 | :or {table default-table 66 | key-ns :default}}]] 67 | (->FaradayDataStore client-opts {:table table 68 | :key-ns key-ns})) 69 | 70 | (comment 71 | (def client-opts creds) 72 | (ensure-table client-opts {:throughput {:read 1 :write 1}}) 73 | (far/describe-table client-opts default-table) 74 | (def dstore (faraday-datastore client-opts)) 75 | (def hardkey (tundra/>urlsafe-str "foo:bar /♡\\:baz ")) 76 | (tundra/put-key dstore hardkey (.getBytes "hello world")) 77 | (String. ^bytes (first (tundra/fetch-keys dstore [hardkey]))) 78 | 79 | (def tstore (tundra/tundra-store dstore)) 80 | (def worker (tundra/worker tstore client-opts {})) 81 | 82 | (require '[taoensso.carmine :as car]) 83 | (car/wcar {} (car/hmset* "ted" {:name "ted"})) 84 | (car/wcar {} (tundra/dirty tstore "ted")) 85 | (car/wcar {} (car/del "ted")) 86 | (car/wcar {} (tundra/ensure-ks tstore "ted")) 87 | (car/wcar {} (car/hget "ted" :name))) 88 | -------------------------------------------------------------------------------- /src/taoensso/carmine_v4/utils.clj: -------------------------------------------------------------------------------- 1 | (ns ^:no-doc taoensso.carmine-v4.utils 2 | "Private ns, implementation detail." 3 | (:require 4 | [taoensso.encore :as enc] 5 | [taoensso.truss :as truss])) 6 | 7 | (comment (remove-ns 'taoensso.carmine-v4.utils)) 8 | 9 | (let [not-found (Object.) 10 | empty? (fn [x] (== (count x) 0)) 11 | merge2 12 | (fn [left right] 13 | (reduce-kv 14 | (fn rf [rm lk lv] 15 | (let [rv (get rm lk not-found)] 16 | (enc/cond 17 | (identical? rv not-found) (assoc rm lk lv) 18 | (map? rv) 19 | (if (map? lv) 20 | (assoc rm lk (reduce-kv rf rv lv)) 21 | (do rm)) 22 | :else rm))) 23 | right left))] 24 | 25 | (defn merge-opts 26 | "Like `enc/nested-merge`, but optimised for merging opts. 27 | Opt vals are used in ascending order of preference: 28 | `o3` > `o2` > `o1`" 29 | ([ o1] o1) 30 | ([ o1 o2] (if (empty? o2) o1 (merge2 o1 o2))) 31 | ([o1 o2 o3] 32 | (if (empty? o3) 33 | (if (empty? o2) 34 | o1 35 | (if (empty? o1) 36 | o2 37 | (merge2 o1 o2))) 38 | 39 | (if (empty? o2) 40 | (if (empty? o1) 41 | o3 42 | (merge2 o1 o3)) 43 | 44 | (if (empty? o1) 45 | (merge2 o2 o3) 46 | (merge2 (merge2 o1 o2) o3))))))) 47 | 48 | (comment (enc/qb 1e6 (merge-opts {:a 1} {:a 2} {:a 3}))) ; 75.67 49 | 50 | (defn dissoc-k [m in-k dissoc-k] 51 | (if-let [in-v (get m in-k)] 52 | (if (map? in-v) 53 | (assoc m in-k (dissoc in-v dissoc-k)) 54 | (do m)) 55 | (do m))) 56 | 57 | (defn dissoc-ks [m in-k dissoc-ks] 58 | (if-let [in-v (get m in-k)] 59 | (if (map? in-v) 60 | (assoc m in-k (reduce dissoc in-v dissoc-ks)) 61 | (do m)) 62 | (do m))) 63 | 64 | (defn get-at 65 | "Optimized `get-in`." 66 | ([m k1 ] (when m (get m k1))) 67 | ([m k1 k2 ] (when m (when-let [m2 (get m k1)] (get m2 k2)))) 68 | ([m k1 k2 k3] (when m (when-let [m2 (get m k1)] (when-let [m3 (get m2 k2)] (get m3 k3)))))) 69 | 70 | (defmacro get-first-contained [m & ks] 71 | (when ks 72 | `(if (contains? ~m ~(first ks)) 73 | (get ~m ~(first ks)) 74 | (get-first-contained ~m ~@(next ks))))) 75 | 76 | (comment (clojure.walk/macroexpand-all '(get-first-contained opts :k1 :k2 :k3))) 77 | 78 | ;;;; 79 | 80 | (defn cb-notify! 81 | "Notifies callbacks by calling them with @data_." 82 | ([cb data_] (when cb (truss/catching (cb (force data_))))) 83 | ([cb1 cb2 data_] 84 | (when cb1 (truss/catching (cb1 (force data_)))) 85 | (when cb2 (truss/catching (cb2 (force data_))))) 86 | 87 | ([cb1 cb2 cb3 data_] 88 | (when cb1 (truss/catching (cb1 (force data_)))) 89 | (when cb2 (truss/catching (cb2 (force data_)))) 90 | (when cb3 (truss/catching (cb3 (force data_)))))) 91 | 92 | (let [get-data_ 93 | (fn [error cbid] 94 | (let [data (assoc (ex-data error) :cbid cbid) 95 | data 96 | (if-let [cause (or (get data :cause) (ex-cause error))] 97 | (assoc data :cause cause) 98 | (do data))] 99 | (delay data)))] 100 | 101 | (defn cb-notify-and-throw! 102 | "Notifies callbacks with error data, then throws error." 103 | ([cbid cb error] (cb-notify! cb (get-data_ error cbid)) (throw error)) 104 | ([cbid cb1 cb2 error] (cb-notify! cb1 cb2 (get-data_ error cbid)) (throw error)) 105 | ([cbid cb1 cb2 cb3 error] (cb-notify! cb1 cb2 cb3 (get-data_ error cbid)) (throw error)))) 106 | 107 | (comment 108 | (cb-notify-and-throw! :cbid1 println 109 | (truss/ex-info "Error msg" {:x :X} (Exception. "Cause")))) 110 | -------------------------------------------------------------------------------- /test/taoensso/carmine/tests/locks.clj: -------------------------------------------------------------------------------- 1 | (ns taoensso.carmine.tests.locks 2 | (:require 3 | [clojure.test :as test :refer [deftest testing is]] 4 | [taoensso.encore :as enc] 5 | [taoensso.truss :as truss] 6 | [taoensso.carmine :as car :refer [wcar]] 7 | [taoensso.carmine.locks :refer [acquire-lock release-lock with-lock]] 8 | [taoensso.carmine.tests.config :as config])) 9 | 10 | (comment 11 | (remove-ns 'taoensso.carmine.tests.locks) 12 | (test/run-tests 'taoensso.carmine.tests.locks)) 13 | 14 | ;;;; Config, etc. 15 | 16 | (def conn-opts config/conn-opts) 17 | (def timeout-ms 2000) 18 | 19 | (defn sleep [n] (Thread/sleep (int n)) (str "slept " n "msecs")) 20 | 21 | ;;;; 22 | 23 | (deftest basic-locking-tests 24 | (let [lock-name "test:1" 25 | act-op1 (acquire-lock conn-opts lock-name timeout-ms 2000) 26 | act-op2 (acquire-lock conn-opts lock-name timeout-ms 200) 27 | act-op3 28 | (do 29 | (sleep 1000) 30 | (acquire-lock conn-opts lock-name timeout-ms 2000))] 31 | 32 | [(is (string? act-op1) "Should acquire lock and return UUID owner string") 33 | (is (nil? act-op2) "Should not acquire lock and return nil") 34 | (is (string? act-op3) "Should acquire lock and return new UUID owner string") 35 | (is (not= act-op1 act-op3) "It should return new owner UUID")])) 36 | 37 | (deftest releasing-lock-tests 38 | (let [lock-name "test:2" 39 | uuid (acquire-lock conn-opts lock-name timeout-ms 2000) 40 | act-op1 (acquire-lock conn-opts lock-name timeout-ms 200) ; Too early 41 | act-op2 (release-lock conn-opts lock-name uuid) 42 | act-op3 (acquire-lock conn-opts lock-name timeout-ms 10)] 43 | 44 | [(is (nil? act-op1) "Can't get lock as its too early") 45 | (is (true? act-op2) "Releasing lock should be successful") 46 | (is (string? act-op3) "Now that its released, we should be able to acquire a lock")])) 47 | 48 | (deftest already-released-tests 49 | (let [lock-name "test:3" 50 | uuid (acquire-lock conn-opts lock-name timeout-ms 2000)] 51 | 52 | (future (release-lock conn-opts lock-name uuid)) 53 | (is 54 | (false? 55 | (do (sleep 200) ; Wait for future to run 56 | (release-lock conn-opts lock-name uuid))) 57 | "Since we already released the lock we can't release it again"))) 58 | 59 | (deftest locking-scope-tests 60 | (let [lock-name "test:4"] 61 | (try (with-lock conn-opts lock-name timeout-ms (throw (Exception.))) 62 | (catch Exception e nil)) 63 | 64 | (is (string? (acquire-lock conn-opts lock-name timeout-ms 2000)) 65 | "Since with-lock threw an exception it came outside the scope and hence we can acquire a lock again."))) 66 | 67 | (deftest locking-failure-tests 68 | (let [lock-name "test:5"] 69 | (acquire-lock conn-opts lock-name 3000 2000) 70 | (is (nil? (with-lock conn-opts lock-name 2000 10)) 71 | "There is already a lock, hence with-lock failed."))) 72 | 73 | (deftest with-lock-expiry-tests 74 | [(testing "Case 1" 75 | (is (truss/throws? clojure.lang.ExceptionInfo 76 | (with-lock conn-opts 9 500 2000 (sleep 1000))) 77 | "Since lock expired before being released, it should throw an exception.")) 78 | 79 | (testing "Case 2" 80 | (let [lock-name "test:6"] 81 | (future (with-lock conn-opts lock-name 500 2000 (sleep 1000))) 82 | (sleep 100) ; Give future time to acquire lock 83 | (is (nil? (with-lock conn-opts lock-name 3000 10 :foo)) 84 | "Since Lock was already acquired we should get nil back"))) 85 | 86 | (testing "Case 3" 87 | (let [lock-name "test:7"] 88 | (future (with-lock conn-opts lock-name 500 2000 (sleep 1000))) 89 | (sleep 600) ; Give future time to acquire + lose lock 90 | (is (= {:result :foo} 91 | (with-lock conn-opts lock-name 3000 10 :foo)) 92 | "Since Lock was expired and then we tried to acquire it, we should get a lock")))]) 93 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Taoensso open source 2 | [**API**][cljdoc] | [**Wiki**][GitHub wiki] | [Slack][] | Latest release: [v3.5.0](../../releases/tag/v3.5.0) (2025-11-06) 3 | 4 | [![Main tests][Main tests SVG]][Main tests URL] 5 | [![Graal tests][Graal tests SVG]][Graal tests URL] 6 | 7 | # Carmine 8 | 9 | ### [Redis](https://en.wikipedia.org/wiki/Redis) client + message queue for Clojure 10 | 11 | Redis and Clojure are individually awesome, and **even better together**. 12 | 13 | Carmine is a mature Redis client for Clojure that offers an idiomatic Clojure API with plenty of **speed**, **power**, and **ease-of-use**. 14 | 15 | ## Why Carmine? 16 | 17 | - High-performance **pure-Clojure** library 18 | - [Fully documented API](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine) with support for the **latest Redis commands and features** 19 | - Easy-to-use, production-ready **connection pooling** 20 | - Auto **de/serialization** of Clojure data types via [Nippy](https://www.taoensso.com/nippy) 21 | - Fast, simple [message queue](../../wiki/3-Message-queue) API 22 | - Fast, simple [distributed lock](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine.locks) API 23 | 24 | ## Compatibility 25 | 26 | Redis is available in a few different flavours depending on your needs: 27 | 28 | | | Features | Support? | 29 | | ----------------------------------------------------------------------------------------- | ----------------------------------- | ------------------- | 30 | | Single node | Simplest setup | Yes | 31 | | Redis [Sentinel](https://redis.io/docs/latest/operate/oss_and_stack/management/sentinel/) | High availability | No (possibly later) | 32 | | Redis [Cluster](https://redis.io/docs/latest/operate/oss_and_stack/management/scaling/) | High availability, sharding | No (possibly later) | 33 | | Redis [Enterprise](https://redis.io/docs/latest/operate/rs/) | High availability, sharding | Yes | 34 | | Redis [Cloud](https://redis.io/cloud/) | High availability, sharding, hosted | Yes | 35 | 36 | ## Documentation 37 | 38 | - [Wiki][GitHub wiki] (getting started, usage, etc.) 39 | - API reference via [cljdoc][cljdoc] 40 | - Support: [Slack][] or [GitHub issues][] 41 | 42 | ## Funding 43 | 44 | You can [help support][sponsor] continued work on this project and [others][my work], thank you!! 🙏 45 | 46 | ## License 47 | 48 | Copyright © 2014-2025 [Peter Taoussanis][]. 49 | Licensed under [EPL 1.0](LICENSE.txt) (same as Clojure). 50 | 51 | 52 | 53 | [GitHub releases]: ../../releases 54 | [GitHub issues]: ../../issues 55 | [GitHub wiki]: ../../wiki 56 | [Slack]: https://www.taoensso.com/carmine/slack 57 | 58 | [Peter Taoussanis]: https://www.taoensso.com 59 | [sponsor]: https://www.taoensso.com/sponsor 60 | [my work]: https://www.taoensso.com/clojure-libraries 61 | 62 | 63 | 64 | [cljdoc]: https://cljdoc.org/d/com.taoensso/carmine/ 65 | 66 | [Clojars SVG]: https://img.shields.io/clojars/v/com.taoensso/carmine.svg 67 | [Clojars URL]: https://clojars.org/com.taoensso/carmine 68 | 69 | [Main tests SVG]: https://github.com/taoensso/carmine/actions/workflows/main-tests.yml/badge.svg 70 | [Main tests URL]: https://github.com/taoensso/carmine/actions/workflows/main-tests.yml 71 | [Graal tests SVG]: https://github.com/taoensso/carmine/actions/workflows/graal-tests.yml/badge.svg 72 | [Graal tests URL]: https://github.com/taoensso/carmine/actions/workflows/graal-tests.yml 73 | -------------------------------------------------------------------------------- /src/taoensso/carmine/lua/mq/enqueue.lua: -------------------------------------------------------------------------------- 1 | -- From msg_status.lua --------------------------------------------------------- 2 | local mid = _:mid; 3 | local now = tonumber(_:now); 4 | 5 | local status = nil; -- base status e/o nil, done, queued, locked 6 | local is_bo = false; -- backoff flag for: done, queued 7 | local is_rq = false; -- requeue flag for: done, locked 8 | -- 8x cases: nil, done(bo/rq), queued(bo), locked(rq) 9 | -- Describe with: {status, $bo, $rq} with $ prefixes e/o: _, +, -, * 10 | 11 | if (redis.call('hexists', _:qk-messages, mid) == 1) then 12 | local exp_lock = tonumber(redis.call('hget', _:qk-locks, mid)) or 0; 13 | local exp_bo = tonumber(redis.call('hget', _:qk-backoffs, mid)) or 0; 14 | 15 | is_bo = (now < exp_bo); 16 | is_rq = (redis.call('sismember', _:qk-requeue, mid) == 1) or -- Deprecated 17 | (redis.call('hexists', _:qk-messages-rq, mid) == 1); 18 | 19 | if (redis.call('sismember', _:qk-done, mid) == 1) then status = 'done'; 20 | elseif (now < exp_lock) then status = 'locked'; 21 | else status = 'queued'; end 22 | else 23 | status = 'nx'; 24 | end 25 | -------------------------------------------------------------------------------- 26 | -- Return {action, error} 27 | 28 | local can_upd = (_:can-upd? == '1'); 29 | local can_rq = (_:can-rq? == '1'); 30 | local reset_ibo = (_:reset-ibo? == '1'); 31 | 32 | local interrupt_sleep = function () 33 | if redis.call('rpoplpush', _:qk-isleep-a, _:qk-isleep-b) then 34 | elseif redis.call('rpoplpush', _:qk-isleep-b, _:qk-isleep-a) then 35 | else redis.call('lpush', _:qk-isleep-a, '_'); end -- Init 36 | end 37 | 38 | local reset_init_backoff = function() 39 | local init_bo = tonumber(_:init-bo); 40 | if (init_bo ~= 0) then 41 | redis.call('hset', _:qk-backoffs, _:mid, now + init_bo); 42 | return true; 43 | else 44 | redis.call('hdel', _:qk-backoffs, _:mid); 45 | return false; 46 | end 47 | end 48 | 49 | local reset_in_queue = function() 50 | redis.call('hset', _:qk-messages, _:mid, _:mcnt); 51 | redis.call('hsetnx', _:qk-udts, _:mid, now); 52 | 53 | local lock_ms = tonumber(_:lock-ms); 54 | if (lock_ms ~= -1) then 55 | redis.call('hset', _:qk-lock-times, _:mid, lock_ms); 56 | else 57 | redis.call('hdel', _:qk-lock-times, _:mid); 58 | end 59 | end 60 | 61 | local reset_in_requeue = function() 62 | redis.call('hset', _:qk-messages-rq, _:mid, _:mcnt); 63 | 64 | local lock_ms = tonumber(_:lock-ms); 65 | if (lock_ms ~= -1) then 66 | redis.call('hset', _:qk-lock-times-rq, _:mid, lock_ms); 67 | else 68 | redis.call('hdel', _:qk-lock-times-rq, _:mid); 69 | end 70 | end 71 | 72 | local ensure_update_in_requeue = function() 73 | if is_rq then 74 | if can_upd then 75 | reset_in_requeue(); 76 | return {'updated'}; 77 | else 78 | return {false, 'already-queued'}; 79 | end 80 | else 81 | reset_in_requeue(); 82 | return {'added'}; 83 | end 84 | end 85 | 86 | if (status == 'nx') then 87 | -- {nil, _bo, _rq} -> add to queue 88 | 89 | -- Ensure that mid-circle is initialized 90 | if redis.call('exists', _:qk-mid-circle) ~= 1 then 91 | redis.call('lpush', _:qk-mid-circle, 'end-of-circle'); 92 | end 93 | 94 | if reset_init_backoff() then 95 | redis.call('lpush', _:qk-mid-circle, _:mid); -- -> Maintenance queue 96 | else 97 | redis.call('lpush', _:qk-mids-ready, _:mid); -- -> Priority queue 98 | end 99 | 100 | reset_in_queue(); 101 | interrupt_sleep(); 102 | return {'added'}; 103 | 104 | elseif (status == 'queued') then 105 | if can_upd then 106 | -- {queued, *bo, _rq} -> update in queue 107 | if reset_ibo then reset_init_backoff(); end 108 | reset_in_queue(); 109 | return {'updated'}; 110 | elseif reset_ibo then 111 | reset_init_backoff(); 112 | return {'updated'}; 113 | else 114 | return {false, 'already-queued'}; 115 | end 116 | 117 | elseif (status == 'locked') then 118 | 119 | if can_rq then 120 | -- {locked, _bo, *rq} -> ensure/update in requeue 121 | return ensure_update_in_requeue(); 122 | else 123 | return {false, 'locked'}; 124 | end 125 | 126 | elseif (status == 'done') then 127 | 128 | if is_bo then 129 | if can_rq then 130 | -- {done, +bo, *rq} -> ensure/update in requeue 131 | return ensure_update_in_requeue(); 132 | else 133 | return {false, 'backoff'}; 134 | end 135 | else 136 | -- {done, -bo, *rq} -> ensure/update in requeue 137 | -- (We're appropriating the requeue mechanism here) 138 | 139 | redis.call('lpush', _:qk-mids-ready, _:mid); -- -> Priority queue 140 | 141 | interrupt_sleep(); 142 | return ensure_update_in_requeue(); 143 | end 144 | 145 | end 146 | 147 | return {false, 'unexpected'}; 148 | -------------------------------------------------------------------------------- /src/taoensso/carmine/lua/mq/dequeue.lua: -------------------------------------------------------------------------------- 1 | -- Return e/o {'sleep' ...}, {'skip' ...}, {'handle' ...}, {'unexpected' ...} 2 | 3 | -- Prioritize mids from ready list 4 | local mid_src = nil; 5 | local mid = nil; 6 | mid = redis.call('rpoplpush', _:qk-mids-ready, _:qk-mid-circle); 7 | if mid then 8 | mid_src = 'ready'; 9 | else 10 | mid = redis.call('rpoplpush', _:qk-mid-circle, _:qk-mid-circle); 11 | if mid then 12 | mid_src = 'circle'; 13 | end 14 | end 15 | 16 | if ((not mid) or (mid == 'end-of-circle')) then -- Uninit'd or eoq 17 | 18 | -- Calculate eoq_backoff_ms 19 | local ndry_runs = tonumber(redis.call('get', _:qk-ndry-runs)) or 0; 20 | local eoq_ms_tab = {_:eoq-bo1, _:eoq-bo2, _:eoq-bo3, _:eoq-bo4, _:eoq-bo5}; 21 | local eoq_backoff_ms = tonumber(eoq_ms_tab[math.min(5, (ndry_runs + 1))]); 22 | redis.call('incr', _:qk-ndry-runs); 23 | 24 | local isleep_on = nil; 25 | if (redis.call('llen', _:qk-isleep-b) > 0) then isleep_on = 'b'; else isleep_on = 'a'; end 26 | 27 | return {'sleep', 'end-of-circle', isleep_on, eoq_backoff_ms}; 28 | end 29 | 30 | -- From msg_status.lua --------------------------------------------------------- 31 | local now = tonumber(_:now); 32 | 33 | local status = nil; -- base status e/o nil, done, queued, locked 34 | local is_bo = false; -- backoff flag for: done, queued 35 | local is_rq = false; -- requeue flag for: done, locked 36 | -- 8x cases: nil, done(bo/rq), queued(bo), locked(rq) 37 | -- Describe with: {status, $bo, $rq} with $ prefixes e/o: _, +, -, * 38 | 39 | if (redis.call('hexists', _:qk-messages, mid) == 1) then 40 | local exp_lock = tonumber(redis.call('hget', _:qk-locks, mid)) or 0; 41 | local exp_bo = tonumber(redis.call('hget', _:qk-backoffs, mid)) or 0; 42 | 43 | is_bo = (now < exp_bo); 44 | is_rq = (redis.call('sismember', _:qk-requeue, mid) == 1) or -- Deprecated 45 | (redis.call('hexists', _:qk-messages-rq, mid) == 1); 46 | 47 | if (redis.call('sismember', _:qk-done, mid) == 1) then status = 'done'; 48 | elseif (now < exp_lock) then status = 'locked'; 49 | else status = 'queued'; end 50 | else 51 | status = 'nx'; 52 | end 53 | -------------------------------------------------------------------------------- 54 | 55 | if (status == 'nx') then 56 | if (mid_src == 'circle') then 57 | redis.call('ltrim', _:qk-mid-circle, 1, -1); -- Remove from circle 58 | return {'skip', 'did-trim'}; 59 | else 60 | return {'skip', 'unexpected'}; 61 | end 62 | elseif (status == 'locked') then return {'skip', 'locked'}; 63 | elseif ((status == 'queued') and is_bo) then return {'skip', 'queued-with-backoff'}; 64 | elseif ((status == 'done') and is_bo) then return {'skip', 'done-with-backoff'}; 65 | end 66 | 67 | redis.call('set', _:qk-ndry-runs, 0); -- Doing useful work 68 | 69 | if (status == 'done') then 70 | if is_rq then 71 | -- {done, -bo, +rq} -> requeue now 72 | local mcontent = 73 | redis.call('hget', _:qk-messages-rq, mid) or 74 | redis.call('hget', _:qk-messages, mid); -- Deprecated (for qk-requeue) 75 | 76 | local lock_ms = 77 | tonumber(redis.call('hget', _:qk-lock-times-rq, mid)) or 78 | tonumber(_:default-lock-ms); 79 | 80 | redis.call('hset', _:qk-messages, mid, mcontent); 81 | redis.call('hset', _:qk-udts, mid, now); 82 | 83 | if lock_ms then 84 | redis.call('hset', _:qk-lock-times, mid, lock_ms); 85 | else 86 | redis.call('hdel', _:qk-lock-times, mid); 87 | end 88 | 89 | redis.call('hdel', _:qk-messages-rq, mid); 90 | redis.call('hdel', _:qk-lock-times-rq, mid); 91 | redis.call('hdel', _:qk-nattempts, mid); 92 | redis.call('srem', _:qk-done, mid); 93 | redis.call('srem', _:qk-requeue, mid); 94 | 95 | redis.call('lpush', _:qk-mids-ready, mid); -- -> Priority queue (>once okay) 96 | 97 | return {'skip', 'did-requeue'}; 98 | else 99 | -- {done, -bo, -rq} -> full GC now 100 | redis.call('hdel', _:qk-messages, mid); 101 | redis.call('hdel', _:qk-messages-rq, mid); 102 | redis.call('hdel', _:qk-lock-times, mid); 103 | redis.call('hdel', _:qk-lock-times-rq, mid); 104 | redis.call('hdel', _:qk-udts, mid); 105 | redis.call('hdel', _:qk-locks, mid); 106 | redis.call('hdel', _:qk-backoffs, mid); 107 | redis.call('hdel', _:qk-nattempts, mid); 108 | redis.call('srem', _:qk-done, mid); 109 | redis.call('srem', _:qk-requeue, mid); 110 | 111 | if (mid_src == 'circle') then 112 | redis.call('ltrim', _:qk-mid-circle, 1, -1); -- Remove from circle 113 | end 114 | 115 | return {'skip', 'did-gc'}; 116 | end 117 | elseif (status == 'queued') then 118 | -- {queued, -bo, _rq} -> handle now 119 | local lock_ms = 120 | tonumber(redis.call('hget', _:qk-lock-times, mid)) or 121 | tonumber(_:default-lock-ms); 122 | 123 | redis.call('hset', _:qk-locks, mid, now + lock_ms); -- Acquire 124 | local mcontent = redis.call('hget', _:qk-messages, mid); 125 | local udt = redis.call('hget', _:qk-udts, mid); 126 | local nattempts = redis.call('hincrby', _:qk-nattempts, mid, 1); 127 | 128 | return {'handle', mid, mcontent, nattempts, lock_ms, tonumber(udt)}; 129 | end 130 | 131 | return {'unexpected'}; 132 | -------------------------------------------------------------------------------- /wiki/1-Getting-started.md: -------------------------------------------------------------------------------- 1 | # Setup 2 | 3 | ## Dependency 4 | 5 | Add the [relevant dependency](../#latest-releases) to your project: 6 | 7 | ```clojure 8 | Leiningen: [com.taoensso/carmine "x-y-z"] ; or 9 | deps.edn: com.taoensso/carmine {:mvn/version "x-y-z"} 10 | ``` 11 | 12 | And setup your namespace imports: 13 | 14 | ```clojure 15 | (ns my-app (:require [taoensso.carmine :as car :refer [wcar]])) 16 | ``` 17 | 18 | ## Configure connections 19 | 20 | You'll usually want to define a single **connection pool**, and one **connection spec** for each of your Redis servers, example: 21 | 22 | ```clojure 23 | (defonce my-conn-pool (car/connection-pool {})) ; Create a new stateful pool 24 | (def my-conn-spec {:uri "redis://redistogo:pass@panga.redistogo.com:9475/"}) 25 | (def my-wcar-opts {:pool my-conn-pool, :spec my-conn-spec}) 26 | ``` 27 | 28 | This `my-wcar-opts` can then be provided to Carmine's `wcar` ("with Carmine") API: 29 | 30 | ```clojure 31 | (wcar my-wcar-opts (car/ping)) ; => "PONG" 32 | ``` 33 | 34 | `wcar` is the main entry-point to Carmine's API. See its [docstring](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine#wcar) for **lots more info** on connection options! 35 | 36 | You can create a `wcar` partial for convenience: 37 | 38 | ```clojure 39 | (defmacro wcar* [& body] `(car/wcar my-wcar-opts ~@body)) 40 | ``` 41 | 42 | # Usage 43 | 44 | ## Command pipelines 45 | 46 | Calling multiple Redis commands in a single `wcar` body uses efficient [Redis pipelining](https://redis.io/docs/manual/pipelining/) under the hood, and returns a pipeline reply (vector) for easy destructuring: 47 | 48 | ```clojure 49 | (wcar* 50 | (car/ping) 51 | (car/set "foo" "bar") 52 | (car/get "foo")) ; => ["PONG" "OK" "bar"] (3 commands -> 3 replies) 53 | ``` 54 | 55 | If the number of commands you'll be calling might vary, it's possible to request that Carmine _always_ return a destructurable pipeline-style reply: 56 | 57 | ```clojure 58 | (wcar* :as-pipeline (car/ping)) ; => ["PONG"] ; Note the pipeline-style reply 59 | ``` 60 | 61 | If the server replies with an error, an exception is thrown: 62 | 63 | ```clojure 64 | (wcar* (car/spop "foo")) 65 | => Exception ERR Operation against a key holding the wrong kind of value 66 | ``` 67 | 68 | But what if we're pipelining? 69 | 70 | ```clojure 71 | (wcar* 72 | (car/set "foo" "bar") 73 | (car/spop "foo") 74 | (car/get "foo")) 75 | => ["OK" # "bar"] 76 | ``` 77 | 78 | ## Serialization 79 | 80 | The only scalar type native to Redis is the [byte string](https://redis.io/docs/data-types/). But Carmine uses [Nippy](https://www.taoensso.com/nippy) under the hood to seamlessly support all of Clojure's rich data types: 81 | 82 | ```clojure 83 | (wcar* 84 | (car/set "clj-key" 85 | {:bigint (bigint 31415926535897932384626433832795) 86 | :vec (vec (range 5)) 87 | :set #{true false :a :b :c :d} 88 | :bytes (byte-array 5) 89 | ;; ... 90 | }) 91 | 92 | (car/get "clj-key")) 93 | 94 | => ["OK" {:bigint 31415926535897932384626433832795N 95 | :vec [0 1 2 3 4] 96 | :set #{true false :a :c :b :d} 97 | :bytes #}] 98 | ``` 99 | 100 | Types are handled as follows: 101 | 102 | Clojure type| Redis type 103 | -- | -- 104 | Strings| Redis strings 105 | Keywords | Redis strings 106 | Simple numbers | Redis strings 107 | Everything else | Auto de/serialized with [Nippy](https://www.taoensso.com/nippy) 108 | 109 | You can force automatic de/serialization for an argument of any type by wrapping it with [`car/freeze`](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine#freeze). 110 | 111 | ## Command coverage 112 | 113 | Carmine uses the [official Redis command spec](https://github.com/redis/redis-doc/blob/master/commands.json) to auto-generate a Clojure function for **every Redis command**. These are accessible from the main Carmine namespace: 114 | 115 | ```clojure 116 | ;; Example: car/sort is a Clojure function for the Redis SORT command 117 | (clojure.repl/doc car/sort) 118 | 119 | => "SORT key [BY pattern] [LIMIT offset count] [GET pattern [GET pattern ...]] [ASC|DESC] [ALPHA] [STORE destination] 120 | 121 | Sort the elements in a list, set or sorted set. 122 | 123 | Available since: 1.0.0. 124 | 125 | Time complexity: O(N+M*log(M)) where N is the number of elements in the list or set to sort, and M the number of returned elements. When the elements are not sorted, complexity is currently O(N) as there is a copy step that will be avoided in next releases." 126 | ``` 127 | 128 | Each Carmine release will always use the latest Redis command spec. 129 | 130 | But if a new Redis command hasn't yet made it to Carmine, or if you want to use a Redis command not in the official spec (a [Redis module](https://redis.io/resources/modules/) command for example) - you can always use Carmine's [`redis-call`](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine#redis-call) function to issue **arbitrary Redis commands**. 131 | 132 | So you're **not constrained** by the commands provided by Carmine. 133 | 134 | ## Commands are functions 135 | 136 | Carmine's auto-generated command functions are **real functions**, which means that you can use them like any other Clojure function: 137 | 138 | ```clojure 139 | (wcar* (doall (repeatedly 5 car/ping))) 140 | => ["PONG" "PONG" "PONG" "PONG" "PONG"] 141 | 142 | (let [first-names ["Salvatore" "Rich"] 143 | surnames ["Sanfilippo" "Hickey"]] 144 | (wcar* (mapv #(car/set %1 %2) first-names surnames) 145 | (mapv car/get first-names))) 146 | => ["OK" "OK" "Sanfilippo" "Hickey"] 147 | 148 | (wcar* (mapv #(car/set (str "key-" %) (rand-int 10)) (range 3)) 149 | (mapv #(car/get (str "key-" %)) (range 3))) 150 | => ["OK" "OK" "OK" "OK" "0" "6" "6" "2"] 151 | ``` 152 | 153 | And since real functions can compose, so can Carmine's. 154 | 155 | By nesting `wcar` calls, you can fully control how composition and pipelining interact: 156 | 157 | ```clojure 158 | (let [hash-key "awesome-people"] 159 | (wcar* 160 | (car/hmset hash-key "Rich" "Hickey" "Salvatore" "Sanfilippo") 161 | (mapv (partial car/hget hash-key) 162 | ;; Execute with own connection & pipeline then return result 163 | ;; for composition: 164 | (wcar* (car/hkeys hash-key))))) 165 | => ["OK" "Sanfilippo" "Hickey"] 166 | ``` 167 | 168 | # Performance 169 | 170 | Redis is probably most famous for being *fast*. Carmine hold up its end and usually performs within ~10% of the official C `redis-benchmark` utility despite offering features like command composition, reply parsing, etc. 171 | 172 | Benchmarks are [included](../blob/master/src/taoensso/carmine/benchmarks.clj) in the Carmine GitHub repo and can be easily run from your own environment. -------------------------------------------------------------------------------- /test/taoensso/carmine/tests/tundra.clj: -------------------------------------------------------------------------------- 1 | (ns taoensso.carmine.tests.tundra 2 | (:require 3 | [clojure.test :as test :refer [is deftest]] 4 | [taoensso.carmine :as car :refer [wcar]] 5 | [taoensso.carmine.message-queue :as mq] 6 | [taoensso.carmine.tundra :as tundra] 7 | [taoensso.carmine.tundra.disk :as tdisk] 8 | [taoensso.carmine.tests.config :as config])) 9 | 10 | (comment 11 | (remove-ns 'taoensso.carmine.tests.tundra) 12 | (test/run-tests 'taoensso.carmine.tests.tundra)) 13 | 14 | (def conn-opts config/conn-opts) 15 | (defmacro wcar* [& body] `(car/wcar conn-opts ~@body)) 16 | 17 | (def tkey (partial car/key :carmine :tundra :test)) 18 | (def tqname "carmine-tundra-tests") 19 | (def mqname (format "tundra:%s" (name tqname))) ; Nb has prefix 20 | 21 | (defn clean-up! [] 22 | (mq/queues-clear!! conn-opts mqname) 23 | (when-let [ks (seq (wcar* (car/keys (tkey :*))))] 24 | (wcar* (apply car/del ks) 25 | (apply car/srem @#'tundra/k-evictable ks))) 26 | (wcar* (car/srem @#'tundra/k-evictable (tkey :invalid-evictable)))) 27 | 28 | (defn cleanup-fixture [f] (clean-up!) (f) (clean-up!)) 29 | (test/use-fixtures :once cleanup-fixture) 30 | 31 | (def dstore (tdisk/disk-datastore "./carmine-tundra-test-temp")) 32 | 33 | (defn- s->ba [^String s] (.getBytes s "UTF-8")) 34 | (defn- ba->s [^bytes ba] (String. ba "UTF-8")) 35 | 36 | (defn sleep [n] 37 | (let [n (case n :c1 400 :c2 1200 :c3 8000 n)] 38 | (Thread/sleep (int n)) (str "slept " n "msecs"))) 39 | 40 | ;;;; 41 | 42 | (deftest datastore-tests 43 | (test/testing "Basic put and fetch case" 44 | (tundra/put-key dstore (tkey :foo) (s->ba "hello world")) 45 | (is (= "hello world" 46 | (-> dstore 47 | (tundra/fetch-keys [(tkey :foo)]) 48 | first 49 | ba->s)))) 50 | 51 | (test/testing "Update val case" 52 | (tundra/put-key dstore (tkey :foo) (s->ba "hello world 1")) 53 | (tundra/put-key dstore (tkey :foo) (s->ba "hello world 2")) 54 | (is (= "hello world 2" 55 | (-> dstore 56 | (tundra/fetch-keys [(tkey :foo)]) 57 | first 58 | ba->s)))) 59 | 60 | (test/testing "Exception Case" 61 | (is (instance? java.nio.file.NoSuchFileException 62 | (first (tundra/fetch-keys dstore [(tkey :invalid)])))))) 63 | 64 | (deftest tundra-api-test-1 65 | (let [tstore (tundra/tundra-store dstore)] 66 | (test/testing "Bad cases" 67 | (is (thrown? Exception (wcar* (tundra/dirty tstore (tkey :invalid))))) 68 | (is (= nil (wcar* (tundra/ensure-ks tstore (tkey :invalid))))) 69 | (is (thrown? Exception 70 | (wcar* (car/sadd @#'tundra/k-evictable (tkey :invalid-evictable)) 71 | (tundra/ensure-ks tstore (tkey :invalid-evictable)))))) 72 | 73 | (test/testing "API never pollutes enclosing pipeline" 74 | (is (= ["OK" "PONG" 1] 75 | (wcar* (car/set (tkey 0) "0") 76 | (car/ping) 77 | ;; Won't throw since `(tkey :invalid)` isn't in evictable set: 78 | (tundra/ensure-ks tstore (tkey 0) (tkey :invalid)) 79 | (tundra/dirty tstore (tkey 0)) 80 | (car/del (tkey 0) "0"))))))) 81 | 82 | 83 | (deftest tundra-api-test-2 84 | (is (= [[:clj-val] [:clj-val] [:clj-val-new]] 85 | (let [_ (clean-up!) 86 | tstore (tundra/tundra-store dstore {:tqname tqname}) 87 | tworker (tundra/worker tstore conn-opts {:eoq-backoff-ms 100 :throttle-ms 100})] 88 | 89 | (sleep :c1) 90 | (wcar* (car/mset (tkey 0) [:clj-val] 91 | (tkey 1) [:clj-val] 92 | (tkey 2) [:clj-val])) ; Reset vals 93 | 94 | ;; Invalid keys don't prevent valid keys from being processed (will still 95 | ;; throw, but only _after_ all possible dirtying): 96 | (wcar* (try (tundra/dirty tstore (tkey :invalid) (tkey :invalid-evictable) 97 | (tkey 0) (tkey 1) (tkey 2)) 98 | (catch Exception _ nil))) 99 | 100 | (sleep :c3) ; Wait for replication 101 | (mq/stop tworker) 102 | 103 | (sleep :c1) 104 | (wcar* (car/del (tkey 0)) 105 | (car/set (tkey 2) [:clj-val-new])) ; Make some local modifications 106 | 107 | ;; Invalid keys don't prevent valid keys from being processed (will still 108 | ;; throw, but only _after_ all possible fetches) 109 | (wcar* (try (tundra/ensure-ks tstore (tkey :invalid) (tkey :invalid-evictable) 110 | (tkey 0) (tkey 1) (tkey 2)) 111 | (catch Exception _ nil))) 112 | 113 | (wcar* (car/mget (tkey 0) (tkey 1) (tkey 2))))))) 114 | 115 | 116 | (deftest tundra-api-test-3 117 | (is (= [-1 -1 -1] ; nil eviction timeout (default) is respected! 118 | (let [tstore (tundra/tundra-store dstore {:tqname tqname}) 119 | tworker (tundra/worker tstore conn-opts {:eoq-backoff-ms 100 :throttle-ms 100})] 120 | 121 | (sleep :c1) 122 | (wcar* (car/mset (tkey 0) "0" (tkey 1) "1" (tkey 2) "1") ; Clears timeouts 123 | (tundra/dirty tstore (tkey 0) (tkey 1) (tkey 2))) 124 | 125 | (sleep :c3) ; Wait for replication 126 | (mq/stop tworker) 127 | 128 | (sleep :c1) 129 | (wcar* (tundra/ensure-ks tstore (tkey 0) (tkey 1) (tkey 2)) 130 | (mapv #(car/ttl (tkey %)) [0 1 2])))))) 131 | 132 | (deftest tundra-api-test-4 133 | (test/testing "nnil eviction timeout is applied & extended correctly" 134 | (is ((fn [[t0 t1 t2]] 135 | (and (= t0 -1) 136 | (> t1 0) 137 | (> t2 t1))) 138 | 139 | (let [_ (clean-up!) 140 | tstore (tundra/tundra-store dstore {:redis-ttl-ms (* 1000 60 60 24) 141 | :tqname tqname}) 142 | tworker (tundra/worker tstore conn-opts {:eoq-backoff-ms 100 :throttle-ms 100 143 | :auto-start false})] 144 | 145 | (sleep :c1) 146 | (wcar* (car/set (tkey 0) "0") ; Clears timeout 147 | (tundra/dirty tstore (tkey 0))) 148 | 149 | [(wcar* (car/pttl (tkey 0))) ; `dirty` doesn't set ttl immediately 150 | (do (mq/start tworker) 151 | (sleep :c3) ; Wait for replication (> exp-backoff) 152 | (mq/stop tworker) 153 | (sleep :c1) 154 | (wcar* (car/pttl (tkey 0)))) ; Worker sets ttl after successful put 155 | (do (sleep :c2) 156 | (wcar* (tundra/ensure-ks tstore (tkey 0)) 157 | (car/pttl (tkey 0))))]))))) 158 | 159 | (comment (clean-up!) 160 | (mq/queue-status conn-opts mqname)) 161 | -------------------------------------------------------------------------------- /wiki/0-Breaking-changes.md: -------------------------------------------------------------------------------- 1 | This page details possible **breaking changes and migration instructions** for Carmine. 2 | 3 | My apologies for the trouble. I'm very mindful of the costs involved in breaking changes, and I try hard to avoid them whenever possible. When there is a very good reason to break, I'll try to batch breaks and to make migration as easy as possible. 4 | 5 | Thanks for your understanding - [Peter Taoussanis](https://www.taoensso.com) 6 | 7 | # Carmine `v3.2.x` to `v3.3.x` 8 | 9 | There are **breaking changes** to the **Carmine message queue API** that **may affect** a small proportion of message queue users. 10 | 11 | - If you **DO NOT** use Carmine's [message queue API](https://github.com/taoensso/carmine/wiki/3-Message-queue), no migration should be necessary. 12 | - If you **DO** use Carmine's message queue API, **please read the below checklist carefully**!! 13 | 14 | ## Migration checklist for message queue users 15 | 16 | 1. **Please be aware**: v3.3 introduces major changes (improvements) to Carmine's message queue architecture and API. 17 | 18 | While the changes have been extensively tested and used for months on private systems, there is always a non-zero chance of unexpected issues when so much code is touched. 19 | 20 | Therefore please **carefully test Carmine v3.3** before deploying to production, and please **carefully monitor your queues** for expected behaviour and performance. 21 | 22 | New tools are provided for queue monitoring. See the [API improvements](#api-improvements) section for details. 23 | 24 | 2. [`enqueue`](https://taoensso.github.io/carmine/taoensso.carmine.message-queue.html#var-enqueue) **return value has changed**. 25 | 26 | This change is relevant to you iff you use the return value of `enqueue` calls (most users do not). Check your `enqueue` call sites to be sure. 27 | 28 | The fn previously returned `` (message id) on success, or `{:carmine.mq/error }` on error. 29 | 30 | The fn now **always returns a map** with `{:keys [success? mid action error]}`. 31 | 32 | See the updated `enqueue` [docstring](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine.message-queue#enqueue) for details. 33 | 34 | 3. [`queue-status`](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine.message-queue#queue-status) **return value has changed**. 35 | 36 | This change is relevant to you iff you use the `queue-status` util. 37 | 38 | The fn previously returned a detailed map of **all queue content** in O(queue-size) and was rarely used. 39 | 40 | The fn now returns a small map in O(1) with `{:keys [nwaiting nlocked nbackoff ntotal]}`. 41 | 42 | If you want the detailed map of all queue content in O(queue-size), 43 | use the new [`queue-content`](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine.message-queue#queue-content) util. 44 | 45 | 4. **The definition of "queue-size" has changed**. 46 | 47 | The old definition: total size of queue. 48 | The new definition: total size of queue, LESS mids that may be locked or in backoff. 49 | 50 | I.e. the new definition now better represents **the number of messages awaiting processing**. 51 | 52 | Most users won't be negatively affected by this change since the new definition better corresponds to how most users actually understood the term before. 53 | 54 | 5. `clear-queues` **has been deprecated**. 55 | 56 | This utility is now called [`queues-clear!!`](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine.message-queue#queues-clear!!) to better match the rest of the API. 57 | 58 | 6. **Handling order of queued messages has changed**. 59 | 60 | To improve latency, queue workers now prioritize handling of **newly queued** messages before **re-visiting old messages** for queue maintenance (re-handling, GC, etc.). 61 | 62 | Most users shouldn't be negatively affected by this change since Carmine's message queue has never given any specific ordering guarantees (and cannot, for various reasons). 63 | 64 | 7. **Queue connections may be held longer**. 65 | 66 | To improve latency, queue sleeping is now **active** rather than **passive**, allowing Redis to immediately signal Carmine's message queue when new messages arrive. 67 | 68 | This change means that each worker thread (default 1) may now *hold a connection open* while sleeping. The maximum hold time is determined by [`:throttle-ms`](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine.message-queue#worker) (usu. in the order of ~10-400 msecs). 69 | 70 | So if you have many queues and/or many queue worker threads, you *may* want to increase the maximum number of connections in your connection pool to ensure that you always have available connections. 71 | 72 | ## Architectural improvements 73 | 74 | - **Significantly improved latency** (esp. worst-case latency) of handling new messages. 75 | 76 | - **Decouple threads for handling and queue maintenance**. Thread counts can now be individually customized. 77 | 78 | - Worker end-of-queue backoff sleeps are now interrupted by new messages. So **sleeping workers will automatically awaken when new messages arrive**. 79 | 80 | - Prioritize requeues (treat as "ready"). So **requeues no longer need to wait for a queue cycle to be reprocessed**. 81 | 82 | - **Smart worker throttling.** The `:throttle-ms` worker option can now be a function of the current queue size, enabling **dynamic worker throttling**. 83 | 84 | The default `:throttle-ms` value is now `:auto`, which uses such a function. 85 | 86 | See the updated `worker` [docstring](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine.message-queue#worker) for details. 87 | 88 | - **Worker threads are now auto desynchronized** to reduce contention. 89 | 90 | ## API improvements 91 | 92 | - Added `enqueue` option: `:lock-ms` to support **per-message lock times** [#223](https://github.com/taoensso/carmine/issues/223). 93 | - Added `enqueue` option: `:can-update?` to support **message updating**. 94 | - Handler fn data now includes `:worker`, `:queue-size` keys. 95 | - Handler fn data now includes `:age-ms` key, enabling **easy integration with Tufte or other profiling tools**. 96 | - Added utils: [`queue-size`](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine.message-queue#queue-size), [`queue-names`](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine.message-queue#queue-names), [`queue-content`](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine.message-queue#queue-content), [`queues-clear!!`](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine.message-queue#queues-clear!!), [`queues-clear-all!!!`](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine.message-queue#queues-clear-all!!!). 97 | - `Worker` object's string/pprint representation is now more useful. 98 | - `Worker` object can now be derefed to get **state and stats useful for monitoring**. 99 | 100 | In particular, the new `:stats` key contains **detailed statistics on queue size, queueing time, handling time, etc**. 101 | 102 | - `Worker` objects can now be invoked as fns to execute common actions. 103 | 104 | Actions [include](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine.message-queue#worker): `:start`, `:stop`, `:queue-size`, `:queue-status`. 105 | 106 | - Various improvements to docstrings, error messages, and logging output. 107 | - Improved message queue [state diagram](https://github.com/taoensso/carmine/blob/master/mq-architecture.svg). 108 | - General improvements to implementation, observability, and tests. 109 | -------------------------------------------------------------------------------- /wiki/2-Further-usage.md: -------------------------------------------------------------------------------- 1 | # Lua scripting 2 | 3 | Redis offers powerful [Lua scripting](https://redis.io/docs/interact/programmability/eval-intro) capabilities. 4 | 5 | As an example, let's write our own version of the `set` command: 6 | 7 | ```clojure 8 | (defn my-set 9 | [key value] 10 | (car/lua "return redis.call('set', _:my-key, 'lua '.. _:my-val)" 11 | {:my-key key} ; Named key variables and their values 12 | {:my-val value} ; Named non-key variables and their values 13 | )) 14 | 15 | (wcar* 16 | (my-set "foo" "bar") 17 | (car/get "foo")) 18 | => ["OK" "lua bar"] 19 | ``` 20 | 21 | Script primitives are also provided: [`eval`](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine#eval), [`eval-sha`](https://taoensso.github.io/carmine/taoensso.carmine.html#var-evalsha), [`eval*`](https://taoensso.github.io/carmine/taoensso.carmine.html#var-eval*), [`eval-sha*`](https://taoensso.github.io/carmine/taoensso.carmine.html#var-evalsha*). 22 | 23 | # Helpers 24 | 25 | The [`lua`](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine#lua) command above is a good example of a Carmine "helper". 26 | 27 | Carmine will never surprise you by interfering with the standard Redis command API. But there are times when it might want to offer you a helping hand (if you want it). Compare: 28 | 29 | ```clojure 30 | (wcar* (car/zunionstore "dest-key" 3 "zset1" "zset2" "zset3" "WEIGHTS" 2 3 5)) 31 | (wcar* (car/zunionstore* "dest-key" ["zset1" "zset2" "zset3"] "WEIGHTS" 2 3 5)) 32 | ``` 33 | 34 | Both of these calls are equivalent but the latter counted the keys for us. [`zunionstore*`](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine#zunionstore*) is another helper: a slightly more convenient version of a standard command, suffixed with a `*` to indicate that it's non-standard. 35 | 36 | Helpers currently include: [`atomic`](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine#atomic), [`eval*`](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine#eval*), [`evalsha*`](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine#evalsha*), [`info*`](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine#info*), [`lua`](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine#lua), [`sort*`](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine#sort*), [`zinterstore*`](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine#zinterstore*), and [`zunionstore*`](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine#zunionstore*). 37 | 38 | # Pub/Sub and Listeners 39 | 40 | Carmine has a flexible **listener API** to support persistent-connection features like [monitoring](https://redis.io/commands/monitor/) and Redis's [Pub/Sub](https://redis.io/docs/interact/pubsub/) facility: 41 | 42 | ```clojure 43 | (def my-listener 44 | (car/with-new-pubsub-listener (:spec server1-conn) 45 | {"channel1" (fn f1 [msg] (println "f1:" msg)) 46 | "channel*" (fn f2 [msg] (println "f2:" msg)) 47 | "ch*" (fn f3 [msg] (println "f3:" msg))} 48 | (car/subscribe "channel1") 49 | (car/psubscribe "channel*" "ch*"))) 50 | ``` 51 | 52 | Exactly 1 handler fn will trigger per published message *exactly* matching each active subscription: 53 | 54 | - `channel1` handler (`f1`) will trigger for messages to `channel1`. 55 | - `channel*` handler (`f2`) will trigger for messages to `channel1`, `channel2`, etc. 56 | - `ch*` handler (`f3`) will trigger for messages to `channel1`, `channel2`, etc. 57 | 58 | So publishing to "channel1" in this example will trigger all 3x handlers: 59 | 60 | ```clojure 61 | (wcar* (car/publish "channel1" "Hello to channel1!")) 62 | 63 | ;; Will trigger: 64 | 65 | (f1 [ "message" "channel1" "Hello to channel1!"]) 66 | (f2 ["pmessage" "channel*" "channel1" "Hello to channel1!"]) 67 | (f3 ["pmessage" "ch*" "channel1" "Hello to channel1!"]) 68 | ``` 69 | 70 | You can adjust subscriptions and/or handlers: 71 | 72 | ```clojure 73 | (car/with-open-listener my-listener 74 | (car/unsubscribe) ; Unsubscribe from every channel (leaving patterns alone) 75 | (car/subscribe "channel3")) 76 | 77 | (swap! (:state my-listener) ; { (fn [msg])} 78 | assoc "channel3" (fn [x] (println "do something"))) 79 | ``` 80 | 81 | **Remember to close listeners** when you're done with them: 82 | 83 | ```clojure 84 | (car/close-listener my-listener) 85 | ``` 86 | 87 | Note that subscriptions are **connection-local**: you can have three different listeners each listening for different messages and using different handlers. 88 | 89 | # Reply parsing 90 | 91 | Want a little more control over how server replies are parsed? See [`parse`](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine#parse): 92 | 93 | ```clojure 94 | (wcar* 95 | (car/ping) 96 | (car/parse clojure.string/lower-case (car/ping) (car/ping)) 97 | (car/ping)) 98 | => ["PONG" "pong" "pong" "PONG"] 99 | ``` 100 | 101 | # Distributed locks 102 | 103 | See the [`locks`](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine.locks) namespace for a simple distributed lock API: 104 | 105 | ```clojure 106 | (:require [taoensso.carmine.locks :as locks]) ; Add to `ns` macro 107 | 108 | (locks/with-lock 109 | {:pool {} :spec {}} ; Connection details 110 | "my-lock" ; Lock name/identifier 111 | 1000 ; Time to hold lock 112 | 500 ; Time to wait (block) for lock acquisition 113 | (println "This was printed under lock!")) 114 | ``` 115 | 116 | # Tundra 117 | 118 | > **Deprecation notice**: Tundra isn't currently being actively maintained, though it will continue to be supported until Carmine 4. If you are using Tundra, [please let me know](https://www.taoensso.com/contact-me)! 119 | 120 | Redis is a beautifully designed datastore that makes some explicit engineering tradeoffs. Probably the most important: your data _must_ fit in memory. Tundra helps relax this limitation: only your **hot** data need fit in memory. How does it work? 121 | 122 | 1. Use Tundra's [`dirty`](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine.tundra#dirty) command **any time you modify/create evictable keys** 123 | 2. Use [`worker`](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine.tundra#ITundraStore) to create a threaded worker that'll **automatically replicate dirty keys to your secondary datastore** 124 | 3. When a dirty key hasn't been used in a specified TTL, it will be **automatically evicted from Redis** (eviction is optional if you just want to use Tundra as a backup/just-in-case mechanism) 125 | 4. Use [`ensure-ks`](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine.tundra#ensure-ks) **any time you want to use evictable keys** - this'll extend their TTL or fetch them from your datastore as necessary 126 | 127 | That's it: two Redis commands, and a worker! 128 | 129 | Tundra uses Redis' own dump/restore mechanism for replication, and Carmine's own [Message queue](./3-Message-queue) to coordinate the replication worker. 130 | 131 | It's possible to easily extend support to **any K/V-capable datastore**. 132 | Implementations are provided out-the-box for: 133 | 134 | - [Disk](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine.tundra.disk) 135 | - [Amazon S3](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine.tundra.s3) 136 | - [Amazon DynamoDB](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine.tundra.faraday) via [Faraday](https://www.taoensso.com/faraday) 137 | 138 | ## Example usage 139 | 140 | ```clojure 141 | (:require [taoensso.carmine.tundra :as tundra :refer (ensure-ks dirty)] 142 | [taoensso.carmine.tundra.s3]) ; Add to ns 143 | 144 | (def my-tundra-store 145 | (tundra/tundra-store 146 | ;; A datastore that implements the necessary (easily-extendable) protocol: 147 | (taoensso.carmine.tundra.s3/s3-datastore {:access-key "" :secret-key ""} 148 | "my-bucket/my-folder"))) 149 | 150 | ;; Now we have access to the Tundra API: 151 | (comment 152 | (worker my-tundra-store {} {}) ; Create a replication worker 153 | (dirty my-tundra-store "foo:bar1" "foo:bar2" ...) ; Queue for replication 154 | (ensure-ks my-tundra-store "foo:bar1" "foo:bar2" ...) ; Fetch from replica when necessary 155 | ) 156 | ``` 157 | 158 | Note that the [Tundra API](https://cljdoc.org/d/com.taoensso/carmine/CURRENT/api/taoensso.carmine.tundra) makes it convenient to use several different datastores simultaneously (perhaps for different purposes with different latency requirements). -------------------------------------------------------------------------------- /src/taoensso/carmine_v4/cluster.clj: -------------------------------------------------------------------------------- 1 | (ns ^:no-doc taoensso.carmine-v4.cluster 2 | "Private ns, implementation detail. 3 | Implementation of the Redis Cluster protocol, 4 | Ref. " 5 | (:require 6 | [taoensso.encore :as enc] 7 | [taoensso.truss :as truss] 8 | ;;[taoensso.carmine-v4.utils :as utils] 9 | ;;[taoensso.carmine-v4.conns :as conns] 10 | [taoensso.carmine-v4.resp.common :as com] 11 | ;;[taoensso.carmine-v4.resp :as resp] 12 | ;;[taoensso.carmine-v4.opts :as opts] 13 | ) 14 | 15 | #_ 16 | (:import [java.util.concurrent.atomic AtomicLong])) 17 | 18 | (comment (remove-ns 'taoensso.carmine-v4.cluster)) 19 | 20 | ;;;; 1st sketch 21 | 22 | ;; Update: might now be best with some sort of 23 | ;; dedicated cluster ConnManager that can delegate 24 | ;; to shard pools, etc. 25 | 26 | ;; Without cluster: 27 | ;; - with-car [conn-opts] 28 | ;; - get-conn [conn-opts], with-conn 29 | ;; - With non-cluster ctx [in out] 30 | ;; - Flush any pending reqs to allow nesting 31 | ;; - New reqs -> pending reqs 32 | ;; - Flush 33 | ;; - Write reqs 34 | ;; - Read replies 35 | 36 | ;; With cluster: 37 | ;; - with-car [conn-opts] 38 | ;; - With cluster ctx [conn-opts] 39 | ;; - Flush any pending reqs to allow nesting 40 | ;; - New reqs -> pending reqs 41 | ;; - Flush 42 | ;; - get-conn [conn-opts], with-conn for each shard 43 | ;; - Write reqs 44 | ;; - Read replies 45 | 46 | ;; [ ] 47 | ;; conn-opts to incl {:keys [cluster-spec cluster-opts]} :server 48 | ;; - => Can use Sentinel or Cluster, not both 49 | ;; - cluster-spec constructor will take initial set of shard addrs 50 | ;; - cluster-opts to contain :conn-opts to use when updating state, etc. 51 | ;; - Ensure Sentinel conn-opts doesn't include Sentinel or Cluster server x 52 | ;; - Ensure Cluster conn-opts doesn't include Sentinel or Cluster server 53 | ;; - Ensure :select-db is nil/0 when using Cluster 54 | 55 | ;; Slots: 56 | ;; - Slot range is divided between different shards (shard-addrs) 57 | ;; - Each Req will have optional slot field 58 | ;; - Slots can be determined automatically for auto-generated commands: 59 | ;; - First arg after command name seems to usu. indicate "key". 60 | ;; - If there's any additional keys, their slots would anyway need to agree 61 | ;; - `rcmd` and co. expect `cluster-key` to be called manually on the appropriate arg 62 | 63 | ;; [ ] 64 | ;; Stateful ClusterSpec: 65 | ;; - shards-state_: sorted map {[ ] {:master :replicas #{s}}} 66 | ;; - "Stable" when no ongoing reconfig (how to detect?) 67 | ;; - ^:private slot->shard-addr [spec parsed-cluster-opts slot] 68 | ;; - Check :prefer-read-replica? in cluster-opts 69 | ;; - Returns (or ?random-replica master) 70 | ;; - Some slots may not have shard-addr, even after updating state 71 | ;; 72 | ;; - ^:public update-shards! [spec parsed-cluster-opts async?] 73 | ;; - Use locking and/or delay with timeout (fire future on CAS state->updating) 74 | ;; - Use :conn-opts in cluster-opts 75 | ;; - Use `SENTINEL SHARDS` or `SENTINEL SLOTS` command (support both) 76 | ;; 77 | ;; - Stats incl.: n-shards, n-reshards, n-moved, n-ask etc. 78 | ;; - Cbs incl.: on-changed-shards, on-key-moved, etc. 79 | 80 | ;; [ ] 81 | ;; Cluster specific flush [conn-opts] implementation: 82 | ;; - [1] First partition reqs into n target shards 83 | ;; - [1b] Check `cluster-slot` and `supports-cluster?` (must be true or nil) 84 | ;; - [2] Acquire conns to n target shards (use wcar conn-opts with injected shard-addr) 85 | ;; - [2b] If :prefer-read-replica? in cluster-opts - 86 | ;; Call READONLY/READWRITE (skipping replies) 87 | ;; - [*] Mention that we _could_ use fp to write & read to each shard simultaneously 88 | ;; - [3] Write to all shards 89 | ;; - [4] Read replies from all shards 90 | ;; - [5] If there's any -MOVED, -ASK, or conn (?) errors: 91 | ;; - Ask ClusterSpec to update-shards! (async?) 92 | ;; - Retry these reqs 93 | ;; - [6] Carefully stitch back replies in correct order 94 | ;; - [7] Ensure that nesting works as expected 95 | 96 | ;; Details on partitioning scheme (could be pure, data-oriented fn): 97 | ;; - Loop through all reqs in order 98 | ;; - If req -> slot -> shard-addr, add [req req-idx] to partition for that shard-addr 99 | ;; - If req -> nil, add [req req-idx] to last non-nil partition, 100 | ;; or buffer until first non-nil partition. 101 | ;; - If never any non-nil partition: choose random shard-addr. 102 | 103 | ;; Conn pooling: 104 | ;; - Pool to operate solely on [host port] servers, injected by slot->addr in flush. 105 | ;; - I.e. pool needs no invalidation or kop-key changes. 106 | 107 | ;;;; Misc 108 | 109 | ;; - Would use Cluster or Sentinel, not both. 110 | ;; - Sentinel provides best availability, and some read perf via replicas. 111 | ;; - Cluster provides some availability, and read+write perf via sharding. 112 | 113 | ;; - Cluster supports all 1-key commands, and >1 key commands iff all keys 114 | ;; in same hash slot. 115 | ;; - Select command not allowed. 116 | ;; - Optional hash tags allow manual control of hash slots. 117 | 118 | ;; - Cluster "stable" when no ongoing reconfig (i.e. hash slots being moved) 119 | ;; - Each node has unique node-id 120 | ;; - Nodes can change host without changing node-id (problematic?) 121 | 122 | ;; - Cluster has internal concept of { } 123 | ;; - Client should store state like 124 | ;; {[ ] {:master :replicas #{s}}}, more 125 | ;; 126 | ;; - To get cluster topology: 127 | ;; - CLUSTER SHARDS (Redis >= v7), 128 | ;; - CLUSTER SLOTS (Redis <= v6), deprecated 129 | ;; - Client cannot assume that all slots will be accounted for, 130 | ;; may need to re-fetch topology or try a random node 131 | ;; 132 | ;; - Update topology when: 133 | ;; - Initially empty 134 | ;; - Any command saw a -MOVED error (use locking?) 135 | 136 | ;; - Possible Cluster errors: 137 | ;; - -MOVED => permanently moved 138 | ;; -MOVED 3999 127.0.0.1:6381 ; (3999 = key slot) => try host:port 139 | ;; -MOVED 3999 :6380 ; => unknown endpoint, try :port 140 | ;; 141 | ;; - On redirection error: 142 | ;; - Either update cache for specific slot, or whole topology 143 | ;; - Prefer whole topology (since one move usu. => more) 144 | ;; 145 | ;; - ASK => 146 | ;; - Send this query (ONCE) to specified endpoint, don't update cache 147 | ;; - Start redirected query with ASKING 148 | ;; 149 | ;; - TRYAGAIN => reshard in progress, wait to retry or throw 150 | 151 | ;; - Possible READONLY / READWRITE commands during :init? 152 | ;; (Nb should affect kop-key) 153 | 154 | ;; - Redis v7+ "Shared pub/sub" implications? 155 | 156 | ;;;; Key slots 157 | 158 | (def ^:private ^:const num-key-slots 16384) 159 | (let [xmodem-crc16-lookup 160 | (long-array 161 | [0x0000,0x1021,0x2042,0x3063,0x4084,0x50a5,0x60c6,0x70e7, 162 | 0x8108,0x9129,0xa14a,0xb16b,0xc18c,0xd1ad,0xe1ce,0xf1ef, 163 | 0x1231,0x0210,0x3273,0x2252,0x52b5,0x4294,0x72f7,0x62d6, 164 | 0x9339,0x8318,0xb37b,0xa35a,0xd3bd,0xc39c,0xf3ff,0xe3de, 165 | 0x2462,0x3443,0x0420,0x1401,0x64e6,0x74c7,0x44a4,0x5485, 166 | 0xa56a,0xb54b,0x8528,0x9509,0xe5ee,0xf5cf,0xc5ac,0xd58d, 167 | 0x3653,0x2672,0x1611,0x0630,0x76d7,0x66f6,0x5695,0x46b4, 168 | 0xb75b,0xa77a,0x9719,0x8738,0xf7df,0xe7fe,0xd79d,0xc7bc, 169 | 0x48c4,0x58e5,0x6886,0x78a7,0x0840,0x1861,0x2802,0x3823, 170 | 0xc9cc,0xd9ed,0xe98e,0xf9af,0x8948,0x9969,0xa90a,0xb92b, 171 | 0x5af5,0x4ad4,0x7ab7,0x6a96,0x1a71,0x0a50,0x3a33,0x2a12, 172 | 0xdbfd,0xcbdc,0xfbbf,0xeb9e,0x9b79,0x8b58,0xbb3b,0xab1a, 173 | 0x6ca6,0x7c87,0x4ce4,0x5cc5,0x2c22,0x3c03,0x0c60,0x1c41, 174 | 0xedae,0xfd8f,0xcdec,0xddcd,0xad2a,0xbd0b,0x8d68,0x9d49, 175 | 0x7e97,0x6eb6,0x5ed5,0x4ef4,0x3e13,0x2e32,0x1e51,0x0e70, 176 | 0xff9f,0xefbe,0xdfdd,0xcffc,0xbf1b,0xaf3a,0x9f59,0x8f78, 177 | 0x9188,0x81a9,0xb1ca,0xa1eb,0xd10c,0xc12d,0xf14e,0xe16f, 178 | 0x1080,0x00a1,0x30c2,0x20e3,0x5004,0x4025,0x7046,0x6067, 179 | 0x83b9,0x9398,0xa3fb,0xb3da,0xc33d,0xd31c,0xe37f,0xf35e, 180 | 0x02b1,0x1290,0x22f3,0x32d2,0x4235,0x5214,0x6277,0x7256, 181 | 0xb5ea,0xa5cb,0x95a8,0x8589,0xf56e,0xe54f,0xd52c,0xc50d, 182 | 0x34e2,0x24c3,0x14a0,0x0481,0x7466,0x6447,0x5424,0x4405, 183 | 0xa7db,0xb7fa,0x8799,0x97b8,0xe75f,0xf77e,0xc71d,0xd73c, 184 | 0x26d3,0x36f2,0x0691,0x16b0,0x6657,0x7676,0x4615,0x5634, 185 | 0xd94c,0xc96d,0xf90e,0xe92f,0x99c8,0x89e9,0xb98a,0xa9ab, 186 | 0x5844,0x4865,0x7806,0x6827,0x18c0,0x08e1,0x3882,0x28a3, 187 | 0xcb7d,0xdb5c,0xeb3f,0xfb1e,0x8bf9,0x9bd8,0xabbb,0xbb9a, 188 | 0x4a75,0x5a54,0x6a37,0x7a16,0x0af1,0x1ad0,0x2ab3,0x3a92, 189 | 0xfd2e,0xed0f,0xdd6c,0xcd4d,0xbdaa,0xad8b,0x9de8,0x8dc9, 190 | 0x7c26,0x6c07,0x5c64,0x4c45,0x3ca2,0x2c83,0x1ce0,0x0cc1, 191 | 0xef1f,0xff3e,0xcf5d,0xdf7c,0xaf9b,0xbfba,0x8fd9,0x9ff8, 192 | 0x6e17,0x7e36,0x4e55,0x5e74,0x2e93,0x3eb2,0x0ed1,0x1ef0])] 193 | 194 | (defn- crc16 195 | "Returns hash for given bytes using the Redis Cluster CRC16 algorithm, 196 | Ref. (Appendix A). 197 | 198 | Thanks to @bpoweski for this implementation." 199 | [^bytes ba] 200 | (let [len (alength ba)] 201 | (loop [n 0 202 | crc 0] ; Inlines faster than `enc/reduce-n` 203 | (if (>= n len) 204 | crc 205 | (recur (unchecked-inc n) 206 | (bit-xor (bit-and (bit-shift-left crc 8) 0xffff) 207 | (aget xmodem-crc16-lookup 208 | (-> (bit-shift-right crc 8) 209 | (bit-xor (aget ba n)) 210 | (bit-and 0xff)))))))))) 211 | 212 | (defn- ba->key-slot [^bytes ba] (mod (crc16 ba) num-key-slots)) 213 | (defn- tag-str->key-slot [^String tag-str] (ba->key-slot (enc/str->utf8-ba tag-str))) 214 | 215 | (defprotocol IClusterKey (^:public cluster-key [redis-key] "TODO: Docstring")) 216 | (deftype ClusterKey [^bytes ba ^long slot] 217 | clojure.lang.IDeref (deref [this] slot) ; For tests 218 | IClusterKey (cluster-key [this] this)) 219 | 220 | (extend-type (Class/forName "[B") 221 | IClusterKey (cluster-key [ba] (ClusterKey. ba (ba->key-slot ba)))) 222 | 223 | (extend-type String 224 | IClusterKey 225 | (cluster-key [s] 226 | (let [s-ba (enc/str->utf8-ba s)] 227 | (if-let [tag-str 228 | (when (enc/str-contains? s "{") 229 | (when-let [match (re-find #"\{(.*?)\}" s)] 230 | (when-let [^String tag (get match 1)] ; "bar" in "foo{bar}{baz}" 231 | (when-not (.isEmpty tag) tag))))] 232 | 233 | (ClusterKey. s-ba (tag-str->key-slot tag-str)) 234 | (ClusterKey. s-ba (ba->key-slot s-ba)))))) 235 | 236 | (defn cluster-slot [x] (when (instance? ClusterKey x) (.-slot ^ClusterKey x))) 237 | 238 | (comment 239 | (enc/qb 1e5 ; [7.59 22.92] 240 | (cluster-key "foo") 241 | (cluster-key "ignore{foo}"))) 242 | 243 | ;;;; 244 | 245 | (comment 246 | (def sm 247 | (sorted-map 248 | [12 30] "a" 249 | [16 18] "b")) 250 | 251 | (defn find-entry [sm ^long n] 252 | (reduce-kv 253 | (fn [acc lohi v] 254 | (if (and 255 | (>= n ^long (get lohi 0)) 256 | (<= n ^long (get lohi 1))) 257 | (reduced v) 258 | nil)) 259 | nil 260 | sm)) 261 | 262 | (comment (find-entry sm 16))) 263 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | This project uses [**Break Versioning**](https://www.taoensso.com/break-versioning). 2 | 3 | --- 4 | 5 | # `v3.5.0` (2025-11-06) 6 | 7 | - **Dependency**: [on Clojars](https://clojars.org/com.taoensso/carmine/versions/3.5.0) 8 | - **Versioning**: [Break Versioning](https://www.taoensso.com/break-versioning) 9 | 10 | This is a **major maintenance release** with many improvements. It includes a [breaking change](https://github.com/taoensso/carmine/commit/ee75844c6515f6e692fd6fdb2c6fe57ee7aaede1) regarding Carmine's logging. Please report any unexpected problems to the [Slack channel](http://taoensso.com/carmine/slack) or [GitHub](https://github.com/taoensso/carmine/issues). 11 | 12 | A big thanks to all contributors and testers on this release! 🙏 13 | 14 | \- [Peter Taoussanis](https://www.taoensso.com) 15 | 16 | ## Since `v3.4.1` (2024-05-30) 17 | 18 | ### Changes 19 | 20 | - **➤ \[mod] [BREAKING]** Switch logging: [Timbre](https://www.taoensso.com/timbre) -> [Trove](https://www.taoensso.com/trove) \[ee75844c] 21 | - \[mod] Drop support for Clojure 1.9 \[70a60652] 22 | 23 | ### New 24 | 25 | - \[new] First publicly available experimental Carmine v4 core for early testers (currently undocumented) 26 | - \[new] Update command spec \[52c0cc0d] 27 | - \[new] Now use [Truss contextual exceptions](https://github.com/taoensso/truss#contextual-exceptions) for all errors \[3c5ede77] 28 | - \[new] \[#322] Tests: use shared config, make benchmarks easier to tune (@kaizer113) \[3a21e98b] 29 | - \[new] \[mq] Add experimental `:reset-init-backoff?` opt for debouncing, etc. \[983fde98] 30 | - \[new] \[mq] [#315] [#316] Allow zero-thread workers (@adam-jijo-swym) \[7e9adce1] 31 | - \[new] \[#313] Replace lua code with native command (@GabrielHVM) \[ed109343] 32 | - \[doc] \[#314] Fix wiki typos (@mardukbp) \[6c20de1d] 33 | - \[doc] \[#317] `wcar` docstring improvements \[d4265aad] 34 | - \[doc] Add migration info re: v3.3 MQ holding connections longer \[8dc6fc14] 35 | 36 | ### Fixes 37 | 38 | - None 39 | 40 | --- 41 | 42 | # `v3.4.1` (2024-05-30) 43 | 44 | > **Dep**: Carmine is [on Clojars](https://clojars.org/com.taoensso/carmine/versions/3.4.1). 45 | > **Versioning**: Carmine uses [Break Versioning](https://www.taoensso.com/break-versioning). 46 | 47 | This is a **hotfix release** that should be **non-breaking**. 48 | 49 | And as always **please report any unexpected problems** - thank you! 🙏 50 | 51 | \- [Peter Taoussanis](https://www.taoensso.com) 52 | 53 | ## Fixes since `v3.4.0` (2024-05-28) 54 | 55 | * 9dd67207 [fix] [mq] [#305] Typo in final loop error handler 56 | * 81f58d80 [fix] [mq] Monitor `:ndry-runs` arg should be an int 57 | * 41e5ed34 [fix] [mq] Properly assume `:success` handler status by default 58 | 59 | --- 60 | 61 | # `v3.4.0` (2024-05-28) 62 | 63 | > **Dep**: Carmine is [on Clojars](https://clojars.org/com.taoensso/carmine/versions/3.4.0). 64 | > **Versioning**: Carmine uses [Break Versioning](https://www.taoensso.com/break-versioning). 65 | 66 | This is a **security and maintenance release** that should be **non-breaking** for most users. 67 | 68 | ⚠️ It addresses a [**security vulnerability**](https://github.com/taoensso/nippy/security/advisories/GHSA-vw78-267v-588h) in [Nippy](https://www.taoensso.com/nippy)'s upstream compression library and is **recommended for all existing users**. Please review the [relevant Nippy release info](https://github.com/taoensso/nippy/releases/tag/v3.4.2), and **ensure adequate testing** in your environment before updating production data. 69 | 70 | And as always **please report any unexpected problems** - thank you! 🙏 71 | 72 | \- [Peter Taoussanis](https://www.taoensso.com) 73 | 74 | ## Changes since `v3.3.2` (2023-10-24) 75 | 76 | - Updated to [latest stable Nippy](https://github.com/taoensso/nippy/releases/tag/v3.4.2). This should be a non-breaking change for most users, but **please ensure adequate testing** in your environment before updating production data. 77 | 78 | ## Fixes since `v3.3.2` (2023-10-24) 79 | 80 | * 105cecb7 [fix] [mq] [#299] Fix `queue-names` and `queues-clear-all!!!` typos 81 | * 95c52aaa [fix] [mq] [#301] Document behaviour on handler errors 82 | * 9c8b9963 [fix] [#307] Fix Pub/Sub channel handlers masking pattern handlers 83 | * Fixed several typos in docs/README (@chage, @J0sueTM, @mohammedamarnah) 84 | 85 | ## New since `v3.3.2` (2023-10-24) 86 | 87 | * 12c4100d [new] Update Redis command spec (2024-05-27) 88 | * Several wiki and docstring improvements 89 | 90 | --- 91 | 92 | # `v3.3.2` (2023-10-24) 93 | 94 | > 📦 [Available on Clojars](https://clojars.org/com.taoensso/carmine/versions/3.3.2), this project uses [Break Versioning](https://www.taoensso.com/break-versioning). 95 | 96 | This is a **minor hotfix release** and should be a safe upgrade for users of `v3.3.0`. 97 | 98 | ## Fixes since `v3.3.1` 99 | 100 | * be9d4cd1 [#289] [fix] Typo in previous hotfix 101 | 102 | --- 103 | 104 | # `v3.3.0` (2023-10-12) 105 | 106 | > 📦 [Available on Clojars](https://clojars.org/com.taoensso/carmine/versions/3.3.0), this project uses [Break Versioning](https://www.taoensso.com/break-versioning). 107 | 108 | This is a **major feature release** focused on significantly improving the performance and observability of Carmine's message queue. 109 | 110 | There are **BREAKING CHANGES** for the message queue API, see [migration info](https://github.com/taoensso/carmine/wiki/0-Breaking-changes#carmine-v32x-to-v33x). 111 | 112 | Please **test carefully and report any unexpected problems**, thank you! 🙏 113 | 114 | ## Changes since `v3.2.0` 115 | 116 | * ⚠️ 1804ef97 [mod] **[BREAKING]** [#278] Carmine message queue API changes (see [migration info](https://github.com/taoensso/carmine/wiki/0-Breaking-changes)) 117 | 118 | ## Fixes since `v3.2.0` 119 | 120 | * 2e6722bf [fix] [#281 #279] `parse-uri`: only provide ACL params when non-empty (@frwdrik) 121 | * c68c995c [fix] [#283] Support new command.json spec format used for `XTRIM`, etc. 122 | 123 | ## New since `v3.2.0` 124 | 125 | * Several major message queue [architectural improvements](https://github.com/taoensso/carmine/wiki/0-Breaking-changes#architectural-improvements) 126 | * Several major message queue [API improvements](https://github.com/taoensso/carmine/wiki/0-Breaking-changes#api-improvements) 127 | * c9c8d810 [new] Update commands to match latest `commands.json` spec 128 | * 19d97ebd [new] [#282] Add support for TLS URIs to URI parser 129 | * 5cfdbbbd [new] Add `scan-keys` util 130 | * 37f0030a [new] Add `set-min-log-level!` util 131 | * GraalVM compatibility is now tested during build 132 | 133 | ## Other improvements since `v3.3.0` 134 | 135 | * Uses latest Nippy ([v3.3.0](https://github.com/taoensso/nippy/releases/tag/v3.3.0)) 136 | * Various internal improvements 137 | 138 | --- 139 | 140 | # `v3.3.0-RC1` (2023-07-18) 141 | 142 | > 📦 [Available on Clojars](https://clojars.org/com.taoensso/carmine/versions/3.3.0-RC1) 143 | 144 | This is a **major feature release** which includes **possible breaking changes** for users of Carmine's message queue. 145 | 146 | The main objective is to introduce a rewrite of Carmine's message queue that *significantly* improves **queue performance and observability**. 147 | 148 | ## If you use Carmine's message queue: 149 | 150 | - Please see the 1804ef97 commit message for important details. 151 | - Please **test this release** carefully before deploying to production, and once deployed **monitor for any unexpected behaviour**. 152 | 153 | My sincere apologies for the possible breaks. My hope is that: 154 | 155 | - Few users will actually be affected by these breaks. 156 | - If affected, migration should be simple. 157 | - Ultimately the changes will be positive - and something that all queue users can benefit from. 158 | 159 | ## Changes since `v3.2.0` 160 | 161 | * ⚠️ 1804ef97 [mod] [BREAK] [#278] Merge work from message queue v2 branch (**see linked commit message for details**) 162 | 163 | ## Fixes since `v3.2.0` 164 | 165 | * 2e6722bf [fix] [#281 #279] `parse-uri`: only provide ACL params when non-empty (@frwdrik) 166 | * c68c995c [fix] [#283] Support new command.json spec format used for `XTRIM`, etc. 167 | 168 | ## New since `v3.2.0` 169 | 170 | * f79a6404 [new] Update commands to match latest `commands.json` spec 171 | * 19d97ebd [new] [#282] Add support for TLS URIs to URI parser 172 | * 5cfdbbbd [new] Add `scan-keys` util 173 | * 37f0030a [new] Add `set-min-log-level!` util 174 | * GraalVM compatibility is now tested during build 175 | 176 | --- 177 | 178 | # `v3.2.0` (2022-12-03) 179 | 180 | ```clojure 181 | [com.taoensso/carmine "3.2.0"] 182 | ``` 183 | 184 | > This is a major feature + maintenance release. It should be **non-breaking**. 185 | > See [here](https://github.com/taoensso/encore#recommended-steps-after-any-significant-dependency-update) for recommended steps when updating any Clojure/Script dependencies. 186 | 187 | ## Changes since `v3.1.0` 188 | 189 | * [#279] [#257] Any username in conn-spec URI will now automatically be used for ACL in AUTH 190 | 191 | ## Fixes since `v3.1.0` 192 | 193 | * Fix warning of parse var replacements (`parse-long`, `parse-double`) 194 | 195 | ## New since `v3.1.0` 196 | 197 | * [#201] [#224] [#266] Improve `wcar` documentation, alias pool constructor in main ns 198 | * [#251] [#257] [#270] Add support for username (ACL) in AUTH (@lsevero @sumeet14) 199 | * Update to latest commands.json (from 2022-09-22) 200 | * [#259] [#261] Ring middleware: add `:extend-on-read?` option (@svdo) 201 | * [Message queue] Print extra debug info on invalid handler status 202 | 203 | ## Other improvements since `v3.1.0` 204 | 205 | * [#264] Remove unnecessary reflection in Tundra (@frwdrik) 206 | * Stop using deprecated Encore vars 207 | * [#271] Refactor tests, fix flakey results 208 | * Update dependencies 209 | * Work [underway](https://github.com/users/ptaoussanis/projects/2) on Carmine v4, which'll introduce support for RESP3, Redis Sentinel, Redis Cluster, and more. 210 | 211 | --- 212 | 213 | # v3.1.0 (2020-11-24) 214 | 215 | ```clojure 216 | [com.taoensso/carmine "3.1.0"] 217 | ``` 218 | 219 | > This is a minor feature release. It should be non-breaking. 220 | 221 | ## New since `v3.0.0` 222 | 223 | * [#244 #245] Message queue: add `initial-backoff-ms` option to `enqueue` (@st3fan) 224 | 225 | --- 226 | 227 | # v3.0.0 (2020-09-22) 228 | 229 | ```clojure 230 | [com.taoensso/carmine "3.0.0"] 231 | ``` 232 | 233 | > This is a major feature + security release. It may be **BREAKING**. 234 | > See [here](https://github.com/taoensso/encore#recommended-steps-after-any-significant-dependency-update) for recommended steps when updating any Clojure/Script dependencies. 235 | 236 | ## Changes since `v2.20.0` 237 | 238 | - **[BREAKING]** Bumped minimum Clojure version from `v1.5` to `v1.7`. 239 | - **[BREAKING]** Bump Nippy to [v3](https://github.com/taoensso/nippy/releases/tag/v3.0.0) for an **important security fix**. See [here](https://github.com/taoensso/nippy/issues/130) for details incl. **necessary upgrade instructions** for folks using Carmine's automatic de/serialization feature. 240 | - Bump Timbre to [v5](https://github.com/taoensso/timbre/releases/tag/v5.0.0). 241 | 242 | ## New since `v2.20.0` 243 | 244 | - Update to [latest Redis commands spec](https://github.com/redis/redis-doc/blob/25555fe05a571454fa0f11dca28cb5796e04112f/commands.json). 245 | - Listeners (incl. Pub/Sub) significantly improved: 246 | - [#15] Allow `conn-alive?` check for listener connections 247 | - [#219] Try ping on socket timeouts for increased reliability (@ylgrgyq) 248 | - [#207] Publish connection and handler errors to handlers, enabling convenient reconnect logic (@aravindbaskaran) 249 | - [#207] Support optional ping-ms keep-alive for increased reliability (@aravindbaskaran) 250 | - Added `parse-listener-msg` util for more easily writing handler fns 251 | - Allow `with-new-pubsub-listener` to take a single direct handler-fn 252 | 253 | --- 254 | 255 | # Earlier releases 256 | 257 | See [here](https://github.com/taoensso/carmine/releases) for earlier releases. 258 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Eclipse Public License - v 1.0 2 | 3 | THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC 4 | LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM 5 | CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. 6 | 7 | 1. DEFINITIONS 8 | 9 | "Contribution" means: 10 | 11 | a) in the case of the initial Contributor, the initial code and documentation 12 | distributed under this Agreement, and 13 | b) in the case of each subsequent Contributor: 14 | i) changes to the Program, and 15 | ii) additions to the Program; 16 | 17 | where such changes and/or additions to the Program originate from and are 18 | distributed by that particular Contributor. A Contribution 'originates' 19 | from a Contributor if it was added to the Program by such Contributor 20 | itself or anyone acting on such Contributor's behalf. Contributions do not 21 | include additions to the Program which: (i) are separate modules of 22 | software distributed in conjunction with the Program under their own 23 | license agreement, and (ii) are not derivative works of the Program. 24 | 25 | "Contributor" means any person or entity that distributes the Program. 26 | 27 | "Licensed Patents" mean patent claims licensable by a Contributor which are 28 | necessarily infringed by the use or sale of its Contribution alone or when 29 | combined with the Program. 30 | 31 | "Program" means the Contributions distributed in accordance with this 32 | Agreement. 33 | 34 | "Recipient" means anyone who receives the Program under this Agreement, 35 | including all Contributors. 36 | 37 | 2. GRANT OF RIGHTS 38 | a) Subject to the terms of this Agreement, each Contributor hereby grants 39 | Recipient a non-exclusive, worldwide, royalty-free copyright license to 40 | reproduce, prepare derivative works of, publicly display, publicly 41 | perform, distribute and sublicense the Contribution of such Contributor, 42 | if any, and such derivative works, in source code and object code form. 43 | b) Subject to the terms of this Agreement, each Contributor hereby grants 44 | Recipient a non-exclusive, worldwide, royalty-free patent license under 45 | Licensed Patents to make, use, sell, offer to sell, import and otherwise 46 | transfer the Contribution of such Contributor, if any, in source code and 47 | object code form. This patent license shall apply to the combination of 48 | the Contribution and the Program if, at the time the Contribution is 49 | added by the Contributor, such addition of the Contribution causes such 50 | combination to be covered by the Licensed Patents. The patent license 51 | shall not apply to any other combinations which include the Contribution. 52 | No hardware per se is licensed hereunder. 53 | c) Recipient understands that although each Contributor grants the licenses 54 | to its Contributions set forth herein, no assurances are provided by any 55 | Contributor that the Program does not infringe the patent or other 56 | intellectual property rights of any other entity. Each Contributor 57 | disclaims any liability to Recipient for claims brought by any other 58 | entity based on infringement of intellectual property rights or 59 | otherwise. As a condition to exercising the rights and licenses granted 60 | hereunder, each Recipient hereby assumes sole responsibility to secure 61 | any other intellectual property rights needed, if any. For example, if a 62 | third party patent license is required to allow Recipient to distribute 63 | the Program, it is Recipient's responsibility to acquire that license 64 | before distributing the Program. 65 | d) Each Contributor represents that to its knowledge it has sufficient 66 | copyright rights in its Contribution, if any, to grant the copyright 67 | license set forth in this Agreement. 68 | 69 | 3. REQUIREMENTS 70 | 71 | A Contributor may choose to distribute the Program in object code form under 72 | its own license agreement, provided that: 73 | 74 | a) it complies with the terms and conditions of this Agreement; and 75 | b) its license agreement: 76 | i) effectively disclaims on behalf of all Contributors all warranties 77 | and conditions, express and implied, including warranties or 78 | conditions of title and non-infringement, and implied warranties or 79 | conditions of merchantability and fitness for a particular purpose; 80 | ii) effectively excludes on behalf of all Contributors all liability for 81 | damages, including direct, indirect, special, incidental and 82 | consequential damages, such as lost profits; 83 | iii) states that any provisions which differ from this Agreement are 84 | offered by that Contributor alone and not by any other party; and 85 | iv) states that source code for the Program is available from such 86 | Contributor, and informs licensees how to obtain it in a reasonable 87 | manner on or through a medium customarily used for software exchange. 88 | 89 | When the Program is made available in source code form: 90 | 91 | a) it must be made available under this Agreement; and 92 | b) a copy of this Agreement must be included with each copy of the Program. 93 | Contributors may not remove or alter any copyright notices contained 94 | within the Program. 95 | 96 | Each Contributor must identify itself as the originator of its Contribution, 97 | if 98 | any, in a manner that reasonably allows subsequent Recipients to identify the 99 | originator of the Contribution. 100 | 101 | 4. COMMERCIAL DISTRIBUTION 102 | 103 | Commercial distributors of software may accept certain responsibilities with 104 | respect to end users, business partners and the like. While this license is 105 | intended to facilitate the commercial use of the Program, the Contributor who 106 | includes the Program in a commercial product offering should do so in a manner 107 | which does not create potential liability for other Contributors. Therefore, 108 | if a Contributor includes the Program in a commercial product offering, such 109 | Contributor ("Commercial Contributor") hereby agrees to defend and indemnify 110 | every other Contributor ("Indemnified Contributor") against any losses, 111 | damages and costs (collectively "Losses") arising from claims, lawsuits and 112 | other legal actions brought by a third party against the Indemnified 113 | Contributor to the extent caused by the acts or omissions of such Commercial 114 | Contributor in connection with its distribution of the Program in a commercial 115 | product offering. The obligations in this section do not apply to any claims 116 | or Losses relating to any actual or alleged intellectual property 117 | infringement. In order to qualify, an Indemnified Contributor must: 118 | a) promptly notify the Commercial Contributor in writing of such claim, and 119 | b) allow the Commercial Contributor to control, and cooperate with the 120 | Commercial Contributor in, the defense and any related settlement 121 | negotiations. The Indemnified Contributor may participate in any such claim at 122 | its own expense. 123 | 124 | For example, a Contributor might include the Program in a commercial product 125 | offering, Product X. That Contributor is then a Commercial Contributor. If 126 | that Commercial Contributor then makes performance claims, or offers 127 | warranties related to Product X, those performance claims and warranties are 128 | such Commercial Contributor's responsibility alone. Under this section, the 129 | Commercial Contributor would have to defend claims against the other 130 | Contributors related to those performance claims and warranties, and if a 131 | court requires any other Contributor to pay any damages as a result, the 132 | Commercial Contributor must pay those damages. 133 | 134 | 5. NO WARRANTY 135 | 136 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN 137 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR 138 | IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, 139 | NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each 140 | Recipient is solely responsible for determining the appropriateness of using 141 | and distributing the Program and assumes all risks associated with its 142 | exercise of rights under this Agreement , including but not limited to the 143 | risks and costs of program errors, compliance with applicable laws, damage to 144 | or loss of data, programs or equipment, and unavailability or interruption of 145 | operations. 146 | 147 | 6. DISCLAIMER OF LIABILITY 148 | 149 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY 150 | CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, 151 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION 152 | LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 153 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 154 | ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE 155 | EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY 156 | OF SUCH DAMAGES. 157 | 158 | 7. GENERAL 159 | 160 | If any provision of this Agreement is invalid or unenforceable under 161 | applicable law, it shall not affect the validity or enforceability of the 162 | remainder of the terms of this Agreement, and without further action by the 163 | parties hereto, such provision shall be reformed to the minimum extent 164 | necessary to make such provision valid and enforceable. 165 | 166 | If Recipient institutes patent litigation against any entity (including a 167 | cross-claim or counterclaim in a lawsuit) alleging that the Program itself 168 | (excluding combinations of the Program with other software or hardware) 169 | infringes such Recipient's patent(s), then such Recipient's rights granted 170 | under Section 2(b) shall terminate as of the date such litigation is filed. 171 | 172 | All Recipient's rights under this Agreement shall terminate if it fails to 173 | comply with any of the material terms or conditions of this Agreement and does 174 | not cure such failure in a reasonable period of time after becoming aware of 175 | such noncompliance. If all Recipient's rights under this Agreement terminate, 176 | Recipient agrees to cease use and distribution of the Program as soon as 177 | reasonably practicable. However, Recipient's obligations under this Agreement 178 | and any licenses granted by Recipient relating to the Program shall continue 179 | and survive. 180 | 181 | Everyone is permitted to copy and distribute copies of this Agreement, but in 182 | order to avoid inconsistency the Agreement is copyrighted and may only be 183 | modified in the following manner. The Agreement Steward reserves the right to 184 | publish new versions (including revisions) of this Agreement from time to 185 | time. No one other than the Agreement Steward has the right to modify this 186 | Agreement. The Eclipse Foundation is the initial Agreement Steward. The 187 | Eclipse Foundation may assign the responsibility to serve as the Agreement 188 | Steward to a suitable separate entity. Each new version of the Agreement will 189 | be given a distinguishing version number. The Program (including 190 | Contributions) may always be distributed subject to the version of the 191 | Agreement under which it was received. In addition, after a new version of the 192 | Agreement is published, Contributor may elect to distribute the Program 193 | (including its Contributions) under the new version. Except as expressly 194 | stated in Sections 2(a) and 2(b) above, Recipient receives no rights or 195 | licenses to the intellectual property of any Contributor under this Agreement, 196 | whether expressly, by implication, estoppel or otherwise. All rights in the 197 | Program not expressly granted under this Agreement are reserved. 198 | 199 | This Agreement is governed by the laws of the State of New York and the 200 | intellectual property laws of the United States of America. No party to this 201 | Agreement will bring a legal action under this Agreement more than one year 202 | after the cause of action arose. Each party waives its rights to a jury trial in 203 | any resulting litigation. 204 | -------------------------------------------------------------------------------- /src/taoensso/carmine/connections.clj: -------------------------------------------------------------------------------- 1 | (ns taoensso.carmine.connections 2 | "Handles socket connection lifecycle. 3 | Pool is implemented with Apache Commons pool. 4 | Originally adapted from `redis-clojure`." 5 | {:author "Peter Taoussanis"} 6 | (:require 7 | [clojure.string :as str] 8 | [taoensso.encore :as enc] 9 | [taoensso.truss :as truss] 10 | [taoensso.carmine.protocol :as protocol]) 11 | 12 | (:import 13 | [java.net InetSocketAddress Socket URI] 14 | [org.apache.commons.pool2 KeyedPooledObjectFactory] 15 | [org.apache.commons.pool2.impl GenericKeyedObjectPool DefaultPooledObject])) 16 | 17 | (enc/declare-remote 18 | taoensso.carmine/ping 19 | taoensso.carmine/auth 20 | taoensso.carmine/select) 21 | 22 | ;;; Outline/nomenclature 23 | ;; connection -> Connection object. 24 | ;; pool -> IConnectionPool implementer (for custom pool types, etc.). 25 | ;; conn spec -> map of processed options describing connection properties. 26 | ;; conn opts -> map of unprocessed options that'll be processed to create a spec. 27 | ;; pool opts -> map of options that'll be used to create a pool (memoized). 28 | ;; conn opts -> {:pool :spec } as taken by `wcar`, etc. 29 | 30 | (defprotocol IConnection 31 | (-conn-error [this]) 32 | (conn-alive? [this]) 33 | (close-conn [this])) 34 | 35 | (defrecord Connection [^Socket socket spec in out] 36 | IConnection 37 | (conn-alive? [this] (nil? (-conn-error this))) 38 | (-conn-error [this] 39 | (try 40 | (let [resp 41 | (protocol/with-context this 42 | (protocol/parse nil 43 | (protocol/with-replies 44 | (taoensso.carmine/ping))))] 45 | 46 | ;; Ref. 47 | (when-not (contains? #{"PONG" ["pong" ""]} resp) 48 | (truss/ex-info! "Unexpected PING response" {:resp resp}))) 49 | 50 | (catch Exception ex 51 | (when-let [f (get-in spec [:instrument :on-conn-error])] 52 | (f {:spec spec :ex ex})) 53 | ex))) 54 | 55 | (close-conn [_] 56 | (when-let [f (get-in spec [:instrument :on-conn-close])] (f {:spec spec})) 57 | (.close socket))) 58 | 59 | (defprotocol IConnectionPool 60 | (get-conn [this spec]) 61 | (release-conn [this conn] [this conn exception])) 62 | 63 | (defrecord ConnectionPool [^GenericKeyedObjectPool pool] 64 | IConnectionPool 65 | (get-conn [_ spec] (.borrowObject pool spec)) 66 | (release-conn [_ conn] (.returnObject pool (:spec conn) conn)) 67 | (release-conn [_ conn exception] (.invalidateObject pool (:spec conn) conn)) 68 | java.io.Closeable 69 | (close [_] (.close pool))) 70 | 71 | ;;; 72 | 73 | (let [factory_ (delay (javax.net.ssl.SSLSocketFactory/getDefault))] 74 | (defn default-ssl-fn 75 | "Takes an unencrypted underlying java.net.Socket and returns an 76 | encrypted java.net.Socket using the environment's default SSLSocketFactory." 77 | [{:keys [socket host port]}] 78 | (.createSocket ^javax.net.ssl.SSLSocketFactory @factory_ 79 | ^Socket socket ^String host ^Integer port true))) 80 | 81 | (defn- get-streams [^Socket socket init-read-buffer-size init-write-buffer-size] 82 | (let [in 83 | (-> 84 | (.getInputStream socket) 85 | (java.io.BufferedInputStream. (int init-read-buffer-size)) 86 | (java.io.DataInputStream.)) 87 | 88 | out 89 | (-> 90 | (.getOutputStream socket) 91 | (java.io.BufferedOutputStream. (int init-write-buffer-size)))] 92 | 93 | ;; Re: choice of stream types: 94 | ;; We need the following: 95 | ;; - Ability to read & write bytes and byte arrays. 96 | ;; - Ability to read lines (chars up to CRLF) as string 97 | ;; - Note that `java.io.DataInputStream/readLine` is deprecated 98 | ;; due to lack of charset support, but is actually fine for our 99 | ;; purposes since we use `readLine` only for simple ascii strings 100 | ;; from Redis server (i.e. no user content). 101 | ;; 102 | ;; Benching also shows ~equal performance to specialised implns. 103 | ;; like `redis.clients.jedis.util.RedisInputStream`. 104 | 105 | [in out])) 106 | 107 | (defn make-new-connection 108 | [{:keys [host port username password db conn-setup-fn 109 | conn-timeout-ms read-timeout-ms timeout-ms ssl-fn] :as spec}] 110 | 111 | (let [;; :timeout-ms controls both :conn-timeout-ms and :read-timeout-ms 112 | ;; unless those are specified individually 113 | ;; :or {conn-timeout-ms (or timeout-ms 4000) 114 | ;; read-timeout-ms timeout-ms} ; Ref. http://goo.gl/XULHCd 115 | conn-timeout-ms (get spec :conn-timeout-ms (or timeout-ms 4000)) 116 | read-timeout-ms (get spec :read-timeout-ms timeout-ms) 117 | 118 | socket-address (InetSocketAddress. ^String host ^Integer port) 119 | ^Socket socket 120 | (enc/doto-cond [expr (Socket.)] 121 | :always (.setTcpNoDelay true) 122 | :always (.setKeepAlive true) 123 | :always (.setReuseAddress true) 124 | ;; :always (.setSoLinger true 0) 125 | read-timeout-ms (.setSoTimeout ^Integer expr)) 126 | 127 | _ (if conn-timeout-ms 128 | (.connect socket socket-address conn-timeout-ms) 129 | (.connect socket socket-address)) 130 | 131 | ^Socket socket 132 | (if ssl-fn 133 | (let [f (if (identical? ssl-fn :default) default-ssl-fn ssl-fn)] 134 | (f {:socket socket :host host :port port})) 135 | socket) 136 | 137 | conn 138 | (let [[in out] (get-streams socket 8192 8192)] 139 | (->Connection socket spec in out)) 140 | 141 | db (when (and db (not (zero? db))) db)] 142 | 143 | (when-let [f (get-in spec [:instrument :on-conn-open])] (f {:spec spec})) 144 | 145 | (when (or username password db conn-setup-fn) 146 | (protocol/with-context conn 147 | (protocol/with-replies ; Discard replies 148 | (when password 149 | (if username 150 | (taoensso.carmine/auth username password) 151 | (taoensso.carmine/auth password))) 152 | 153 | (when db (taoensso.carmine/select (str db))) 154 | (when conn-setup-fn 155 | (conn-setup-fn {:conn conn :spec spec}))))) 156 | conn)) 157 | 158 | ;; A degenerate connection pool: gives pool-like interface for non-pooled conns 159 | (defrecord NonPooledConnectionPool [] 160 | IConnectionPool 161 | (get-conn [_ spec] (make-new-connection spec)) 162 | (release-conn [_ conn] (close-conn conn)) 163 | (release-conn [_ conn exception] (close-conn conn)) 164 | java.io.Closeable 165 | (close [_] nil)) 166 | 167 | (defn make-connection-factory [] 168 | (reify KeyedPooledObjectFactory 169 | (makeObject [_ spec] (DefaultPooledObject. (make-new-connection spec))) 170 | (activateObject [_ spec pooled-obj]) 171 | (validateObject [_ spec pooled-obj] (let [conn (.getObject pooled-obj)] 172 | (conn-alive? conn))) 173 | (passivateObject [_ spec pooled-obj]) 174 | (destroyObject [_ spec pooled-obj] (let [conn (.getObject pooled-obj)] 175 | (close-conn conn))))) 176 | 177 | (defn- set-pool-option [^GenericKeyedObjectPool pool k v] 178 | (case k 179 | 180 | ;;; org.apache.commons.pool2.impl.GenericKeyedObjectPool 181 | :min-idle-per-key (.setMinIdlePerKey pool v) ; 0 182 | :max-idle-per-key (.setMaxIdlePerKey pool v) ; 8 183 | :max-total-per-key (.setMaxTotalPerKey pool v) ; 8 184 | 185 | ;;; org.apache.commons.pool2.impl.BaseGenericObjectPool 186 | :block-when-exhausted? (.setBlockWhenExhausted pool v) ; true 187 | :lifo? (.setLifo pool v) ; true 188 | :max-total (.setMaxTotal pool v) ; -1 189 | :max-wait-ms (.setMaxWaitMillis pool v) ; -1 190 | :min-evictable-idle-time-ms (.setMinEvictableIdleTimeMillis pool v) ; 1800000 191 | :num-tests-per-eviction-run (.setNumTestsPerEvictionRun pool v) ; 3 192 | :soft-min-evictable-idle-time-ms (.setSoftMinEvictableIdleTimeMillis pool v) ; -1 193 | :swallowed-exception-listener (.setSwallowedExceptionListener pool v) 194 | :test-on-borrow? (.setTestOnBorrow pool v) ; false 195 | :test-on-return? (.setTestOnReturn pool v) ; false 196 | :test-while-idle? (.setTestWhileIdle pool v) ; false 197 | :time-between-eviction-runs-ms (.setTimeBetweenEvictionRunsMillis pool v) ; -1 198 | 199 | :instrument nil ; noop (ignore) 200 | 201 | (truss/ex-info! (str "Unknown pool option: " k) {:option k})) 202 | pool) 203 | 204 | (def conn-pool 205 | (enc/memoize 206 | (fn [pool-opts] 207 | 208 | (when-let [f (get-in pool-opts [:instrument :on-pool-init])] 209 | (f {:pool-opts pool-opts})) 210 | 211 | (cond 212 | (identical? pool-opts :none) (->NonPooledConnectionPool) 213 | ;; Pass through pre-made pools (note that test reflects): 214 | (satisfies? IConnectionPool pool-opts) pool-opts 215 | :else 216 | (let [pool-opts (dissoc pool-opts :id) ; Support >1 pool with same opts 217 | jedis-defaults ; Ref. http://goo.gl/y1mDbE 218 | {:test-while-idle? true ; from false 219 | :num-tests-per-eviction-run -1 ; from 3 220 | :min-evictable-idle-time-ms 60000 ; from 1800000 221 | :time-between-eviction-runs-ms 30000 ; from -1 222 | } 223 | carmine-defaults 224 | {:max-total-per-key 16 ; from 8 225 | :max-idle-per-key 16 ; Same as above to avoid early connection closing 226 | }] 227 | (->ConnectionPool 228 | (reduce-kv set-pool-option 229 | (GenericKeyedObjectPool. (make-connection-factory)) 230 | (merge jedis-defaults carmine-defaults pool-opts)))))))) 231 | 232 | ;; (defn uncached-conn-pool [pool-opts] (conn-pool :mem/fresh pool-opts)) 233 | (comment (conn-pool :none) (conn-pool {})) 234 | 235 | (defn- parse-uri [uri] 236 | (when uri 237 | (let [^URI uri (if (instance? URI uri) uri (URI. uri)) 238 | [username password] (.split (str (.getUserInfo uri)) ":") 239 | port (.getPort uri) 240 | db (when-let [[_ db-str] (re-matches #"/(\d+)$" (.getPath uri))] 241 | (Integer. ^String db-str)) 242 | 243 | ssl-fn 244 | (when-let [scheme (.getScheme uri)] 245 | (when (contains? #{"rediss" "https"} (str/lower-case scheme)) 246 | :default))] 247 | 248 | (-> {:host (.getHost uri)} 249 | (#(if (pos? port) (assoc % :port port) %)) 250 | (#(if (and db (pos? (long db))) (assoc % :db db) %)) 251 | (#(if (enc/nempty-str? username) (assoc % :username username) %)) 252 | (#(if (enc/nempty-str? password) (assoc % :password password) %)) 253 | (#(if ssl-fn (assoc % :ssl-fn ssl-fn) %)))))) 254 | 255 | (comment 256 | [(parse-uri "redis://user:pass@x.y.com:9475/3") 257 | (parse-uri "redis://:pass@x.y.com.com:9475/3") 258 | (parse-uri "redis://user:@x.y.com:9475/3") 259 | (parse-uri "rediss://user:@x.y.com:9475/3")]) 260 | 261 | (def conn-spec 262 | (enc/memoize 263 | (fn [{:keys [uri host port username password timeout-ms db 264 | conn-setup-fn ; nb must be var-level for fn equality 265 | ] :as spec-opts}] 266 | (let [defaults {:host "127.0.0.1" :port 6379} 267 | spec-opts (if-let [timeout (:timeout spec-opts)] ; Deprecated opt 268 | (assoc spec-opts :timeout-ms timeout) 269 | spec-opts)] 270 | (merge defaults spec-opts (parse-uri uri)))))) 271 | 272 | (defn pooled-conn "Returns [ ]" 273 | [{:as conn-opts pool-opts :pool spec-opts :spec}] 274 | (let [spec (conn-spec spec-opts) 275 | pool (conn-pool pool-opts)] 276 | (try 277 | (try 278 | [pool (get-conn pool spec)] 279 | (catch IllegalStateException e ; Cached pool's gone bad 280 | (let [pool (conn-pool :mem/fresh pool-opts)] 281 | [pool (get-conn pool spec)]))) 282 | (catch Exception e 283 | (truss/ex-info! "Carmine connection error" {} e))))) 284 | -------------------------------------------------------------------------------- /test/taoensso/carmine_v4/tests/main.clj: -------------------------------------------------------------------------------- 1 | (ns taoensso.carmine-v4.tests.main 2 | "High-level Carmine tests. 3 | These need a running Redis server." 4 | (:require 5 | [clojure.test :as test :refer [deftest testing is]] 6 | [taoensso.encore :as enc] 7 | [taoensso.truss :as truss :refer [throws?]] 8 | [taoensso.carmine :as v3-core] 9 | [taoensso.carmine-v4 :as car :refer [wcar with-replies]] 10 | [taoensso.carmine-v4.resp :as resp] 11 | [taoensso.carmine-v4.utils :as utils] 12 | [taoensso.carmine-v4.opts :as opts] 13 | [taoensso.carmine-v4.conns :as conns] 14 | [taoensso.carmine-v4.sentinel :as sentinel] 15 | [taoensso.carmine-v4.cluster :as cluster])) 16 | 17 | (comment 18 | (remove-ns 'taoensso.carmine-v4.tests.main) 19 | (test/run-tests 'taoensso.carmine-v4.tests.main) 20 | (test/run-all-tests #"taoensso\.carmine-v4.*")) 21 | 22 | ;;;; TODO 23 | ;; - Interactions between systems (read-opts, parsers, etc.) 24 | ;; - Test conns 25 | ;; - Callbacks, closing data, etc. 26 | ;; - Sentinel, resolve changes 27 | 28 | ;;;; Setup, etc. 29 | 30 | (defn tk "Test key" [key] (str "__:carmine:test:" (enc/as-qname key))) 31 | (def tc "Unparsed test conn-opts" {}) 32 | (def tc+ "Parsed test conn-opts" (opts/parse-conn-opts false tc)) 33 | 34 | (defonce mgr_ (delay (conns/conn-manager-pooled {:conn-opts tc}))) 35 | 36 | (let [delete-test-keys 37 | (fn [] 38 | (when-let [ks (seq (wcar mgr_ (resp/rcmd "KEYS" (tk "*"))))] 39 | (wcar mgr_ (doseq [k ks] (resp/rcmd "DEL" k)))))] 40 | 41 | (test/use-fixtures :once 42 | (enc/test-fixtures 43 | {:before delete-test-keys 44 | :after delete-test-keys}))) 45 | 46 | ;;;; Utils 47 | 48 | (deftest _merge-opts 49 | [(is (= (utils/merge-opts {:a 1 :b 1} {:a 2}) {:a 2, :b 1})) 50 | (is (= (utils/merge-opts {:a {:a1 1} :b 1} {:a {:a1 2}}) {:a {:a1 2}, :b 1})) 51 | (is (= (utils/merge-opts {:a {:a1 1} :b 1} {:a nil}) {:a nil, :b 1})) 52 | 53 | (is (= (utils/merge-opts {:a 1} {:a 2} {:a 3}) {:a 3})) 54 | 55 | (is (= (utils/merge-opts {:a 1} {:a 2} { }) {:a 2})) 56 | (is (= (utils/merge-opts {:a 1} { } {:a 3}) {:a 3})) 57 | (is (= (utils/merge-opts { } {:a 2} {:a 3}) {:a 3}))]) 58 | 59 | (deftest _dissoc-utils 60 | [(is (= (utils/dissoc-k {:a {:b :B :c :C :d :D}} :a :b) {:a {:c :C, :d :D}})) 61 | (is (= (utils/dissoc-ks {:a {:b :B :c :C :d :D}} :a [:b :d]) {:a {:c :C}}))]) 62 | 63 | (deftest _get-first-contained 64 | [(is (= (let [m {:a :A :b :B}] (utils/get-first-contained m :q :r :a :b)) :A)) 65 | (is (= (let [m {:a false :b :B}] (utils/get-first-contained m :q :r :a :b)) false))]) 66 | 67 | ;;;; Opts 68 | 69 | (deftest _sock-addrs 70 | [(is (= (opts/descr-sock-addr (opts/parse-sock-addr "ip" "80")) ["ip" 80 ])) 71 | (is (= (opts/descr-sock-addr (opts/parse-sock-addr ^:my-meta ["ip" "80"])) ["ip" 80 {:my-meta true}]))]) 72 | 73 | (deftest _parse-string-server 74 | [(is (= (#'opts/parse-string-server "redis://user:pass@x.y.com:9475/3") {:server ["x.y.com" 9475], :init {:auth {:username "user", :password "pass"}, :select-db 3}})) 75 | (is (= (#'opts/parse-string-server "redis://:pass@x.y.com.com:9475/3") {:server ["x.y.com.com" 9475], :init {:auth { :password "pass"}, :select-db 3}} )) 76 | (is (= (#'opts/parse-string-server "redis://user:@x.y.com:9475/3") {:server ["x.y.com" 9475], :init {:auth {:username "user" }, :select-db 3}})) 77 | (is (= (#'opts/parse-string-server "rediss://user:@x.y.com:9475/3") {:server ["x.y.com" 9475], :init {:auth {:username "user" }, :select-db 3}, 78 | :socket-opts {:ssl true}}))]) 79 | (deftest _parse-conn-opts 80 | [(is (enc/submap? (opts/parse-conn-opts false {:server [ "127.0.0.1" "80"]}) {:server ["127.0.0.1" 80]})) 81 | (is (enc/submap? (opts/parse-conn-opts false {:server {:host "127.0.0.1" :port "80"}}) {:server ["127.0.0.1" 80]})) 82 | (is (enc/submap? (opts/parse-conn-opts false {:server {:host "127.0.0.1" :port "80"}}) {:server ["127.0.0.1" 80]})) 83 | (is (enc/submap? (opts/parse-conn-opts false {:server "rediss://user:pass@x.y.com:9475/3"}) 84 | {:server ["x.y.com" 9475], :init {:auth {:username "user", :password "pass"}, :select-db 3, :resp3? true}, 85 | :socket-opts {:ssl true}})) 86 | 87 | (is (->> (opts/parse-conn-opts false {:server ^:my-meta ["127.0.0.1" "6379"]}) :server (meta) :my-meta) "Retains metadata") 88 | 89 | (is (->> (opts/parse-conn-opts false {:server ["127.0.0.1" "invalid-port"]}) (throws? :any {:eid :carmine.conn-opts/invalid-server}))) 90 | (is (->> (opts/parse-conn-opts false {:server {:host "127.0.0.1" :port "80" :invalid "foo"}}) (throws? :any {:eid :carmine.conn-opts/invalid-server}))) 91 | 92 | (is (enc/submap? 93 | (opts/parse-conn-opts false 94 | {:server {:sentinel-spec (sentinel/sentinel-spec {:foo/bar [["127.0.0.1" 26379]]}) 95 | :master-name :foo/bar}}) 96 | {:server {:master-name "foo/bar", :sentinel-opts {:retry-delay-ms 250}}})) 97 | 98 | (is (enc/submap? 99 | (opts/parse-conn-opts false 100 | {:server {:sentinel-spec (sentinel/sentinel-spec {:foo/bar [["127.0.0.1" 26379]]}) 101 | :master-name :foo/bar, :sentinel-opts {:retry-delay-ms 100}}}) 102 | {:server {:master-name "foo/bar", :sentinel-opts {:retry-delay-ms 100}}}))]) 103 | 104 | ;;;; Sentinel 105 | 106 | (deftest _addr-utils 107 | [(let [sm (#'sentinel/add-addrs->back nil [["ip1" 1] ["ip2" "2"] ^{:server-name "server3"} ["ip3" 3]]) 108 | sm (#'sentinel/add-addr->front sm ["ip2" 2]) 109 | sm (#'sentinel/add-addrs->back sm [["ip3" 3] ["ip6" 6]])] 110 | 111 | [(is (= sm [["ip2" 2] ["ip1" 1] ["ip3" 3] ["ip6" 6]])) 112 | (is (= (mapv opts/descr-sock-addr sm) 113 | [["ip2" 2] ["ip1" 1] ["ip3" 3 {:server-name "server3"}] ["ip6" 6]]))]) 114 | 115 | (let [sm (#'sentinel/add-addrs->back nil [["ip4" 4] ["ip5" "5"]]) 116 | sm (#'sentinel/remove-addr sm ["ip4" 4])] 117 | [(is (= sm [["ip5" 5]]))])]) 118 | 119 | (deftest _unique-addrs 120 | [(is (= (#'sentinel/unique-addrs 121 | {:m1 {:master [1 1] :sentinels [[1 1] [1 2] [2 2]]} 122 | :m2 {:master [1 1] :sentinels [[3 3]] :replicas #{[1 1] [3 3]}}}) 123 | 124 | {:masters #{[1 1]}, 125 | :replicas #{[3 3] [1 1]}, 126 | :sentinels #{[1 1] [1 2] [2 2] [3 3]}}))]) 127 | 128 | (deftest _parse-nodes-info->addrs 129 | [(is (= (#'sentinel/parse-nodes-info->addrs 130 | [{"host" "host1" "port" "port1" "x1" "y1"} 131 | {"host" "host2" "port" "port2"} 132 | ["host" "host3" "port" "port3" "x2" "y2"]]) 133 | 134 | [["host1" "port1"] ["host2" "port2"] ["host3" "port3"]]))]) 135 | 136 | ;;;; Cluster 137 | 138 | (deftest _cluster 139 | [(is (= @(cluster/cluster-key "foo") 12182)) 140 | (is (= @(cluster/cluster-key "ignore{foo}") 12182)) 141 | (is (= @(cluster/cluster-key (cluster/cluster-key "ignore{foo}")) 12182))]) 142 | 143 | 144 | ;;;; Conns 145 | 146 | (defn- test-manager [mgr_] 147 | (let [v (volatile! []) 148 | v+ #(vswap! v conj %)] 149 | 150 | (with-open [mgr ^java.io.Closeable (force mgr_)] 151 | (v+ 152 | (car/with-car mgr 153 | (fn [conn] 154 | [(v+ (#'conns/conn? conn)) 155 | (v+ (#'conns/conn-ready? conn)) 156 | (v+ (resp/ping)) 157 | (v+ (car/with-replies (resp/rcmd "ECHO" "x")))]))) 158 | 159 | [@v mgr]))) 160 | 161 | (deftest _basic-conns 162 | [(is (= (conns/with-new-conn tc+ 163 | (fn [conn in out] 164 | [(#'conns/conn? conn) 165 | (#'conns/conn-ready? conn) 166 | (resp/basic-ping! in out) 167 | (resp/with-replies in out false false 168 | (fn [] (resp/rcmd "ECHO" "x")))])) 169 | [true true "PONG" "x"]) 170 | "Unmanaged conn") 171 | 172 | (let [[v mgr] (test-manager (delay (conns/conn-manager-unpooled {})))] 173 | [(is (= [true true nil "x" "PONG"])) 174 | (is (enc/submap? @mgr {:ready? false, :stats {:counts {:active 0, :created 1, :failed 0}}}))]) 175 | 176 | (let [[v mgr] (test-manager (delay (conns/conn-manager-pooled {})))] 177 | [(is (= [true true nil "x" "PONG"])) 178 | (is (enc/submap? @mgr 179 | {:ready? false, 180 | :stats {:counts {:idle 0, :returned 1, :created 1, :waiting 0, :active 0, :cleared 0, 181 | :destroyed {:total 1}, :borrowed 1, :failed 0}}}))])]) 182 | 183 | (deftest _conn-manager-interrupt 184 | (let [mgr (conns/conn-manager-unpooled {}) 185 | k1 (tk "tlist") 186 | f 187 | (future 188 | (wcar mgr 189 | (resp/rcmds 190 | ["DEL" k1] 191 | ["LPUSH" k1 "x"] 192 | ["LPOP" k1] 193 | ["BLPOP" k1 5] ; Block for 5 secs 194 | )))] 195 | 196 | (Thread/sleep 1000) ; Wait for wcar to start but not complete 197 | [(is (true? (car/conn-manager-close! mgr 0 {}))) ; Interrupt pool conns 198 | (is (instance? java.net.SocketException (ex-cause (truss/throws @f))) 199 | "Close with zero timeout interrupts blocking blpop")])) 200 | 201 | (deftest _wcar-basics 202 | [(is (= (wcar mgr_ (resp/ping)) "PONG")) 203 | (is (= (wcar mgr_ {:as-vec? true} (resp/ping)) ["PONG"])) 204 | (is (= (wcar mgr_ (resp/local-echo "hello")) "hello") "Local echo") 205 | 206 | (let [k1 (tk "k1") 207 | v1 (str (rand-int 1e6))] 208 | (is 209 | (= (wcar mgr_ 210 | (resp/ping) 211 | (resp/rset k1 v1) 212 | (resp/echo (wcar mgr_ (resp/rget k1))) 213 | (resp/rset k1 "0")) 214 | 215 | ["PONG" "OK" v1 "OK"]) 216 | 217 | "Flush triggered by `wcar` in `wcar`")) 218 | 219 | (let [k1 (tk "k1") 220 | v1 (str (rand-int 1e6))] 221 | (is 222 | (= (wcar mgr_ 223 | (resp/ping) 224 | (resp/rset k1 v1) 225 | (resp/echo (with-replies (resp/rget k1))) 226 | (resp/echo (str (= (with-replies (resp/rget k1)) v1))) 227 | (resp/rset k1 "0")) 228 | 229 | ["PONG" "OK" v1 "true" "OK"]) 230 | 231 | "Flush triggered by `with-replies` in `wcar`")) 232 | 233 | (is (= (wcar mgr_ (resp/ping) (wcar mgr_)) "PONG") "Parent replies not swallowed by `wcar`") 234 | (is (= (wcar mgr_ (resp/ping) (with-replies)) "PONG") "Parent replies not swallowed by `with-replies`") 235 | 236 | (is (= (let [k1 (tk "k1")] 237 | (wcar mgr_ 238 | (resp/rset k1 "v1") 239 | (resp/echo 240 | (with-replies 241 | (car/skip-replies (resp/rset k1 "v2")) 242 | (resp/echo 243 | (with-replies (resp/rget k1))))))) 244 | ["OK" "v2"])) 245 | 246 | (is (= 247 | (wcar mgr_ 248 | (resp/ping) 249 | (resp/echo (first (with-replies {:as-vec? true} (resp/ping)))) 250 | (resp/local-echo (first (with-replies {:as-vec? true} (resp/ping))))) 251 | 252 | ["PONG" "PONG" "PONG"]) 253 | 254 | "Nested :as-vec")]) 255 | 256 | ;;;; Benching 257 | 258 | (deftest _benching 259 | (do 260 | (println) 261 | (println "Benching times (1e4 laps)...") 262 | (with-open [mgr-unpooled (conns/conn-manager-unpooled {}) 263 | mgr-default (conns/conn-manager-pooled {}) 264 | mgr-untested (conns/conn-manager-pooled {:pool-opts {:test-on-create? false 265 | :test-on-borrow? false 266 | :test-on-return? false}})] 267 | 268 | (println " - wcar/unpooled:" (enc/round0 (* (enc/qb 1e3 (wcar mgr-unpooled)) 10))) 269 | (println " - wcar/default: " (enc/round0 (enc/qb 1e4 (wcar mgr-default)))) 270 | (println " - wcar/untested:" (enc/round0 (enc/qb 1e4 (wcar mgr-untested)))) 271 | (println " - ping/default: " (enc/round0 (enc/qb 1e4 (wcar mgr-default (resp/ping))))) 272 | (println " - ping/untested:" (enc/round0 (enc/qb 1e4 (wcar mgr-untested (resp/ping)))))))) 273 | -------------------------------------------------------------------------------- /src/taoensso/carmine/commands.clj: -------------------------------------------------------------------------------- 1 | (ns ^:no-doc taoensso.carmine.commands 2 | "Macros to define an up-to-date, fully documented function for every Redis 3 | command as specified in the official json command spec." 4 | (:require 5 | [clojure.string :as str] 6 | [taoensso.encore :as enc] 7 | [taoensso.truss :as truss] 8 | [taoensso.carmine.protocol :as protocol]) 9 | 10 | (:import 11 | [taoensso.carmine.protocol EnqueuedRequest Context])) 12 | 13 | ;;;; Cluster keyslots 14 | 15 | (def ^:private ^:const num-keyslots 16384) 16 | (let [xmodem-crc16-lookup 17 | (long-array 18 | [0x0000,0x1021,0x2042,0x3063,0x4084,0x50a5,0x60c6,0x70e7, 19 | 0x8108,0x9129,0xa14a,0xb16b,0xc18c,0xd1ad,0xe1ce,0xf1ef, 20 | 0x1231,0x0210,0x3273,0x2252,0x52b5,0x4294,0x72f7,0x62d6, 21 | 0x9339,0x8318,0xb37b,0xa35a,0xd3bd,0xc39c,0xf3ff,0xe3de, 22 | 0x2462,0x3443,0x0420,0x1401,0x64e6,0x74c7,0x44a4,0x5485, 23 | 0xa56a,0xb54b,0x8528,0x9509,0xe5ee,0xf5cf,0xc5ac,0xd58d, 24 | 0x3653,0x2672,0x1611,0x0630,0x76d7,0x66f6,0x5695,0x46b4, 25 | 0xb75b,0xa77a,0x9719,0x8738,0xf7df,0xe7fe,0xd79d,0xc7bc, 26 | 0x48c4,0x58e5,0x6886,0x78a7,0x0840,0x1861,0x2802,0x3823, 27 | 0xc9cc,0xd9ed,0xe98e,0xf9af,0x8948,0x9969,0xa90a,0xb92b, 28 | 0x5af5,0x4ad4,0x7ab7,0x6a96,0x1a71,0x0a50,0x3a33,0x2a12, 29 | 0xdbfd,0xcbdc,0xfbbf,0xeb9e,0x9b79,0x8b58,0xbb3b,0xab1a, 30 | 0x6ca6,0x7c87,0x4ce4,0x5cc5,0x2c22,0x3c03,0x0c60,0x1c41, 31 | 0xedae,0xfd8f,0xcdec,0xddcd,0xad2a,0xbd0b,0x8d68,0x9d49, 32 | 0x7e97,0x6eb6,0x5ed5,0x4ef4,0x3e13,0x2e32,0x1e51,0x0e70, 33 | 0xff9f,0xefbe,0xdfdd,0xcffc,0xbf1b,0xaf3a,0x9f59,0x8f78, 34 | 0x9188,0x81a9,0xb1ca,0xa1eb,0xd10c,0xc12d,0xf14e,0xe16f, 35 | 0x1080,0x00a1,0x30c2,0x20e3,0x5004,0x4025,0x7046,0x6067, 36 | 0x83b9,0x9398,0xa3fb,0xb3da,0xc33d,0xd31c,0xe37f,0xf35e, 37 | 0x02b1,0x1290,0x22f3,0x32d2,0x4235,0x5214,0x6277,0x7256, 38 | 0xb5ea,0xa5cb,0x95a8,0x8589,0xf56e,0xe54f,0xd52c,0xc50d, 39 | 0x34e2,0x24c3,0x14a0,0x0481,0x7466,0x6447,0x5424,0x4405, 40 | 0xa7db,0xb7fa,0x8799,0x97b8,0xe75f,0xf77e,0xc71d,0xd73c, 41 | 0x26d3,0x36f2,0x0691,0x16b0,0x6657,0x7676,0x4615,0x5634, 42 | 0xd94c,0xc96d,0xf90e,0xe92f,0x99c8,0x89e9,0xb98a,0xa9ab, 43 | 0x5844,0x4865,0x7806,0x6827,0x18c0,0x08e1,0x3882,0x28a3, 44 | 0xcb7d,0xdb5c,0xeb3f,0xfb1e,0x8bf9,0x9bd8,0xabbb,0xbb9a, 45 | 0x4a75,0x5a54,0x6a37,0x7a16,0x0af1,0x1ad0,0x2ab3,0x3a92, 46 | 0xfd2e,0xed0f,0xdd6c,0xcd4d,0xbdaa,0xad8b,0x9de8,0x8dc9, 47 | 0x7c26,0x6c07,0x5c64,0x4c45,0x3ca2,0x2c83,0x1ce0,0x0cc1, 48 | 0xef1f,0xff3e,0xcf5d,0xdf7c,0xaf9b,0xbfba,0x8fd9,0x9ff8, 49 | 0x6e17,0x7e36,0x4e55,0x5e74,0x2e93,0x3eb2,0x0ed1,0x1ef0])] 50 | 51 | (defn- crc16 52 | "Thanks to Ben Poweski for our implementation here." 53 | [^bytes ba] 54 | (let [len (alength ba)] 55 | (loop [n 0 56 | crc 0] 57 | (if (>= n len) 58 | crc 59 | (recur (inc n) 60 | (bit-xor (bit-and (bit-shift-left crc 8) 0xffff) 61 | (aget xmodem-crc16-lookup 62 | (-> (bit-shift-right crc 8) 63 | (bit-xor (aget ba n)) 64 | (bit-and 0xff)))))))))) 65 | 66 | (defprotocol IKeySlot 67 | "Returns the Redis Cluster key slot ℕ∈[0,num-keyslots) for given key arg 68 | using the CRC16 algorithm, Ref. http://redis.io/topics/cluster-spec (Appendix A)." 69 | (keyslot [redis-key])) 70 | 71 | (extend-type (Class/forName "[B") 72 | IKeySlot (keyslot [rk] (mod (crc16 rk) num-keyslots))) 73 | 74 | (extend-type String 75 | IKeySlot 76 | (keyslot [rk] 77 | (let [match 78 | (when (enc/str-contains? rk "{") 79 | (re-find #"\{(.*?)\}" rk)) 80 | 81 | ^String to-hash 82 | (if match 83 | (let [tag (nth match 1)] ; "bar" in "foo{bar}{baz}" 84 | (if (.isEmpty ^String tag) rk tag)) 85 | rk)] 86 | 87 | (mod (crc16 (.getBytes to-hash "UTF-8")) num-keyslots)))) 88 | 89 | (comment 90 | [(re-find #"\{(.*?)\}" "foo{bar}{baz}") 91 | (re-find #"\{(.*?)\}" "foo")] 92 | [(keyslot "foobar") (keyslot "ignore-this{foobar}")] 93 | (enc/qb 1e5 94 | (keyslot "hello") 95 | (keyslot "hello{world}")) ; [14.69 56.41] 96 | ) 97 | 98 | ;;;; Command specs 99 | 100 | (comment ; Generate commands.edn 101 | (require '[clojure.data.json :as json]) 102 | (defn- get-redis-command-spec 103 | [source] 104 | (let [json 105 | (case source 106 | ;; Ref. 107 | :online (slurp (java.net.URL. "https://raw.githubusercontent.com/redis/docs/refs/heads/main/data/commands.json")) 108 | :local (enc/slurp-resource "redis-commands.json"))] 109 | 110 | {:as-map (clojure.data.json/read-str json :key-fn keyword), 111 | :as-json json})) 112 | 113 | (comment (= (get-redis-command-spec :local) (get-redis-command-spec :online))) 114 | 115 | (defn- get-fixed-params [^long n-min arguments] 116 | (let [simple-params 117 | (reduce 118 | (fn [acc in] 119 | (let [{:keys [optional multiple arguments name]} (truss/have map? in)] 120 | (cond 121 | optional (reduced acc) 122 | arguments (reduced acc) 123 | multiple (reduced (conj acc (symbol name))) 124 | :else (conj acc (symbol name))))) 125 | [] 126 | arguments) 127 | 128 | n-additional (- n-min (count simple-params))] 129 | 130 | (if (pos? n-additional) 131 | (into simple-params (mapv #(symbol (str "arg" %)) (range 1 (inc n-additional)))) 132 | (do simple-params)))) 133 | 134 | (defn- get-carmine-command-spec 135 | [redis-command-spec] 136 | (try 137 | (let [as-map 138 | (persistent! 139 | (reduce-kv 140 | (fn [m k v] 141 | (let [cmd-name (name k) ; "CONFIG SET" 142 | cmd-args (-> cmd-name (str/split #" ")) ; ["CONFIG" "SET"] 143 | fn-name (-> cmd-name str/lower-case (str/replace #" " "-")) ; "config-set" 144 | 145 | {:keys [summary since complexity arguments arity _group]} v 146 | 147 | ;; Ref. https://redis.io/commands/command/ 148 | [fn-params-fixed fn-params-more req-args-fixed] 149 | (let [n (long arity) 150 | more? (neg? n) 151 | n 152 | (if more? 153 | (+ n (count cmd-args)) 154 | (- n (count cmd-args))) 155 | 156 | n-min (Math/abs n) 157 | fixed (get-fixed-params n-min arguments)] 158 | [fixed (when more? (into fixed '[& args])) (into cmd-args fixed)]) 159 | 160 | fn-docstring 161 | (let [docs-url (str "https://redis.io/commands/" fn-name "/")] 162 | (enc/into-str 163 | "`" cmd-name "` - Redis command function.\n" 164 | (when since [" Available since: Redis " since "\n"]) 165 | (when complexity [" Complexity: " complexity "\n"]) 166 | "\n" summary 167 | "\n" 168 | "Ref. " docs-url " for more info.")) 169 | 170 | ;; Assuming for now that cluster key always follows 171 | ;; right after command args (seems to hold?). 172 | ;; Can adjust later if needed. 173 | cluster-key-idx (count cmd-args)] 174 | 175 | (truss/have? pos? cluster-key-idx) 176 | (assoc! m cmd-name 177 | {:fn-name fn-name 178 | :cluster-key-idx cluster-key-idx 179 | :fn-params-more fn-params-more 180 | :fn-params-fixed fn-params-fixed 181 | :req-args-fixed req-args-fixed ; ["CONFIG" "SET" 'key 'value] 182 | :fn-docstring fn-docstring}))) 183 | 184 | (transient {}) 185 | (get redis-command-spec :as-map)))] 186 | 187 | {:as-map as-map 188 | :as-edn 189 | (str 190 | "{\n" 191 | (reduce 192 | (fn [acc k] 193 | (let [v (get as-map k)] 194 | (str acc (enc/pr-edn k) " " (enc/pr-edn v) "\n"))) 195 | "" 196 | (sort (keys as-map))) 197 | "}")}) 198 | 199 | (catch Throwable t 200 | (truss/ex-info! "Failed to generate Carmine command spec" 201 | {:redis-command-spec redis-command-spec} 202 | t)))) 203 | 204 | (comment 205 | (get-in (get-redis-command-spec :local) [:as-map :XTRIM]) 206 | (get-in (get-carmine-command-spec (get-redis-command-spec :local)) [:as-map "XTRIM"])) 207 | 208 | (defn update-commands! [json-source] 209 | (let [redis-command-spec (get-redis-command-spec json-source) 210 | carmine-command-spec (get-carmine-command-spec redis-command-spec)] 211 | 212 | (spit "resources/redis-commands.json" (truss/have (:as-json redis-command-spec))) 213 | (spit "resources/carmine-commands.edn" (truss/have (:as-edn carmine-command-spec))))) 214 | 215 | (update-commands! :local) 216 | (update-commands! :online)) 217 | 218 | ;;;; 219 | 220 | (defn enqueue-request 221 | "Implementation detail. 222 | Takes a request like [\"SET\" \"my-key\" \"my-val\"] and adds it to 223 | dynamic context's request queue." 224 | ([cluster-key-idx request more-args] 225 | (enqueue-request cluster-key-idx 226 | (reduce conj request more-args) ; Avoid transients 227 | #_(into request more-args))) 228 | 229 | ([cluster-key-idx request] 230 | ;; (truss/have? vector? request) 231 | (let [context protocol/*context* 232 | _ (when (nil? context) (throw protocol/no-context-ex)) 233 | parser protocol/*parser* 234 | ^Context context context 235 | conn (.-conn context) 236 | req-queue_ (.-req-queue_ context) 237 | ;; cluster-mode? (.-cluster-mode? context) 238 | cluster-mode? false #_(get-in conn [:spec :cluster]) 239 | 240 | request-bs (mapv protocol/byte-str request) 241 | cluster-keyslot 242 | (if cluster-mode? 243 | (let [ck (nth request cluster-key-idx)] 244 | (if (string? ck) 245 | (keyslot ck) 246 | (keyslot (nth request-bs cluster-key-idx)))) 247 | 0) 248 | 249 | ereq (EnqueuedRequest. cluster-keyslot parser request request-bs)] 250 | 251 | (swap! req-queue_ conj ereq)))) 252 | 253 | ;;;; 254 | 255 | (def ^:private skip-fns "#{}" #{}) 256 | (def ^:private rename-fns "{ }" {}) 257 | 258 | (defmacro defcommand [cmd-name spec] 259 | (let [{:keys [fn-name fn-docstring fn-params-fixed fn-params-more 260 | req-args-fixed cluster-key-idx]} spec] 261 | 262 | ;; TODO Optimization: could pre-generate raw byte-strings for req 263 | ;; cmd args, e.g. ["CONFIG" "SET"]? 264 | 265 | (when-not (skip-fns fn-name) 266 | (let [fn-name (get rename-fns fn-name fn-name)] 267 | (if fn-params-more 268 | `(defn ~(symbol fn-name) ~fn-docstring {:redis-api true} 269 | ~`(~fn-params-fixed (enqueue-request ~cluster-key-idx ~req-args-fixed)) 270 | ~`(~fn-params-more (enqueue-request ~cluster-key-idx ~req-args-fixed ~'args))) 271 | 272 | `(defn ~(symbol fn-name) ~fn-docstring {:redis-api true} 273 | ~fn-params-fixed 274 | (enqueue-request ~cluster-key-idx ~req-args-fixed))))))) 275 | 276 | (comment 277 | (count command-spec) ; 197 278 | (get command-spec "HMGET") 279 | (macroexpand 280 | '(defcommand "HMGET" 281 | {:fn-name "hmget", :fn-docstring "doc", :fn-params-fixed [key field], 282 | :fn-params-more [key field & args], :req-args-fixed ["HMGET" key field], 283 | :cluster-key-idx 1}))) 284 | 285 | (defonce ^:private command-spec 286 | (if-let [edn (enc/slurp-resource "carmine-commands.edn")] 287 | (try 288 | (enc/read-edn edn) 289 | (catch Exception e (truss/ex-info! "Failed to read Carmine commands edn" {} e))) 290 | (do (truss/ex-info! "Failed to find Carmine commands edn" {})))) 291 | 292 | (defmacro defcommands [] 293 | `(do ~@(map (fn [[k v]] `(defcommand ~k ~v)) command-spec))) 294 | 295 | (comment (defcommands)) 296 | -------------------------------------------------------------------------------- /src/taoensso/carmine_v4/opts.clj: -------------------------------------------------------------------------------- 1 | (ns ^:no-doc taoensso.carmine-v4.opts 2 | "Private ns, implementation detail. 3 | Carmine has a lot of options, so we do our best to: 4 | - Coerce and validate early when possible. 5 | - Throw detailed error messages when issues occur." 6 | (:require 7 | [clojure.string :as str] 8 | [taoensso.encore :as enc] 9 | [taoensso.truss :as truss :refer [have have?]] 10 | [taoensso.carmine-v4.utils :as utils]) 11 | 12 | (:import 13 | [java.net Socket] 14 | [org.apache.commons.pool2.impl 15 | BaseGenericObjectPool 16 | GenericObjectPool GenericKeyedObjectPool])) 17 | 18 | (comment (remove-ns 'taoensso.carmine-v4.opts)) 19 | 20 | (enc/declare-remote 21 | taoensso.carmine-v4/default-conn-opts 22 | taoensso.carmine-v4/default-sentinel-opts 23 | taoensso.carmine-v4.conns/conn-manager? 24 | taoensso.carmine-v4.sentinel/sentinel-spec? 25 | taoensso.carmine-v4.sentinel/sentinel-opts) 26 | 27 | (do 28 | (alias 'core 'taoensso.carmine-v4) 29 | (alias 'conns 'taoensso.carmine-v4.conns) 30 | (alias 'sentinel 'taoensso.carmine-v4.sentinel)) 31 | 32 | ;;;; Mutators 33 | 34 | (defn set-socket-opts! 35 | ^Socket [^Socket s socket-opts] 36 | (enc/run-kv! 37 | (fn [k v] 38 | (case k 39 | ;; Carmine options, noop and pass through 40 | (:ssl :connect-timeout-ms :ready-timeout-ms) nil 41 | 42 | (:setKeepAlive :keep-alive?) (.setKeepAlive s (boolean v)) 43 | (:setOOBInline :oob-inline?) (.setOOBInline s (boolean v)) 44 | (:setTcpNoDelay :tcp-no-delay?) (.setTcpNoDelay s (boolean v)) 45 | (:setReuseAddress :reuse-address?) (.setReuseAddress s (boolean v)) 46 | 47 | (:setReceiveBufferSize :receive-buffer-size) (.setReceiveBufferSize s (int v)) 48 | (:setSendBufferSize :send-buffer-size) (.setSendBufferSize s (int v)) 49 | (:setSoTimeout :so-timeout :read-timeout-ms) (.setSoTimeout s (int (or v 0))) 50 | 51 | ;; (:setSocketImplFactory :socket-impl-factory) (.setSocketImplFactory s v) 52 | (:setTrafficClass :traffic-class) (.setTrafficClass s v) 53 | 54 | (:setSoLinger :so-linger) 55 | (let [[on? linger] (have vector? v)] 56 | (.setSoLinger s (boolean on?) (int linger))) 57 | 58 | (:setPerformancePreferences :performance-preferences) 59 | (let [[conn-time latency bandwidth] (have vector? v)] 60 | (.setPerformancePreferences s (int conn-time) (int latency) (int bandwidth))) 61 | 62 | (truss/ex-info! "[Carmine] Unknown socket option specified" 63 | {:eid :carmine.conns/unknown-socket-option 64 | :opt-key (enc/typed-val k) 65 | :opt-val (enc/typed-val v) 66 | :all-opts socket-opts}))) 67 | socket-opts) 68 | s) 69 | 70 | (defn set-pool-opts! 71 | [^BaseGenericObjectPool p pool-opts] 72 | (let [neg-duration (java.time.Duration/ofSeconds -1)] 73 | (enc/run-kv! 74 | (fn [k v] 75 | (case k 76 | ;;; org.apache.commons.pool2.impl.GenericObjectPool 77 | (:setMinIdle :min-idle) (.setMinIdle ^GenericObjectPool p (int (or v -1))) 78 | (:setMaxIdle :max-idle) (.setMaxIdle ^GenericObjectPool p (int (or v -1))) 79 | 80 | ;;; org.apache.commons.pool2.impl.GenericKeyedObjectPool 81 | (:setMinIdlePerKey :min-idle-per-key) (.setMinIdlePerKey ^GenericKeyedObjectPool p (int (or v -1))) 82 | (:setMaxIdlePerKey :max-idle-per-key) (.setMaxIdlePerKey ^GenericKeyedObjectPool p (int (or v -1))) 83 | (:setMaxTotalPerKey :max-total-per-key) (.setMaxTotalPerKey ^GenericKeyedObjectPool p (int (or v -1))) 84 | 85 | ;;; org.apache.commons.pool2.impl.BaseGenericObjectPool 86 | (:setBlockWhenExhausted :block-when-exhausted?) (.setBlockWhenExhausted p (boolean v)) 87 | (:setLifo :lifo?) (.setLifo p (boolean v)) 88 | 89 | (:setMaxTotal :max-total) (.setMaxTotal p (int (or v -1))) 90 | (:setMaxWaitMillis :max-wait-ms) (.setMaxWaitMillis p (long (or v -1))) 91 | (:setMaxWait :max-wait) (.setMaxWait p (or v neg-duration)) 92 | 93 | (:setMinEvictableIdleTimeMillis :min-evictable-idle-time-ms) (.setMinEvictableIdleTimeMillis p (long (or v -1))) 94 | (:setMinEvictableIdle :min-evictable-idle) (.setMinEvictableIdle p (or v neg-duration)) 95 | (:setSoftMinEvictableIdleTimeMillis :soft-min-evictable-idle-time-ms) (.setSoftMinEvictableIdleTimeMillis p (long (or v -1))) 96 | (:setSoftMinEvictableIdle :soft-min-evictable-idle) (.setSoftMinEvictableIdle p (or v neg-duration)) 97 | (:setNumTestsPerEvictionRun :num-tests-per-eviction-run) (.setNumTestsPerEvictionRun p (int (or v 0))) 98 | (:setTimeBetweenEvictionRunsMillis :time-between-eviction-runs-ms) (.setTimeBetweenEvictionRunsMillis p (long (or v -1))) 99 | (:setTimeBetweenEvictionRuns :time-between-eviction-runs) (.setTimeBetweenEvictionRuns p (or v neg-duration)) 100 | 101 | (:setEvictorShutdownTimeoutMillis :evictor-shutdown-timeout-ms) (.setEvictorShutdownTimeoutMillis p (long v)) 102 | (:setEvictorShutdownTimeout :evictor-shutdown-timeout) (.setEvictorShutdownTimeout p v) 103 | 104 | (:setTestOnCreate :test-on-create?) (.setTestOnCreate p (boolean v)) 105 | (:setTestWhileIdle :test-while-idle?) (.setTestWhileIdle p (boolean v)) 106 | (:setTestOnBorrow :test-on-borrow?) (.setTestOnBorrow p (boolean v)) 107 | (:setTestOnReturn :test-on-return?) (.setTestOnReturn p (boolean v)) 108 | 109 | (:setSwallowedExceptionListener :swallowed-exception-listener) 110 | (.setSwallowedExceptionListener p v) 111 | (truss/ex-info! "[Carmine] Unknown pool option specified" 112 | {:eid :carmine.conns/unknown-pool-option 113 | :opt-key (enc/typed-val k) 114 | :opt-val (enc/typed-val v) 115 | :all-opts pool-opts}))) 116 | pool-opts)) 117 | p) 118 | 119 | ;;;; Misc 120 | 121 | (defn parse-sock-addr 122 | "Returns valid [ ] socket address pair, or throws. 123 | Retains metadata (server name, comments, etc.)." 124 | ( [host port ] [(have string? host) (enc/as-int port)]) 125 | ( [host port metadata] (with-meta [(have string? host) (enc/as-int port)] metadata)) 126 | ([[host port :as addr]] 127 | (have? string? host) 128 | (assoc addr 1 (enc/as-int port)))) 129 | 130 | (defn descr-sock-addr 131 | "Returns [ ] socket address." 132 | [addr] (if-let [m (meta addr)] (conj addr m) addr)) 133 | 134 | (defn get-sentinel-server [conn-opts] 135 | (let [{:keys [server]} conn-opts] 136 | (when (and (map? server) (get server :sentinel-spec)) 137 | server))) 138 | 139 | ;;;; 140 | 141 | (declare ^:private parse-string-server ^:private parse-sentinel-server) 142 | 143 | (defn parse-conn-opts 144 | "Returns valid parsed conn-opts, or throws." 145 | [in-sentinel-opts? conn-opts] 146 | (try 147 | (have? [:or nil? map?] conn-opts) 148 | (let [default-conn-opts 149 | (if in-sentinel-opts? 150 | (dissoc core/default-conn-opts :server) 151 | (do core/default-conn-opts)) 152 | 153 | conn-opts (utils/merge-opts default-conn-opts conn-opts) 154 | {:keys [server cbs socket-opts buffer-opts init]} conn-opts 155 | {:keys [auth]} init] 156 | 157 | (if in-sentinel-opts? 158 | ;; [host port] of Sentinel server will be auto added by resolver 159 | (have? [:ks<= #{:id #_:server :cbs :socket-opts :buffer-opts :init}] conn-opts) 160 | (have? [:ks<= #{:id :server :cbs :socket-opts :buffer-opts :init}] conn-opts)) 161 | 162 | (have? [:ks<= #{:on-conn-close :on-conn-error}] cbs) 163 | (have? [:or nil? fn?] :in (vals cbs)) 164 | 165 | (when socket-opts (set-socket-opts! (java.net.Socket.) socket-opts)) ; Dry run 166 | (have? [:ks<= #{:init-size-in :init-size-out}] buffer-opts) 167 | 168 | (if in-sentinel-opts? 169 | (have? [:ks<= #{:commands :auth :resp3? #_:client-name #_:select-db}] init) 170 | (have? [:ks<= #{:commands :auth :resp3? :client-name :select-db}] init)) 171 | 172 | (have? [:ks<= #{:username :password}] auth) 173 | 174 | (if in-sentinel-opts? 175 | (do conn-opts) ; Doesn't have :server 176 | (utils/merge-opts conn-opts 177 | (try 178 | (enc/cond 179 | (vector? server) {:server (parse-sock-addr server)} 180 | (string? server) (have map? (parse-string-server server)) 181 | (map? server) 182 | (case (set (keys server)) 183 | #{:host :port} 184 | (let [{:keys [host port]} server] {:server (parse-sock-addr host port (meta server))}) 185 | (#{:master-name :sentinel-spec } 186 | #{:master-name :sentinel-spec :sentinel-opts}) {:server (parse-sentinel-server server)} 187 | 188 | (do (truss/ex-info! "Unexpected `:server` keys" {:keys (keys server)}))) 189 | :else (truss/ex-info! "Unexpected `:server` type" {:type (type server)})) 190 | 191 | (catch Throwable t 192 | (truss/ex-info! "[Carmine] Invalid Redis server specification in connection options" 193 | {:eid :carmine.conn-opts/invalid-server 194 | :server (enc/typed-val server) 195 | :expected '(or uri-string [host port] {:keys [host port]} 196 | {:keys [master-name sentinel-spec sentinel-opts]})} 197 | t)))))) 198 | 199 | (catch Throwable t 200 | (truss/ex-info! "[Carmine] Invalid connection options" 201 | {:eid :carmine.conn-opts/invalid 202 | :conn-opts (assoc (enc/typed-val conn-opts) :id (get conn-opts :id)) 203 | :purpose 204 | (if in-sentinel-opts? 205 | :conn-to-sentinel-server 206 | :conn-to-redis-server)} 207 | t)))) 208 | 209 | ;;;; 210 | 211 | (defn- parse-string-server 212 | "\"rediss://user:pass@x.y.com:9475/3\" -> 213 | {:keys [server init socket-opts]}, etc." 214 | [s] 215 | (let [uri (java.net.URI. (have string? s)) 216 | server [(.getHost uri) (.getPort uri)] 217 | init 218 | (enc/assoc-some nil 219 | :auth 220 | (let [[username password] (.split (str (.getUserInfo uri)) ":")] 221 | (enc/assoc-some nil 222 | :username (enc/as-?nempty-str username) 223 | :password (enc/as-?nempty-str password))) 224 | 225 | :select-db 226 | (when-let [[_ db-str] (re-matches #"/(\d+)$" (.getPath uri))] 227 | (Integer. ^String db-str))) 228 | 229 | socket-opts 230 | (when-let [scheme (.getScheme uri)] 231 | (when (contains? #{"rediss" "https"} (str/lower-case scheme)) 232 | {:ssl true}))] 233 | 234 | (enc/assoc-some {:server server} 235 | :init init 236 | :socket-opts socket-opts))) 237 | 238 | (comment 239 | [(parse-string-server "redis://user:pass@x.y.com:9475/3") 240 | (parse-string-server "redis://:pass@x.y.com.com:9475/3") 241 | (parse-string-server "redis://user:@x.y.com:9475/3") 242 | (parse-string-server "rediss://user:@x.y.com:9475/3")]) 243 | 244 | (defn- parse-sentinel-server [server] 245 | (have? map? server) 246 | (let [{:keys [master-name sentinel-spec sentinel-opts]} server 247 | 248 | master-name (enc/as-qname (have [:or string? enc/named?] master-name)) 249 | sentinel-opts 250 | (let [sentinel-spec (have sentinel/sentinel-spec? sentinel-spec) 251 | sentinel-opts 252 | (utils/merge-opts core/default-sentinel-opts 253 | (sentinel/sentinel-opts sentinel-spec) 254 | sentinel-opts)] 255 | 256 | (try 257 | (have? map? sentinel-opts) 258 | (have? [:ks<= #{:id :conn-opts :cbs 259 | :retry-delay-ms :resolve-timeout-ms :clear-timeout-ms 260 | :update-sentinels? :update-replicas? :prefer-read-replica?}] 261 | sentinel-opts) 262 | 263 | (let [{:keys [cbs]} sentinel-opts] 264 | (have? [:ks<= #{:on-resolve-success :on-resolve-error 265 | :on-changed-master :on-changed-replicas :on-changed-sentinels}] cbs) 266 | (have? [:or nil? fn?] :in (vals cbs))) 267 | 268 | (if-let [conn-opts (not-empty (get sentinel-opts :conn-opts))] 269 | (assoc sentinel-opts :conn-opts (parse-conn-opts true conn-opts)) 270 | (do sentinel-opts)) 271 | 272 | (catch Throwable t 273 | (truss/ex-info! "[Carmine] Invalid Sentinel options" 274 | {:eid :carmine.sentinel-opts/invalid 275 | :sentinel-opts 276 | (assoc (enc/typed-val sentinel-opts) 277 | :id (get sentinel-opts :id))} 278 | t))))] 279 | 280 | (assoc server 281 | :master-name master-name 282 | :sentinel-opts sentinel-opts))) 283 | -------------------------------------------------------------------------------- /src/taoensso/carmine_v4/resp/write.clj: -------------------------------------------------------------------------------- 1 | (ns ^:no-doc taoensso.carmine-v4.resp.write 2 | "Private ns, implementation detail." 3 | (:refer-clojure :exclude [bytes]) 4 | (:require 5 | [taoensso.encore :as enc] 6 | [taoensso.truss :as truss] 7 | [taoensso.nippy :as nippy] 8 | [taoensso.carmine-v4.resp.common :as com 9 | :refer [with-out with-out->str]]) 10 | 11 | (:import 12 | [java.nio.charset StandardCharsets] 13 | [java.io BufferedOutputStream])) 14 | 15 | (enc/declare-remote 16 | ^:dynamic taoensso.carmine-v4/*auto-freeze?* 17 | ^:dynamic taoensso.carmine-v4/*freeze-opts*) 18 | 19 | (alias 'core 'taoensso.carmine-v4) 20 | 21 | (comment (remove-ns 'taoensso.carmine-v4.resp.write)) 22 | 23 | ;;;; Bulk byte strings 24 | 25 | (do 26 | (def ^:const min-num-to-cache (long Short/MIN_VALUE)) 27 | (def ^:const max-num-to-cache (long Short/MAX_VALUE))) 28 | 29 | ;; Cache ba representation of common number bulks, etc. 30 | (let [long->bytes (fn [n] (.getBytes (Long/toString n) StandardCharsets/UTF_8)) 31 | create-cache ; { ((fn [n])->ba)} 32 | (fn [n-cast from-n to-n f] 33 | (java.util.concurrent.ConcurrentHashMap. ^java.util.Map 34 | (persistent! 35 | (enc/reduce-n 36 | (fn [m n] (let [n (n-cast n)] (assoc! m n (f n)))) 37 | (transient {}) from-n to-n)))) 38 | 39 | b* (int \*) 40 | b$ (int \$) 41 | ba-crlf com/ba-crlf] 42 | 43 | (let [;; { *} for common lengths 44 | ^java.util.concurrent.ConcurrentHashMap cache 45 | (create-cache long 0 256 46 | (fn [n] 47 | (let [n-as-ba (long->bytes n)] 48 | (com/xs->ba \* n-as-ba "\r\n"))))] 49 | 50 | (defn write-array-len 51 | [^BufferedOutputStream out n] 52 | (let [n (long n)] 53 | (if-let [^bytes cached-ba (.get cache n)] 54 | (.write out cached-ba 0 (alength cached-ba)) 55 | 56 | (let [^bytes n-as-ba (long->bytes n)] 57 | (.write out b*) 58 | (.write out n-as-ba 0 (alength n-as-ba)) 59 | (.write out ba-crlf 0 2)))))) 60 | 61 | (let [;; { $} for common lengths 62 | ^java.util.concurrent.ConcurrentHashMap cache 63 | (create-cache long 0 256 64 | (fn [n] 65 | (let [n-as-ba (long->bytes n)] 66 | (com/xs->ba \$ n-as-ba "\r\n"))))] 67 | 68 | (defn- write-bulk-len 69 | [^BufferedOutputStream out n] 70 | (let [n (long n)] 71 | (if-let [^bytes cached-ba (.get cache n)] 72 | (.write out cached-ba 0 (alength cached-ba)) 73 | 74 | (let [^bytes n-as-ba (long->bytes n)] 75 | (.write out b$) 76 | (.write out n-as-ba 0 (alength n-as-ba)) 77 | (.write out ba-crlf 0 2)))))) 78 | 79 | (let [b-colon (int \:) 80 | ;; { :} for common longs 81 | ^java.util.concurrent.ConcurrentHashMap cache 82 | (create-cache long min-num-to-cache (inc max-num-to-cache) 83 | (fn [n] (com/xs->ba \: (long->bytes n) "\r\n")))] 84 | 85 | (defn- write-simple-long 86 | [^BufferedOutputStream out n] 87 | (let [n (long n)] 88 | (if-let [^bytes cached-ba (.get cache n)] 89 | (.write out cached-ba 0 (alength cached-ba)) 90 | 91 | (let [^bytes n-as-ba (long->bytes n) 92 | len (alength n-as-ba) 93 | ^bytes len-as-ba (long->bytes len)] 94 | 95 | (.write out b-colon) 96 | (.write out n-as-ba 0 len) 97 | (.write out ba-crlf 0 2)))))) 98 | 99 | (let [;; { $bytes n) 104 | len (alength n-as-ba) 105 | ^bytes len-as-ba (long->bytes len)] 106 | 107 | (com/xs->ba \$ len-as-ba "\r\n" n-as-ba "\r\n"))))] 108 | 109 | (defn- write-bulk-long 110 | [^BufferedOutputStream out n] 111 | (let [n (long n)] 112 | (if-let [^bytes cached-ba (.get cache n)] 113 | (.write out cached-ba 0 (alength cached-ba)) 114 | 115 | (let [^bytes n-as-ba (long->bytes n) 116 | len (alength n-as-ba) 117 | ^bytes len-as-ba (long->bytes len)] 118 | 119 | (.write out b$) 120 | (.write out len-as-ba 0 (alength len-as-ba)) 121 | (.write out ba-crlf 0 2) 122 | 123 | (.write out n-as-ba 0 len) 124 | (.write out ba-crlf 0 2)))))) 125 | 126 | (let [double->bytes (fn [n] (.getBytes (Double/toString n) StandardCharsets/UTF_8)) 127 | 128 | ;; { $bytes n) 133 | len (alength n-as-ba) 134 | ^bytes len-as-ba (long->bytes len)] 135 | 136 | (com/xs->ba \$ len-as-ba "\r\n" n-as-ba "\r\n"))))] 137 | 138 | (defn- write-bulk-double 139 | [^BufferedOutputStream out n] 140 | (let [n (double n)] 141 | (if-let [^bytes cached-ba (.get cache n)] 142 | (.write out cached-ba 0 (alength cached-ba)) 143 | 144 | (let [^bytes n-as-ba (double->bytes n) 145 | len (alength n-as-ba) 146 | ^bytes len-as-ba (long->bytes len)] 147 | 148 | (.write out b$) 149 | (.write out len-as-ba 0 (alength len-as-ba)) 150 | (.write out ba-crlf 0 2) 151 | 152 | (.write out n-as-ba 0 len) 153 | (.write out ba-crlf 0 2))))))) 154 | 155 | (let [write-bulk-len write-bulk-len 156 | ba-crlf com/ba-crlf] 157 | 158 | (defn- write-bulk-ba 159 | "$" 160 | ([^BufferedOutputStream out ^bytes ba] 161 | (let [len (alength ba)] 162 | (write-bulk-len out len) 163 | (.write out ba 0 len) 164 | (.write out ba-crlf 0 2))) 165 | 166 | ([^BufferedOutputStream out ^bytes ba-marker ^bytes ba-payload] 167 | (let [marker-len (alength ba-marker) 168 | payload-len (alength ba-payload) 169 | total-len (+ marker-len payload-len)] 170 | (write-bulk-len out total-len) 171 | (.write out ba-marker 0 marker-len) 172 | (.write out ba-payload 0 payload-len) 173 | (.write out ba-crlf 0 2))))) 174 | 175 | (defn- reserve-null! 176 | "This is a Carmine (not Redis) limitation to support auto null-prefixed 177 | blob markers with special semantics (`ba-npy`, etc.)." 178 | [^String s] 179 | (when (and (not (.isEmpty s)) (== ^int (.charAt s 0) 0)) 180 | (truss/ex-info! "[Carmine] String args can't begin with null (char 0)" 181 | {:eid :carmine.write/null-reserved 182 | :arg s}))) 183 | 184 | (defn- write-bulk-str [^BufferedOutputStream out s] 185 | (reserve-null! s) 186 | (write-bulk-ba out (enc/str->utf8-ba s))) 187 | 188 | ;;;; Wrapper types 189 | ;; Influence `IRedisArg` behaviour by wrapping arguments. 190 | ;; Wrapping must capture any relevant dynamic config at wrap time. 191 | ;; 192 | ;; Implementation detail: 193 | ;; We try to avoid lazily converting arguments to Redis byte strings 194 | ;; (i.e. while writing to out) if there's a chance the conversion 195 | ;; could fail (e.g. Nippy freeze). 196 | 197 | (deftype WriteBytes [ba]) 198 | (defn ^:public bytes 199 | "Wraps given byte array to ensure that it'll be written to Redis 200 | without any modifications (serialization, blob markers, etc.)." 201 | (^WriteBytes [ba] 202 | (if (instance? WriteBytes ba) 203 | ba 204 | (if (enc/bytes? ba) 205 | (WriteBytes. ba) 206 | (truss/ex-info! "[Carmine] `bytes` expects a byte-array argument" 207 | {:eid :carmine.write/unsupported-arg-type 208 | :arg (enc/typed-val ba)})))) 209 | 210 | ;; => Vector for destructuring (undocumented) 211 | ([ba & more] (mapv bytes (cons ba more)))) 212 | 213 | (deftype WriteFrozen [unfrozen-val freeze-opts ?frozen-ba]) 214 | (defn ^:public freeze 215 | "Wraps given arb Clojure value to ensure that it'll be written to Redis 216 | using Nippy serialization [1]. 217 | 218 | Options: 219 | See `taoensso.nippy/freeze` for `freeze-opts` docs. 220 | By default, `*freeze-opts*` value will be used. 221 | 222 | See also `thaw` for thawing (deserialization). 223 | [1] Ref. " 224 | 225 | (^WriteFrozen [ clj-val] (freeze core/*freeze-opts* clj-val)) 226 | (^WriteFrozen [freeze-opts clj-val] 227 | ;; We do eager freezing here since we can, and we'd prefer to 228 | ;; catch freezing errors early (rather than while writing to out). 229 | (if (instance? WriteFrozen clj-val) 230 | (let [^WriteFrozen wrapper clj-val] 231 | (if (= freeze-opts (.-freeze-opts wrapper)) 232 | wrapper 233 | ;; Re-freeze (expensive) 234 | (let [clj-val (.-unfrozen-val wrapper)] 235 | (WriteFrozen. clj-val freeze-opts 236 | (nippy/freeze clj-val freeze-opts))))) 237 | 238 | (WriteFrozen. clj-val freeze-opts 239 | (nippy/freeze clj-val freeze-opts)))) 240 | 241 | ;; => Vector for destructuring (undocumented) 242 | ([freeze-opts clj-val & more] 243 | (let [freeze-opts 244 | (truss/have [:or nil? map?] 245 | (if (identical? freeze-opts :dynamic) 246 | core/*freeze-opts* 247 | freeze-opts))] 248 | 249 | (mapv #(freeze freeze-opts %) (cons clj-val more))))) 250 | 251 | ;;;; IRedisArg 252 | 253 | (defprotocol ^:private IRedisArg 254 | "Internal protocol, not for public use or extension." 255 | (write-bulk-arg [x ^BufferedOutputStream out] 256 | "Writes given arbitrary Clojure argument to `out` as a Redis byte string.")) 257 | 258 | (def ^:private bulk-nil 259 | (with-out 260 | (write-bulk-len out 2) 261 | (.write out com/ba-nil 0 2) 262 | (.write out com/ba-crlf 0 2))) 263 | 264 | (comment (enc/utf8-ba->str bulk-nil)) 265 | 266 | (let [write-bulk-str write-bulk-str 267 | ba-bin com/ba-bin 268 | ba-npy com/ba-npy 269 | bulk-nil bulk-nil 270 | bulk-nil-len (alength ^bytes bulk-nil) 271 | kw->str 272 | (fn [kw] 273 | (if-let [ns (namespace kw)] 274 | (str ns "/" (name kw)) 275 | (do (name kw)))) 276 | 277 | non-native-type! 278 | (fn [arg] 279 | (truss/ex-info! "[Carmine] Trying to send argument of non-native type to Redis while `*auto-freeze?` is false" 280 | {:eid :carmine.write/non-native-arg-type 281 | :arg (enc/typed-val arg)}))] 282 | 283 | (extend-protocol IRedisArg 284 | String (write-bulk-arg [s out] (write-bulk-str out s)) 285 | Character (write-bulk-arg [c out] (write-bulk-str out (.toString c))) 286 | clojure.lang.Keyword (write-bulk-arg [kw out] (write-bulk-str out (kw->str kw))) 287 | 288 | ;; Redis doesn't currently seem to accept `write-simple-long` (at least 289 | ;; without RESP3 mode?) though this seems an unnecessary limitation? 290 | Long (write-bulk-arg [n out] (write-bulk-long out n)) 291 | Integer (write-bulk-arg [n out] (write-bulk-long out n)) 292 | Short (write-bulk-arg [n out] (write-bulk-long out n)) 293 | Byte (write-bulk-arg [n out] (write-bulk-long out n)) 294 | Double (write-bulk-arg [n out] (write-bulk-double out n)) 295 | Float (write-bulk-arg [n out] (write-bulk-double out n)) 296 | WriteBytes (write-bulk-arg [w out] (write-bulk-ba out (.-ba w))) 297 | WriteFrozen 298 | (write-bulk-arg [w out] 299 | (let [ba (or (.-?frozen-ba w) (nippy/freeze (.-unfrozen-val w) (.-freeze-opts w)))] 300 | (if core/*auto-freeze?* 301 | (write-bulk-ba out ba-npy ba) 302 | (write-bulk-ba out ba)))) 303 | 304 | Object 305 | (write-bulk-arg [x out] 306 | (if core/*auto-freeze?* 307 | (write-bulk-ba out ba-npy (nippy/freeze x)) 308 | (non-native-type! x))) 309 | 310 | nil 311 | (write-bulk-arg [x ^BufferedOutputStream out] 312 | (if core/*auto-freeze?* 313 | (.write out bulk-nil 0 bulk-nil-len) 314 | (non-native-type! x)))) 315 | 316 | (extend-type (Class/forName "[B") ; Extra `extend` needed due to CLJ-1381 317 | IRedisArg 318 | (write-bulk-arg [ba out] 319 | (if core/*auto-freeze?* 320 | (write-bulk-ba out ba-bin ba) ; Write marked bytes 321 | (write-bulk-ba out ba) ; Write unmarked bytes 322 | )))) 323 | 324 | ;;;; 325 | 326 | (defn- write-requests ; Used only for REPL/testing 327 | "Sends pipelined requests to Redis server using its byte string protocol: 328 | * crlf 329 | [$ crlf 330 | crlf ...]" 331 | [^BufferedOutputStream out reqs] 332 | (enc/run! 333 | (fn [req-args] 334 | (let [n-args (count req-args)] 335 | (when-not (== n-args 0) 336 | (write-array-len out n-args) 337 | (enc/run! 338 | (fn [arg] (write-bulk-arg arg out)) 339 | req-args)))) 340 | reqs) 341 | (.flush out)) 342 | -------------------------------------------------------------------------------- /src/taoensso/carmine_v4.clj: -------------------------------------------------------------------------------- 1 | (ns ^:no-doc taoensso.carmine-v4 2 | "Experimental modern Clojure Redis client prototype. 3 | Still private, not yet intended for public use!" 4 | {:author "Peter Taoussanis (@ptaoussanis)"} 5 | (:refer-clojure :exclude [bytes]) 6 | (:require 7 | [taoensso.encore :as enc] 8 | [taoensso.truss :as truss] 9 | [taoensso.carmine :as v3-core] 10 | [taoensso.carmine 11 | [connections :as v3-conns] 12 | [protocol :as v3-protocol] 13 | [commands :as v3-cmds]] 14 | 15 | [taoensso.carmine-v4.resp.common :as com] 16 | [taoensso.carmine-v4.resp.read :as read] 17 | [taoensso.carmine-v4.resp.write :as write] 18 | [taoensso.carmine-v4.resp :as resp] 19 | ;; 20 | [taoensso.carmine-v4.utils :as utils] 21 | [taoensso.carmine-v4.opts :as opts] 22 | [taoensso.carmine-v4.conns :as conns] 23 | [taoensso.carmine-v4.sentinel :as sentinel] 24 | [taoensso.carmine-v4.cluster :as cluster])) 25 | 26 | (enc/assert-min-encore-version [3 145 0]) 27 | 28 | (comment (remove-ns 'taoensso.carmine-v4)) 29 | 30 | ;;;; Aliases 31 | 32 | (enc/defaliases 33 | enc/get-env 34 | 35 | ;;; Read opts 36 | com/skip-replies 37 | com/normal-replies 38 | com/natural-replies 39 | com/as-bytes 40 | com/thaw 41 | 42 | ;;; Reply parsing 43 | com/reply-error? 44 | com/unparsed 45 | com/parse 46 | com/parse-aggregates 47 | com/parsing-rf 48 | ;; 49 | com/as-?long 50 | com/as-?double 51 | com/as-?kw 52 | ;; 53 | com/as-long 54 | com/as-double 55 | com/as-kw 56 | 57 | ;;; Write wrapping 58 | write/bytes 59 | write/freeze 60 | 61 | ;;; RESP3 62 | resp/rcmd 63 | resp/rcmd* 64 | resp/rcmds 65 | resp/rcmds* 66 | resp/local-echo 67 | resp/local-echos 68 | resp/local-echos* 69 | 70 | ;;; Connections 71 | #_conns/conn? 72 | #_conns/conn-ready? 73 | #_conns/conn-close! 74 | ;; 75 | sentinel/sentinel-spec 76 | sentinel/sentinel-spec? 77 | ;; 78 | conns/conn-manager? 79 | conns/conn-manager-unpooled 80 | conns/conn-manager-pooled 81 | {:alias conn-manager-ready? :src conns/mgr-ready?} 82 | {:alias conn-manager-clear! :src conns/mgr-clear!} 83 | {:alias conn-manager-close! :src conns/mgr-close!} 84 | 85 | ;;; Cluster 86 | cluster/cluster-key) 87 | 88 | ;;;; Config 89 | 90 | (def default-conn-opts 91 | "TODO Docstring incl. env config." 92 | (let [from-env (enc/get-env {:as :edn} :taoensso.carmine.default-conn-opts) 93 | base 94 | {:server ["127.0.0.1" 6379] 95 | #_{:host "127.0.0.1" :port "6379"} 96 | #_{:master-name "my-master" 97 | :sentinel-spec my-spec 98 | :sentinel-opts {}} 99 | 100 | :cbs {:on-conn-close nil, :on-conn-error nil} 101 | :buffer-opts {:init-size-in 8192, :init-size-out 8192} 102 | :socket-opts {:ssl false, :connect-timeout-ms 400, :read-timeout-ms nil 103 | :ready-timeout-ms 200} 104 | :init 105 | {;; :commands [["HELLO" 3 "AUTH" "default" "my-password" "SETNAME" "client-name"] 106 | ;; ["auth" "default" "my-password"]] 107 | :resp3? true 108 | :auth {:username "default" :password nil} 109 | ;; :client-name "carmine" 110 | ;; :select-db 5 111 | }}] 112 | 113 | (enc/nested-merge base from-env))) 114 | 115 | (def default-pool-opts 116 | "TODO Docstring incl. env config." 117 | (let [from-env (enc/get-env {:as :edn} :taoensso.carmine.default-pool-opts) 118 | base 119 | {:test-on-create? true 120 | :test-while-idle? true 121 | :test-on-borrow? true 122 | :test-on-return? false 123 | :num-tests-per-eviction-run -1 124 | :min-evictable-idle-time-ms 60000 125 | :time-between-eviction-runs-ms 30000 126 | :max-total 16 127 | :max-idle 16}] 128 | 129 | (enc/nested-merge base from-env))) 130 | 131 | (def default-sentinel-opts 132 | "TODO Docstring incl. env config." 133 | (let [from-env (enc/get-env {:as :edn} :taoensso.carmine.default-sentinel-opts) 134 | base 135 | {:cbs 136 | {:on-resolve-success nil 137 | :on-resolve-error nil 138 | :on-changed-master nil 139 | :on-changed-replicas nil 140 | :on-changed-sentinels nil} 141 | 142 | :update-sentinels? true 143 | :update-replicas? false 144 | :prefer-read-replica? false 145 | 146 | :retry-delay-ms 250 147 | :resolve-timeout-ms 2000 148 | :clear-timeout-ms 10000 149 | 150 | :conn-opts 151 | {:cbs {:on-conn-close nil, :on-conn-error nil} 152 | :buffer-opts {:init-size-in 512, :init-size-out 256} 153 | :socket-opts {:ssl false, :connect-timeout-ms 200, :read-timeout-ms 200 154 | :ready-timeout-ms 200}}}] 155 | 156 | (enc/nested-merge base from-env))) 157 | 158 | ;;;; 159 | 160 | (def ^:dynamic *auto-freeze?* 161 | "TODO Docstring incl. env config. 162 | Should Carmine automatically serialize arguments sent to Redis 163 | that are non-native to Redis? 164 | 165 | Affects non-(string, keyword, simple long/double) types. 166 | 167 | Default is true. If false, an exception will be thrown when trying 168 | to send such arguments. 169 | 170 | See also `*auto-freeze?`*." 171 | (enc/get-env {:as :bool, :default true} 172 | :taoensso.carmine.auto-freeze)) 173 | 174 | (def ^:dynamic *auto-thaw?* 175 | "TODO Docstring incl. env config. 176 | Should Carmine automatically deserialize Redis replies that 177 | contain data previously serialized by `*auto-thaw?*`? 178 | 179 | Affects non-(string, keyword, simple long/double) types. 180 | 181 | Default is true. If false, such replies will by default look like 182 | malformed strings. 183 | TODO: Mention utils, bindings. 184 | 185 | See also `*auto-thaw?`*." 186 | (enc/get-env {:as :bool, :default true} 187 | :taoensso.carmine.auto-thaw)) 188 | 189 | ;; TODO Docstrings incl. env config. 190 | (def ^:dynamic *raw-verbatim-strings?* false) 191 | (def ^:dynamic *keywordize-maps?* true) 192 | (def ^:dynamic *freeze-opts* nil) 193 | 194 | (def ^:dynamic *issue-83-workaround?* 195 | "TODO Docstring incl. env config. 196 | Only relevant if `*auto-thaw?` is true. 197 | 198 | A bug in Carmine v2.6.0 to v2.6.1 (2014-04-01 to 2014-05-01) caused Nippy blobs 199 | to be marked incorrectly, Ref. 200 | 201 | When enabled, this workaround will cause Carmine to automatically try thaw any 202 | reply byte data that starts with a valid Nippy header. 203 | 204 | Enable iff you might read data written by Carmine < v2.6.1 (2014-05-01). 205 | Disabled by default." 206 | (enc/get-env {:as :bool, :default false} 207 | :taoensso.carmine.issue-83-workaround)) 208 | 209 | (def ^:dynamic *conn-cbs* 210 | "Map of any additional callback fns, as in `conn-opts` or `sentinel-opts`. 211 | Useful for REPL/debugging/tests/etc. 212 | 213 | Possible keys: 214 | `:on-conn-close` 215 | `:on-conn-error` 216 | `:on-resolve-success` 217 | `:on-resolve-error` 218 | `:on-changed-master` 219 | `:on-changed-replicas` 220 | `:on-changed-sentinels` 221 | 222 | Values should be unary callback fns of a single data map." 223 | nil) 224 | 225 | ;;;; Core API (main entry point to Carmine) 226 | 227 | (def ^:private default-conn-manager 228 | (delay (conns/conn-manager-pooled {:mgr-name :default}))) 229 | 230 | (comment (force default-conn-manager)) 231 | 232 | (defn with-car 233 | "TODO Docstring" 234 | ([conn-mgr body-fn] (with-car conn-mgr nil body-fn)) 235 | ([conn-mgr {:keys [as-vec?] :as reply-opts} body-fn] 236 | (let [{:keys [natural-replies?]} reply-opts] ; Undocumented 237 | (conns/mgr-borrow! (force (or conn-mgr default-conn-manager)) 238 | (fn [conn in out] 239 | (resp/with-replies in out natural-replies? as-vec? 240 | (fn [] (body-fn conn)))))))) 241 | 242 | (defmacro wcar 243 | "TODO Docstring" 244 | {:arglists 245 | '([conn-mgr & body] 246 | [conn-mgr {:keys [as-vec?]} & body])} 247 | 248 | [conn-mgr & body] 249 | (let [[reply-opts body] (resp/parse-body-reply-opts body)] 250 | `(with-car ~conn-mgr ~reply-opts 251 | (fn [~'__wcar-conn] ~@body)))) 252 | 253 | (comment 254 | (let [mgr1 (conns/conn-manager-unpooled {}) 255 | mgr2 (conns/conn-manager-pooled {}) 256 | mgr3 (conns/conn-manager-pooled 257 | {:pool-opts 258 | {:test-on-create? false 259 | :test-on-borrow? false 260 | :test-on-return? false}})] 261 | 262 | (try 263 | (enc/qb 1e3 ; [22.33 97.37 38.87 19.86] 264 | (v3-core/wcar {} (v3-core/ping)) 265 | (with-car mgr1 (fn [conn] (resp/ping))) 266 | (with-car mgr2 (fn [conn] (resp/ping))) 267 | (with-car mgr3 (fn [conn] (resp/ping)))) 268 | 269 | (finally 270 | (doseq [mgr [mgr1 mgr2 mgr3]] 271 | (conns/mgr-close! mgr 5000 nil))))) 272 | 273 | [(wcar nil (resp/rcmd :PING)) 274 | (wcar nil (resp/rcmd :SET "k1" 3)) 275 | (wcar nil (resp/rcmd :GET "k1")) 276 | 277 | (wcar nil (resp/ping)) 278 | (wcar nil {:as-vec? true} (resp/ping)) 279 | (wcar nil :as-vec (resp/ping))]) 280 | 281 | (defmacro with-replies 282 | "TODO Docstring 283 | Expects to be called within the body of `wcar` or `with-car`." 284 | {:arglists '([& body] [{:keys [as-vec?]} & body])} 285 | [& body] 286 | (let [[reply-opts body] (resp/parse-body-reply-opts body) 287 | {:keys [natural-replies? as-vec?]} reply-opts] 288 | `(resp/with-replies ~natural-replies? ~as-vec? 289 | (fn [] ~@body)))) 290 | 291 | ;;;; Push API ; TODO 292 | 293 | (defmulti push-handler (fn [state [data-type :as data-vec]] data-type)) 294 | (defmethod push-handler :default [state data-vec] #_(println data-vec) nil) 295 | 296 | (enc/defonce push-agent_ 297 | (delay (agent nil :error-mode :continue))) 298 | 299 | (def ^:dynamic *push-fn* 300 | "TODO Docstring: this and push-handler, etc. 301 | ?(fn [data-vec]) => ?effects. 302 | If provided (non-nil), this fn should never throw." 303 | (fn [data-vec] 304 | (send-off @push-agent_ 305 | (fn [state] 306 | (try 307 | (push-handler state data-vec) 308 | (catch Throwable t 309 | ;; TODO Try publish error message? 310 | )))))) 311 | 312 | ;;;; 313 | 314 | (defn ^:no-doc dummy-scan-fn 315 | "Private, don't use. For unit tests, etc." 316 | [num-steps elements-fn] 317 | (let [c (java.util.concurrent.atomic.AtomicLong. (long num-steps))] 318 | (fn [cursor] 319 | (let [idx (.addAndGet c -1)] 320 | (if (<= idx 0) 321 | ["0" (elements-fn idx)] 322 | ["x" (elements-fn idx)]))))) 323 | 324 | (defn ^:no-doc scan-reduce-elements 325 | "Private, don't use. 326 | Low-level, for use with `scan`, `hscan`, `zscan`, etc. 327 | Takes: 328 | - (fn scan-fn [cursor]) -> next scan result 329 | - (fn rf [acc elements]) -> next accumulator 330 | Acts as `transduce` when given `xform`." 331 | ([ init scan-fn rf] (scan-reduce-elements nil init scan-fn rf)) 332 | ([xform init scan-fn rf] 333 | (let [rf (if xform (xform rf) rf) 334 | rv 335 | (loop [cursor "0" acc init] 336 | (let [[next-cursor next-in] (scan-fn cursor)] 337 | (if (= next-cursor "0") 338 | (rf acc next-in) 339 | (let [result (rf acc next-in)] 340 | (if (reduced? result) 341 | @result 342 | (recur next-cursor result))))))] 343 | (if xform (rf rv) rv)))) 344 | 345 | (defn scan-reduce 346 | "TODO Docstring, example, tests. 347 | Takes: 348 | - (fn scan-fn [cursor]) -> next scan result 349 | - (fn rf [acc element]) -> next accumulator 350 | Acts as `transduce` when given `xform`." 351 | ([ init scan-fn rf] (scan-reduce nil init scan-fn rf)) 352 | ([xform init scan-fn rf] 353 | (let [rf (if xform (xform rf) rf) 354 | rv 355 | (scan-reduce-elements nil init scan-fn 356 | (fn wrapped-rf [acc elements] 357 | (reduce 358 | (fn [acc element] (enc/convey-reduced (rf acc element))) 359 | acc elements)))] 360 | (if xform (rf rv) rv)))) 361 | 362 | (comment 363 | (count 364 | (scan-reduce (distinct) [] 365 | (fn scan-fn [cursor] (wcar default-conn-manager (rcmd :SCAN cursor :MATCH "*"))) 366 | (completing (fn rf [acc in] (conj acc in))))) 367 | 368 | (count 369 | (scan-reduce (distinct) [] 370 | (dummy-scan-fn 8 (fn [_] (repeatedly 8 #(rand-int 10)))) 371 | (completing (fn rf [acc in] (conj acc in)))))) 372 | 373 | (defn scan-reduce-kv 374 | "TODO Docstring, example, tests. 375 | Takes: 376 | - (fn scan-fn [cursor]) -> next scan result 377 | - (fn rf [acc k v]) -> next accumulator 378 | Auto de-duplicates so that `rf` won't be called >once 379 | for the same key." 380 | [init scan-fn rf] 381 | (let [seen_ (volatile! (transient #{}))] 382 | (scan-reduce-elements nil init scan-fn 383 | (fn wrapped-rf [acc kvs] 384 | (enc/reduce-kvs 385 | (fn [acc k v] 386 | (if (contains? @seen_ k) 387 | acc 388 | (do 389 | (vswap! seen_ conj! k) 390 | (enc/convey-reduced (rf acc k v))))) 391 | acc kvs))))) 392 | 393 | (comment 394 | (wcar default-conn-manager (rcmd :HMSET "my-hash" "k1" "v1" "k2" "v2")) 395 | (scan-reduce-kv {} 396 | (fn scan-fn [cursor] (wcar default-conn-manager (rcmd :HSCAN "my-hash" cursor))) 397 | (fn rf [acc k v] (assoc acc k v)))) 398 | 399 | ;;;; Scratch 400 | 401 | ;; TODO For command docstrings 402 | ;; As with all Carmine Redis command fns: expects to be called within a `wcar` 403 | ;; body, and returns nil. The server's reply to this command will be included 404 | ;; in the replies returned by the enclosing `wcar`. 405 | -------------------------------------------------------------------------------- /test/taoensso/carmine/tests/message_queue.clj: -------------------------------------------------------------------------------- 1 | (ns taoensso.carmine.tests.message-queue 2 | (:require 3 | [clojure.test :as test :refer [deftest testing is]] 4 | [taoensso.encore :as enc] 5 | [taoensso.carmine :as car :refer [wcar]] 6 | [taoensso.carmine.message-queue :as mq] 7 | [taoensso.carmine.tests.config :as config])) 8 | 9 | (comment 10 | (remove-ns 'taoensso.carmine.tests.message-queue) 11 | (test/run-tests 'taoensso.carmine.tests.message-queue)) 12 | 13 | ;;;; Utils, etc. 14 | 15 | (defn subvec? [v sub] 16 | (enc/reduce-indexed 17 | (fn [acc idx in] 18 | (if (= in (get v idx ::nx)) 19 | acc 20 | (reduced false))) 21 | true 22 | sub)) 23 | 24 | (comment 25 | [(subvec? [:a :b :c] [:a :b]) 26 | (subvec? [:a :b] [:a :b :c])]) 27 | 28 | ;;;; Config, etc. 29 | 30 | (def conn-opts config/conn-opts) 31 | (defmacro wcar* [& body] `(car/wcar conn-opts ~@body)) 32 | 33 | (def tq "carmine-test-queue") 34 | (defn clear-tq! [] (mq/queues-clear!! conn-opts [tq])) 35 | 36 | (defn test-fixture [f] (f) (clear-tq!)) 37 | (test/use-fixtures :once test-fixture) ; Just for final teardown 38 | 39 | (def ^:const default-lock-ms (enc/ms :mins 60)) 40 | (def ^:const eoq-backoff-ms 100) 41 | 42 | (do 43 | (def enqueue mq/enqueue) 44 | (def msg-status mq/message-status) 45 | 46 | (let [default-opts {:eoq-backoff-ms eoq-backoff-ms}] 47 | (defn- dequeue [qname & [opts]] 48 | (#'mq/dequeue qname (conj default-opts opts))))) 49 | 50 | (defn sleep 51 | ([ n] (sleep nil n)) 52 | ([isleep-on n] 53 | (let [n (int (case n :eoq (* 2.5 eoq-backoff-ms) n))] 54 | (if-let [on isleep-on] 55 | (#'mq/interruptible-sleep conn-opts tq on n) 56 | (Thread/sleep n)) 57 | 58 | (if-let [on isleep-on] 59 | (str "islept " n "msecs on " isleep-on) 60 | (str "slept " n "msecs"))))) 61 | 62 | ;;;; 63 | 64 | (defn throw! [] (throw (Exception.))) 65 | (defn handle-end-of-circle [isleep-on] 66 | (let [reply (wcar* (dequeue tq))] 67 | (every? identity 68 | [(is (= reply ["sleep" "end-of-circle" isleep-on eoq-backoff-ms])) 69 | (is (subvec? (#'mq/handle1 conn-opts tq (fn hf [_] (throw!)) reply {}) 70 | [:slept "end-of-circle" isleep-on #_eoq-backoff-ms])) 71 | (sleep isleep-on :eoq)]))) 72 | 73 | ;;;; 74 | 75 | (deftest basics 76 | (testing "Basic enqueue & dequeue" 77 | (clear-tq!) 78 | [(is (= (wcar* (dequeue tq)) ["sleep" "end-of-circle" "a" eoq-backoff-ms])) 79 | (sleep "a" :eoq) 80 | 81 | (is (= (wcar* (enqueue tq :msg1a {:mid :mid1})) {:success? true, :action :added, :mid :mid1})) 82 | (is (= (wcar* (enqueue tq :msg1b {:mid :mid1})) {:success? false, :error :already-queued}) "Dupe mid") 83 | (is (= (wcar* (enqueue tq :msg1b {:mid :mid1 :can-update? true})) {:success? true, :action :updated, :mid :mid1})) 84 | 85 | (is (= (wcar* (msg-status tq :mid1)) :queued)) 86 | (is (enc/submap? (#'mq/queue-mids conn-opts tq) 87 | {:ready ["mid1"] 88 | :circle ["end-of-circle"]})) 89 | 90 | (is (subvec? (wcar* (dequeue tq)) ["handle" "mid1" :msg1b 1 default-lock-ms #_udt])) 91 | (is (= (wcar* (msg-status tq :mid1)) :locked)) 92 | (is (= (wcar* (dequeue tq)) ["sleep" "end-of-circle" "a" eoq-backoff-ms])) 93 | (is (contains? (mq/queue-names conn-opts) tq))])) 94 | 95 | (deftest init-backoff 96 | (testing "Enqueue with initial backoff" 97 | (clear-tq!) 98 | [(is (= (wcar* (dequeue tq)) ["sleep" "end-of-circle" "a" eoq-backoff-ms])) 99 | (is (= (wcar* (enqueue tq :msg1 {:mid :mid1 :init-backoff-ms 500})) {:success? true, :action :added, :mid :mid1})) 100 | (is (= (wcar* (enqueue tq :msg2 {:mid :mid2 :init-backoff-ms 100})) {:success? true, :action :added, :mid :mid2})) 101 | 102 | (is (enc/submap? (#'mq/queue-mids conn-opts tq) 103 | {:ready [] 104 | :circle ["mid2" "mid1" "end-of-circle"]})) 105 | 106 | (is (enc/submap? (mq/queue-content conn-opts tq) 107 | {"mid1" {:message :msg1} 108 | "mid2" {:message :msg2}})) 109 | 110 | ;; Dupes before the backoff expired 111 | (is (= (wcar* (enqueue tq :msg1 {:mid :mid1})) {:success? false, :error :already-queued})) 112 | (is (= (wcar* (enqueue tq :msg2 {:mid :mid2})) {:success? false, :error :already-queued})) 113 | 114 | ;; Both should be queued with backoff before the backoff expires 115 | (is (= (wcar* (msg-status tq :mid1)) :queued-with-backoff)) 116 | (is (= (wcar* (msg-status tq :mid2)) :queued-with-backoff)) 117 | 118 | (sleep 150) ; > 2nd msg 119 | (is (= (wcar* (msg-status tq :mid1)) :queued-with-backoff)) 120 | (is (= (wcar* (msg-status tq :mid2)) :queued)) 121 | 122 | (sleep 750) ; > 1st msg 123 | (is (= (wcar* (msg-status tq :mid1)) :queued)) 124 | (is (= (wcar* (msg-status tq :mid2)) :queued)) 125 | 126 | ;; Dupes after backoff expired 127 | (is (= (wcar* (enqueue tq :msg1 {:mid :mid1})) {:success? false, :error :already-queued})) 128 | (is (= (wcar* (enqueue tq :msg2 {:mid :mid2})) {:success? false, :error :already-queued})) 129 | 130 | (is (= (wcar* (enqueue tq :msg2 {:mid :mid2 :init-backoff-ms 500 :reset-init-backoff? true})) 131 | {:success? true, :action :updated, :mid :mid2}) "Reset init backoff") 132 | 133 | (handle-end-of-circle "b") 134 | 135 | (is (subvec? (wcar* (dequeue tq)) ["handle" "mid1" :msg1 1 default-lock-ms #_udt])) 136 | (is (= (wcar* (msg-status tq :mid1)) :locked)) 137 | 138 | (is (subvec? (wcar* (dequeue tq)) ["skip" "queued-with-backoff"])) 139 | (is (= (wcar* (msg-status tq :mid2)) :queued-with-backoff))])) 140 | 141 | (defn test-handler 142 | "Returns [ ]" 143 | ([ hf] (test-handler false hf)) 144 | ([async? hf] 145 | (let [poll-reply (wcar* (dequeue tq)) 146 | handler-arg_ (promise) 147 | handle1 148 | (fn [] 149 | (#'mq/handle1 conn-opts tq 150 | (fn [m] (deliver handler-arg_ m) (hf m)) 151 | poll-reply {})) 152 | 153 | handle1-result 154 | (if async? 155 | (future-call handle1) 156 | (do (handle1)))] 157 | 158 | [poll-reply (deref handler-arg_ 5000 :timeout) handle1-result]))) 159 | 160 | (deftest handlers 161 | [(testing "Handler => success" 162 | (clear-tq!) 163 | [(is (= (wcar* (enqueue tq :msg1 {:mid :mid1})) {:success? true, :action :added, :mid :mid1})) 164 | 165 | (let [[pr ha hr] (test-handler (fn [_m] {:status :success}))] 166 | [(is (subvec? pr ["handle" "mid1" :msg1 1 default-lock-ms #_udt])) 167 | (is (enc/submap? ha 168 | {:qname "carmine-test-queue", :mid "mid1", :message :msg1, 169 | :attempt 1, :lock-ms default-lock-ms})) 170 | (is (= hr [:handled :success]))]) 171 | 172 | (is (= (wcar* (msg-status tq :mid1)) :done-awaiting-gc)) 173 | (handle-end-of-circle "a") 174 | (is (= (wcar* (dequeue tq)) ["skip" "did-gc"])) 175 | (is (= (wcar* (msg-status tq :mid1)) nil))]) 176 | 177 | (testing "Handler => throws" 178 | (clear-tq!) 179 | [(is (= (wcar* (enqueue tq :msg1 {:mid :mid1})) {:success? true, :action :added, :mid :mid1})) 180 | 181 | (let [[pr ha hr] (test-handler (fn [_m] (throw!)))] 182 | [(is (subvec? pr ["handle" "mid1" :msg1 1 default-lock-ms #_udt])) 183 | (is (= hr [:handled :error]))]) 184 | 185 | (is (= (wcar* (msg-status tq :mid1)) :done-awaiting-gc )) 186 | (handle-end-of-circle "a") 187 | (is (= (wcar* (dequeue tq)) ["skip" "did-gc"])) 188 | (is (= (wcar* (msg-status tq :mid1)) nil))]) 189 | 190 | (testing "Handler => success with backoff (dedupe)" 191 | (clear-tq!) 192 | [(is (= (wcar* (enqueue tq :msg1 {:mid :mid1})) {:success? true, :action :added, :mid :mid1})) 193 | 194 | (let [[pr ha hr] (test-handler (fn [_m] {:status :success :backoff-ms 2000}))] 195 | [(is (subvec? pr ["handle" "mid1" :msg1 1 default-lock-ms #_udt])) 196 | (is (= hr [:handled :success]))]) 197 | 198 | (is (= (wcar* (msg-status tq :mid1)) :done-with-backoff)) 199 | (handle-end-of-circle "a") 200 | (is (= (wcar* (dequeue tq)) ["skip" "done-with-backoff"])) 201 | 202 | (sleep 2500) ; > handler backoff 203 | (is (= (wcar* (msg-status tq :mid1)) :done-awaiting-gc)) 204 | (handle-end-of-circle "b") 205 | 206 | (is (= (wcar* (dequeue tq)) ["skip" "did-gc"]))]) 207 | 208 | (testing "Handler => retry with backoff" 209 | (clear-tq!) 210 | [(is (= (wcar* (enqueue tq :msg1 {:mid :mid1})) {:success? true, :action :added, :mid :mid1})) 211 | 212 | (let [[pr ha hr] (test-handler (fn [_m] {:status :retry :backoff-ms 2000}))] 213 | [(is (subvec? pr ["handle" "mid1" :msg1 1 default-lock-ms #_udt])) 214 | (is (= hr [:handled :retry]))]) 215 | 216 | (is (= (wcar* (msg-status tq :mid1)) :queued-with-backoff)) 217 | (handle-end-of-circle "a") 218 | (is (= (wcar* (dequeue tq)) ["skip" "queued-with-backoff"])) 219 | 220 | (sleep 2500) ; > handler backoff 221 | (is (= (wcar* (msg-status tq :mid1)) :queued)) 222 | (handle-end-of-circle "b") 223 | 224 | (is (subvec? (wcar* (dequeue tq)) ["handle" "mid1" :msg1 2 default-lock-ms #_udt]))]) 225 | 226 | (testing "Handler => lock timeout" 227 | 228 | (testing "Default lock time" 229 | (clear-tq!) 230 | [(is (= (wcar* (enqueue tq :msg1 {:mid :mid1})) {:success? true, :action :added, :mid :mid1})) 231 | 232 | ;; Simulate bad handler 233 | (is (subvec? (wcar* (dequeue tq {:default-lock-ms 1000})) ["handle" "mid1" :msg1 1 1000 #_udt])) 234 | 235 | (is (= (wcar* (msg-status tq :mid1)) :locked)) 236 | (handle-end-of-circle "a") 237 | 238 | (sleep 1500) ; Wait for lock to expire 239 | (is (subvec? (wcar* (dequeue tq {:default-lock-ms 1000})) ["handle" "mid1" :msg1 2 1000 #_udt]))]) 240 | 241 | (testing "Custom lock time" 242 | (clear-tq!) 243 | [(is (= (wcar* (enqueue tq :msg1 {:mid :mid1 :lock-ms 2000})) {:success? true, :action :added, :mid :mid1})) 244 | 245 | ;; Simulate bad handler 246 | (is (subvec? (wcar* (dequeue tq {:default-lock-ms 500})) ["handle" "mid1" :msg1 1 2000 #_udt])) 247 | 248 | (is (= (wcar* (msg-status tq :mid1)) :locked)) 249 | (handle-end-of-circle "a") 250 | 251 | (sleep 2500) ; Wait for lock to expire 252 | (is (subvec? (wcar* (dequeue tq {:default-lock-ms 500})) ["handle" "mid1" :msg1 2 2000 #_udt]))]))]) 253 | 254 | (deftest requeue 255 | [(testing "Enqueue while :locked" 256 | (clear-tq!) 257 | [(is (= (wcar* (enqueue tq :msg1a {:mid :mid1})) {:success? true, :action :added, :mid :mid1})) 258 | 259 | (do (test-handler :async (fn [_m] (Thread/sleep 2000) {:status :success})) :async-handler-running) 260 | 261 | (is (= (wcar* (msg-status tq :mid1)) :locked)) 262 | (is (= (wcar* (enqueue tq :msg1b {:mid :mid1})) {:success? false, :error :locked})) 263 | 264 | (is (= (wcar* (enqueue tq :msg1c {:mid :mid1, :can-requeue? true})) {:success? true, :action :added, :mid :mid1})) 265 | (is (= (wcar* (enqueue tq :msg1d {:mid :mid1, :can-requeue? true})) {:success? false, :error :already-queued})) 266 | (is (= (wcar* (enqueue tq :msg1e {:mid :mid1, :can-requeue? true, 267 | :can-update? true, :lock-ms 500})) {:success? true, :action :updated, :mid :mid1})) 268 | 269 | (is (= (wcar* (msg-status tq :mid1)) :locked-with-requeue)) 270 | (sleep 2500) ; > handler lock 271 | (is (= (wcar* (msg-status tq :mid1)) :done-with-requeue) "Not :done-awaiting-gc") 272 | (handle-end-of-circle "a") 273 | 274 | (is (= (wcar* (dequeue tq)) ["skip" "did-requeue"])) 275 | (is (subvec? (wcar* (dequeue tq)) ["handle" "mid1" :msg1e 1 500 #_udt]))]) 276 | 277 | (testing "Enqueue while :done-with-backoff" 278 | (clear-tq!) 279 | [(is (= (wcar* (enqueue tq :msg1a {:mid :mid1})) {:success? true, :action :added, :mid :mid1})) 280 | 281 | (do (test-handler (fn [_m] {:status :success :backoff-ms 2000})) :ran-handler) 282 | 283 | (is (= (wcar* (msg-status tq :mid1)) :done-with-backoff)) 284 | (is (= (wcar* (enqueue tq :msg1b {:mid :mid1})) {:success? false, :error :backoff})) 285 | (is (= (wcar* (enqueue tq :msg1c {:mid :mid1, :can-requeue? true, 286 | :lock-ms 500})) {:success? true, :action :added, :mid :mid1})) 287 | (is (= (wcar* (msg-status tq :mid1)) :done-with-requeue)) 288 | 289 | (handle-end-of-circle "a") 290 | (sleep 2500) ; > handler backoff 291 | 292 | (is (= (wcar* (dequeue tq)) ["skip" "did-requeue"])) 293 | (is (subvec? (wcar* (dequeue tq)) ["handle" "mid1" :msg1c 1 500 #_udt]))])]) 294 | 295 | (deftest workers 296 | (testing "Basic worker functionality" 297 | (clear-tq!) 298 | (let [msgs_ (atom []) 299 | handler-fn 300 | (fn [{:keys [mid message] :as in}] 301 | (swap! msgs_ conj message) 302 | {:status :success})] 303 | 304 | (with-open [^java.io.Closeable worker 305 | (mq/worker conn-opts tq 306 | {:auto-start false, 307 | :handler handler-fn 308 | :throttle-ms 10 309 | :eoq-backoff-ms 10})] 310 | 311 | [(is (enc/submap? (wcar* (enqueue tq :msg1 {:mid :mid1})) {:success? true, :action :added})) 312 | (is (enc/submap? (wcar* (enqueue tq :msg2 {:mid :mid2})) {:success? true, :action :added})) 313 | 314 | (is (enc/submap? (worker :queue-mids) 315 | {:ready ["mid2" "mid1"] 316 | :circle ["end-of-circle"]})) 317 | 318 | (is (mq/start worker)) 319 | (is (:running? @worker)) 320 | 321 | (sleep 1000) 322 | (is (= @msgs_ [:msg1 :msg2])) 323 | (is (enc/submap? (worker :queue-mids) 324 | {:ready [] 325 | :circle ["end-of-circle"]})) 326 | 327 | (is (mq/stop worker))])))) 328 | --------------------------------------------------------------------------------