├── tests ├── config.nims ├── bench.nim └── test.nim ├── config.nims ├── .gitignore ├── ready.nimble ├── examples ├── basic.nim ├── pool.nim ├── pool_connect_callback.nim ├── mummy_server.nim ├── pipeline.nim ├── pool_borrow_callback.nim └── pubsub.nim ├── src ├── ready.nim └── ready │ ├── pools.nim │ └── connections.nim ├── .github └── workflows │ └── docs.yml ├── LICENSE └── README.md /tests/config.nims: -------------------------------------------------------------------------------- 1 | --path:"../src" 2 | -------------------------------------------------------------------------------- /config.nims: -------------------------------------------------------------------------------- 1 | when (NimMajor, NimMinor, NimPatch) < (2, 0, 0): 2 | --threads:on 3 | --mm:orc 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # ignore files with no extention: 2 | * 3 | !*/ 4 | !*.* 5 | 6 | # normal ignores: 7 | *.exe 8 | nimcache 9 | *.pdb 10 | *.ilk 11 | .* 12 | -------------------------------------------------------------------------------- /ready.nimble: -------------------------------------------------------------------------------- 1 | version = "0.1.9" 2 | author = "Ryan Oldenburg" 3 | description = "A Redis client for multi-threaded servers" 4 | license = "MIT" 5 | 6 | srcDir = "src" 7 | 8 | requires "nim >= 1.6.10" 9 | -------------------------------------------------------------------------------- /examples/basic.nim: -------------------------------------------------------------------------------- 1 | import ready, std/options 2 | 3 | ## This example demonstrates opening a Redis connection, sending a command and 4 | ## then receiving the reply. 5 | 6 | let redis = newRedisConn() # Defaults to localhost:6379 7 | 8 | let reply = redis.command("GET", "mykey").to(Option[string]) 9 | 10 | echo reply 11 | -------------------------------------------------------------------------------- /src/ready.nim: -------------------------------------------------------------------------------- 1 | import ready/connections 2 | export connections 3 | 4 | when compileOption("threads"): 5 | when not defined(nimdoc): 6 | when not defined(gcArc) and not defined(gcOrc): 7 | {.error: "Ready requires --mm:arc or --mm:orc when --threads:on.".} 8 | 9 | ## Using the connection pool requires --threads:on 10 | import ready/pools 11 | export pools 12 | -------------------------------------------------------------------------------- /examples/pool.nim: -------------------------------------------------------------------------------- 1 | import ready 2 | 3 | ## This example demonstrates creating a pool Redis connections, then using 4 | ## one of those connections to execute a command before it is returned 5 | ## to the pool. 6 | 7 | ## nim c --threads:on --mm:orc -r examples/pool.nim 8 | 9 | let pool = newRedisPool(2) # Defaults to localhost:6379 10 | 11 | echo pool.command("INCR", "number").to(int) 12 | -------------------------------------------------------------------------------- /examples/pool_connect_callback.nim: -------------------------------------------------------------------------------- 1 | import ready 2 | 3 | ## This example demonstrates creating a pool Redis connections and setting them 4 | ## up using the onConnect callback. 5 | 6 | ## nim c --threads:on --mm:orc -r examples/pool_connect_callback.nim 7 | 8 | proc setUpRedisConn(conn: RedisConn) = 9 | ## This proc is called for every Redis connection opened in the pool. 10 | ## Use this callback to set up the connection, such as sending AUTH 11 | ## and SELECT commands as needed. 12 | # discard conn.command("AUTH", "password") 13 | discard conn.command("SELECT", "0") 14 | 15 | let pool = newRedisPool(2, onConnect = setUpRedisConn) 16 | -------------------------------------------------------------------------------- /examples/mummy_server.nim: -------------------------------------------------------------------------------- 1 | import mummy, mummy/routers, ready 2 | 3 | ## This example shows how to use Ready and Mummy for an HTTP server. 4 | ## 5 | ## We set up a simple server and use Redis to maintain a simple request counter. 6 | 7 | let pool = newRedisPool(2) # Defaults to localhost:6379 8 | 9 | proc indexHandler(request: Request) = 10 | let count = pool.command("INCR", "index_request_counter").to(int) 11 | 12 | var headers: HttpHeaders 13 | headers["Content-Type"] = "text/plain" 14 | request.respond( 15 | 200, 16 | headers, 17 | "Hello, World! This is request " & $count & " to this server." 18 | ) 19 | 20 | var router: Router 21 | router.get("/", indexHandler) 22 | 23 | let server = newServer(router) 24 | echo "Serving on http://localhost:8080" 25 | server.serve(Port(8080)) 26 | -------------------------------------------------------------------------------- /tests/bench.nim: -------------------------------------------------------------------------------- 1 | import ready, std/times 2 | 3 | const 4 | iterations = 10_000 5 | testKey = "test_key" 6 | 7 | var testValue: string 8 | for i in 0 ..< 10_000: 9 | testValue.add('a') 10 | 11 | block: 12 | let ready = newRedisConn() 13 | discard ready.command("SET", testKey, testValue) 14 | ready.close() 15 | 16 | block: 17 | let start = epochTime() 18 | 19 | let ready = newRedisConn() 20 | 21 | for _ in 0 ..< iterations: 22 | let r = ready.command("GET", testKey).to(string) 23 | doAssert r.len == testValue.len 24 | 25 | echo epochTime() - start 26 | 27 | import redis 28 | 29 | block: 30 | let start = epochTime() 31 | 32 | let nim = open() 33 | 34 | for _ in 0 ..< iterations: 35 | let r = nim.get(testKey) 36 | doAssert r.len == testValue.len 37 | 38 | echo epochTime() - start 39 | -------------------------------------------------------------------------------- /examples/pipeline.nim: -------------------------------------------------------------------------------- 1 | import ready 2 | 3 | ## This example demonstrates pipelining mulitple Redis commands before 4 | ## calling receive to get the replies. 5 | ## 6 | ## Remember to call receive for every command you sent~ 7 | 8 | let redis = newRedisConn() # Defaults to localhost:6379 9 | 10 | redis.send("MULTI") 11 | redis.send("INCR", "mycount") 12 | redis.send("SET", "mykey", "myvalue") 13 | redis.send("EXEC") 14 | 15 | ## OR: 16 | 17 | # redis.send([ 18 | # ("MULTI", @[]), 19 | # ("INCR", @["mycount"]), 20 | # ("SET", @["mykey", "myvalue"]), 21 | # ("EXEC", @[]) 22 | #]) 23 | 24 | # Match the number of `receive` calls to the number of commands sent 25 | 26 | discard redis.receive() # OK 27 | discard redis.receive() # QUEUED 28 | discard redis.receive() # QUEUED 29 | let (num, _) = redis.receive().to((int, string)) 30 | 31 | echo num 32 | -------------------------------------------------------------------------------- /examples/pool_borrow_callback.nim: -------------------------------------------------------------------------------- 1 | import ready, std/times 2 | 3 | ## This example demonstrates using the borrow callback to verify a Redis 4 | ## connection is open before issuing commands. 5 | 6 | ## nim c --threads:on --mm:orc -r examples/pool_borrow_callback.nim 7 | 8 | proc onBorrow(conn: RedisConn, lastReturned: float) = 9 | ## This proc is called each time a Redis connection is borrowed from the pool. 10 | ## You can use it to verify a connection is open. 11 | ## Raising an exception in this callback will close this Redis connection and 12 | ## a new Redis connection will be opened. 13 | if epochTime() - lastReturned > 4 * 60: 14 | # If this PING fails, a RedisError exception will be raised. 15 | discard conn.command("PING") 16 | 17 | let pool = newRedisPool(2, onBorrow = onBorrow) 18 | 19 | pool.withConnection conn: 20 | echo "Borrowed ", conn 21 | -------------------------------------------------------------------------------- /examples/pubsub.nim: -------------------------------------------------------------------------------- 1 | import ready, std/os 2 | 3 | ## This example shows how to do a simple multi-threaded PubSub where 4 | ## a thread is dedicated to receiving incoming messages and the other thread 5 | ## is free to send commands to manage the connection, such as more SUBSCRIBE 6 | ## commands or UNSUBSCRIBE commands. 7 | 8 | ## nim c --threads:on --mm:orc -r examples/pubsub.nim 9 | 10 | let pubsub = newRedisConn() # Defaults to localhost:6379 11 | 12 | proc receiveThreadProc() = 13 | try: 14 | while true: 15 | let reply = pubsub.receive() 16 | echo "Event: ", reply[0].to(string) 17 | echo "Channel: ", reply[1].to(string) 18 | echo "Raw: ", reply 19 | except RedisError as e: 20 | echo e.msg 21 | 22 | var receiveThread: Thread[void] 23 | createThread(receiveThread, receiveThreadProc) 24 | 25 | pubsub.send("SUBSCRIBE", "mychannel") 26 | 27 | sleep(5000) 28 | -------------------------------------------------------------------------------- /.github/workflows/docs.yml: -------------------------------------------------------------------------------- 1 | name: docs 2 | on: 3 | push: 4 | branches: 5 | - master 6 | env: 7 | nim-version: 'stable' 8 | nim-src: src/${{ github.event.repository.name }}.nim 9 | deploy-dir: .gh-pages 10 | jobs: 11 | docs: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v3 15 | - uses: jiro4989/setup-nim-action@v1 16 | with: 17 | nim-version: ${{ env.nim-version }} 18 | - run: nimble install -Y 19 | - run: nimble doc --index:on --project --git.url:https://github.com/${{ github.repository }} --git.commit:master --out:${{ env.deploy-dir }} ${{ env.nim-src }} 20 | - name: "Copy to index.html" 21 | run: cp ${{ env.deploy-dir }}/${{ github.event.repository.name }}.html ${{ env.deploy-dir }}/index.html 22 | - name: Deploy documents 23 | uses: peaceiris/actions-gh-pages@v3 24 | with: 25 | github_token: ${{ secrets.GITHUB_TOKEN }} 26 | publish_dir: ${{ env.deploy-dir }} 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2023 Ryan Oldenburg 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 10 | -------------------------------------------------------------------------------- /src/ready/pools.nim: -------------------------------------------------------------------------------- 1 | import connections, std/locks, std/random, std/tables, std/times 2 | 3 | type 4 | RedisPoolObj = object 5 | address: string 6 | port: Port 7 | conns: seq[RedisConn] 8 | lock: Lock 9 | cond: Cond 10 | r: Rand 11 | lastReturned: Table[RedisConn, float] 12 | onConnect: proc(conn: RedisConn) {.gcsafe.} 13 | onBorrow: proc(conn: RedisConn, lastReturned: float) {.gcsafe.} 14 | 15 | RedisPool* = ptr RedisPoolObj 16 | 17 | proc close*(pool: RedisPool) = 18 | ## Closes the Redis connections in the pool then deallocates the pool. 19 | ## All connections should be returned to the pool before it is closed. 20 | withLock pool.lock: 21 | for conn in pool.conns: 22 | conn.close() 23 | `=destroy`(pool[]) 24 | deallocShared(pool) 25 | 26 | proc openNewConnection(pool: RedisPool): RedisConn = 27 | result = newRedisConn(pool.address, pool.port) 28 | if pool.onConnect != nil: 29 | pool.onConnect(result) 30 | 31 | proc recycle*(pool: RedisPool, conn: RedisConn) {.raises: [], gcsafe.} = 32 | ## Returns a Redis connection to the pool. 33 | withLock pool.lock: 34 | pool.conns.add(conn) 35 | pool.r.shuffle(pool.conns) 36 | pool.lastReturned[conn] = epochTime() 37 | signal(pool.cond) 38 | 39 | proc newRedisPool*( 40 | size: int, 41 | address = "localhost", 42 | port = Port(6379), 43 | onConnect: proc(conn: RedisConn) {.gcsafe.} = nil, 44 | onBorrow: proc(conn: RedisConn, lastReturned: float) {.gcsafe.} = nil 45 | ): RedisPool = 46 | ## Creates a new thead-safe pool of Redis connections. 47 | if size <= 0: 48 | raise newException(CatchableError, "Invalid pool size") 49 | result = cast[RedisPool](allocShared0(sizeof(RedisPoolObj))) 50 | result.port = port 51 | result.address = address 52 | initLock(result.lock) 53 | initCond(result.cond) 54 | result.r = initRand(2023) 55 | result.onConnect = onConnect 56 | result.onBorrow = onBorrow 57 | try: 58 | for _ in 0 ..< size: 59 | result.recycle(result.openNewConnection()) 60 | except: 61 | try: 62 | result.close() 63 | except: 64 | discard 65 | raise getCurrentException() 66 | 67 | proc borrow*(pool: RedisPool): RedisConn {.gcsafe.} = 68 | ## Removes a Redis connection from the pool. This call blocks until it can take 69 | ## a connection. Remember to add the connection back to the pool with recycle 70 | ## when you're finished with it. 71 | 72 | var lastReturned: float 73 | acquire(pool.lock) 74 | while pool.conns.len == 0: 75 | wait(pool.cond, pool.lock) 76 | result = pool.conns.pop() 77 | lastReturned = pool.lastReturned.getOrDefault(result, 0) 78 | release(pool.lock) 79 | 80 | if pool.onBorrow != nil: 81 | try: 82 | pool.onBorrow(result, lastReturned) 83 | except: 84 | # Close this connection and open a new one 85 | withLock pool.lock: 86 | pool.lastReturned.del(result) 87 | result.close() 88 | result = pool.openNewConnection() 89 | if pool.onBorrow != nil: 90 | pool.onBorrow(result, epochTime()) 91 | 92 | template withConnection*(pool: RedisPool, conn, body) = 93 | block: 94 | let conn = pool.borrow() 95 | try: 96 | body 97 | finally: 98 | pool.recycle(conn) 99 | 100 | proc command*( 101 | pool: RedisPool, 102 | command: string, 103 | args: varargs[string] 104 | ): RedisReply = 105 | ## Borrows a Redis connection from the pool, sends a command to the 106 | ## server and receives the reply. 107 | pool.withConnection conn: 108 | result = conn.command(command, args) 109 | -------------------------------------------------------------------------------- /tests/test.nim: -------------------------------------------------------------------------------- 1 | import ready, std/os, std/options 2 | 3 | block: 4 | let redis = newRedisConn() 5 | discard redis.command("SET", "foo", "bar") 6 | doAssert redis.command("GET", "foo").to(string) == "bar" 7 | redis.close() 8 | 9 | block: 10 | let redis = newRedisConn() 11 | doAssertRaises RedisError: 12 | discard redis.command("STE", "foo", "bar") 13 | discard redis.command("SET", "foo", "bar") 14 | redis.close() 15 | 16 | block: 17 | let redis = newRedisConn() 18 | doAssert not redis.command("GET", "does_not_exist").to(Option[string]).isSome 19 | discard redis.command("SET", "empty", "") 20 | doAssert redis.command("GET", "empty").to(string) == "" 21 | redis.close() 22 | 23 | block: 24 | let redis = newRedisConn() 25 | redis.send([ 26 | ("SET", @["key1", "value1"]), 27 | ("SET", @["key2", "value2"]), 28 | ("SET", @["key3", "value3"]) 29 | ]) 30 | discard redis.receive() 31 | discard redis.receive() 32 | discard redis.receive() 33 | let values = redis.command("MGET", "key1", "key2", "key3").to(seq[string]) 34 | doAssert values == @["value1", "value2", "value3"] 35 | redis.close() 36 | 37 | block: 38 | let redis = newRedisConn() 39 | redis.send([ 40 | ("SET", @["key1", "value1"]), 41 | ("SET", @["key2", "value2"]), 42 | ("SET", @["key3", "value3"]) 43 | ]) 44 | discard redis.receive() 45 | discard redis.receive() 46 | discard redis.receive() 47 | let values = redis.command("MGET", "key3", "key4").to(seq[Option[string]]) 48 | doAssert values == @[some("value3"), none(string)] 49 | redis.close() 50 | 51 | block: 52 | let redis = newRedisConn() 53 | redis.send([ 54 | ("SET", @["count1", "1"]), 55 | ("SET", @["count2", "2"]), 56 | ("SET", @["count3", "3"]) 57 | ]) 58 | discard redis.receive() 59 | discard redis.receive() 60 | discard redis.receive() 61 | let values = redis.command("MGET", "count2", "count4").to(seq[Option[int]]) 62 | doAssert values == @[some(2), none(int)] 63 | redis.close() 64 | 65 | block: 66 | proc onConnect(conn: RedisConn) = 67 | echo "onConnect" 68 | 69 | proc onBorrow(conn: RedisConn, lastReturned: float) = 70 | echo "onBorrow" 71 | discard conn.command("PING") 72 | 73 | let pool = newRedisPool(1, onConnect = onConnect, onBorrow = onBorrow) 74 | pool.withConnection redis: 75 | discard redis.command("SET", "mynumber", "0") 76 | redis.send("INCR", "mynumber") 77 | redis.send("INCR", "mynumber") 78 | redis.send("INCR", "mynumber") 79 | redis.send("INCR", "mynumber") 80 | doAssert redis.receive().to(int) == 1 81 | doAssert redis.receive().to(int) == 2 82 | doAssert redis.receive().to(int) == 3 83 | doAssert redis.receive().to(int) == 4 84 | pool.close() 85 | 86 | block: 87 | let pubsub = newRedisConn() 88 | 89 | var received: seq[RedisReply] 90 | 91 | proc receiveThreadProc() = 92 | try: 93 | while true: 94 | {.gcsafe.}: 95 | received.add(pubsub.receive()) 96 | except RedisError as e: 97 | echo e.msg 98 | 99 | var receiveThread: Thread[void] 100 | createThread(receiveThread, receiveThreadProc) 101 | 102 | pubsub.send("SUBSCRIBE", "mychannel") 103 | 104 | proc publishThreadProc() = 105 | let publisher = newRedisConn() 106 | 107 | for i in 0 ..< 10: 108 | discard publisher.command("PUBLISH", "mychannel", $i) 109 | 110 | publisher.close() 111 | 112 | var publishThread: Thread[void] 113 | createThread(publishThread, publishThreadProc) 114 | 115 | joinThread(publishThread) 116 | 117 | sleep(100) 118 | 119 | pubsub.close() 120 | 121 | doAssert received[0].to((string, string, int)) == ("subscribe", "mychannel", 1) 122 | for i in 0 ..< 10: 123 | doAssert received[i + 1].to((string, string, string)) == 124 | ("message", "mychannel", $i) 125 | 126 | sleep(1000) 127 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Ready 2 | 3 | `nimble install ready` 4 | 5 | [API reference](https://guzba.github.io/ready/) 6 | 7 | Ready is a Redis client that is built to work well in multi-threaded programs. A great use-case for Ready is in a multi-threaded HTTP server like [Mummy](https://github.com/guzba/mummy). 8 | 9 | Check out the [examples/](https://github.com/guzba/ready/tree/master/examples) folder for more sample code using Ready. 10 | 11 | ## Using Ready 12 | 13 | First you'll need to open a Redis connection. By default Ready connects to the default Redis server at localhost:6379. You can easily specify a different address and port in `newRedisConn` when needed. 14 | 15 | ```nim 16 | import ready 17 | 18 | let redis = newRedisConn() # Defaults to localhost:6379 19 | ``` 20 | 21 | After opening a connection you can start sending commands. You can send any of Redis's vast set of commands. 22 | 23 | ```nim 24 | import ready, std/options 25 | 26 | let redis = newRedisConn() # Defaults to localhost:6379 27 | 28 | let value = redis.command("GET", "key").to(Option[string]) 29 | ``` 30 | 31 | We use `Option[string]` above since the reply may be nil if the key is not present. Alternatively, if you know the key exists, you could just use `string`. 32 | 33 | You can also easily work with replies to more complex commands: 34 | 35 | ```nim 36 | import ready 37 | 38 | let redis = newRedisConn() # Defaults to localhost:6379 39 | 40 | let values = redis.command("MGET", "key1", "key2", "key3").to(seq[string]) 41 | ``` 42 | 43 | Here we are using `MGET` to request multiple keys in one command. Since we expect multiple reply entries, we can use `to` to convert the reply to a `seq[string]`. 44 | 45 | ## Working with replies 46 | 47 | A call to `command` or `receive` will return a `RedisReply` object. You'll want to convert that into the types you expect. Ready makes that easy by providing the `to` proc. 48 | 49 | ```nim 50 | # Basic conversions: 51 | 52 | echo reply.to(int) 53 | echo reply.to(string) 54 | echo reply.to(Option[string]) # If the reply can be nil 55 | 56 | # Convert array replies to seq: 57 | 58 | echo reply.to(seq[int]) 59 | echo reply.to(seq[string]) 60 | echo reply.to(seq[Option[string]]) 61 | 62 | # Convert array replies to tuples: 63 | 64 | echo reply.to((int, string)) 65 | echo reply.to((int, Option[string])) 66 | echo reply.to((string, Option[string], int)) 67 | 68 | # Mix and match: 69 | 70 | echo reply.to((string, Option[string], seq[int])) 71 | 72 | # Index access, if you know the reply is an array you can access its elements 73 | 74 | echo reply[0].to(string) 75 | 76 | ``` 77 | 78 | A call to `reply.to` for a type Ready does not know how to convert to will fail at compile time. 79 | 80 | If Ready is unable to convert the reply from Redis to your requested type, a `RedisError` is raised. 81 | 82 | ## Connection pooling 83 | 84 | Ready includes a built-in connection pool when compiled with `--threads:on`: 85 | 86 | ```nim 87 | import ready 88 | 89 | let redisPool = newRedisPool(3) # Defaults to localhost:6379 90 | 91 | # This automatically removes a connection from the pool, runs the command 92 | # and then returns it back to the pool 93 | redisPool.command("PING") 94 | ``` 95 | 96 | Or, if you want to run more than one command with the same connection: 97 | 98 | ```nim 99 | import ready 100 | 101 | let redisPool = newRedisPool(3) # Defaults to localhost:6379 102 | 103 | redisPool.withConnection conn: 104 | # `conn` is automatically recycled back into the pool after this block 105 | discard conn.command("PING") 106 | ``` 107 | 108 | Reusing Redis connections is much faster and more efficient than opening new connections for every command. 109 | 110 | ## Pipelining commands and transactions 111 | 112 | Ready also includes separate `send` and `receive` calls as an alternative to the `command` call. These commands make pipelining commands easy: 113 | 114 | ```nim 115 | import ready 116 | 117 | let redis = newRedisConn() 118 | 119 | redis.send("MULTI") 120 | redis.send("INCR", "mycount") 121 | redis.send("SET", "mykey", "myvalue") 122 | redis.send("EXEC") 123 | 124 | ## OR: 125 | 126 | # redis.send([ 127 | # ("MULTI", @[]), 128 | # ("INCR", @["mycount"]), 129 | # ("SET", @["mykey", "myvalue"]), 130 | # ("EXEC", @[]) 131 | #]) 132 | 133 | # Match the number of `receive` calls to the number of commands sent 134 | 135 | discard redis.receive() # OK 136 | discard redis.receive() # QUEUED 137 | discard redis.receive() # QUEUED 138 | let (num, _) = redis.receive().to((int, string)) 139 | ``` 140 | 141 | Pipelining as an advanced technique when using Redis that can drastically increase performance when possible. 142 | 143 | Important! Remember to match the number of `receive` calls to the number of commands sent. 144 | 145 | ## Publish and subscribe (PubSub) 146 | 147 | Ready makes it easy to use Redis's [PubSub](https://redis.io/docs/manual/pubsub/) functionality. 148 | 149 | Here we dedicate a thread to receiving messages on a PubSub connection while our other thread is free to send commands like `SUBSCRIBE` and `UNSUBSCRIBE` to manage the PubSub connection. 150 | 151 | ```nim 152 | let pubsub = newRedisConn() # Defaults to localhost:6379 153 | 154 | proc receiveThreadProc() = 155 | try: 156 | while true: 157 | let reply = pubsub.receive() 158 | echo "Event: ", reply[0].to(string) 159 | echo "Channel: ", reply[1].to(string) 160 | echo "Raw: ", reply 161 | except RedisError as e: 162 | echo e.msg 163 | 164 | var receiveThread: Thread[void] 165 | createThread(receiveThread, receiveThreadProc) 166 | 167 | pubsub.send("SUBSCRIBE", "mychannel") 168 | ``` 169 | 170 | Note that using PubSub with Ready requires threads. 171 | 172 | ## Pro Tips 173 | 174 | You can use Ready in two ways, either by calling `command` or by calling `send` and `receive`. Calling `command` is the equivalent of calling `send` and then calling `receive` immediately. 175 | 176 | Whenever a `command` or `receive` call gets an error reply from Redis a `RedisError` is raised. This means discarding the reply in `discard redis.command("PING")` is perfectly ok. If the reply was an error an exception would have been raised. 177 | 178 | If you open a short-lived Redis connection, remember to call `close` when you no longer need it. The connections are not garbage collected. (For HTTP servers this is unlikely, see [#1](https://github.com/guzba/ready/issues/1#issuecomment-1586255510) for a brief discussion.) 179 | 180 | ## Why use `send` and `receive` separately? Two reasons: 181 | 182 | First, where possible, it is more efficient to pipeline many Redis commands. This is easy to do with Ready, just call `send` multiple times (or ideally call `send` with a seq of commands). 183 | 184 | Second, you may want to have a separate thread be sending vs receiving. A common use of this is [PubSub](https://redis.io/docs/manual/pubsub/), where one thread is dedicated to receiving messages and the sending thread manages what channels are subscribed to. See [this example](https://github.com/guzba/ready/blob/master/examples/pubsub.nim). 185 | -------------------------------------------------------------------------------- /src/ready/connections.nim: -------------------------------------------------------------------------------- 1 | import std/nativesockets, std/os, std/strutils, std/parseutils, 2 | std/options, std/typetraits, std/atomics 3 | 4 | when not defined(nimdoc): 5 | # nimdoc produces bizarre and annoying errors 6 | when defined(windows): 7 | import winlean 8 | elif defined(posix): 9 | import posix 10 | 11 | export Port 12 | 13 | ## RESP https://redis.io/docs/reference/protocol-spec/ 14 | 15 | const 16 | initialRecvBufLen = (32 * 1024) - 9 # 8 byte cap field + null terminator 17 | 18 | type 19 | RedisError* = object of CatchableError 20 | 21 | RedisConnObj = object 22 | socket: SocketHandle 23 | recvBuf: string 24 | bytesReceived, recvPos: int 25 | poisoned: Atomic[bool] 26 | 27 | RedisConn* = ptr RedisConnObj 28 | 29 | RedisReplyKind = enum 30 | SimpleStringReply, BulkStringReply, IntegerReply, ArrayReply 31 | 32 | RedisReply* = object 33 | case kind: RedisReplyKind 34 | of SimpleStringReply: 35 | simple: string 36 | of BulkStringReply: 37 | bulk: Option[string] 38 | of IntegerReply: 39 | value: int 40 | of ArrayReply: 41 | elements: seq[RedisReply] 42 | 43 | func `$`*(conn: RedisConn): string = 44 | "RedisConn " & $cast[uint](conn) 45 | 46 | func `$`*(reply: RedisReply): string = 47 | case reply.kind: 48 | of IntegerReply: 49 | $reply.value 50 | of SimpleStringReply: 51 | reply.simple 52 | of BulkStringReply: 53 | $reply.bulk 54 | of ArrayReply: 55 | '[' & join(reply.elements, ", ") & ']' 56 | 57 | proc close*(conn: RedisConn) {.raises: [], gcsafe.} = 58 | ## Closes and deallocates the connection. 59 | if conn.socket.int > 0: 60 | when not defined(nimdoc): 61 | discard conn.socket.shutdown(when defined(windows): 2 else: SHUT_RDWR) 62 | conn.socket.close() 63 | `=destroy`(conn[]) 64 | deallocShared(conn) 65 | 66 | template raisePoisonedConnError() = 67 | raise newException(RedisError, "Redis connection is in a broken state") 68 | 69 | proc send*( 70 | conn: RedisConn, 71 | commands: openarray[(string, seq[string])] 72 | ) {.raises: [RedisError].} = 73 | ## Sends commands to the Redis server. 74 | 75 | if conn.poisoned.load(moRelaxed): 76 | raisePoisonedConnError() 77 | 78 | var msg: string 79 | for (command, args) in commands: 80 | msg.add '*' & $(1 + args.len) & "\r\n" 81 | msg.add '$' & $command.len & "\r\n" & command & "\r\n" 82 | for arg in args: 83 | msg.add '$' & $arg.len & "\r\n" & arg & "\r\n" 84 | 85 | if conn.socket.send( 86 | msg[0].addr, 87 | msg.len.cint, 88 | when defined(MSG_NOSIGNAL): MSG_NOSIGNAL else: 0 89 | ) < 0: 90 | raise newException(RedisError, osErrorMsg(osLastError())) 91 | 92 | proc send*( 93 | conn: RedisConn, 94 | command: string, 95 | args: varargs[string] 96 | ) {.inline, raises: [RedisError].} = 97 | ## Sends a command to the Redis server. 98 | 99 | # conn.send([(command, toSeq(args))]) 100 | 101 | if conn.poisoned.load(moRelaxed): 102 | raisePoisonedConnError() 103 | 104 | var msg: string 105 | msg.add '*' & $(1 + args.len) & "\r\n" 106 | msg.add '$' & $command.len & "\r\n" & command & "\r\n" 107 | for arg in args: 108 | msg.add '$' & $arg.len & "\r\n" & arg & "\r\n" 109 | 110 | if conn.socket.send( 111 | msg[0].addr, 112 | msg.len.cint, 113 | when defined(MSG_NOSIGNAL): MSG_NOSIGNAL else: 0 114 | ) < 0: 115 | raise newException(RedisError, osErrorMsg(osLastError())) 116 | 117 | proc recvBytes(conn: RedisConn) {.raises: [RedisError].} = 118 | # Expand the buffer if it is full 119 | if conn.bytesReceived == conn.recvBuf.len: 120 | conn.recvBuf.setLen(conn.recvBuf.len * 2) 121 | 122 | # Read more response data 123 | let bytesReceived = conn.socket.recv( 124 | conn.recvBuf[conn.bytesReceived].addr, 125 | (conn.recvBuf.len - conn.bytesReceived).cint, 126 | 0 127 | ) 128 | if bytesReceived > 0: 129 | conn.bytesReceived += bytesReceived 130 | else: 131 | raise newException(RedisError, osErrorMsg(osLastError())) 132 | 133 | proc redisParseInt(conn: RedisConn): int = 134 | try: 135 | discard parseInt(conn.recvBuf, result, conn.recvPos) 136 | except ValueError: 137 | conn.poisoned.store(true, moRelaxed) 138 | raise newException(RedisError, "Error parsing number") 139 | 140 | proc receive*( 141 | conn: RedisConn 142 | ): RedisReply {.raises: [RedisError].} = 143 | ## Receives a single reply from the Redis server. 144 | 145 | if conn.poisoned.load(moRelaxed): 146 | raisePoisonedConnError() 147 | 148 | if conn.recvPos == conn.bytesReceived: 149 | # If we haven't received any response data yet do an initial recv 150 | conn.recvBytes() 151 | 152 | let dataType = conn.recvBuf[conn.recvPos] 153 | inc conn.recvPos 154 | case dataType: 155 | of '-': 156 | while true: 157 | let simpleEnd = conn.recvBuf.find("\r\n", conn.recvPos) 158 | if simpleEnd > 0: 159 | var msg = conn.recvBuf[conn.recvPos ..< simpleEnd] 160 | conn.recvPos = simpleEnd + 2 161 | raise newException(RedisError, move msg) 162 | conn.recvBytes() 163 | of '+': 164 | result = RedisReply(kind: SimpleStringReply) 165 | while true: 166 | let simpleEnd = conn.recvBuf.find("\r\n", conn.recvPos) 167 | if simpleEnd > 0: 168 | result.simple = conn.recvBuf[conn.recvPos ..< simpleEnd] 169 | conn.recvPos = simpleEnd + 2 170 | break 171 | conn.recvBytes() 172 | of ':': 173 | result = RedisReply(kind: IntegerReply) 174 | while true: 175 | let simpleEnd = conn.recvBuf.find("\r\n", conn.recvPos) 176 | if simpleEnd > 0: 177 | result.value = redisParseInt(conn) 178 | conn.recvPos = simpleEnd + 2 179 | break 180 | conn.recvBytes() 181 | of '$': 182 | result = RedisReply(kind: BulkStringReply) 183 | while true: 184 | let lenEnd = conn.recvBuf.find("\r\n", conn.recvPos) 185 | if lenEnd > 0: 186 | let strLen = redisParseInt(conn) 187 | if strLen >= 0: 188 | if conn.bytesReceived >= lenEnd + 2 + strLen + 2: 189 | var bulk = newString(strLen) 190 | copyMem(bulk.cstring, conn.recvBuf[lenEnd + 2].addr, strLen) 191 | result.bulk = some(move bulk) 192 | conn.recvPos = lenEnd + 2 + strLen + 2 193 | break 194 | else: 195 | conn.recvPos = lenEnd + 2 196 | result.bulk = none(string) 197 | break 198 | conn.recvBytes() 199 | of '*': 200 | result = RedisReply(kind: ArrayReply) 201 | while true: 202 | let numEnd = conn.recvBuf.find("\r\n", conn.recvPos) 203 | if numEnd > 0: 204 | let numElements = redisParseInt(conn) 205 | conn.recvPos = numEnd + 2 206 | for i in 0 ..< numElements: 207 | result.elements.add(conn.receive()) 208 | break 209 | conn.recvBytes() 210 | else: 211 | conn.poisoned.store(true, moRelaxed) 212 | raise newException( 213 | RedisError, 214 | "Unexpected RESP data type " & dataType & " (" & $dataType.uint8 & ")" 215 | ) 216 | 217 | # If we've read to the end of the recv buffer, reset 218 | if conn.recvPos > 0 and conn.bytesReceived == conn.recvPos: 219 | conn.bytesReceived = 0 220 | conn.recvPos = 0 221 | 222 | proc command*( 223 | conn: RedisConn, 224 | command: string, 225 | args: varargs[string] 226 | ): RedisReply {.raises: [RedisError]} = 227 | ## Sends a command to the Redis server and receives the reply. 228 | conn.send(command, args) 229 | conn.receive() 230 | 231 | proc newRedisConn*( 232 | address = "localhost", 233 | port = Port(6379) 234 | ): RedisConn {.raises: [OSError].} = 235 | result = cast[RedisConn](allocShared0(sizeof(RedisConnObj))) 236 | result.recvBuf.setLen(initialRecvBufLen) 237 | 238 | try: 239 | result.socket = createNativeSocket( 240 | Domain.AF_INET, 241 | SockType.SOCK_STREAM, 242 | Protocol.IPPROTO_TCP, 243 | false 244 | ) 245 | if result.socket == osInvalidSocket: 246 | raiseOSError(osLastError()) 247 | 248 | let ai = getAddrInfo( 249 | address, 250 | port, 251 | Domain.AF_INET, 252 | SockType.SOCK_STREAM, 253 | Protocol.IPPROTO_TCP, 254 | ) 255 | try: 256 | if result.socket.connect(ai.ai_addr, ai.ai_addrlen.SockLen) < 0: 257 | raiseOSError(osLastError()) 258 | finally: 259 | freeAddrInfo(ai) 260 | except OSError as e: 261 | result.close() 262 | raise e 263 | 264 | proc to*[T](reply: RedisReply, t: typedesc[T]): T = 265 | when t is SomeInteger: 266 | case reply.kind: 267 | of SimpleStringReply: 268 | raise newException(RedisError, "Cannot convert string to " & $t) 269 | of IntegerReply: 270 | cast[T](reply.value) 271 | of BulkStringReply: 272 | if reply.bulk.isSome: 273 | cast[T](parseInt(reply.bulk.get)) 274 | else: 275 | raise newException(RedisError, "Reply is nil") 276 | of ArrayReply: 277 | raise newException(RedisError, "Cannot convert array to " & $t) 278 | elif t is string: 279 | case reply.kind: 280 | of SimpleStringReply: 281 | reply.simple 282 | of BulkStringReply: 283 | if reply.bulk.isSome: 284 | reply.bulk.get 285 | else: 286 | raise newException(RedisError, "Reply is nil") 287 | of IntegerReply: 288 | $reply.value 289 | of ArrayReply: 290 | raise newException(RedisError, "Cannot convert array to " & $t) 291 | elif t is Option[string]: 292 | case reply.kind: 293 | of SimpleStringReply: 294 | some(reply.simple) 295 | of BulkStringReply: 296 | reply.bulk 297 | of IntegerReply: 298 | some($reply.value) 299 | of ArrayReply: 300 | raise newException(RedisError, "Cannot convert array to " & $t) 301 | elif t is Option[int]: 302 | case reply.kind: 303 | of SimpleStringReply: 304 | raise newException(RedisError, "Cannot convert string to " & $t) 305 | of BulkStringReply: 306 | if reply.bulk.isSome: 307 | some(parseInt(reply.bulk.get)) 308 | else: 309 | none(int) 310 | of IntegerReply: 311 | some(reply.value) 312 | of ArrayReply: 313 | raise newException(RedisError, "Cannot convert array to " & $t) 314 | elif t is seq[int]: 315 | case reply.kind: 316 | of ArrayReply: 317 | for element in reply.elements: 318 | result.add(element.to(int)) 319 | else: 320 | raise newException(RedisError, "Cannot convert non-array reply to " & $t) 321 | elif t is seq[string]: 322 | case reply.kind: 323 | of ArrayReply: 324 | for element in reply.elements: 325 | result.add(element.to(string)) 326 | else: 327 | raise newException(RedisError, "Cannot convert non-array reply to " & $t) 328 | elif t is seq[Option[string]]: 329 | case reply.kind: 330 | of ArrayReply: 331 | for element in reply.elements: 332 | case element.kind: 333 | of SimpleStringReply: 334 | result.add(some(element.simple)) 335 | of BulkStringReply: 336 | result.add(element.bulk) 337 | of IntegerReply: 338 | result.add(some($element.value)) 339 | of ArrayReply: 340 | raise newException(RedisError, "Cannot convert array to " & $t) 341 | else: 342 | raise newException(RedisError, "Cannot convert non-array reply to " & $t) 343 | elif t is seq[Option[int]]: 344 | case reply.kind: 345 | of ArrayReply: 346 | for element in reply.elements: 347 | case element.kind: 348 | of SimpleStringReply: 349 | raise newException(RedisError, "Cannot convert string to " & $t) 350 | of BulkStringReply: 351 | if element.bulk.isSome: 352 | result.add(some(parseInt(element.bulk.get))) 353 | else: 354 | result.add(none(int)) 355 | of IntegerReply: 356 | result.add(some(element.value)) 357 | of ArrayReply: 358 | raise newException(RedisError, "Cannot convert array to " & $t) 359 | else: 360 | raise newException(RedisError, "Cannot convert non-array reply to " & $t) 361 | elif t is tuple: 362 | case reply.kind: 363 | of ArrayReply: 364 | var i: int 365 | for name, value in result.fieldPairs: 366 | if i == reply.elements.len: 367 | raise newException(RedisError, "Reply array len < tuple len") 368 | when value is SomeInteger: 369 | value = reply.elements[i].to(typeof(value)) 370 | elif value is string: 371 | value = reply.elements[i].to(string) 372 | elif value is Option[string]: 373 | value = reply.elements[i].to(Option[string]) 374 | elif value is seq[int]: 375 | value = reply.elements[i].to(seq[int]) 376 | elif value is seq[string]: 377 | value = reply.elements[i].to(seq[string]) 378 | inc i 379 | if i != reply.elements.len: 380 | raise newException(RedisError, "Reply array len > tuple len") 381 | else: 382 | raise newException(RedisError, "Cannot convert non-array reply to " & $t) 383 | else: 384 | {.error: "Coverting to " & $t & " not supported.".} 385 | 386 | proc `[]`*(reply: RedisReply, index: int): RedisReply = 387 | if reply.kind == ArrayReply: 388 | reply.elements[index] 389 | else: 390 | raise newException(RedisError, "Reply is not an array") 391 | --------------------------------------------------------------------------------