├── .swift-version ├── .periphery.yml ├── .vscode ├── settings.json └── launch.json ├── .swiftpm └── xcode │ └── package.xcworkspace │ └── contents.xcworkspacedata ├── .pre-commit-config.yaml ├── .swiftformat ├── Sources ├── RedisConnection │ ├── RedisError.swift │ ├── Scratch.swift │ ├── RedisProtocol.swift │ ├── RESPValue.swift │ ├── RedisConnection.swift │ └── RedisStreamingParser.swift └── CLI │ ├── Support.swift │ └── cli.swift ├── Package.swift ├── LICENSE.md ├── README.md ├── Tests └── RedisConnectionTests │ ├── RedisConnectionTests.swift │ └── foo.dat ├── .gitignore ├── .swiftlint.yml └── Documentation └── spec.md /.swift-version: -------------------------------------------------------------------------------- 1 | 5.9 2 | -------------------------------------------------------------------------------- /.periphery.yml: -------------------------------------------------------------------------------- 1 | retain_public: true 2 | targets: 3 | - RedisConnection 4 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "cSpell.words": [ 3 | "RESP" 4 | ] 5 | } 6 | -------------------------------------------------------------------------------- /.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # See https://pre-commit.com for more information 2 | # See https://pre-commit.com/hooks.html for more hooks 3 | repos: 4 | - repo: https://github.com/pre-commit/pre-commit-hooks 5 | rev: v3.2.0 6 | hooks: 7 | - id: trailing-whitespace 8 | - id: end-of-file-fixer 9 | - id: check-yaml 10 | - id: check-added-large-files 11 | -------------------------------------------------------------------------------- /.swiftformat: -------------------------------------------------------------------------------- 1 | --disable andOperator 2 | --disable emptyBraces 3 | --disable fileHeader 4 | --disable redundantParens 5 | --disable trailingClosures 6 | --enable isEmpty 7 | 8 | --elseposition next-line 9 | --ifdef indent 10 | --patternlet inline 11 | --stripunusedargs closure-only 12 | --closingparen balanced 13 | --wraparguments preserve 14 | --wrapcollections before-first 15 | -------------------------------------------------------------------------------- /Sources/RedisConnection/RedisError.swift: -------------------------------------------------------------------------------- 1 | public enum RedisError: Error { 2 | case authenticationFailure 3 | case unexpectedState 4 | case parseError 5 | case messageReceiveFailure 6 | case typeMismatch 7 | case stringDecodingError 8 | case unknownHeader(Character) 9 | case partialSubscribe 10 | 11 | @available(*, deprecated, message: "Create a better error") 12 | case undefined(String) 13 | } 14 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "configurations": [ 3 | { 4 | "type": "lldb", 5 | "request": "launch", 6 | "name": "Debug CLI", 7 | "program": "${workspaceFolder:RedisConnection}/.build/debug/CLI", 8 | "args": [], 9 | "cwd": "${workspaceFolder:RedisConnection}", 10 | "preLaunchTask": "swift: Build Debug CLI" 11 | }, 12 | { 13 | "type": "lldb", 14 | "request": "launch", 15 | "name": "Release CLI", 16 | "program": "${workspaceFolder:RedisConnection}/.build/release/CLI", 17 | "args": [], 18 | "cwd": "${workspaceFolder:RedisConnection}", 19 | "preLaunchTask": "swift: Build Release CLI" 20 | } 21 | ] 22 | } 23 | -------------------------------------------------------------------------------- /Package.swift: -------------------------------------------------------------------------------- 1 | // swift-tools-version: 5.9 2 | 3 | import PackageDescription 4 | 5 | let package = Package( 6 | name: "RedisConnection", 7 | platforms: [ 8 | .iOS("17.0"), 9 | .macOS("12.0"), 10 | .macCatalyst("17.0"), 11 | ], 12 | products: [ 13 | .library(name: "RedisConnection", targets: ["RedisConnection"]), 14 | ], 15 | targets: [ 16 | .executableTarget( 17 | name: "CLI", 18 | dependencies: ["RedisConnection"], 19 | swiftSettings: [.unsafeFlags(["-Xfrontend", "-warn-concurrency", "-Xfrontend", "-enable-actor-data-race-checks"])] 20 | ), 21 | .target( 22 | name: "RedisConnection", 23 | swiftSettings: [.unsafeFlags(["-Xfrontend", "-warn-concurrency", "-Xfrontend", "-enable-actor-data-race-checks"])] 24 | ), 25 | .testTarget( 26 | name: "RedisConnectionTests", 27 | dependencies: ["RedisConnection"], 28 | resources: [.copy("foo.dat")] 29 | ), 30 | ] 31 | ) 32 | -------------------------------------------------------------------------------- /Sources/CLI/Support.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import os 3 | import RedisConnection 4 | 5 | actor Timeit { 6 | @TaskLocal 7 | static var shared = Timeit() 8 | 9 | private var starts: [String: CFAbsoluteTime] = [:] 10 | 11 | nonisolated 12 | func start(_ label: String) { 13 | let current = CFAbsoluteTimeGetCurrent() 14 | Task { 15 | await update(label: label, start: current) 16 | } 17 | } 18 | 19 | func finish(_ label: String) { 20 | let current = CFAbsoluteTimeGetCurrent() 21 | guard let start = starts[label] else { 22 | return 23 | } 24 | starts[label] = nil 25 | 26 | let s = FloatingPointFormatStyle().format(current - start) 27 | print("⌛ \(label): \(s)") 28 | } 29 | 30 | private func update(label: String, start: CFAbsoluteTime) { 31 | starts[label] = start 32 | } 33 | } 34 | 35 | func log(_ connection: RedisConnection, _ values: Any...) { 36 | let message = values.map(String.init(describing:)).joined(separator: " ") 37 | print("\(connection.label ?? "-"): \(message)") 38 | } 39 | 40 | extension Task where Success == () { 41 | func wait() async throws { 42 | try await value 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2022, Jonathan Wight 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | 3. Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /Sources/RedisConnection/Scratch.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import Network 3 | 4 | extension NWConnection.State: CustomStringConvertible { 5 | public var description: String { 6 | switch self { 7 | case .preparing: 8 | "preparing" 9 | case .ready: 10 | "ready" 11 | case .cancelled: 12 | "cancelled" 13 | case .setup: 14 | "setup" 15 | default: 16 | "?" 17 | } 18 | } 19 | } 20 | 21 | public struct AnyAsyncIterator: AsyncIteratorProtocol { 22 | let body: () async throws -> Element? 23 | 24 | public init(_ body: @escaping () async throws -> Element?) { 25 | self.body = body 26 | } 27 | 28 | public mutating func next() async throws -> Element? { 29 | try await body() 30 | } 31 | } 32 | 33 | public struct AnyAsyncSequence: AsyncSequence, Sendable { 34 | public typealias Iterator = AnyAsyncIterator 35 | 36 | public let makeUnderlyingIterator: @Sendable () -> Iterator 37 | 38 | public init(_ makeUnderlyingIterator: @Sendable @escaping () -> I) where Element == I.Element, I: AsyncIteratorProtocol { 39 | self.makeUnderlyingIterator = { 40 | var i = makeUnderlyingIterator() 41 | return AnyAsyncIterator { 42 | try await i.next() 43 | } 44 | } 45 | } 46 | 47 | public func makeAsyncIterator() -> Iterator { 48 | makeUnderlyingIterator() 49 | } 50 | } 51 | 52 | extension [UInt8] { 53 | // I will fight you. 54 | static let crlf = [UInt8(ascii: "\r"), UInt8(ascii: "\n")] 55 | } 56 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Redis Connection 2 | 3 | A Swift Concurrency-based implementation of the Redis (3) Protocol. This project uses Apple's Network framework to implement the Redis Protocol. 4 | 5 | The project supports the Redis streaming (RESP-3) protocol. It has built-in support for Redis pub-sub using Swift AsyncIterators. 6 | 7 | ## Basic Example 8 | 9 | ```swift 10 | let connection = RedisConnection(host: host) 11 | try await connection.connect() 12 | try await connection.hello(password: password) 13 | _ = try await connection.send("SET", "foo", "bar") 14 | print(try await connection.send("GET", "foo").stringValue) 15 | ``` 16 | 17 | ## Basic Pubsub Listener Example 18 | 19 | ```swift 20 | let connection = RedisConnection(host: host) 21 | try await connection.connect() 22 | try await connection.hello(password: password) 23 | for try await message in try await connection.subscribe(channels: channel) { 24 | print(message.value) 25 | } 26 | ``` 27 | 28 | ## Notes 29 | 30 | This project is a hobby project. Caveat emptor and all that. 31 | 32 | A lot of effort was spent on the Network framework protocol implementation. The Redis protocol (or rather the RESP3 protocol) isn't very well designed (there's no framing data and you have to parse the entire message to know how big it is...) and a lot of implementation issues were encountered. I feel I've addressed all these issues and that the protocol works well now. 33 | 34 | Performance seems excellent publishing and subscribing to 10,000 messages per second on a remote Redis server (accessed via Wireguard). 35 | 36 | The project doesn't know anything about any Redis command types except the handful of commands that are used to implement login/authentication and the pub-sub functionality. 37 | -------------------------------------------------------------------------------- /Tests/RedisConnectionTests/RedisConnectionTests.swift: -------------------------------------------------------------------------------- 1 | import class Foundation.Bundle 2 | import RedisConnection 3 | import XCTest 4 | 5 | final class RedisConnectionTests: XCTestCase { 6 | // let url = Bundle.module.url(forResource: "foo", withExtension: "dat")! 7 | // let data = try Data(contentsOf: url) 8 | // var scanner = CollectionScanner(elements: data) 9 | // let value = scanner.scanRESPValue() 10 | 11 | func testNull() throws { 12 | XCTAssert(try RESPValueParser.parse(bytes: "_\r\n".utf8) == .null) 13 | XCTAssertThrowsError(try RESPValueParser.parse(bytes: "_x\r\n".utf8)) 14 | } 15 | 16 | func testBool() throws { 17 | XCTAssert(try RESPValueParser.parse(bytes: "_\r\n".utf8) == .null) 18 | XCTAssertThrowsError(try RESPValueParser.parse(bytes: "_x\r\n".utf8)) 19 | } 20 | 21 | func testBlobError() throws { 22 | XCTAssertEqual(try RESPValueParser.parse(bytes: "!11\r\nHello world\r\n".utf8)?.description, "Hello world") 23 | } 24 | 25 | func testVerbatimString() throws { 26 | XCTAssertEqual(try RESPValueParser.parse(bytes: "=11\r\nHello world\r\n".utf8)?.description, "Hello world") 27 | } 28 | 29 | func testMap() throws { 30 | XCTAssertEqual(try RESPValueParser.parse(bytes: "%1\r\n+Hello\r\n+world\r\n".utf8)?.description, ".map([Hello: world])") 31 | } 32 | 33 | func testSet() throws { 34 | XCTAssertEqual(try RESPValueParser.parse(bytes: "~1\r\n+Hello World\r\n".utf8)?.description, ".set([Hello World])") 35 | } 36 | 37 | func testAttribute() throws { 38 | XCTAssertEqual(try RESPValueParser.parse(bytes: "|1\r\n+Hello\r\n+world\r\n".utf8)?.description, ".attribute([Hello: world])") 39 | } 40 | } 41 | 42 | // TODO: Null bulk string 43 | 44 | // var p = RESPValueParser() 45 | // print(try p.parse(bytes: Array("+Hello world\r\n".utf8))) 46 | // print(try p.parse(bytes: Array("-Hello world\r\n".utf8))) 47 | // print(try p.parse(bytes: Array(":1234\r\n".utf8))) 48 | // print(try p.parse(bytes: Array("$3\r\nfoo\r\n".utf8))) 49 | // print(try p.parse(bytes: Array("$0\r\n\r\n".utf8))) 50 | // print(try p.parse(bytes: Array("$-1\r\n".utf8))) 51 | // print(try p.parse(bytes: Array("*-1\r\n".utf8))) 52 | // print(try p.parse(bytes: Array("*0\r\n".utf8))) 53 | // print(try p.parse(bytes: Array("*2\r\n+A\r\n+B\r\n".utf8))) 54 | // 55 | // 56 | // let u = URL(fileURLWithPath: "/Users/schwa/Library/Mobile Documents/com~apple~CloudDocs/Projects/RedisConnection/Tests/RedisConnectionTests/foo.dat") 57 | // let bytes = Array(try Data(contentsOf: u)) 58 | // print(try p.parse(bytes: bytes)) 59 | // 60 | // 61 | // 62 | //// var parser = RESPStreamingParser() 63 | ////// try parser.feed(Array("+hello world\r\n".utf8)) 64 | ////// try parser.feed(Array("-hello world\r\n".utf8)) 65 | ////// try parser.feed(Array(":123\r\n".utf8)) 66 | ////// try parser.feed(Array("$6\r\nfoobar\r\n".utf8)) 67 | ////// try parser.feed(Array("*0\r\n".utf8)) 68 | ////// try parser.feed(Array("*0\r\n".utf8)) 69 | //// try parser.feed(Array("*2\r\n$3\r\nfoo\r\n$3\r\nbar\r\n".utf8)) 70 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by https://www.toptal.com/developers/gitignore/api/swift,swiftpm,macos 2 | # Edit at https://www.toptal.com/developers/gitignore?templates=swift,swiftpm,macos 3 | 4 | ### macOS ### 5 | # General 6 | .DS_Store 7 | .AppleDouble 8 | .LSOverride 9 | 10 | # Icon must end with two \r 11 | Icon 12 | 13 | # Thumbnails 14 | ._* 15 | 16 | # Files that might appear in the root of a volume 17 | .DocumentRevisions-V100 18 | .fseventsd 19 | .Spotlight-V100 20 | .TemporaryItems 21 | .Trashes 22 | .VolumeIcon.icns 23 | .com.apple.timemachine.donotpresent 24 | 25 | # Directories potentially created on remote AFP share 26 | .AppleDB 27 | .AppleDesktop 28 | Network Trash Folder 29 | Temporary Items 30 | .apdisk 31 | 32 | ### macOS Patch ### 33 | # iCloud generated files 34 | *.icloud 35 | 36 | ### Swift ### 37 | # Xcode 38 | # 39 | # gitignore contributors: remember to update Global/Xcode.gitignore, Objective-C.gitignore & Swift.gitignore 40 | 41 | ## User settings 42 | xcuserdata/ 43 | 44 | ## compatibility with Xcode 8 and earlier (ignoring not required starting Xcode 9) 45 | *.xcscmblueprint 46 | *.xccheckout 47 | 48 | ## compatibility with Xcode 3 and earlier (ignoring not required starting Xcode 4) 49 | build/ 50 | DerivedData/ 51 | *.moved-aside 52 | *.pbxuser 53 | !default.pbxuser 54 | *.mode1v3 55 | !default.mode1v3 56 | *.mode2v3 57 | !default.mode2v3 58 | *.perspectivev3 59 | !default.perspectivev3 60 | 61 | ## Obj-C/Swift specific 62 | *.hmap 63 | 64 | ## App packaging 65 | *.ipa 66 | *.dSYM.zip 67 | *.dSYM 68 | 69 | ## Playgrounds 70 | timeline.xctimeline 71 | playground.xcworkspace 72 | 73 | # Swift Package Manager 74 | # Add this line if you want to avoid checking in source code from Swift Package Manager dependencies. 75 | # Packages/ 76 | # Package.pins 77 | # Package.resolved 78 | # *.xcodeproj 79 | # Xcode automatically generates this directory with a .xcworkspacedata file and xcuserdata 80 | # hence it is not needed unless you have added a package configuration file to your project 81 | # .swiftpm 82 | 83 | .build/ 84 | 85 | # CocoaPods 86 | # We recommend against adding the Pods directory to your .gitignore. However 87 | # you should judge for yourself, the pros and cons are mentioned at: 88 | # https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control 89 | # Pods/ 90 | # Add this line if you want to avoid checking in source code from the Xcode workspace 91 | # *.xcworkspace 92 | 93 | # Carthage 94 | # Add this line if you want to avoid checking in source code from Carthage dependencies. 95 | # Carthage/Checkouts 96 | 97 | Carthage/Build/ 98 | 99 | # Accio dependency management 100 | Dependencies/ 101 | .accio/ 102 | 103 | # fastlane 104 | # It is recommended to not store the screenshots in the git repo. 105 | # Instead, use fastlane to re-generate the screenshots whenever they are needed. 106 | # For more information about the recommended setup visit: 107 | # https://docs.fastlane.tools/best-practices/source-control/#source-control 108 | 109 | fastlane/report.xml 110 | fastlane/Preview.html 111 | fastlane/screenshots/**/*.png 112 | fastlane/test_output 113 | 114 | # Code Injection 115 | # After new code Injection tools there's a generated folder /iOSInjectionProject 116 | # https://github.com/johnno1962/injectionforxcode 117 | 118 | iOSInjectionProject/ 119 | 120 | ### SwiftPM ### 121 | Packages 122 | xcuserdata 123 | *.xcodeproj 124 | 125 | 126 | # End of https://www.toptal.com/developers/gitignore/api/swift,swiftpm,macos 127 | -------------------------------------------------------------------------------- /Sources/CLI/cli.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import os 3 | import RedisConnection 4 | 5 | @main 6 | struct Main { 7 | static let host = "localhost" 8 | static let password = "notagoodpassword" 9 | 10 | static func main() async throws { 11 | // try! await basic1Test() 12 | // try! await basic2Test() 13 | try! await pubSubTest() 14 | } 15 | 16 | static func basic1Test() async throws { 17 | let connection = RedisConnection(label: "preamble", host: host) 18 | try await connection.connect() 19 | try await connection.hello(password: password) 20 | _ = try await connection.send("SET", "foo", "bar") 21 | try await print(connection.send("GET", "foo").stringValue) 22 | } 23 | 24 | static func basic2Test() async throws { 25 | // Connect to a REDIS server and ask it a bunch of questions... 26 | Timeit.shared.start("PREAMBLE") 27 | let connection = RedisConnection(label: "preamble", host: host) 28 | try await connection.connect() 29 | try await connection.hello(username: "default", password: password, clientName: "example-client") 30 | try await log(connection, connection.send("PING")) 31 | try await log(connection, connection.send("CLIENT", "INFO")) 32 | try await log(connection, connection.send("CLIENT", "GETNAME")) 33 | try await log(connection, connection.send("ACL", "WHOAMI")) 34 | try await log(connection, connection.send("ACL", "USERS")) 35 | try await log(connection, connection.send("ACL", "LIST")) 36 | // log(connection, try await connection.send("CONFIG", "GET", "*")) 37 | // log(connection, try await connection.send("COMMAND")) 38 | try await log(connection, connection.send("QUIT")) 39 | try await connection.disconnect() 40 | await Timeit.shared.finish("PREAMBLE") 41 | } 42 | 43 | static func pubSubTest() async throws { 44 | let channel = "my-example-channel" 45 | 46 | let listenerTask = Task { 47 | Timeit.shared.start("Listening") 48 | let connection = RedisConnection(label: "listener", host: host) 49 | try await connection.connect() 50 | try await connection.hello(password: password) 51 | var values = Set() 52 | for try await message in try await connection.subscribe(channels: channel) { 53 | if try message.kind == .message && message.value.stringValue == "STOP" { 54 | log(connection, "STOPPING") 55 | break 56 | } 57 | try values.insert(Int(message.value.stringValue)!) 58 | } 59 | await Timeit.shared.finish("Listening") 60 | return values 61 | } 62 | 63 | let publisherTask = Task { 64 | // We're going to publish some things. 65 | let values = 0 ..< 10000 66 | Timeit.shared.start("Sending") 67 | let connection = RedisConnection(label: "sender", host: host) 68 | try await connection.connect() 69 | try await connection.hello(password: password) 70 | try await connection.send(values: values.map { ["PUBLISH", channel, "\($0)"] }) 71 | try await log(connection, connection.publish(channel: channel, value: "STOP")) 72 | await Timeit.shared.finish("Sending") 73 | return Set(values) 74 | } 75 | 76 | let receivedValues = try await listenerTask.value 77 | let publishedValues = try await publisherTask.value 78 | 79 | assert(publishedValues == receivedValues) 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /Sources/RedisConnection/RedisProtocol.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import Network 3 | import os 4 | 5 | // https://developer.apple.com/forums/thread/118686 6 | // https://developer.apple.com/forums/thread/132575 7 | // https://developer.apple.com/documentation/network/building_a_custom_peer-to-peer_protocol 8 | 9 | // The value you return from parseInput is "move the cursor ahead by N bytes, I've consumed them". The value you return from handleInput is "I need N bytes to present before I want to be woken up again". The value you pass to deliverInputNoCopy is "move the cursor ahead by N bytes, and deliver those N bytes". 10 | 11 | public class RedisProtocol: NWProtocolFramerImplementation { 12 | public static let label: String = "REDIS" 13 | public static let definition = NWProtocolFramer.Definition(implementation: RedisProtocol.self) 14 | 15 | var debugLogger: Logger? // Logger() 16 | let logger = Logger() 17 | var parser = RESPValueParser() 18 | 19 | public required init(framer _: NWProtocolFramer.Instance) {} 20 | 21 | public func start(framer _: NWProtocolFramer.Instance) -> NWProtocolFramer.StartResult { 22 | .ready 23 | } 24 | 25 | public func handleInput(framer: NWProtocolFramer.Instance) -> Int { 26 | debugLogger?.debug(#function) 27 | 28 | var value: RESPValue? 29 | var error: Error? 30 | 31 | while error == nil { 32 | // We always need minimumIncompleteLength to be at least 1. 33 | let parsed = framer.parseInput(minimumIncompleteLength: 1, maximumLength: Int.max) { buffer, isComplete in 34 | do { 35 | debugLogger?.debug("Buffer \(String(describing: buffer)), isComplete: \(isComplete)") 36 | // Remember how many bytes we've passed so far 37 | let startBytesParsed = parser.bytesParsed 38 | debugLogger?.debug("startBytesParsed: \(startBytesParsed)") 39 | debugLogger?.debug("\(String(describing: buffer)) \(isComplete)") 40 | guard let buffer, !buffer.isEmpty else { 41 | debugLogger?.debug("Zero bytes. Skipping") 42 | return 0 43 | } 44 | value = try parser.parse(bytes: buffer) 45 | debugLogger?.debug("parsed: \(self.parser.bytesParsed - startBytesParsed) / \(buffer.count)") 46 | // We may not have parsed the entire buffer so compute what we did pass 47 | return parser.bytesParsed - startBytesParsed 48 | } 49 | catch let localError { 50 | logger.error("\(String(describing: localError))") 51 | error = localError 52 | return 0 53 | } 54 | } 55 | if !parsed { 56 | debugLogger?.debug("NOT PARSED") 57 | return 0 58 | } 59 | guard let value else { 60 | debugLogger?.debug("CONTINUE") 61 | continue 62 | } 63 | let message = NWProtocolFramer.Message(instance: framer) 64 | message["message"] = value 65 | let result = framer.deliverInputNoCopy(length: 0, message: message, isComplete: true) 66 | if result { 67 | return 0 68 | } 69 | } 70 | 71 | return 0 72 | } 73 | 74 | public func handleOutput(framer: NWProtocolFramer.Instance, message _: NWProtocolFramer.Message, messageLength: Int, isComplete _: Bool) { 75 | do { 76 | try framer.writeOutputNoCopy(length: messageLength) 77 | } 78 | catch { 79 | logger.error("\(String(describing: error))") 80 | } 81 | } 82 | 83 | public func wakeup(framer _: NWProtocolFramer.Instance) {} 84 | 85 | public func stop(framer _: NWProtocolFramer.Instance) -> Bool { 86 | true 87 | } 88 | 89 | public func cleanup(framer _: NWProtocolFramer.Instance) {} 90 | } 91 | -------------------------------------------------------------------------------- /.swiftlint.yml: -------------------------------------------------------------------------------- 1 | analyzer_rules: 2 | - capture_variable 3 | - explicit_self 4 | - unused_declaration 5 | - unused_import 6 | only_rules: 7 | # - anonymous_argument_in_multiline_closure 8 | - anyobject_protocol 9 | - array_init 10 | - attributes 11 | - balanced_xctest_lifecycle 12 | - block_based_kvo 13 | - class_delegate_protocol 14 | - closing_brace 15 | - closure_body_length 16 | # - closure_end_indentation 17 | - closure_parameter_position 18 | - closure_spacing 19 | - collection_alignment 20 | - colon 21 | - comma 22 | - comment_spacing 23 | - compiler_protocol_init 24 | - computed_accessors_order 25 | - conditional_returns_on_newline 26 | - contains_over_filter_count 27 | - contains_over_filter_is_empty 28 | - contains_over_first_not_nil 29 | - contains_over_range_nil_comparison 30 | - control_statement 31 | # - convenience_type 32 | - custom_rules 33 | # - cyclomatic_complexity 34 | - deployment_target 35 | - discarded_notification_center_observer 36 | - discouraged_assert 37 | - discouraged_direct_init 38 | - discouraged_none_name 39 | - discouraged_object_literal 40 | - discouraged_optional_boolean 41 | - discouraged_optional_collection 42 | - duplicate_enum_cases 43 | - duplicate_imports 44 | - duplicated_key_in_dictionary_literal 45 | - dynamic_inline 46 | - empty_collection_literal 47 | # - empty_count 48 | - empty_enum_arguments 49 | - empty_parameters 50 | - empty_parentheses_with_trailing_closure 51 | - empty_string 52 | - empty_xctest_method 53 | - enum_case_associated_values_count 54 | - expiring_todo 55 | # - explicit_acl 56 | - explicit_enum_raw_value 57 | - explicit_init 58 | # - explicit_top_level_acl 59 | # - explicit_type_interface 60 | # - extension_access_modifier 61 | - fallthrough 62 | # - fatal_error_message 63 | - file_header 64 | - file_length 65 | # - file_name 66 | # - file_name_no_space 67 | # - file_types_order 68 | - first_where 69 | - flatmap_over_map_reduce 70 | - for_where 71 | - force_cast 72 | # - force_try 73 | # - force_unwrapping 74 | # - function_body_length 75 | # - function_default_parameter_at_end 76 | - function_parameter_count 77 | - generic_type_name 78 | - ibinspectable_in_extension 79 | - identical_operands 80 | # - identifier_name 81 | - implicit_getter 82 | # - implicit_return 83 | # - implicitly_unwrapped_optional 84 | - inclusive_language 85 | # - indentation_width 86 | - inert_defer 87 | - is_disjoint 88 | - joined_default_parameter 89 | - large_tuple 90 | - last_where 91 | - leading_whitespace 92 | - legacy_cggeometry_functions 93 | - legacy_constant 94 | - legacy_constructor 95 | - legacy_hashing 96 | - legacy_multiple 97 | - legacy_nsgeometry_functions 98 | - legacy_objc_type 99 | - legacy_random 100 | - let_var_whitespace 101 | # - line_length 102 | - literal_expression_end_indentation 103 | - lower_acl_than_parent 104 | - mark 105 | # - missing_docs 106 | - modifier_order 107 | # - multiline_arguments 108 | # - multiline_arguments_brackets 109 | - multiline_function_chains 110 | - multiline_literal_brackets 111 | - multiline_parameters 112 | - multiline_parameters_brackets 113 | - multiple_closures_with_trailing_closure 114 | - nesting 115 | - nimble_operator 116 | # - no_extension_access_modifier 117 | - no_fallthrough_only 118 | # - no_grouping_extension 119 | - no_space_in_method_call 120 | - notification_center_detachment 121 | - nslocalizedstring_key 122 | - nslocalizedstring_require_bundle 123 | - nsobject_prefer_isequal 124 | # - number_separator 125 | - object_literal 126 | - opening_brace 127 | - operator_usage_whitespace 128 | - operator_whitespace 129 | - optional_enum_case_matching 130 | # - orphaned_doc_comment 131 | - overridden_super_call 132 | - override_in_extension 133 | - pattern_matching_keywords 134 | # - prefer_nimble 135 | - prefer_self_in_static_references 136 | - prefer_self_type_over_type_of_self 137 | - prefer_zero_over_explicit_init 138 | # - prefixed_toplevel_constant 139 | - private_action 140 | - private_outlet 141 | - private_over_fileprivate 142 | - private_subject 143 | - private_unit_test 144 | - prohibited_interface_builder 145 | - prohibited_super_call 146 | - protocol_property_accessors_order 147 | - quick_discouraged_call 148 | - quick_discouraged_focused_test 149 | - quick_discouraged_pending_test 150 | - raw_value_for_camel_cased_codable_enum 151 | - reduce_boolean 152 | - reduce_into 153 | - redundant_discardable_let 154 | - redundant_nil_coalescing 155 | - redundant_objc_attribute 156 | - redundant_optional_initialization 157 | - redundant_set_access_control 158 | - redundant_string_enum_value 159 | - redundant_type_annotation 160 | - redundant_void_return 161 | # - required_deinit 162 | - required_enum_case 163 | - return_arrow_whitespace 164 | - self_in_property_initialization 165 | - shorthand_operator 166 | - single_test_class 167 | - sorted_first_last 168 | - sorted_imports 169 | - statement_position 170 | - static_operator 171 | - strict_fileprivate 172 | - strong_iboutlet 173 | - superfluous_disable_command 174 | - switch_case_alignment 175 | - switch_case_on_newline 176 | - syntactic_sugar 177 | - test_case_accessibility 178 | # - todo 179 | - toggle_bool 180 | # - trailing_closure 181 | - trailing_comma 182 | - trailing_newline 183 | - trailing_semicolon 184 | - trailing_whitespace 185 | - type_body_length 186 | # - type_contents_order 187 | - type_name 188 | - unavailable_function 189 | - unneeded_break_in_switch 190 | - unneeded_parentheses_in_closure_argument 191 | - unowned_variable_capture 192 | - untyped_error_in_catch 193 | - unused_capture_list 194 | - unused_closure_parameter 195 | - unused_control_flow_label 196 | - unused_enumerated 197 | - unused_optional_binding 198 | - unused_setter_value 199 | - valid_ibinspectable 200 | - vertical_parameter_alignment 201 | - vertical_parameter_alignment_on_call 202 | - vertical_whitespace 203 | # - vertical_whitespace_between_cases 204 | # - vertical_whitespace_closing_braces 205 | # - vertical_whitespace_opening_braces 206 | - void_return 207 | - weak_delegate 208 | - xct_specific_matcher 209 | - xctfail_message 210 | - yoda_condition 211 | -------------------------------------------------------------------------------- /Sources/RedisConnection/RESPValue.swift: -------------------------------------------------------------------------------- 1 | public indirect enum RESPValue: Sendable, Hashable { 2 | // //\r\n` 4 | case errorString(String) // ✅🔲🔲 RESP 2+: `-\r\n` 5 | case integer(Int) // ✅🔲🔲 RESP 2+: `:\r\n` 6 | case blobString([UInt8]) // ✅🔲🔲 RESP 2+: `$\r\n\r\n` 7 | case nullBulkString // ✅🔲🔲 RESP 2: `$-1\r\n 8 | case nullArray // ✅🔲🔲 RESP 2: `*-1\r\n` 9 | case null // ✅✅🔲 RESP 3: `_\r\n` 10 | case double(Double) // ✅🔲🔲 RESP 3: `,\r\n` 11 | case boolean(Bool) // ✅✅🔲 RESP 3: `#t\r\n` / `#f\r\n` 12 | case blobError([UInt8]) // ✅✅🔲 RESP 3: `!\r\n\r\n` 13 | case verbatimString([UInt8]) // ✅✅🔲 RESP 3: `=\r\n` 14 | case bigNumber([UInt8]) // ✅🔲🔲 RESP 3: `(\r\n` 15 | case array([Self]) // ✅🔲🔲 RESP 2+: `*\r\n` 16 | case map([Self: Self]) // ✅✅🔲 RESP 3+: `%\r\n` 17 | case set(Set) // ✅✅🔲 RESP 3+: `~\r\n` 18 | case attribute([Self: Self]) // ✅🔲🔲 RESP 3+: `|\r\n` 19 | case pubsub(Pubsub) // ✅🔲🔲 RESP 3+: `>\r\n` // TODO - this may not be exactly how this works 20 | } 21 | 22 | public struct Pubsub: Sendable, Hashable { 23 | public enum Kind: String, Sendable { 24 | case message 25 | case subscribe 26 | case unsubscribe 27 | } 28 | 29 | public var kind: Kind 30 | public var channel: String 31 | public var value: RESPValue 32 | } 33 | 34 | public extension RESPValue { 35 | static func blobString(_ string: String) -> RESPValue { 36 | blobString(Array(string.utf8)) 37 | } 38 | 39 | var integerValue: Int { 40 | get throws { 41 | guard case .integer(let value) = self else { 42 | throw RedisError.typeMismatch 43 | } 44 | return value 45 | } 46 | } 47 | 48 | var stringValue: String { 49 | get throws { 50 | switch self { 51 | case .simpleString(let value), .errorString(let value): 52 | return value 53 | case .blobString(let value), .blobError(let value), .verbatimString(let value): 54 | // TODO: encoding is safe to assume? 55 | guard let value = String(bytes: value, encoding: .utf8) else { 56 | throw RedisError.stringDecodingError 57 | } 58 | return value 59 | default: 60 | throw RedisError.typeMismatch 61 | } 62 | } 63 | } 64 | 65 | var arrayValue: [RESPValue] { 66 | get throws { 67 | switch self { 68 | case .array(let array): 69 | return array 70 | default: 71 | throw RedisError.typeMismatch 72 | } 73 | } 74 | } 75 | 76 | var pubsubValue: Pubsub { 77 | get throws { 78 | switch self { 79 | case .pubsub(let value): 80 | return value 81 | default: 82 | throw RedisError.typeMismatch 83 | } 84 | } 85 | } 86 | } 87 | 88 | public extension RESPValue { 89 | func encode() throws -> [UInt8] { 90 | switch self { 91 | case .simpleString(let value): 92 | return Array("+\(value)\r\n".utf8) 93 | case .errorString(let value): 94 | return Array("-\(value)\r\n".utf8) 95 | case .integer(let value): 96 | return Array(":\(value)\r\n".utf8) 97 | case .blobString(let value): 98 | return Array("$\(value.count)\r\n".utf8) + value + Array("\r\n".utf8) 99 | case .nullBulkString: 100 | return Array("$-1\r\n".utf8) 101 | case .array(let values): 102 | let encodedValues = try values.flatMap { try $0.encode() } 103 | return Array("*\(values.count)\r\n".utf8) + encodedValues 104 | case .nullArray: 105 | return Array("*-1\r\n".utf8) 106 | case .null: 107 | return Array("_\r\n".utf8) 108 | case .boolean(let value): 109 | return Array("#\(value ? "t" : "f")\r\n".utf8) 110 | case .blobError(let value): 111 | return Array("!\(value.count)\r\n".utf8) + value + Array("\r\n".utf8) 112 | case .verbatimString(let value): 113 | return Array("=\(value.count)\r\n".utf8) + value + Array("\r\n".utf8) 114 | case .bigNumber(let value): 115 | return Array("+\(value)\r\n".utf8) 116 | case .double: 117 | fatalError("Inimplemented") 118 | case .map: 119 | fatalError("Inimplemented") 120 | case .set: 121 | fatalError("Inimplemented") 122 | case .attribute: 123 | fatalError("Inimplemented") 124 | case .pubsub: 125 | fatalError("Inimplemented") 126 | } 127 | } 128 | } 129 | 130 | extension RESPValue: CustomStringConvertible { 131 | public var description: String { 132 | switch self { 133 | case .simpleString(let value): 134 | value 135 | case .errorString(let value): 136 | value 137 | case .integer(let value): 138 | "\(value)" 139 | case .blobString(let value): 140 | String(bytes: value, encoding: .utf8)! 141 | case .nullBulkString: 142 | "" 143 | case .array(let values): 144 | "[" + values.map(\.description).joined(separator: ", ") + "]" 145 | case .nullArray: 146 | "" 147 | case .null: 148 | "" 149 | case .double(let value): 150 | "\(value)" 151 | case .boolean(let value): 152 | "\(value)" 153 | case .blobError(let value): 154 | String(bytes: value, encoding: .utf8)! 155 | case .verbatimString(let value): 156 | String(bytes: value, encoding: .utf8)! 157 | case .bigNumber(let value): 158 | ".bigNumber(\(value))" 159 | case .map(let values): 160 | ".map([" + values.map { "\($0.key.description): \($0.value.description)" }.joined(separator: ", ") + "])" 161 | case .set(let values): 162 | ".set([" + values.map(\.description).joined(separator: ", ") + "])" 163 | case .attribute(let values): 164 | ".attribute([" + values.map { "\($0.key.description): \($0.value.description)" }.joined(separator: ", ") + "])" 165 | case .pubsub(let value): 166 | "pubsub(\(String(describing: value))" 167 | } 168 | } 169 | } 170 | -------------------------------------------------------------------------------- /Sources/RedisConnection/RedisConnection.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import Network 3 | @preconcurrency import os 4 | 5 | public actor RedisConnection { 6 | public nonisolated 7 | let label: String? 8 | 9 | let connection: NWConnection 10 | let stateStream: AsyncStream 11 | 12 | let logger: Logger? = nil // Logger() 13 | 14 | enum Mode { 15 | case normal 16 | case subscriber 17 | } 18 | 19 | var mode = Mode.normal 20 | 21 | public init(label: String? = nil, host: String? = nil, port: Int? = nil) { 22 | self.label = label 23 | let host = NWEndpoint.Host(host ?? "localhost") 24 | let port = NWEndpoint.Port(rawValue: NWEndpoint.Port.RawValue(port ?? 6379))! 25 | let params = NWParameters(tls: nil, tcp: .init()) 26 | params.defaultProtocolStack.applicationProtocols.insert(NWProtocolFramer.Options(definition: RedisProtocol.definition), at: 0) 27 | let connection = NWConnection(host: host, port: port, using: params) 28 | let stateStream = AsyncStream { continuation in 29 | connection.stateUpdateHandler = { state in 30 | continuation.yield(state) 31 | } 32 | } 33 | self.connection = connection 34 | self.stateStream = stateStream 35 | } 36 | 37 | public func connect() async throws { 38 | logger?.debug("\(#function)") 39 | let queue = DispatchQueue(label: "redis-connection", qos: .default, attributes: []) 40 | connection.start(queue: queue) 41 | loop: for await state in stateStream { 42 | logger?.debug("State change: \(state)") 43 | switch state { 44 | case .preparing: 45 | break 46 | case .ready: 47 | break loop 48 | case .failed(let error): 49 | throw error 50 | case .waiting(let error): 51 | throw error 52 | case .setup: 53 | throw RedisError.unexpectedState 54 | case .cancelled: 55 | throw RedisError.unexpectedState 56 | @unknown default: 57 | throw RedisError.unexpectedState 58 | } 59 | } 60 | if let logger { 61 | Task { 62 | for await state in stateStream { 63 | logger.debug("State change: \(state)") 64 | } 65 | } 66 | } 67 | } 68 | 69 | public func disconnect() async throws { 70 | logger?.debug("\(#function)") 71 | connection.cancel() 72 | } 73 | 74 | // MARK: - 75 | 76 | private func receiveHelper(resumeThrowing: @Sendable @escaping (Error) -> Void, resumeReturning: @Sendable @escaping (RESPValue) -> Void) { 77 | let logger = logger 78 | logger?.debug("\(#function)") 79 | connection.receive(minimumIncompleteLength: 0, maximumLength: Int.max) { _, context, _, error in 80 | logger?.debug("\(#function) (closure)") 81 | if let error { 82 | resumeThrowing(error) 83 | return 84 | } 85 | guard let context else { 86 | fatalError("No context") 87 | } 88 | guard let message = context.protocolMetadata(definition: RedisProtocol.definition) as? NWProtocolFramer.Message else { 89 | resumeThrowing(RedisError.messageReceiveFailure) 90 | return 91 | } 92 | guard let value = message["message"] as? RESPValue else { 93 | fatalError("No message/message of wrong type.") 94 | } 95 | 96 | resumeReturning(value) 97 | } 98 | } 99 | 100 | // MARK: - 101 | 102 | public func send(value: RESPValue) async throws -> (RESPValue) { 103 | assert(mode == .normal) 104 | logger?.debug("\(#function)") 105 | let encodedValue = try value.encode() 106 | return try await withCheckedThrowingContinuation { continuation in 107 | connection.batch { 108 | receiveHelper { error in 109 | continuation.resume(throwing: error) 110 | } resumeReturning: { value in 111 | continuation.resume(returning: value) 112 | } 113 | connection.send(content: encodedValue, completion: .contentProcessed { error in 114 | if let error { 115 | continuation.resume(throwing: error) 116 | } 117 | }) 118 | } 119 | } 120 | } 121 | 122 | public func sendNoReceive(value: RESPValue) async throws { 123 | assert(mode == .subscriber) 124 | logger?.debug("\(#function)") 125 | let encodedValue = try value.encode() 126 | return try await withCheckedThrowingContinuation { continuation in 127 | connection.send(content: encodedValue, completion: .contentProcessed { error in 128 | if let error { 129 | continuation.resume(throwing: error) 130 | } 131 | continuation.resume() 132 | }) 133 | } 134 | } 135 | 136 | public func receive() async throws -> RESPValue { 137 | assert(mode == .subscriber) 138 | logger?.debug("\(#function)") 139 | return try await withCheckedThrowingContinuation { continuation in 140 | receiveHelper { error in 141 | continuation.resume(throwing: error) 142 | } resumeReturning: { value in 143 | continuation.resume(returning: value) 144 | } 145 | } 146 | } 147 | } 148 | 149 | // MARK: - 150 | 151 | public extension RedisConnection { 152 | func send(value: [String]) async throws -> (RESPValue) { 153 | logger?.debug("\(#function)") 154 | let value = RESPValue.array(value.map { .blobString($0) }) 155 | return try await send(value: value) 156 | } 157 | 158 | func sendNoReceive(value: [String]) async throws { 159 | logger?.debug("\(#function)") 160 | let value = RESPValue.array(value.map { .blobString($0) }) 161 | try await sendNoReceive(value: value) 162 | } 163 | 164 | func send(_ value: String...) async throws -> (RESPValue) { 165 | try await send(value: value) 166 | } 167 | 168 | func send(values: [[String]]) async throws { 169 | logger?.debug("\(#function)") 170 | let encodedValue = try values.flatMap { 171 | try RESPValue.array($0.map { .blobString($0) }).encode() 172 | } 173 | return try await withCheckedThrowingContinuation { continuation in 174 | connection.send(content: encodedValue, completion: .contentProcessed { error in 175 | if let error { 176 | continuation.resume(throwing: error) 177 | } 178 | continuation.resume() 179 | }) 180 | } 181 | } 182 | } 183 | 184 | // MARK: - 185 | 186 | public extension RedisConnection { 187 | func subscribe(channels: String...) async throws -> AnyAsyncSequence { 188 | mode = .subscriber 189 | 190 | logger?.debug("\(#function)") 191 | try await sendNoReceive(value: ["SUBSCRIBE"] + channels) 192 | var confirmedChannels: Set = [] 193 | for _ in 0 ..< channels.count { 194 | try await confirmedChannels.insert(receive().pubsubValue.channel) 195 | } 196 | if confirmedChannels != Set(channels) { 197 | throw RedisError.partialSubscribe 198 | } 199 | return AnyAsyncSequence { 200 | AnyAsyncIterator { 201 | let value = try await self.receive() 202 | switch value { 203 | case .pubsub(let pubsub): 204 | return pubsub 205 | default: 206 | return nil 207 | } 208 | } 209 | } 210 | } 211 | 212 | func publish(channel: String, value: String) async throws -> Int { 213 | let response = try await send(value: ["PUBLISH", channel, value]) 214 | return try response.integerValue 215 | } 216 | } 217 | 218 | // MARK: - 219 | 220 | public extension RedisConnection { 221 | func hello(username: String = "default", password: String, clientName: String? = nil) async throws { 222 | logger?.debug("\(#function)") 223 | 224 | var request = ["HELLO", "3", "AUTH", username, password] 225 | if let clientName { 226 | request += ["SETNAME", clientName] 227 | } 228 | 229 | let response = try await send(value: request) 230 | guard case .map(let response) = response else { 231 | throw RedisError.authenticationFailure 232 | } 233 | guard let respVersion = try response[.blobString("proto")]?.integerValue, respVersion == 3 else { 234 | throw RedisError.authenticationFailure 235 | } 236 | } 237 | 238 | func authenticate(password: String) async throws { 239 | logger?.debug("\(#function)") 240 | let response = try await send("AUTH", password) 241 | 242 | guard try response.stringValue == "OK" else { 243 | throw RedisError.authenticationFailure 244 | } 245 | } 246 | } 247 | 248 | // MARK: - 249 | -------------------------------------------------------------------------------- /Tests/RedisConnectionTests/foo.dat: -------------------------------------------------------------------------------- 1 | *336 2 | $11 3 | rdbchecksum 4 | $3 5 | yes 6 | $9 7 | daemonize 8 | $2 9 | no 10 | $19 11 | io-threads-do-reads 12 | $2 13 | no 14 | $22 15 | lua-replicate-commands 16 | $3 17 | yes 18 | $16 19 | always-show-logo 20 | $2 21 | no 22 | $14 23 | protected-mode 24 | $2 25 | no 26 | $14 27 | rdbcompression 28 | $3 29 | yes 30 | $18 31 | rdb-del-sync-files 32 | $2 33 | no 34 | $15 35 | activerehashing 36 | $3 37 | yes 38 | $27 39 | stop-writes-on-bgsave-error 40 | $3 41 | yes 42 | $14 43 | set-proc-title 44 | $3 45 | yes 46 | $10 47 | dynamic-hz 48 | $3 49 | yes 50 | $22 51 | lazyfree-lazy-eviction 52 | $2 53 | no 54 | $20 55 | lazyfree-lazy-expire 56 | $2 57 | no 58 | $24 59 | lazyfree-lazy-server-del 60 | $2 61 | no 62 | $22 63 | lazyfree-lazy-user-del 64 | $2 65 | no 66 | $24 67 | lazyfree-lazy-user-flush 68 | $2 69 | no 70 | $24 71 | repl-disable-tcp-nodelay 72 | $2 73 | no 74 | $18 75 | repl-diskless-sync 76 | $2 77 | no 78 | $14 79 | gopher-enabled 80 | $2 81 | no 82 | $29 83 | aof-rewrite-incremental-fsync 84 | $3 85 | yes 86 | $25 87 | no-appendfsync-on-rewrite 88 | $2 89 | no 90 | $29 91 | cluster-require-full-coverage 92 | $3 93 | yes 94 | $26 95 | rdb-save-incremental-fsync 96 | $3 97 | yes 98 | $18 99 | aof-load-truncated 100 | $3 101 | yes 102 | $20 103 | aof-use-rdb-preamble 104 | $3 105 | yes 106 | $27 107 | cluster-replica-no-failover 108 | $2 109 | no 110 | $25 111 | cluster-slave-no-failover 112 | $2 113 | no 114 | $18 115 | replica-lazy-flush 116 | $2 117 | no 118 | $16 119 | slave-lazy-flush 120 | $2 121 | no 122 | $24 123 | replica-serve-stale-data 124 | $3 125 | yes 126 | $22 127 | slave-serve-stale-data 128 | $3 129 | yes 130 | $17 131 | replica-read-only 132 | $3 133 | yes 134 | $15 135 | slave-read-only 136 | $3 137 | yes 138 | $24 139 | replica-ignore-maxmemory 140 | $3 141 | yes 142 | $22 143 | slave-ignore-maxmemory 144 | $3 145 | yes 146 | $18 147 | jemalloc-bg-thread 148 | $3 149 | yes 150 | $12 151 | activedefrag 152 | $2 153 | no 154 | $14 155 | syslog-enabled 156 | $2 157 | no 158 | $15 159 | cluster-enabled 160 | $2 161 | no 162 | $10 163 | appendonly 164 | $3 165 | yes 166 | $29 167 | cluster-allow-reads-when-down 168 | $2 169 | no 170 | $17 171 | crash-log-enabled 172 | $3 173 | yes 174 | $22 175 | crash-memcheck-enabled 176 | $3 177 | yes 178 | $17 179 | use-exit-on-panic 180 | $2 181 | no 182 | $11 183 | disable-thp 184 | $3 185 | yes 186 | $31 187 | cluster-allow-replica-migration 188 | $3 189 | yes 190 | $17 191 | replica-announced 192 | $3 193 | yes 194 | $7 195 | aclfile 196 | $0 197 | 198 | $10 199 | unixsocket 200 | $0 201 | 202 | $7 203 | pidfile 204 | $0 205 | 206 | $19 207 | replica-announce-ip 208 | $0 209 | 210 | $17 211 | slave-announce-ip 212 | $0 213 | 214 | $10 215 | masteruser 216 | $0 217 | 218 | $19 219 | cluster-announce-ip 220 | $0 221 | 222 | $12 223 | syslog-ident 224 | $5 225 | redis 226 | $10 227 | dbfilename 228 | $8 229 | dump.rdb 230 | $14 231 | appendfilename 232 | $14 233 | appendonly.aof 234 | $14 235 | server_cpulist 236 | $0 237 | 238 | $11 239 | bio_cpulist 240 | $0 241 | 242 | $19 243 | aof_rewrite_cpulist 244 | $0 245 | 246 | $14 247 | bgsave_cpulist 248 | $0 249 | 250 | $15 251 | ignore-warnings 252 | $0 253 | 254 | $19 255 | proc-title-template 256 | $35 257 | {title} {listen-addr} {server-mode} 258 | $10 259 | masterauth 260 | $0 261 | 262 | $11 263 | requirepass 264 | $16 265 | notagoodpassword 266 | $10 267 | supervised 268 | $2 269 | no 270 | $15 271 | syslog-facility 272 | $6 273 | local0 274 | $18 275 | repl-diskless-load 276 | $8 277 | disabled 278 | $8 279 | loglevel 280 | $6 281 | notice 282 | $16 283 | maxmemory-policy 284 | $10 285 | noeviction 286 | $11 287 | appendfsync 288 | $8 289 | everysec 290 | $13 291 | oom-score-adj 292 | $2 293 | no 294 | $18 295 | acl-pubsub-default 296 | $11 297 | allchannels 298 | $21 299 | sanitize-dump-payload 300 | $2 301 | no 302 | $9 303 | databases 304 | $2 305 | 16 306 | $4 307 | port 308 | $4 309 | 6379 310 | $10 311 | io-threads 312 | $1 313 | 1 314 | $27 315 | auto-aof-rewrite-percentage 316 | $3 317 | 100 318 | $31 319 | cluster-replica-validity-factor 320 | $2 321 | 10 322 | $29 323 | cluster-slave-validity-factor 324 | $2 325 | 10 326 | $21 327 | list-max-ziplist-size 328 | $2 329 | -2 330 | $13 331 | tcp-keepalive 332 | $3 333 | 300 334 | $25 335 | cluster-migration-barrier 336 | $1 337 | 1 338 | $23 339 | active-defrag-cycle-min 340 | $1 341 | 1 342 | $23 343 | active-defrag-cycle-max 344 | $2 345 | 25 346 | $29 347 | active-defrag-threshold-lower 348 | $2 349 | 10 350 | $29 351 | active-defrag-threshold-upper 352 | $3 353 | 100 354 | $14 355 | lfu-log-factor 356 | $2 357 | 10 358 | $14 359 | lfu-decay-time 360 | $1 361 | 1 362 | $16 363 | replica-priority 364 | $3 365 | 100 366 | $14 367 | slave-priority 368 | $3 369 | 100 370 | $24 371 | repl-diskless-sync-delay 372 | $1 373 | 5 374 | $17 375 | maxmemory-samples 376 | $1 377 | 5 378 | $27 379 | maxmemory-eviction-tenacity 380 | $2 381 | 10 382 | $7 383 | timeout 384 | $1 385 | 0 386 | $21 387 | replica-announce-port 388 | $1 389 | 0 390 | $19 391 | slave-announce-port 392 | $1 393 | 0 394 | $11 395 | tcp-backlog 396 | $3 397 | 511 398 | $25 399 | cluster-announce-bus-port 400 | $1 401 | 0 402 | $21 403 | cluster-announce-port 404 | $1 405 | 0 406 | $25 407 | cluster-announce-tls-port 408 | $1 409 | 0 410 | $12 411 | repl-timeout 412 | $2 413 | 60 414 | $24 415 | repl-ping-replica-period 416 | $2 417 | 10 418 | $22 419 | repl-ping-slave-period 420 | $2 421 | 10 422 | $19 423 | list-compress-depth 424 | $1 425 | 0 426 | $18 427 | rdb-key-save-delay 428 | $1 429 | 0 430 | $14 431 | key-load-delay 432 | $1 433 | 0 434 | $20 435 | active-expire-effort 436 | $1 437 | 1 438 | $2 439 | hz 440 | $2 441 | 10 442 | $21 443 | min-replicas-to-write 444 | $1 445 | 0 446 | $19 447 | min-slaves-to-write 448 | $1 449 | 0 450 | $20 451 | min-replicas-max-lag 452 | $2 453 | 10 454 | $18 455 | min-slaves-max-lag 456 | $2 457 | 10 458 | $10 459 | maxclients 460 | $5 461 | 10000 462 | $29 463 | active-defrag-max-scan-fields 464 | $4 465 | 1000 466 | $15 467 | slowlog-max-len 468 | $3 469 | 128 470 | $14 471 | acllog-max-len 472 | $3 473 | 128 474 | $14 475 | lua-time-limit 476 | $4 477 | 5000 478 | $20 479 | cluster-node-timeout 480 | $5 481 | 15000 482 | $23 483 | slowlog-log-slower-than 484 | $5 485 | 10000 486 | $25 487 | latency-monitor-threshold 488 | $1 489 | 0 490 | $18 491 | proto-max-bulk-len 492 | $9 493 | 536870912 494 | $23 495 | stream-node-max-entries 496 | $3 497 | 100 498 | $17 499 | repl-backlog-size 500 | $7 501 | 1048576 502 | $9 503 | maxmemory 504 | $1 505 | 0 506 | $24 507 | hash-max-ziplist-entries 508 | $3 509 | 512 510 | $22 511 | set-max-intset-entries 512 | $3 513 | 512 514 | $24 515 | zset-max-ziplist-entries 516 | $3 517 | 128 518 | $26 519 | active-defrag-ignore-bytes 520 | $9 521 | 104857600 522 | $22 523 | hash-max-ziplist-value 524 | $2 525 | 64 526 | $21 527 | stream-node-max-bytes 528 | $4 529 | 4096 530 | $22 531 | zset-max-ziplist-value 532 | $2 533 | 64 534 | $20 535 | hll-sparse-max-bytes 536 | $4 537 | 3000 538 | $23 539 | tracking-table-max-keys 540 | $7 541 | 1000000 542 | $25 543 | client-query-buffer-limit 544 | $10 545 | 1073741824 546 | $16 547 | repl-backlog-ttl 548 | $4 549 | 3600 550 | $25 551 | auto-aof-rewrite-min-size 552 | $8 553 | 67108864 554 | $8 555 | tls-port 556 | $1 557 | 0 558 | $22 559 | tls-session-cache-size 560 | $5 561 | 20480 562 | $25 563 | tls-session-cache-timeout 564 | $3 565 | 300 566 | $11 567 | tls-cluster 568 | $2 569 | no 570 | $15 571 | tls-replication 572 | $2 573 | no 574 | $16 575 | tls-auth-clients 576 | $3 577 | yes 578 | $25 579 | tls-prefer-server-ciphers 580 | $2 581 | no 582 | $19 583 | tls-session-caching 584 | $3 585 | yes 586 | $13 587 | tls-cert-file 588 | $0 589 | 590 | $12 591 | tls-key-file 592 | $0 593 | 594 | $17 595 | tls-key-file-pass 596 | $0 597 | 598 | $20 599 | tls-client-cert-file 600 | $0 601 | 602 | $19 603 | tls-client-key-file 604 | $0 605 | 606 | $24 607 | tls-client-key-file-pass 608 | $0 609 | 610 | $18 611 | tls-dh-params-file 612 | $0 613 | 614 | $16 615 | tls-ca-cert-file 616 | $0 617 | 618 | $15 619 | tls-ca-cert-dir 620 | $0 621 | 622 | $13 623 | tls-protocols 624 | $0 625 | 626 | $11 627 | tls-ciphers 628 | $0 629 | 630 | $16 631 | tls-ciphersuites 632 | $0 633 | 634 | $7 635 | logfile 636 | $0 637 | 638 | $15 639 | watchdog-period 640 | $1 641 | 0 642 | $3 643 | dir 644 | $5 645 | /data 646 | $4 647 | save 648 | $23 649 | 3600 1 300 100 60 10000 650 | $26 651 | client-output-buffer-limit 652 | $67 653 | normal 0 0 0 slave 268435456 67108864 60 pubsub 33554432 8388608 60 654 | $14 655 | unixsocketperm 656 | $1 657 | 0 658 | $7 659 | slaveof 660 | $0 661 | 662 | $22 663 | notify-keyspace-events 664 | $0 665 | 666 | $4 667 | bind 668 | $0 669 | 670 | $20 671 | oom-score-adj-values 672 | $9 673 | 0 200 800 674 | -------------------------------------------------------------------------------- /Sources/RedisConnection/RedisStreamingParser.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | // https://redis.io/topics/protocol 4 | // https://github.com/antirez/RESP3/blob/master/spec.md 5 | 6 | // NOT IMPLEMENTED: STREAM TYPES 7 | 8 | public protocol RESPParser { 9 | mutating func parse(byte: UInt8) throws -> RESPValue? 10 | } 11 | 12 | public extension RESPParser { 13 | mutating func parse(bytes: some Collection) throws -> RESPValue? { 14 | for byte in bytes { 15 | if let value = try parse(byte: byte) { 16 | return value 17 | } 18 | } 19 | return nil 20 | } 21 | } 22 | 23 | // MARK: - 24 | 25 | public struct RESPSimpleValueParserHelper: RESPParser { 26 | enum State { 27 | case waiting 28 | case headerConsumed 29 | } 30 | 31 | let header: UInt8 32 | let value: ([UInt8]) throws -> RESPValue? 33 | var state: State = .waiting 34 | var bytes: [UInt8] = [] 35 | 36 | public init(header: UInt8, value: @escaping ([UInt8]) throws -> RESPValue?) { 37 | self.header = header 38 | self.value = value 39 | } 40 | 41 | public mutating func parse(byte: UInt8) throws -> RESPValue? { 42 | bytes.append(byte) 43 | switch (state, byte, Array(bytes.suffix(2))) { 44 | case (.waiting, header, _): 45 | state = .headerConsumed 46 | bytes = [] 47 | return nil 48 | case (.headerConsumed, _, .crlf): 49 | defer { 50 | self.state = .waiting 51 | self.bytes = [] 52 | } 53 | return try value(bytes.dropLast(2)) 54 | case (.headerConsumed, _, _): 55 | return nil 56 | default: 57 | throw RedisError.parseError 58 | } 59 | } 60 | } 61 | 62 | public struct RESPFramedValueParserHelper: RESPParser { 63 | enum State { 64 | case waiting 65 | case headerConsumed 66 | case lengthConsumed 67 | } 68 | 69 | let header: UInt8 70 | let value: (Int, [UInt8]) -> RESPValue? 71 | var state: State = .waiting 72 | var bytes: [UInt8] = [] 73 | var length: Int? 74 | 75 | public init(header: UInt8, value: @escaping (Int, [UInt8]) -> RESPValue?) { 76 | self.header = header 77 | self.value = value 78 | } 79 | 80 | public mutating func parse(byte: UInt8) throws -> RESPValue? { 81 | bytes.append(byte) 82 | switch (state, byte, Array(bytes.suffix(2)), bytes.count) { 83 | case (.waiting, header, _, _): 84 | state = .headerConsumed 85 | bytes = [] 86 | return nil 87 | case (.headerConsumed, _, .crlf, _): 88 | state = .lengthConsumed 89 | guard let s = String(bytes: bytes.dropLast(2), encoding: .ascii) else { 90 | throw RedisError.stringDecodingError 91 | } 92 | guard let i = Int(s) else { 93 | throw RedisError.parseError 94 | } 95 | length = i 96 | bytes = [] 97 | return nil 98 | case (.headerConsumed, _, _, _): 99 | return nil 100 | case (.lengthConsumed, _, .crlf, length! + 2): 101 | defer { 102 | state = .waiting 103 | bytes = [] 104 | } 105 | return value(length!, bytes.dropLast(2)) 106 | case (.lengthConsumed, _, _, _): 107 | return nil 108 | default: 109 | throw RedisError.parseError 110 | } 111 | } 112 | } 113 | 114 | public struct RESPCollectionParserHelper: RESPParser { 115 | enum State { 116 | case waiting 117 | case headerConsumed 118 | case countConsumed 119 | } 120 | 121 | let header: UInt8 122 | let valueCount: (Int) -> Int 123 | let value: (Int, [RESPValue]) throws -> RESPValue? 124 | var state: State = .waiting 125 | var bytes: [UInt8] = [] 126 | var count: Int? 127 | var values: [RESPValue] = [] 128 | var valueParser = RESPValueParser() 129 | 130 | public init(header: UInt8, valueCount: @escaping (Int) -> Int = { $0 }, value: @escaping (Int, [RESPValue]) throws -> RESPValue?) { 131 | self.header = header 132 | self.valueCount = valueCount 133 | self.value = value 134 | } 135 | 136 | public mutating func parse(byte: UInt8) throws -> RESPValue? { 137 | bytes.append(byte) 138 | switch (state, byte, Array(bytes.suffix(2)), bytes.count) { 139 | case (.waiting, header, _, _): 140 | state = .headerConsumed 141 | bytes = [] 142 | return nil 143 | case (.headerConsumed, _, .crlf, _): 144 | state = .countConsumed 145 | guard let s = String(bytes: bytes.dropLast(2), encoding: .ascii) else { 146 | throw RedisError.stringDecodingError 147 | } 148 | guard let i = Int(s) else { 149 | throw RedisError.parseError 150 | } 151 | count = valueCount(i) 152 | bytes = [] 153 | if count == 0 { 154 | return try value(0, []) 155 | } 156 | return nil 157 | case (.headerConsumed, _, _, _): 158 | return nil 159 | case (.countConsumed, _, _, _): 160 | if let value = try valueParser.parse(byte: byte) { 161 | values.append(value) 162 | valueParser = RESPValueParser() 163 | if values.count == count { 164 | return try self.value(count!, values) 165 | } 166 | } 167 | return nil 168 | default: 169 | throw RedisError.parseError 170 | } 171 | } 172 | } 173 | 174 | // MARK: - 175 | 176 | public struct RESPValueParser: RESPParser { 177 | private var helper: RESPParser! 178 | 179 | // private var bytes: [UInt8] = [] 180 | public private(set) var bytesParsed: Int = 0 181 | 182 | public init() {} 183 | 184 | public mutating func parse(byte: UInt8) throws -> RESPValue? { 185 | // bytes.append(byte) 186 | if helper == nil { 187 | switch Character(UnicodeScalar(byte)) { 188 | case "+": 189 | helper = RESPSimpleValueParserHelper(header: UInt8(ascii: "+")) { 190 | .simpleString(String(bytes: $0, encoding: .utf8)!) 191 | } 192 | case "-": 193 | helper = RESPSimpleValueParserHelper(header: UInt8(ascii: "-")) { 194 | .errorString(String(bytes: $0, encoding: .utf8)!) 195 | } 196 | case ":": 197 | helper = RESPSimpleValueParserHelper(header: UInt8(ascii: ":")) { 198 | .integer(Int(String(bytes: $0, encoding: .utf8)!)!) 199 | } 200 | case "$": 201 | helper = RESPFramedValueParserHelper(header: UInt8(ascii: "$")) { length, bytes in 202 | if length == -1 { 203 | .nullBulkString 204 | } 205 | else { 206 | .blobString(bytes) 207 | } 208 | } 209 | case "*": 210 | helper = RESPCollectionParserHelper(header: UInt8(ascii: "*")) { count, values in 211 | if count == -1 { 212 | return .nullArray 213 | } 214 | else { 215 | guard count == values.count else { 216 | throw RedisError.parseError 217 | } 218 | return .array(values) 219 | } 220 | } 221 | case "_": 222 | helper = RESPSimpleValueParserHelper(header: UInt8(ascii: "_")) { 223 | guard $0.isEmpty else { 224 | throw RedisError.parseError 225 | } 226 | return .null 227 | } 228 | case "#": 229 | helper = RESPSimpleValueParserHelper(header: UInt8(ascii: "#")) { 230 | switch String(bytes: $0, encoding: .utf8) { 231 | case "t": 232 | return .boolean(true) 233 | case "f": 234 | return .boolean(false) 235 | default: 236 | throw RedisError.parseError 237 | } 238 | } 239 | case "!": 240 | helper = RESPFramedValueParserHelper(header: UInt8(ascii: "!")) { _, bytes in 241 | .blobError(bytes) 242 | } 243 | case "=": 244 | helper = RESPFramedValueParserHelper(header: UInt8(ascii: "=")) { _, bytes in 245 | .verbatimString(bytes) 246 | } 247 | case "(": 248 | helper = RESPSimpleValueParserHelper(header: UInt8(ascii: "(")) { 249 | .bigNumber($0) 250 | } 251 | case "%": 252 | helper = RESPCollectionParserHelper(header: UInt8(ascii: "%")) { 253 | $0 * 2 254 | } 255 | value: { _, values in 256 | let pairs = try stride(from: 0, to: values.count, by: 2).map { 257 | Array(values[$0 ..< Swift.min($0 + 2, values.count)]) 258 | } 259 | .map { (values: [RESPValue]) -> (RESPValue, RESPValue) in 260 | guard values.count == 2 else { 261 | throw RedisError.parseError 262 | } 263 | return (values[0], values[1]) 264 | } 265 | return .map(Dictionary(uniqueKeysWithValues: pairs)) 266 | } 267 | case "~": 268 | helper = RESPCollectionParserHelper(header: UInt8(ascii: "~")) { _, values in 269 | .set(Set(values)) 270 | } 271 | case "|": 272 | helper = RESPCollectionParserHelper(header: UInt8(ascii: "|")) { 273 | $0 * 2 274 | } 275 | value: { _, values in 276 | let pairs = stride(from: 0, to: values.count, by: 2).map { 277 | Array(values[$0 ..< Swift.min($0 + 2, values.count)]) 278 | } 279 | .map { 280 | ($0[0], $0[1]) 281 | } 282 | return .attribute(Dictionary(uniqueKeysWithValues: pairs)) 283 | } 284 | case ",": 285 | helper = RESPSimpleValueParserHelper(header: UInt8(ascii: ",")) { 286 | .double(Double(String(bytes: $0, encoding: .utf8)!)!) 287 | } 288 | case ">": 289 | helper = RESPCollectionParserHelper(header: UInt8(ascii: ">")) { _, values in 290 | guard let kind = try Pubsub.Kind(rawValue: values[0].stringValue.lowercased()) else { 291 | throw RedisError.parseError 292 | } 293 | let pubsub = try Pubsub( 294 | kind: kind, 295 | channel: values[1].stringValue, 296 | value: values[2] 297 | ) 298 | return .pubsub(pubsub) 299 | } 300 | default: 301 | // throw RedisError.undefined("Unknown header character, \(Character(UnicodeScalar(byte))), bytes: \(String(bytes: bytes, encoding: .utf8)!.replacingOccurrences(of: "\r\n", with: "\\r\\n"))") 302 | throw RedisError.unknownHeader(Character(UnicodeScalar(byte))) 303 | } 304 | } 305 | defer { 306 | bytesParsed += 1 307 | } 308 | if let value = try helper.parse(byte: byte) { 309 | helper = nil 310 | return value 311 | } 312 | return nil 313 | } 314 | } 315 | 316 | public extension RESPValueParser { 317 | static func parse(bytes: some Collection) throws -> RESPValue? { 318 | var parser = RESPValueParser() 319 | return try parser.parse(bytes: bytes) 320 | } 321 | } 322 | -------------------------------------------------------------------------------- /Documentation/spec.md: -------------------------------------------------------------------------------- 1 | # RESP3 specification 2 | 3 | Versions history: 4 | * 1.0, 2 May 2018, Initial draft to get community feedbacks. 5 | * 1.1, 5 Nov 2018, Leave CRLF as terminators + improved "hello reply" section. 6 | * 1.2, 5 Nov 2018, A few things are now better specified in the document 7 | thanks to developers reading the specification and 8 | sending their feedbacks. No actual change to the protocol 9 | was made. 10 | * 1.3, 11 Mar 2019, Streamed strings and streamed aggregated types. 11 | 12 | ## Background 13 | 14 | The Redis protocol has served us well in the past years, showing that, if carefully designed, a simple human readable protocol is not the bottleneck in the client server communication, and that the simplicity of the design is a major advantage in creating a healthy client libraries ecosystem. 15 | 16 | Yet the Redis experience has shown that after about six years from its introduction (when it replaced the initial Redis protocol), the current RESP protocol could be improved, especially in order to make client implementations simpler and to support new features. 17 | 18 | At the same time the basic structure is tested and sounding, and there are several things to save from the old design: 19 | 20 | * RESP is human readable. This confers to the protocol the important quality of being simple to observe and debug. 21 | * Despite being human readable, RESP is no slower than a binary protocol with fixed length fields, and in many cases can be more compact. Often Redis commands have few arguments composed of few bytes each. Similarly many replies are short. The protocol prefixed lengths represented as decimal digits will save space on the wire, compared to 64 bit integers, and will not be slower to parse. It is possible to design a more efficient binary protocol only introducing the complexity of a variable length encoding, defeating the goal of simplicity. 22 | * The design is very simple: this makes the specification and the resulting implementations short and easy to understand. 23 | 24 | At the same time, RESP has flaws. One of the most obvious is the fact that it has not enough semantic power to allow the client to implicitly understand what kind of conversion is appropriate. For instance the Redis commands `LRANGE`, `SMEMBERS` and `HGETALL` will all reply an array, called in RESP v2 terms a *multi bulk reply*. However the three commands actually return an array, a set and a map. 25 | Currently, from the point of view of the client, the conversion of the reply to the programming language type is command-dependent. Clients need to remember what command was called in order to turn the Redis reply into a reply object of some kind. What is worse is that clients need to know about each command, or alternatively provide a lower level interface letting users select the appropriate conversion. 26 | 27 | Similarly RESP lacks important data types: floating point numbers and boolean values were returned respectively as strings and integers. Null values had a double representation, called *null bulk* and *null multi bulk*, which is useless, since the semantic value of telling apart null arrays from null strings is non existent. 28 | 29 | Finally there was no way to return binary safe errors. When implementing generic APIs producing the protocol, implementations must check and remove potential newlines from the error string. 30 | 31 | The limitations stated so far are the main motivations for a new version of RESP. However the redesign gave us a chance to consider different other improvements that may be worthwhile and make the protocol both more powerful and less dependent on the implicit state of the connection. 32 | 33 | The gist of such additional features are to provide the protocol with the ability to support a more generic *push mode* compared to the Pub/Sub mode used by Redis, which is not really built-in in the protocol, but is an agreement between the server and the client about the fact that the connection will consume replies as they arrive. Moreover it was useful to modify the protocol to return *out of band* data, such as attributes that augment the reply. Also the protocol was sometimes abused by the internals of Redis, like for example in the case of *replicaless replication*, in order to support streaming of large strings whose length is initially not known. This specification makes this special mode part of the RESP protocol itself. 34 | 35 | This specification describes RESP3 from scratch, and not just as a change from RESP v2. However differences between the two will be noted while describing the new protocol. 36 | 37 | ## Why existing serialization protocols are not good enough? 38 | 39 | In theory instead of improving RESP v2, we could use an existing serialization 40 | protocol like MessagePack, BSON or others, which are widely available 41 | and implemented. This is a viable approach, and could be a potential solution, 42 | however there are certain design goals that RESP has that are not aligned 43 | with using such serialization formats. 44 | 45 | The first and most important is the fact that such serialization protocols 46 | are not specifically designed for a request-response server-client chat, so 47 | clients and servers would be required to agree on an additional protocol on 48 | top of the underlying serialization protocol. Basically it is not possible 49 | to escape the problem of designing a Redis specific protocol, so 50 | to bind together the serialization and the semantic looks a more effective 51 | way to reach the goals of RESP3. 52 | 53 | A different problem is the fact that such serialization protocols are more 54 | complex than RESP, so client libraries would have to rely on a separated 55 | library implementing the serialization protocol. The library may not have 56 | support for streaming directly to a socket, and may return a buffer instead, 57 | leading to potential inefficiencies. Relying on a library in order to perform 58 | the serialization has the other disadvantage of having more moving parts 59 | where things may go wrong, or where different versions of Redis and 60 | serialization libraries may not work well together. 61 | 62 | Finally certain features like transferring large strings with initially 63 | unknown length, or streaming of arrays, two features that RESP3 supports 64 | and that this specification describes, are not easy to found among 65 | existing serialization protocols. 66 | 67 | In short this specification was written believing that designing a good 68 | serialization format is different compared to designing a protocol 69 | specifically suited in order to support the chat between a client and 70 | its server. RESP3 aims to create a protocol which is not just suitable 71 | for Redis, but more broadly to solve the general problem of client-server 72 | communication in many scenarios even outside Redis and outside the 73 | database space. 74 | 75 | ## Conventions used in this document 76 | 77 | In order to illustrate the RESP3 protocol this specification will show 78 | fragments of the protocol in many sections. In addition of using the escaped 79 | string format, like "foo\r\n", we'll use a more readable format where 80 | "\r\n" will be presented as `` followed by an actual newline. Other 81 | special characters will be displayed as `<\xff>`, where `ff` is the 82 | hex code of the byte. 83 | 84 | So for instance the string `"*1\r\n$1\r\nA\r\n"` representing a RESP3 array with 85 | a single string `"A"` as unique element, will be presented like this: 86 | 87 | *1 88 | $1 89 | A 90 | 91 | However for the first part of this specification, both this format and 92 | the escaped string format will be used in order to make sure there is 93 | no confusion. In the latter sections however only one or the other 94 | will be used. 95 | 96 | When nested aggregate data structures are shown, indentation is used in 97 | order to make it more clear the actual structure of the protocol. For 98 | instance instead of writing: 99 | 100 | *2 101 | *2 102 | :1 103 | :2 104 | #t 105 | 106 | We'll write: 107 | 108 | *2 109 | *2 110 | :1 111 | :2 112 | #t 113 | 114 | Both the indentation and the newlines are hence only used in order to improve 115 | human readability and are not semantical in respect of the actual protocol. 116 | 117 | ## RESP3 overview 118 | 119 | RESP3 is an updated version of RESP v2, which is the protocol used in Redis 120 | starting with roughly version 2.0 (1.2 already supported it, but Redis 2.0 121 | was the first version to talk only this protocol). The name of this protocol 122 | is just **RESP3** and not RESP v3 or RESP 3.0. 123 | 124 | The protocol is designed to handle request-response chats between clients 125 | and servers, where the client performs some kind of request, and the server 126 | replies with some data. The protocol is especially suitable for databases due 127 | to its ability to return complex data types and associated information to 128 | augment the returned data (for instance the popularity index of a given 129 | information). 130 | 131 | The RESP3 protocol can be used asymmetrically, as it is in Redis: only a subset 132 | can be sent by the client to the server, while the server can return the full set 133 | of types available. This is due to the fact that RESP is designed to send non 134 | structured commands like `SET mykey somevalue` or `SADD myset a b c d`. Such 135 | commands can be represented as arrays, where each argument is an array element, 136 | so this is the only type the client needs to send to a server. However different 137 | applications willing to use RESP3 for other goals may just allow the protocol 138 | to be used in a "full duplex" fashion where both the ends can use the full set 139 | of types available. 140 | 141 | Not all parts of RESP3 are mandatory for clients and servers. In the specific 142 | case of Redis, RESP3 describes certain functionalities that will be useful 143 | in the future and likely will not be initially implemented. Other optional parts 144 | of RESP3 may be implemented by Redis only in specific situations, like the 145 | link between the primary database and its replicas, or client connections in 146 | a specific state. 147 | 148 | ## RESP3 types 149 | 150 | RESP3 abandons the confusing wording of the second version of RESP, and uses 151 | a much simpler to understand name for types, so you'll see no mention of 152 | *bulk reply* or *multi bulk reply* in this document. 153 | 154 | The following are the types implemented by RESP3: 155 | 156 | **Types equivalent to RESP version 2** 157 | 158 | * Array: an ordered collection of N other types 159 | * Blob string: binary safe strings 160 | * Simple string: a space efficient non binary safe string 161 | * Simple error: a space efficient non binary safe error code and message 162 | * Number: an integer in the signed 64 bit range 163 | 164 | **Types introduced by RESP3** 165 | 166 | * Null: a single null value replacing RESP v2 `*-1` and `$-1` null values. 167 | * Double: a floating point number 168 | * Boolean: true or false 169 | * Blob error: binary safe error code and message. 170 | * Verbatim string: a binary safe string that should be displayed to humans without any escaping or filtering. For instance the output of `LATENCY DOCTOR` in Redis. 171 | * Map: an ordered collection of key-value pairs. Keys and values can be any other RESP3 type. 172 | * Set: an unordered collection of N other types. 173 | * Attribute: Like the Map type, but the client should keep reading the reply ignoring the attribute type, and return it to the client as additional information. 174 | * Push: Out of band data. The format is like the Array type, but the client should just check the first string element, stating the type of the out of band data, a call a callback if there is one registered for this specific type of push information. Push types are not related to replies, since they are information that the server may push at any time in the connection, so the client should keep reading if it is reading the reply of a command. 175 | * Hello: Like the Map type, but is sent only when the connection between the client and the server is established, in order to welcome the client with different information like the name of the server, its version, and so forth. 176 | * Big number: a large number non representable by the Number type 177 | 178 | ## Simple types 179 | 180 | This section describes all the RESP3 types which are not aggregate types. 181 | They consist of just a single typed element. 182 | 183 | **Blob string** 184 | 185 | The general form is `$\r\n\r\n`. It is basically exactly like 186 | in the previous version of RESP. 187 | 188 | The string `"hello world"` is represented by the following protocol: 189 | 190 | $11 191 | helloworld 192 | 193 | Or as an escaped string: 194 | 195 | "$11\r\nhelloworld\r\n" 196 | 197 | The length field is limited to the range of an unsigned 64 bit 198 | integer. Zero is a valid length, so the empty string is represented by: 199 | 200 | "$0\r\n\r\n" 201 | 202 | **Simple string** 203 | 204 | The general form is `+\r\n`, so "hello world" is encoded as 205 | 206 | +hello world 207 | 208 | Or as an escaped string: 209 | 210 | "+hello world\r\n" 211 | 212 | Simple strings cannot contain the `` nor the `` characters inside. 213 | 214 | **Simple error** 215 | 216 | This is exactly like a simple string, but the initial byte is `-` instead 217 | of `+`: 218 | 219 | -ERR this is the error description 220 | 221 | Or as an escaped string: 222 | 223 | "-ERR this is the error description\r\n" 224 | 225 | The first word in the error is in upper case and describes the error 226 | code. The remaining string is the error message itself. 227 | The `ERR` error code is the generic one. The error code is useful for 228 | clients to distinguish among different error conditions without having 229 | to do pattern matching in the error message, that may change. 230 | 231 | **Number** 232 | 233 | The general form is `:\r\n`, so the number 1234 is encoded as 234 | 235 | :1234 236 | 237 | Or as an escaped string: 238 | 239 | ":1234\r\n" 240 | 241 | Valid numbers are in the range of the signed 64 bit integer. 242 | Larger numbers should use the Big Number type instead. 243 | 244 | **Null** 245 | 246 | The null type is encoded just as `_\r\n`, which is just the underscore 247 | character followed by the `CR` and `LF` characters. 248 | 249 | **Double** 250 | 251 | The general form is `,\r\n`. For instance 1.23 is 252 | encoded as: 253 | 254 | ,1.23 255 | 256 | Or as an escaped string: 257 | 258 | ",1.23\r\n" 259 | 260 | To just start with `.` assuming an initial zero is invalid. 261 | Exponential format is invalid. 262 | To completely miss the decimal part, that is, the point followed by other 263 | digits, is valid, so the number 10 may be returned both using the number 264 | or double format: 265 | 266 | ":10\r\n" 267 | ",10\r\n" 268 | 269 | However the client library should return a floating point number in one 270 | case and an integer in the other case, if the programming language in which 271 | the client is implemented has a clear distinction between the two types. 272 | 273 | In addition the double reply may return positive or negative infinity 274 | as the following two stings: 275 | 276 | ",inf\r\n" 277 | ",-inf\r\n" 278 | 279 | So client implementations should be able to handle this correctly. 280 | 281 | **Boolean** 282 | 283 | True and false values are just represented using `#t\r\n` and `#f\r\n` 284 | sequences. Client libraries implemented in programming languages without 285 | the boolean type should return to the client the canonical values used 286 | to represent true and false in such languages. For instance a C program 287 | should likely return an integer type with a value of 0 or 1. 288 | 289 | **Blob error** 290 | 291 | The general form is `!\r\n\r\n`. It is exactly like the String 292 | type. However like the Simple error type, the first uppercase word represents 293 | the error code. 294 | 295 | The error `"SYNTAX invalid syntax"` is represented by the following protocol: 296 | 297 | !21 298 | SYNTAX invalid syntax 299 | 300 | Or as an escaped string: 301 | 302 | "!21\r\nSYNTAX invalid syntax\r\n" 303 | 304 | **Verbatim string** 305 | 306 | This is exactly like the Blob string type, but the initial byte is `=` instead 307 | of `$`. Moreover the first three bytes provide information about the format 308 | of the following string, which can be `txt` for plain text, or `mkd` for 309 | markdown. The fourth byte is always `:`. Then the real string follows. 310 | 311 | For instance this is a valid verbatim string: 312 | 313 | =15 314 | txt:Some string 315 | 316 | Normal client libraries may ignore completely the difference between this 317 | type and the String type, and return a string in both cases. However interactive 318 | clients such as command line interfaces (for instance `redis-cli`), knows that 319 | the output must be presented to the human user as it is, without quoting 320 | the string. 321 | 322 | For example the Redis command `LATENCY DOCTOR` outputs a report that includes 323 | newlines. It's basically a plain text document. However currently `redis-cli` 324 | requires special handling to avoid quoting the resulting string as it normally 325 | does when a string is received. 326 | 327 | From the `redis-cli` source code: 328 | 329 | ```c 330 | output_raw = 0; 331 | if (!strcasecmp(command,"info") || 332 | ... [snip] ... 333 | (argc == 3 && !strcasecmp(command,"latency") && 334 | !strcasecmp(argv[1],"graph")) || 335 | (argc == 2 && !strcasecmp(command,"latency") && 336 | !strcasecmp(argv[1],"doctor"))) 337 | { 338 | output_raw = 1; 339 | } 340 | ``` 341 | 342 | With the introduction of verbatim strings clients can be simplified not 343 | having to remember if the output must be escaped or not. 344 | 345 | **Big number** 346 | 347 | This type represents very large numbers that are out of the range of 348 | the signed 64 bit numbers that can be represented by the Number type. 349 | 350 | The general form is `(\r\n`, like in the following example: 351 | 352 | (3492890328409238509324850943850943825024385 353 | 354 | Or as an escaped string: 355 | 356 | "(3492890328409238509324850943850943825024385\r\n" 357 | 358 | Big numbers can be positive or negative, but they must not include a 359 | decimal part. Client libraries written in languages with support for big 360 | numbers should just return a big number. When big numbers are not available 361 | the client should return a string, signaling however that the reply is 362 | a big integer when possible (it depends on the API used by the client 363 | library). 364 | 365 | ## Aggregate data types overview 366 | 367 | The types described so far are simple types that just define a single 368 | item of a given type. However the core of RESP3 is the ability to represent 369 | different kinds of aggregate data types having different semantic meaning, 370 | both from the type perspective, and from the protocol perspective. 371 | 372 | In general an aggregate type has a given format stating what is the type 373 | of the aggregate, and how many elements there are inside the aggregate. 374 | Then the single elements follow. Elements of the aggregate type can be, in turn, 375 | other aggregate types, so it is possible to have an array of arrays, or 376 | a map of sets, and so forth. Normally Redis commands will use just a subset 377 | of the possibilities. However with Lua scripts or using Redis modules any 378 | combination is possible. From the point of view of the client library however 379 | this is not complex to handle: every type fully specifies how the client 380 | should translate it to report it to the user, so all the aggregated data types 381 | are implemented as recursive functions that then read N other types. 382 | 383 | The format for every aggregate type in RESP3 is always the same: 384 | 385 | ... numelements other types ... 386 | 387 | The aggregate type char for the Array is `*`, so to represent an array 388 | of three Numbers 1, 2, 3, the following protocol will be emitted: 389 | 390 | *3 391 | :1 392 | :2 393 | :3 394 | 395 | Or as an escaped string: 396 | 397 | "*3\r\n:1\r\n:2\r\n:3\r\n" 398 | 399 | Of course an array can also contain other nested arrays: 400 | 401 | *2 402 | *3 403 | :1 404 | $5 405 | hello 406 | :2 407 | #f 408 | 409 | The above represents the array `[[1,"hello",2],false]` 410 | 411 | Client libraries should return the arrays with a sensible type representing 412 | ordered sequences of elements, accessible at random indexes in constant 413 | or logarithmic time. For instance a Ruby client should return a 414 | *Ruby array* type, while Python should use a *Python list*, and so forth. 415 | 416 | ## Map type 417 | 418 | Maps are represented exactly as arrays, but instead of using the `*` 419 | byte, the encoded value starts with a `%` byte. Moreover the number of 420 | following elements must be even. Maps represent a sequence of field-value 421 | items, basically what we could call a dictionary data structure, or in 422 | other terms, an hash. 423 | 424 | For instance the dictionary represented in JSON by: 425 | 426 | { 427 | "first":1, 428 | "second":2 429 | } 430 | 431 | Is represented in RESP3 as: 432 | 433 | %2 434 | +first 435 | :1 436 | +second 437 | :2 438 | 439 | Note that after the `%` character, what follows is not, like in the array, 440 | the number of single items, but the number of field-value pairs. 441 | 442 | Maps can have any other type as field and value, however Redis will use 443 | only a subset of the available possibilities. For instance it is very unlikely 444 | that Redis commands would return an Array as a key, however Lua scripts 445 | and modules will likely be able to do so. 446 | 447 | Client libraries should return Maps using the idiomatic dictionary type 448 | available. However low level languages like C will likely still return 449 | an array of items, but with type information so that the user can tell 450 | the reply is actually a dictionary. 451 | 452 | ## Set reply 453 | 454 | Sets are exactly like the Array type, but the first byte is `~` instead 455 | of `*`: 456 | 457 | ~5 458 | +orange 459 | +apple 460 | #t 461 | :100 462 | :999 463 | 464 | However they are semantically different because the represented 465 | items are *unordered* collections of elements, so the client library should 466 | return a type that, while not necessarily ordered, has a *test for existence* 467 | operation running in constant or logarithmic time. 468 | 469 | Since many programming languages lack a native set type, a sensible 470 | choice is to return an Hash where the fields are the elements inside the 471 | Set type, and the values are just *true* values or any other value. 472 | 473 | In lower level programming languages such as C, the type should be still 474 | reported as a linear array, together with type information to signal the 475 | user it is a Set type. 476 | 477 | Normally Set replies should not contain the same element emitted multiple 478 | times, but the protocol does not enforce that: client libraries should try 479 | to handle such case, and in case of repeated elements, do some effort to 480 | avoid returning duplicated data, at least if some form of hash is used in 481 | order to return the reply. Otherwise when returning an array just reading 482 | what the protocol contains, duplicated items if present could be passed 483 | by client libraries to the caller. Many implementations will find it very 484 | natural to avoid duplicates. For instance they'll try to add every read 485 | element in some Map or Hash or Set data type, and adding the same element 486 | again will either replace the old copy or will fail silently, retaining the 487 | old copy. 488 | 489 | ## Attribute type 490 | 491 | The attribute type is exactly like the Map type, but instead of the `%` 492 | first byte, the `|` byte is used. Attributes describe a dictionary exactly 493 | like the Map type, however the client should not consider such a dictionary 494 | part of the reply, but *just auxiliary data* that is used in order to 495 | augment the reply. 496 | 497 | For instance newer versions of Redis may include the ability to report, for 498 | every executed command, the popularity of keys. So the reply to the command 499 | `MGET a b` may be the following: 500 | 501 | |1 502 | +key-popularity 503 | %2 504 | $1 505 | a 506 | ,0.1923 507 | $1 508 | b 509 | ,0.0012 510 | *2 511 | :2039123 512 | :9543892 513 | 514 | The actual reply to `MGET` was just the two items array `[2039123,9543892]`, 515 | however the attributes specify the popularity (frequency of requests) of the 516 | keys mentioned in the original command, as floating point numbers from 0 517 | to 1 (at least in the example, the actual Redis implementation may differ). 518 | 519 | When a client reads a reply and encounters an attribute type, it should read 520 | the attribute, and continue reading the reply. The attribute reply should 521 | be accumulated separately, and the user should have a way to access such 522 | attributes. For instance, if we imagine a session in an higher level language, 523 | something like that could happen: 524 | 525 | > r = Redis.new 526 | # 527 | 528 | > r.mget("a","b") 529 | # 530 | 531 | > r 532 | [2039123,9543892] 533 | 534 | > r.attribs 535 | {:key-popularity => {:a => 0.1923, :b => 0.0012}} 536 | 537 | Attributes can appear anywhere before a valid part of the protocol identifying 538 | a given type, and will inform only the part of the reply that immediately 539 | follows, like in the following example: 540 | 541 | *3 542 | :1 543 | :2 544 | |1 545 | +ttl 546 | :3600 547 | :3 548 | 549 | In the above example the third element of the array has an associated 550 | auxiliary information of `{ttl:3600}`. Note that it's not up to the client 551 | library to interpret the attributes, they'll just be passed to the caller 552 | in a sensible way. 553 | 554 | ## Push type 555 | 556 | A push connection is one where the usual *request-response* mode of the 557 | protocol is no longer true, and the server may send to the client asynchronous 558 | data which was not explicitly requested. 559 | 560 | In Redis there is already the concept of a connection pushing data in at least 561 | three different parts of the Redis protocol: 562 | 563 | 1. Pub/Sub is a push-mode connection, where clients receive published data. 564 | 2. The `MONITOR` command implements an *ad-hoc* push mode with a kinda unspecified protocol which is obvious to parse, but still, unspecified. 565 | 3. The Master-Replica link may, at a first glance, be considered a push mode connection. However actually in this case the client (which is the replica), even if is the entity starting the connection, will configure the connection like if the master is a client sending commands, so in practical terms, it is unfair to call this a push mode connection. 566 | 567 | Let's ignore the master-replica link since it is an internal protocol, and 568 | as already noted, is an edge case, and focus on the Pub/Sub and `MONITOR` 569 | modes. They have a common problem: 570 | 571 | 1. The fact that the connection is in push mode is a private *state* of the connection. Otherwise the data we get from Pub/Sub do not contain anything that at the protocol level to make them distinguishable from other replies. 572 | 2. The connection can only be used for Pub/Sub or `MONITOR` once setup in this way, because there is no way (because of the previous problem) in order to tell apart replies from commands and push data. 573 | 574 | Moreover a connection for Pub/Sub cannot be used also for `MONITOR` or any other kind of push notifications. For this reasons RESP3 introduces an explicit 575 | push data type, attempting to solve the above issues. 576 | 577 | RESP3 push data is represented from the point of view of the protocol exactly 578 | like the Array type. However the first byte is `>` instead of `*`, and the 579 | first element of the array is always a String item, representing the kind 580 | of push data the server is sending to the client. All the other fields in the 581 | push array are type dependent, which means that depending on the type string 582 | as first argument, the remaining items will be interpreted following different 583 | conventions. The existing push data in RESP version 2 will be represented 584 | in RESP3 by the push types `pubsub` and `monitor`. 585 | 586 | This is an example of push data: 587 | 588 | >4 589 | +pubsub 590 | +message 591 | +somechannel 592 | +this is the message 593 | 594 | *Note that the above format uses simple strings for simplicity, the 595 | actual Redis implementation would use blob strings instead* 596 | 597 | In the above example the push data type is `pubsub`. For this type, if 598 | the next element is `message` we know that it's a Pub/Sub message (other 599 | sub types may be subscribe, unsubscribe, and so forth), which is followed 600 | by the channel name and the message itself. 601 | 602 | Push data may be interleaved with any protocol data, but always at the top 603 | level, so the client will never find push data in the middle of a Map reply 604 | for instance. 605 | 606 | Clients should normally react to the publication of a push data by invoking 607 | a callback associated with the push data type. Synchronous clients may 608 | instead enter a loop and return every new message or command reply. 609 | 610 | Note that in this mode it is possible to get both replies and push messages 611 | at the same time, interleaved in any way, however the order of the commands 612 | and their replies is not affected: if a command is called, the next reply 613 | (that is not a push reply) received will be the one relative of this command, 614 | and so forth, even if it is possible that there will be push data items to 615 | consume before. 616 | 617 | For instance after a `GET key` command, it is possible to get the two following 618 | valid replies: 619 | 620 | >4 621 | +pubsub 622 | +message 623 | +somechannel 624 | +this is the message 625 | $9 626 | Get-Reply 627 | 628 | Or in inverse order: 629 | 630 | $9 631 | Get-Reply 632 | >4 633 | +pubsub 634 | +message 635 | +somechannel 636 | +this is the message 637 | 638 | Still the client will know that the first non push type reply processed 639 | will be the actual reply to GET. 640 | 641 | However synchronous clients may of course miss for a long time that there is 642 | something to read in the socket, because they only read after a command is 643 | executed, so the client library should still allow the user to specify if the 644 | connection should be monitored for new messages in some way (usually by 645 | entering some loop) or not. For asynchronous clients the implementation is a 646 | lot more obvious. 647 | 648 | ## Streamed strings 649 | 650 | Normally RESP strings have a prefixed length: 651 | 652 | $1234 653 | .... 1234 bytes of data here ... 654 | 655 | Unfortunately this is not always optimal. 656 | 657 | Sometimes it is very useful to transfer a large string from the server 658 | to the client, or the other way around, without knowing in advance the 659 | size of such string. Redis already uses this feature internally, however it 660 | is a private extension of the protocol in the case of RESP2. For RESP3 661 | we want it to be part of the specification, because we found other uses 662 | for the feature where it is crucial that the client has support for it. 663 | 664 | For instance in diskless replication the Redis master sends the RDB file 665 | for the first synchronization to its replica without generating the file 666 | to disk: instead the output of the RDB file, that is incrementally generated 667 | from the keyspace data in memory, is directly sent to the socket. In this 668 | case we don't have any way to know in advance the final length of the 669 | string we are transferring. 670 | 671 | The protocol we used internally was something like that: 672 | 673 | $EOF:<40 bytes marker> 674 | ... any number of bytes of data here not containing the marker ... 675 | <40 bytes marker> 676 | 677 | As we already specified, this was just a *private extension* only known 678 | by the server itself. It uses an EOF marker that is generated in a pseudo 679 | random way, and is practically impossible to collide with normal data. However 680 | such approach, we found, have certain limits when extended to be a known, 681 | well documented mechanism that Redis can use when talking with other clients. 682 | We were worried expecially by the following issues: 683 | 684 | 1. Generating the EOF: failing at that makes the protocol very fragile, and often even experienced developers don't know much about probability, counting, and randomness. 685 | 2. Parsing the EOF: while not so hard, is non trivial. The client need to implement an algorithm that can detect the EOF even while reading it in two separated calls. 686 | 687 | For this reason instead the final version of this specification proposes 688 | a chunked encoding approach, that is often used in order protocols. 689 | 690 | The protocol can be easily explained by a small example, in which the 691 | string "Hello world" is transmitted without knowing its size in advance: 692 | 693 | $? 694 | ;4 695 | Hell 696 | ;5 697 | o wor 698 | ;1 699 | d 700 | ;0 701 | 702 | Basically the transfer starts with `$?`. We use the same prefix as normal 703 | strings, that is `$`, but later instead of the count we use a question mark 704 | in order to communicate the client that this is a chunked encoding transfer, 705 | and we don't know the final size yet. 706 | 707 | Later the differnet parts are transferred like that: 708 | 709 | ; 710 | ... coun bytes of data ... 711 | 712 | The transferring program can send as many parts as it likes, there are no 713 | limits. Finally in order to signal that the transfer has ended, a part 714 | with length zero, without any data, is transferred: 715 | 716 | ;0 717 | 718 | Note that right now the Redis server does not implement such protocol, that 719 | is, there is no command that will reply like that, however it is likely that 720 | we'll soon implement this ability at least for modules. However it is currently 721 | not planned to have support to send streamed strings to the server, as 722 | part of a command. 723 | 724 | ## Streamed aggregated data types 725 | 726 | Sometimes it is useful to send an aggregated data type whose size is not 727 | known in advance. Imagine a search application written as a Redis module 728 | that collects the data it finds in the inverted index using multiple threads 729 | and, as it finds results, it sends such results to the client. 730 | 731 | In this, and many other situations, the size of the result set is not known 732 | in advance. So far the only possibility to solve the issue has been to 733 | buffer the result in memory, and at the end send everything, because the 734 | old RESP2 protocol had no mechanism in order to send aggregated data types 735 | without specifying the final number of elements immediately. 736 | 737 | For instance the Array type is like that: 738 | 739 | *123 (number of items) 740 | :1 (items...) 741 | :2 742 | ... 743 | 744 | RESP3 extends this mechanism, allowing to send all the aggregated data 745 | types of type Array, Set and Map, not specifying the length, but instead 746 | using an explicit terminator. The transfer is initiated like that 747 | (in the case of the Array type): 748 | 749 | *? 750 | 751 | So instead of the length, we just use a '?' character. Then we can 752 | continue to reply with other RESP3 types: 753 | 754 | :1 755 | :2 756 | :3 757 | 758 | Finally we can terminate the Array using a special **END type**, that 759 | has the following format: 760 | 761 | . 762 | 763 | Unbound Sets are exactly like Arrays with the difference that the transfer 764 | wills tart with `~` instead of `*` character. However with the Map type 765 | things are marginally different: the program emitting the protocol *must* 766 | make sure that it emits an even number of elements, since every couple 767 | represents a field-value pair: 768 | 769 | %? 770 | +a 771 | :1 772 | +b 773 | :2 774 | . 775 | 776 | Currently there is no Redis 6 command that uses such extension to the protocol, 777 | however it is possible that modules running in Redis 6 will use such feature 778 | so it is suggested for client libraries to implement this part of 779 | the specification ASAP, and if this is not the case, to clearly document that 780 | this part of RESP3 is not supported. 781 | 782 | ## The HELLO command and connection handshake 783 | 784 | RESP connections should always start sending a special command called HELLO. 785 | This step accomplishes two things: 786 | 787 | 1. It allows servers to be backward compatible with RESP2 versions. This is needed in Redis in order to switch more gently to the version 3 of the protocol. 788 | 2. The `HELLO` command is useful in order to return information about the server and the protocol, that the client can use for different goals. 789 | 790 | The HELLO command has the following format and arguments: 791 | 792 | HELLO [AUTH ] 793 | 794 | Currently only the AUTH option is available, in order to authenticate the client 795 | in case the server is configured in order to be password protected. Redis 6 796 | will support ACLs, however in order to just use a general password like in 797 | Redis 5 clients should use "default" as username (all lower case). 798 | 799 | The first argument of the command is the protocol version we want the connection 800 | to be set. By default the connection starts in RESP2 mode. If we specify a 801 | connection version which is too big and is not supported by the server, it should 802 | reply with a -NOPROTO error. Example: 803 | 804 | Client: HELLO 4 805 | Server: -NOPROTO sorry this protocol version is not supported 806 | 807 | Then the client may retry with a lower protocol version. 808 | 809 | Similarly the client can easily detect a server that is only able to speak 810 | RESP2: 811 | 812 | Client: HELLO 3 AUTH default mypassword 813 | Server: -ERR unknown command 'HELLO' 814 | 815 | It can then proceed by sending the AUTH command directly and continue talking 816 | RESP2 to the server. 817 | 818 | Note that even if the protocol is supported, the HELLO command may return an 819 | error and perform no action (so the connection will remain in RESP2 mode) in case 820 | the authentication credentials are wrong: 821 | 822 | Client: HELLO 3 AUTH default mypassword 823 | Server: -ERR invalid password 824 | (the connection remains in RESP2 mode) 825 | 826 | The successful reply to the HELLO command is just a map reply. 827 | The information in the Hello reply is in part server dependent, but there are 828 | certain fields that are mandatory for all the RESP3 implementations: 829 | 830 | * server: "redis" (or other software name) 831 | * version: the server version 832 | * proto: the maximum version of the RESP protocol supported 833 | 834 | In addition, in the case of the RESP3 implementation of Redis, the following 835 | fields will also be emitted: 836 | 837 | * id: the client connection ID 838 | * mode: "standalone", "sentinel" or "cluster" 839 | * role: "master" or "replica" 840 | * modules: list of loaded modules as an array of strings 841 | 842 | The exact number and value of fields emitted by Redis is however currently 843 | a work in progress, you should not rely on the above list. 844 | 845 | ## Acknowledgements 846 | 847 | This specification was written by Salvatore Sanfilippo, however the design was informed by multiple people that contributed worthwhile ideas and improvements. A special thank to: 848 | 849 | * Dvir Volk 850 | * Yao Yue 851 | * Yossi Gottlieb 852 | * Marc Gravell 853 | * Nick Craver 854 | 855 | For the conversation and ideas to make this specification better. 856 | 857 | ## FAQ 858 | 859 | * **Why the RESP3 line break was not changed to a single character?** 860 | 861 | Because that would require a client to send command, and parse replies, in 862 | a different way based on the fact the client is in RESP v2 or v3 mode. 863 | Even a client supporting only RESP3, would start sending the `HELLO` command 864 | with CRLF as separators, since initially the connection is in RESP v2 mode. 865 | The parsing code would also be designed in order to accept the different line 866 | break based on the conditions. All in all the saving of one byte did not made 867 | enough sense in light of a more complex client implementation, especially since 868 | the way RESP3 is designed, it is mostly a superset of RESP2, so most clients 869 | will just have to add the parsing of the new data types supported without 870 | touching the implementation of the old ones. 871 | 872 | ## TODOs in this specification 873 | 874 | * Document the optional "inline" protocol. 875 | * Document pipelining 876 | --------------------------------------------------------------------------------