├── .gitattributes ├── .github ├── FUNDING.yml └── workflows │ └── test.yml ├── .gitignore ├── README.md ├── amazon-msk-sasl-lib ├── LICENSE ├── aws-msk-iam.rkt └── info.rkt ├── bench ├── go │ ├── .gitignore │ ├── go.mod │ ├── go.sum │ └── main.go ├── python │ ├── main.py │ └── requirements.txt └── rkt │ └── publish-1M.rkt ├── confluent-schema-registry-lib ├── LICENSE ├── info.rkt ├── schema-registry.rkt └── schema-registry │ ├── api.rkt │ ├── client.rkt │ ├── error.rkt │ └── record.rkt ├── example ├── amazon-msk-auth.rkt ├── consumer-offsets.rkt ├── consumer.rkt ├── producer.rkt ├── proxy.rkt ├── sasl-ssl.rkt ├── scram-sha256-auth.rkt └── scram-sha512-auth.rkt ├── kafka-lib ├── LICENSE ├── consumer.rkt ├── info.rkt ├── internal.rkt ├── iterator.rkt ├── logger.rkt ├── main.rkt ├── private │ ├── assignor.rkt │ ├── batch-native.rkt │ ├── batch.bnf │ ├── batch.rkt │ ├── client.rkt │ ├── common.rkt │ ├── connection.rkt │ ├── crc.rkt │ ├── error.rkt │ ├── help.rkt │ ├── logger.rkt │ ├── protocol-consumer.bnf │ ├── protocol-internal.bnf │ ├── protocol-native.rkt │ ├── protocol.bnf │ ├── proxy.rkt │ ├── record.rkt │ ├── serde.rkt │ └── serde │ │ ├── alter-configs.rkt │ │ ├── authorized-operation.rkt │ │ ├── commit.rkt │ │ ├── contract.rkt │ │ ├── core.rkt │ │ ├── create-topics.rkt │ │ ├── delete-groups.rkt │ │ ├── delete-topics.rkt │ │ ├── describe-cluster.rkt │ │ ├── describe-configs.rkt │ │ ├── describe-groups.rkt │ │ ├── describe-producers.rkt │ │ ├── fetch-offsets.rkt │ │ ├── fetch.rkt │ │ ├── find-coordinator.rkt │ │ ├── group.rkt │ │ ├── heartbeat.rkt │ │ ├── internal.rkt │ │ ├── join-group.rkt │ │ ├── leave-group.rkt │ │ ├── list-groups.rkt │ │ ├── list-offsets.rkt │ │ ├── metadata.rkt │ │ ├── produce.rkt │ │ ├── resource.rkt │ │ ├── sasl-authenticate.rkt │ │ ├── sasl-handshake.rkt │ │ └── sync-group.rkt └── producer.rkt ├── kafka-test ├── info.rkt └── kafka │ ├── fixtures │ ├── truncated-records.bin │ └── zstd-frame-no-length.bin │ └── records.rkt ├── kafka ├── LICENSE ├── info.rkt └── scribblings │ ├── info.rkt │ └── kafka.scrbl └── tests ├── 00-wait-for-kafka.rkt ├── 01-publish-consume.rkt ├── 02-commit-after-rebalance.rkt ├── 03-delete-groups.rkt └── 99-publish-consume-for-a-while.rkt /.gitattributes: -------------------------------------------------------------------------------- 1 | *.bin filter=lfs diff=lfs merge=lfs -text 2 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: Bogdanp 2 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | on: [push] 3 | 4 | jobs: 5 | test: 6 | runs-on: ubuntu-latest 7 | strategy: 8 | matrix: 9 | kafka-version: ["2", "3"] 10 | 11 | name: Test (Kafka ${{ matrix.kafka-version }}) 12 | services: 13 | zookeeper: 14 | image: bitnami/zookeeper:latest 15 | ports: 16 | - 2181:2181 17 | env: 18 | ALLOW_ANONYMOUS_LOGIN: "yes" 19 | kafka: 20 | image: bitnami/kafka:${{ matrix.kafka-version }} 21 | ports: 22 | - 9092:9092 23 | env: 24 | KAFKA_BROKER_ID: "1" 25 | KAFKA_CFG_LISTENERS: "PLAINTEXT://:9092" 26 | KAFKA_CFG_ADVERTISED_LISTENERS: "PLAINTEXT://127.0.0.1:9092" 27 | KAFKA_CFG_ZOOKEEPER_CONNECT: "zookeeper:2181" 28 | ALLOW_PLAINTEXT_LISTENER: "yes" 29 | options: "--link zookeeper" 30 | 31 | steps: 32 | - uses: actions/checkout@master 33 | - uses: Bogdanp/setup-racket@v1.9 34 | with: 35 | variant: 'CS' 36 | version: '8.6' 37 | - run: git lfs pull 38 | - run: sudo raco pkg update --catalog https://pkgs.racket-lang.org sasl-lib 39 | - run: raco pkg install --auto amazon-msk-sasl-lib/ kafka-lib/ kafka-test/ kafka/ 40 | - run: env PLTSTDERR='error warning@kafka' raco test --drdr kafka-lib/ kafka-test/ 41 | - run: env PLTSTDERR='error warning@kafka' raco test amazon-msk-sasl-lib/ tests/ 42 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | compiled/ 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # racket-kafka 2 | 3 | A Racket client for [Apache Kafka][kafka] versions 0.11 and up. You 4 | can find the [docs] on the Racket package server. 5 | 6 | ## License 7 | 8 | racket-kafka is licensed under the 3-Clause BSD license. 9 | 10 | 11 | [kafka]: https://kafka.apache.org/ 12 | [docs]: https://docs.racket-lang.org/kafka/index.html 13 | -------------------------------------------------------------------------------- /amazon-msk-sasl-lib/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2022 Bogdan Popa 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | 1. Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in the 12 | documentation and/or other materials provided with the distribution. 13 | 14 | 3. Neither the name of the copyright holder nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 24 | TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 25 | PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 26 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 27 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 28 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | -------------------------------------------------------------------------------- /amazon-msk-sasl-lib/aws-msk-iam.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require crypto 4 | crypto/libcrypto 5 | json 6 | racket/format 7 | racket/port 8 | racket/string 9 | sasl 10 | threading) 11 | 12 | (provide 13 | make-aws-msk-iam-ctx) 14 | 15 | ;; https://github.com/aws/aws-msk-iam-auth/blob/2a0e1ef5cb9c08f4526b5642be060b5bfb84d50b/src/main/java/software/amazon/msk/auth/iam/internals/IAMSaslClient.java#L85 16 | (define (make-aws-msk-iam-ctx 17 | #:region region 18 | #:access-key-id access-key-id 19 | #:secret-access-key secret-access-key 20 | #:session-token [session-token #f] 21 | #:server-name server-name) 22 | (define payload 23 | (make-authentication-payload 24 | #:region region 25 | #:access-key-id access-key-id 26 | #:secret-access-key secret-access-key 27 | #:session-token session-token 28 | #:server-name server-name)) 29 | (make-sasl-ctx #f payload aws-msk-iam-receive)) 30 | 31 | (define (aws-msk-iam-receive aux msg) 32 | (if (bytes=? msg #"") 33 | (error "authentication failed") 34 | (values aux 'done))) 35 | 36 | (module+ test 37 | (require rackunit) 38 | 39 | (test-case "happy path" 40 | (define ctx 41 | (make-aws-msk-iam-ctx 42 | #:region "us-east-1" 43 | #:access-key-id "ACCESSEXAMPLE" 44 | #:secret-access-key "supercalifragilisticexpialidocious" 45 | #:server-name "example.com")) 46 | (check-equal? (sasl-state ctx) 'send/receive) 47 | (let ([data (bytes->jsexpr (sasl-next-message ctx))]) 48 | (define-simple-check (check-ref k e) 49 | (equal? (hash-ref data k) e)) 50 | (check-ref 'action "kafka-cluster:Connect") 51 | (check-ref 'host "example.com") 52 | (check-ref 'user-agent "racket-kafka") 53 | (check-ref 'version "2020_10_22") 54 | (check-ref 'x-amz-algorithm "AWS4-HMAC-SHA256") 55 | (check-regexp-match #rx"ACCESSEXAMPLE/......../us-east-1/kafka-cluster/aws4_request" (hash-ref data 'x-amz-credential)) 56 | (check-regexp-match #rx"........T......Z" (hash-ref data 'x-amz-date)) 57 | (check-ref 'x-amz-expires "900") 58 | (check-false (hash-has-key? data 'x-amz-security-token)) 59 | (check-true (hash-has-key? data 'x-amz-signature)) 60 | (check-ref 'x-amz-signedheaders "host")) 61 | (sasl-receive-message ctx #"ok") 62 | (check-equal? (sasl-state ctx) 'done)) 63 | 64 | (test-case "happy path with session token" 65 | (define ctx 66 | (make-aws-msk-iam-ctx 67 | #:region "us-east-1" 68 | #:access-key-id "ACCESSEXAMPLE" 69 | #:secret-access-key "supercalifragilisticexpialidocious" 70 | #:session-token "sekret" 71 | #:server-name "example.com")) 72 | (check-equal? (sasl-state ctx) 'send/receive) 73 | (let ([data (bytes->jsexpr (sasl-next-message ctx))]) 74 | (define-simple-check (check-ref k e) 75 | (equal? (hash-ref data k) e)) 76 | (check-ref 'action "kafka-cluster:Connect") 77 | (check-ref 'host "example.com") 78 | (check-ref 'user-agent "racket-kafka") 79 | (check-ref 'version "2020_10_22") 80 | (check-ref 'x-amz-algorithm "AWS4-HMAC-SHA256") 81 | (check-regexp-match #rx"ACCESSEXAMPLE/......../us-east-1/kafka-cluster/aws4_request" (hash-ref data 'x-amz-credential)) 82 | (check-regexp-match #rx"........T......Z" (hash-ref data 'x-amz-date)) 83 | (check-ref 'x-amz-expires "900") 84 | (check-ref 'x-amz-security-token "sekret") 85 | (check-true (hash-has-key? data 'x-amz-signature)) 86 | (check-ref 'x-amz-signedheaders "host")) 87 | (sasl-receive-message ctx #"ok") 88 | (check-equal? (sasl-state ctx) 'done)) 89 | 90 | (test-case "fail" 91 | (define ctx 92 | (make-aws-msk-iam-ctx 93 | #:region "us-east-1" 94 | #:access-key-id "ACCESSEXAMPLE" 95 | #:secret-access-key "supercalifragilisticexpialidocious" 96 | #:server-name "example.com")) 97 | (check-not-false (sasl-next-message ctx)) 98 | (check-exn 99 | #rx"failed" 100 | (λ () (sasl-receive-message ctx #""))) 101 | (check-equal? (sasl-state ctx) 'error))) 102 | 103 | 104 | ;; AWSV4 Auth ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 105 | 106 | ;; https://github.com/aws/aws-msk-iam-auth/blob/2a0e1ef5cb9c08f4526b5642be060b5bfb84d50b/src/main/java/software/amazon/msk/auth/iam/internals/AWS4SignedPayloadGenerator.java 107 | ;; https://github.com/aws/aws-msk-iam-auth/blob/2a0e1ef5cb9c08f4526b5642be060b5bfb84d50b/src/main/java/software/amazon/msk/auth/iam/internals/AuthenticationRequestParams.java 108 | (define (make-authentication-payload 109 | #:region region 110 | #:access-key-id access-key-id 111 | #:secret-access-key secret-access-key 112 | #:session-token [session-token #f] 113 | #:server-name server-name) 114 | (define presigned-params 115 | (make-presigned-request 116 | #:region region 117 | #:access-key-id access-key-id 118 | #:secret-access-key secret-access-key 119 | #:session-token session-token 120 | #:scope "kafka-cluster" 121 | #:expires-in (* 15 60) 122 | #:params (hasheq 'Action "kafka-cluster:Connect") 123 | #:headers (hasheq 'Host server-name) 124 | #:payload-signature (sha256-hex-bytes #""))) 125 | (define augmented-params 126 | (~> presigned-params 127 | (hash-set 'host server-name) 128 | (hash-set 'version "2020_10_22") 129 | (hash-set 'user-agent "racket-kafka"))) 130 | (jsexpr->bytes 131 | (for/hasheq ([(k v) (in-hash augmented-params)]) 132 | (values (string->symbol (string-downcase (~a k))) v)))) 133 | 134 | ;; https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html 135 | (define (make-presigned-request 136 | #:region region 137 | #:access-key-id access-key-id 138 | #:secret-access-key secret-access-key 139 | #:session-token [session-token #f] 140 | #:date [the-date (seconds->date (current-seconds) #f)] 141 | #:expires-in [expires 3600] 142 | #:scope scope 143 | #:method [method "GET"] 144 | #:uri [uri "/"] 145 | #:params [params (hasheq)] 146 | #:headers [headers (hasheq)] 147 | #:payload-signature [payload-signature #"UNSIGNED-PAYLOAD"]) 148 | (define credentials 149 | (string-join 150 | (list access-key-id (~date the-date) region scope "aws4_request") "/")) 151 | (define params* 152 | (~> params 153 | (hash-set 'X-Amz-Algorithm "AWS4-HMAC-SHA256") 154 | (hash-set 'X-Amz-Credential credentials) 155 | (hash-set 'X-Amz-Date (~datetime the-date)) 156 | (hash-set 'X-Amz-Expires (~a expires)) 157 | (when~> session-token 158 | (hash-set 'X-Amz-Security-Token session-token)) 159 | (hash-set 'X-Amz-SignedHeaders (make-canonical-headers headers)))) 160 | (define request 161 | (make-canonical-request method uri params* headers payload-signature)) 162 | (define string-to-sign 163 | (make-string-to-sign the-date region scope request)) 164 | (define signing-key 165 | (make-signing-key secret-access-key the-date region scope)) 166 | (define signature 167 | (bytes->string/utf-8 168 | (make-signature signing-key string-to-sign))) 169 | (hash-set params* 'X-Amz-Signature signature)) 170 | 171 | ;; https://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html 172 | (define (make-signature signing-key string-to-sign) 173 | (call-with-hmac-context 174 | (lambda (hmac) 175 | (bytes->hex-bytes 176 | (hmac signing-key string-to-sign))))) 177 | 178 | (define (make-signing-key secret-access-key the-date region scope) 179 | (call-with-hmac-context 180 | (lambda (hmac) 181 | (~> (~a "AWS4" secret-access-key) 182 | (string->bytes/utf-8) 183 | (hmac (~date the-date)) 184 | (hmac region) 185 | (hmac scope) 186 | (hmac "aws4_request"))))) 187 | 188 | (define (call-with-hmac-context proc) 189 | (parameterize ([crypto-factories (list libcrypto-factory)]) 190 | (define (hmac-proc k d) 191 | (hmac 'sha256 k d)) 192 | (proc hmac-proc))) 193 | 194 | ;; https://docs.aws.amazon.com/general/latest/gr/sigv4-create-string-to-sign.html 195 | (define (make-string-to-sign the-date region scope canonical-request) 196 | (call-with-output-bytes 197 | (lambda (out) 198 | (displayln "AWS4-HMAC-SHA256" out) 199 | (displayln (~datetime the-date) out) 200 | (displayln (string-join (list (~date the-date) region scope "aws4_request") "/") out) 201 | (display (sha256-hex-bytes canonical-request) out)))) 202 | 203 | (define (~date d) 204 | (~a (date-year d) 205 | (pad (date-month d)) 206 | (pad (date-day d)))) 207 | 208 | (define (~datetime d) 209 | (~a (~date d) 210 | "T" 211 | (pad (date-hour d)) 212 | (pad (date-minute d)) 213 | (pad (date-second d)) 214 | "Z")) 215 | 216 | (define (pad n) 217 | (if (< n 10) 218 | (~a "0" n) 219 | n)) 220 | 221 | ;; https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html 222 | (define (make-canonical-request method uri params headers [payload-signature (sha256-hex-bytes #"")]) 223 | (call-with-output-bytes 224 | (lambda (out) 225 | (displayln method out) 226 | (displayln uri out) 227 | (for* ([(k idx) (in-indexed (in-list (sort (hash-keys params) symbolstring k)) out) 232 | (display "=" out) 233 | (display (urlencode v) out)) 234 | (newline out) 235 | (for* ([k (in-list (sort (hash-keys headers) symbolstring k)) out) 238 | (display ":" out) 239 | (displayln (string-trim v) out)) 240 | (newline out) 241 | (displayln (make-canonical-headers headers) out) 242 | (display payload-signature out)))) 243 | 244 | (define (make-canonical-headers headers) 245 | (string-join 246 | (for/list ([k (in-list (sort (hash-keys headers) symbolstring k))) 248 | ";")) 249 | 250 | (define (sha256-hex-bytes s) 251 | (bytes->hex-bytes 252 | (sha256-bytes (open-input-bytes s)))) 253 | 254 | (define (bytes->hex-bytes bs) 255 | (string->bytes/utf-8 256 | (string-downcase 257 | (bytes->hex-string bs)))) 258 | 259 | (define (urlencode s) 260 | (call-with-output-string 261 | (lambda (out) 262 | (for ([b (in-bytes (string->bytes/utf-8 s))]) 263 | (cond 264 | [(or (and (>= b 48) (<= b 57)) ;; 0-9 265 | (and (>= b 65) (<= b 90)) ;; a-z 266 | (and (>= b 97) (<= b 122)) ;; A-Z 267 | (memv b '(45 46 95 128))) ;; - . _ ~ 268 | (write-char (integer->char b) out)] 269 | [else 270 | (write-char #\% out) 271 | (when (< b #x10) 272 | (write-char #\0 out)) 273 | (write-string (string-upcase (number->string b 16)) out)]))))) 274 | 275 | (module+ test 276 | (require racket/date) 277 | 278 | (define canon 279 | (make-canonical-request 280 | "GET" "/" 281 | (hasheq 282 | 'Action "ListUsers" 283 | 'Version "2010-05-08") 284 | (hasheq 285 | 'Content-Type "application/x-www-form-urlencoded; charset=utf-8" 286 | 'Host "iam.amazonaws.com" 287 | 'X-Amz-Date "20150830T123600Z"))) 288 | 289 | (check-equal? 290 | (bytes->string/utf-8 canon) 291 | #<date (find-seconds 0 36 12 30 8 2015 #f) #f) 307 | "us-east-1" 308 | "iam" 309 | canon)) 310 | (check-equal? 311 | (bytes->string/utf-8 string-to-sign) 312 | #<date (find-seconds 0 36 12 30 8 2015 #f) #f) 324 | "us-east-1" 325 | "iam")) 326 | (check-equal? 327 | (bytes->string/utf-8 (bytes->hex-bytes signing-key)) 328 | "c4afb1cc5771d871763a393e44b703571b55cc28424d1a5e86da6ed3c154a4b9") 329 | 330 | (define signed-canon 331 | (make-signature signing-key string-to-sign)) 332 | (check-equal? 333 | (bytes->string/utf-8 signed-canon) 334 | "5d672d79c15b13162d9279b0855cfba6789a8edb4c82c400e06b5924a6f2b5d7") 335 | 336 | (define psr 337 | (make-presigned-request 338 | #:region "us-east-1" 339 | #:access-key-id "AKIAIOSFODNN7EXAMPLE" 340 | #:secret-access-key "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" 341 | #:date (seconds->date (find-seconds 0 0 0 24 5 2013 #f) #f) 342 | #:expires-in 86400 343 | #:scope "s3" 344 | #:uri "/test.txt" 345 | #:headers (hasheq 'Host "examplebucket.s3.amazonaws.com"))) 346 | (check-equal? 347 | (hash-ref psr 'X-Amz-Signature) 348 | "aeeed9bbccd4d02ee5c0109b86d86835f995330da4c265957d157751f604d404")) 349 | -------------------------------------------------------------------------------- /amazon-msk-sasl-lib/info.rkt: -------------------------------------------------------------------------------- 1 | #lang info 2 | 3 | (define license 'BSD-3-Clause) 4 | (define version "0.3") 5 | (define collection "sasl") 6 | (define deps '("base" 7 | "crypto-lib" 8 | ["sasl-lib" #:version "1.3"] 9 | "threading-lib")) 10 | (define build-deps '("rackunit-lib")) 11 | -------------------------------------------------------------------------------- /bench/go/.gitignore: -------------------------------------------------------------------------------- 1 | vendor/ 2 | -------------------------------------------------------------------------------- /bench/go/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/Bogdanp/racket-kafka-bench 2 | 3 | go 1.19 4 | 5 | require github.com/confluentinc/confluent-kafka-go v1.9.2 6 | -------------------------------------------------------------------------------- /bench/go/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "time" 7 | 8 | "github.com/confluentinc/confluent-kafka-go/kafka" 9 | ) 10 | 11 | const N = 1000000 12 | const P = 8 13 | 14 | func main() { 15 | p, err := kafka.NewProducer(&kafka.ConfigMap{ 16 | "bootstrap.servers": "127.0.0.1", 17 | "compression.codec": "none", 18 | "linger.ms": "10000", 19 | }) 20 | if err != nil { 21 | panic(err) 22 | } 23 | defer p.Close() 24 | 25 | go func() { 26 | for e := range p.Events() { 27 | switch ev := e.(type) { 28 | case *kafka.Message: 29 | if ev.TopicPartition.Error != nil { 30 | fmt.Printf("Error: %v\n", ev.TopicPartition.Error) 31 | } 32 | } 33 | } 34 | }() 35 | 36 | start := time.Now() 37 | topic := "bench-publish-1M" 38 | for i := 0; i < N; i++ { 39 | p.Produce(&kafka.Message{ 40 | TopicPartition: kafka.TopicPartition{ 41 | Topic: &topic, 42 | Partition: int32(i % P), 43 | }, 44 | Key: []byte("k"), 45 | Value: []byte("v"), 46 | }, nil) 47 | } 48 | 49 | p.Flush(60000) 50 | log.Println(time.Since(start)) 51 | } 52 | -------------------------------------------------------------------------------- /bench/python/main.py: -------------------------------------------------------------------------------- 1 | from kafka import KafkaProducer 2 | 3 | producer = KafkaProducer( 4 | bootstrap_servers="localhost:9092", 5 | compression_type=None, 6 | linger_ms=10000, 7 | ) 8 | for i in range(1_000_000): 9 | producer.send("bench-publish-1M", key=b"k", value=b"v", partition=i % 8) 10 | producer.flush() 11 | -------------------------------------------------------------------------------- /bench/python/requirements.txt: -------------------------------------------------------------------------------- 1 | kafka-python 2 | -------------------------------------------------------------------------------- /bench/rkt/publish-1M.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require kafka 4 | (submod kafka/producer unsafe) 5 | profile) 6 | 7 | (define (bench) 8 | (define N 1000000) 9 | (define t "bench-publish-1M") 10 | (define k (make-client)) 11 | (define p (make-producer k 12 | #:compression 'none 13 | #:flush-interval 10000 14 | #:max-batch-size 10000)) 15 | (create-topics k (make-CreateTopic #:name t #:partitions 8)) 16 | (time 17 | (for ([n (in-range N)]) 18 | (produce p t #"k" #"v" #:partition (modulo n 8))) 19 | (producer-stop p)) 20 | (disconnect-all k)) 21 | 22 | (define (bench/sync) 23 | (define N 1000000) 24 | (define t "bench-publish-1M") 25 | (define k (make-client)) 26 | (define p (make-producer k 27 | #:compression 'none 28 | #:flush-interval 1000 29 | #:max-batch-size 1000)) 30 | (create-topics k (make-CreateTopic #:name t #:partitions 8)) 31 | (time 32 | (for/fold ([evts null]) 33 | ([n (in-range N)]) 34 | (define evt (produce p t #"k" #"v" #:partition (modulo n 8))) 35 | (define evts* (cons evt evts)) 36 | (cond 37 | [(= (length evts*) 1000) 38 | (begin0 null 39 | (for-each sync evts*))] 40 | [else 41 | evts*])) 42 | (producer-stop p)) 43 | (delete-topics k t) 44 | (disconnect-all k)) 45 | 46 | (module+ main 47 | (require racket/cmdline) 48 | (define benchmark bench) 49 | (define profile? #f) 50 | (command-line 51 | #:once-each 52 | ["--check-results" "synchronize publish results" (set! benchmark bench/sync)] 53 | ["--profile" "turn on profiling" (set! profile? #t)]) 54 | (if profile? 55 | (profile-thunk 56 | #:use-errortrace? #t 57 | #:threads #t 58 | #:delay 0.001 59 | benchmark) 60 | (benchmark))) 61 | -------------------------------------------------------------------------------- /confluent-schema-registry-lib/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2022 Bogdan Popa 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | 1. Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in the 12 | documentation and/or other materials provided with the distribution. 13 | 14 | 3. Neither the name of the copyright holder nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 24 | TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 25 | PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 26 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 27 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 28 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | -------------------------------------------------------------------------------- /confluent-schema-registry-lib/info.rkt: -------------------------------------------------------------------------------- 1 | #lang info 2 | 3 | (define license 'BSD-3-Clause) 4 | (define version "0.2.3") 5 | (define collection "confluent") 6 | (define deps '("base" 7 | "http-easy-lib" 8 | "threading-lib")) 9 | -------------------------------------------------------------------------------- /confluent-schema-registry-lib/schema-registry.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require (only-in net/http-easy auth-procedure/c) 4 | racket/contract/base 5 | "schema-registry/api.rkt" 6 | "schema-registry/client.rkt" 7 | "schema-registry/error.rkt") 8 | 9 | (provide 10 | client? 11 | exn:fail:schema-registry? 12 | exn:fail:schema-registry:client? 13 | exn:fail:schema-registry:client-code 14 | exn:fail:schema-registry:client-message 15 | exn:fail:schema-registry:server? 16 | exn:fail:schema-registry:server-code 17 | exn:fail:schema-registry:server-text 18 | schema-type/c 19 | 20 | (contract-out 21 | [make-client (->* (string?) (auth-procedure/c) client?)] 22 | 23 | ;; schema 24 | [Reference? (-> any/c boolean?)] 25 | [make-Reference (->* (#:name string? 26 | #:subject string? 27 | #:version exact-integer?) 28 | () 29 | Reference?)] 30 | [Reference-name (-> Reference? string?)] 31 | [Reference-subject (-> Reference? string?)] 32 | [Reference-version (-> Reference? exact-integer?)] 33 | [Schema? (-> any/c boolean?)] 34 | [make-Schema (->* (#:schema string?) 35 | (#:id exact-integer? 36 | #:type schema-type/c 37 | #:subject string? 38 | #:version exact-integer? 39 | #:references (listof Reference?)) 40 | Schema?)] 41 | [Schema-id (-> Schema? (or/c #f exact-integer?))] 42 | [Schema-subject (-> Schema? (or/c #f string?))] 43 | [Schema-version (-> Schema? (or/c #f exact-integer?))] 44 | [Schema-schema (-> Schema? string?)] 45 | [Schema-type (-> Schema? schema-type/c)] 46 | [Schema-references (-> Schema? (listof Reference?))] 47 | 48 | [get-schema (-> client? exact-integer? Schema?)] 49 | [get-schema-types (-> client? (listof symbol?))] 50 | [get-schema-versions (-> client? exact-integer? (listof (cons/c string? exact-integer?)))] 51 | 52 | ;; subject 53 | [get-all-subjects (->* (client?) 54 | (#:prefix string? 55 | #:include-deleted? boolean?) 56 | (listof string?))] 57 | [get-subject-versions (-> client? string? (listof exact-integer?))] 58 | [get-subject-version (-> client? string? (or/c 'latest exact-integer?) Schema?)] 59 | [get-subject-version-referenced-by (-> client? string? exact-integer? (listof exact-integer?))] 60 | [delete-subject (->* (client? string?) 61 | (#:permanently? boolean?) 62 | (listof exact-integer?))] 63 | [delete-subject-version (->* (client? string? exact-integer?) 64 | (#:permanently? boolean?) 65 | exact-integer?)] 66 | [register-schema (-> client? string? Schema? exact-integer?)] 67 | [check-schema (-> client? string? Schema? boolean?)] 68 | 69 | ;; mode 70 | [get-mode (->* (client?) 71 | (#:subject string?) 72 | mode/c)] 73 | [set-mode (->* (client? mode/c) 74 | (#:subject string? 75 | #:force? boolean?) 76 | mode/c)] 77 | [delete-mode (-> client? string? mode/c)] 78 | 79 | ;; config 80 | [Config? (-> any/c boolean?)] 81 | [make-Config (->* () 82 | (#:compatibility-level compatibility-level/c) 83 | Config?)] 84 | [Config-compatibility-level (-> Config? compatibility-level/c)] 85 | [get-config (->* (client?) 86 | (#:subject string?) 87 | Config?)] 88 | [set-config (->* (client? Config?) 89 | (#:subject string?) 90 | Config?)] 91 | [delete-config (-> client? string? Config?)])) 92 | 93 | (define schema-type/c 94 | (or/c 'avro 'protobuf 'json)) 95 | 96 | (define mode/c 97 | (or/c 'import 'readonly 'readwrite)) 98 | 99 | (define compatibility-level/c 100 | (or/c 'backward 'backward-transitive 'forward 'forward-transitive 'full 'full-transitive 'none)) 101 | -------------------------------------------------------------------------------- /confluent-schema-registry-lib/schema-registry/api.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require json 4 | racket/string 5 | threading 6 | "client.rkt" 7 | "record.rkt") 8 | 9 | ;; data ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 10 | 11 | (define-record Reference 12 | ([name] 13 | [subject] 14 | [version])) 15 | 16 | (define-record Schema 17 | ([id #:default (json-null)] 18 | [subject #:default (json-null)] 19 | [version #:default (json-null)] 20 | [schema] 21 | [type #:key 'schemaType 22 | #:default "AVRO" 23 | #:decode variant->symbol 24 | #:encode symbol->variant] 25 | [references #:default null 26 | #:decode (λ (rs) (map jsexpr->Reference rs)) 27 | #:encode (λ (rs) (map Reference->jsexpr rs))])) 28 | 29 | ;; schema ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 30 | ;; https://docs.confluent.io/platform/current/schema-registry/develop/api.html#schemas 31 | 32 | (provide 33 | get-schema 34 | get-schema-types 35 | get-schema-versions) 36 | 37 | (define (get-schema c id) 38 | (jsexpr->Schema 39 | (get c (format "/schemas/ids/~a" id)))) 40 | 41 | (define (get-schema-types c) 42 | (map variant->symbol (get c "/schemas/types/"))) 43 | 44 | (define (get-schema-versions c id) 45 | (for/list ([p (in-list (get c (format "/schemas/ids/~a/versions" id)))]) 46 | (cons (hash-ref p 'subject) 47 | (hash-ref p 'version)))) 48 | 49 | ;; subject ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 50 | ;; https://docs.confluent.io/platform/current/schema-registry/develop/api.html#subjects 51 | 52 | (provide 53 | get-all-subjects 54 | get-subject-versions 55 | get-subject-version 56 | get-subject-version-referenced-by 57 | delete-subject 58 | delete-subject-version 59 | register-schema 60 | check-schema) 61 | 62 | (define (get-all-subjects c 63 | #:prefix [subject-prefix #f] 64 | #:include-deleted? [include-deleted? #f]) 65 | (get 66 | #:params (~> (maybe-add-param null include-deleted? 'deleted "true") 67 | (maybe-add-param subject-prefix 'subjectPrefix subject-prefix)) 68 | c "/subjects")) 69 | 70 | (define (get-subject-versions c subject) 71 | (get c (format "/subjects/~a/versions" subject))) 72 | 73 | (define (get-subject-version c subject version-id) 74 | (jsexpr->Schema 75 | (get c (format "/subjects/~a/versions/~a" subject version-id)))) 76 | 77 | (define (get-subject-version-referenced-by c subject version-id) 78 | (get c (format "/subjects/~a/versions/~a/referencedby" subject version-id))) 79 | 80 | (define (delete-subject c subject #:permanently? [permanent? #f]) 81 | (delete 82 | #:params (maybe-add-param null permanent? 'permanent "true") 83 | c (format "/subjects/~a" subject))) 84 | 85 | (define (delete-subject-version c subject version-id #:permanently? [permanent? #f]) 86 | (delete 87 | #:params (maybe-add-param null permanent? 'permanent "true") 88 | c (format "/subjects/~a/versions/~a" subject version-id))) 89 | 90 | (define (register-schema c subject schema) 91 | (define res 92 | (post 93 | #:headers (hasheq 'content-type "application/json") 94 | #:data (jsexpr->bytes (Schema->jsexpr schema)) 95 | c (format "/subjects/~a/versions" subject))) 96 | (hash-ref res 'id)) 97 | 98 | (define (check-schema c subject schema) 99 | (hash-ref 100 | (post 101 | #:headers (hasheq 'content-type "application/json") 102 | #:data (jsexpr->bytes (Schema->jsexpr schema)) 103 | c (format "/compatibility/subjects/~a/versions/latest" subject)) 104 | 'is_compatible)) 105 | 106 | 107 | ;; mode ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 108 | ;; https://docs.confluent.io/platform/current/schema-registry/develop/api.html#mode 109 | 110 | (provide 111 | get-mode 112 | set-mode 113 | delete-mode) 114 | 115 | (define (get-mode c #:subject [subject #f]) 116 | (extract-mode (get c (if subject (format "/mode/~a" subject) "/mode")))) 117 | 118 | (define (set-mode c mode 119 | #:subject [subject #f] 120 | #:force? [force? #f]) 121 | (extract-mode 122 | (put 123 | #:params (maybe-add-param null force? 'force "true") 124 | #:data (jsexpr->bytes (hasheq 'mode (symbol->variant mode))) 125 | c (if subject (format "/mode/~a" subject) "/mode")))) 126 | 127 | (define (delete-mode c subject) 128 | (extract-mode 129 | (delete c (format "/mode/~a" subject)))) 130 | 131 | 132 | ;; config ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 133 | ;; https://docs.confluent.io/platform/current/schema-registry/develop/api.html#config 134 | 135 | (provide 136 | get-config 137 | set-config 138 | delete-config) 139 | 140 | (define-record Config 141 | ([compatibility-level #:key 'compatibilityLevel 142 | #:to-key 'compatibility 143 | #:default "FULL" 144 | #:decode variant->symbol 145 | #:encode symbol->variant])) 146 | 147 | (define (get-config c #:subject [subject #f]) 148 | (jsexpr->Config (get c (if subject (format "/config/~a" subject) "/config")))) 149 | 150 | (define (set-config c conf #:subject [subject #f]) 151 | (jsexpr->Config 152 | (put 153 | #:data (jsexpr->bytes (Config->jsexpr conf)) 154 | c (if subject (format "/config/~a" subject) "/config")))) 155 | 156 | (define (delete-config c subject) 157 | (jsexpr->Config (delete c (format "/config/~a" subject)))) 158 | 159 | 160 | ;; help ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 161 | 162 | (define (extract-mode res) 163 | (variant->symbol (hash-ref res 'mode))) 164 | 165 | (define (maybe-add-param params add? k v) 166 | (if add? (cons (cons k v) params) params)) 167 | 168 | (define (variant->symbol t) 169 | (string->symbol (string-downcase (string-replace t "_" "-")))) 170 | 171 | (define (symbol->variant s) 172 | (string-replace (string-upcase (symbol->string s)) "-" "_")) 173 | -------------------------------------------------------------------------------- /confluent-schema-registry-lib/schema-registry/client.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require (for-syntax racket/base) 4 | (prefix-in http: net/http-easy) 5 | racket/string 6 | "error.rkt") 7 | 8 | (provide 9 | client? 10 | make-client) 11 | 12 | (struct client (session uri auth-proc)) 13 | 14 | (define (make-client uri [auth-proc (λ (_url headers params) 15 | (values headers params))]) 16 | (client 17 | (http:make-session) 18 | (string-trim uri "/") 19 | auth-proc)) 20 | 21 | (define ((make-requester method) c path 22 | #:headers [headers (hasheq)] 23 | #:params [params null] 24 | #:data [data #f]) 25 | (define res 26 | (http:session-request 27 | #:method method 28 | #:headers (hash-set headers 'accept "application/vnd.schemaregistry.v1+json") 29 | #:params params 30 | #:data data 31 | #:auth (client-auth-proc c) 32 | (client-session c) 33 | (string-append (client-uri c) path))) 34 | (define status-code 35 | (http:response-status-code res)) 36 | (cond 37 | [(>= status-code 500) 38 | (raise (server-error status-code (http:response-body res)))] 39 | [(>= status-code 400) 40 | (define data (http:response-json res)) 41 | (define code (hash-ref data 'error_code status-code)) 42 | (define message (hash-ref data 'message "")) 43 | (raise (client-error code message))] 44 | [else 45 | (http:response-json res)])) 46 | 47 | (define-syntax-rule (define-requesters [id ...]) 48 | (begin 49 | (provide id ...) 50 | (define id (make-requester 'id)) ...)) 51 | 52 | (define-requesters 53 | [delete get post put]) 54 | -------------------------------------------------------------------------------- /confluent-schema-registry-lib/schema-registry/error.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (provide 4 | (all-defined-out)) 5 | 6 | (struct exn:fail:schema-registry exn:fail ()) 7 | (struct exn:fail:schema-registry:client exn:fail:schema-registry (code message)) 8 | (struct exn:fail:schema-registry:server exn:fail:schema-registry (code text)) 9 | 10 | (define (client-error code message) 11 | (exn:fail:schema-registry:client 12 | (format "client error~n code: ~a~n message: ~a" code message) 13 | (current-continuation-marks) 14 | code message)) 15 | 16 | (define (server-error code [text #f]) 17 | (exn:fail:schema-registry:server "server error" (current-continuation-marks) code text)) 18 | -------------------------------------------------------------------------------- /confluent-schema-registry-lib/schema-registry/record.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require (for-syntax racket/base 4 | racket/syntax 5 | syntax/parse/pre)) 6 | 7 | (provide 8 | define-record) 9 | 10 | (define-syntax (define-record stx) 11 | (define-syntax-class field 12 | (pattern [id:id {~alt 13 | {~optional {~seq #:key key:expr}} 14 | {~optional {~seq #:to-key to-key:expr}} 15 | {~optional {~seq #:default default:expr}} 16 | {~optional {~seq #:decode decode-proc:expr}} 17 | {~optional {~seq #:encode encode-proc:expr}}} ...] 18 | #:with kwd (string->keyword (symbol->string (syntax-e #'id))) 19 | #:with (ctor-arg ...) #'{~? (kwd [id {~? (decode-proc default) default}]) 20 | (kwd id)})) 21 | 22 | (syntax-parse stx 23 | [(_ id:id (fld:field ...)) 24 | #:with make-id (format-id #'id "make-~a" #'id) 25 | #:with id? (format-id #'id "~a?" #'id) 26 | #:with jsexpr->id-id (format-id #'id "jsexpr->~a" #'id) 27 | #:with id->jsexpr-id (format-id #'id "~a->jsexpr" #'id) 28 | #:with (fld-accessor ...) (for/list ([fld-id-stx (in-list (syntax-e #'(fld.id ...)))]) 29 | (format-id #'id "~a-~a" #'id fld-id-stx)) 30 | #'(begin 31 | (provide id? make-id fld-accessor ...) 32 | (struct id (fld.id ...) 33 | #:transparent) 34 | (define (make-id fld.ctor-arg ... ...) 35 | (id fld.id ...)) 36 | (define (jsexpr->id-id e) 37 | (id 38 | ({~? fld.decode-proc values} 39 | {~? (hash-ref e {~? fld.key 'fld.id} fld.default) 40 | (hash-ref e {~? fld.key 'fld.id})}) ...)) 41 | (define (id->jsexpr-id v) 42 | (define ks (list {~? {~? fld.to-key fld.key} 'fld.id} ...)) 43 | (define vs (list ({~? fld.encode-proc values} (fld-accessor v)) ...)) 44 | (for/hasheq ([k (in-list ks)] 45 | [v (in-list vs)]) 46 | (values k v))))])) 47 | -------------------------------------------------------------------------------- /example/amazon-msk-auth.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require kafka 4 | openssl 5 | sasl/aws-msk-iam) 6 | 7 | (define (make-auth-ctx host _port) 8 | (make-aws-msk-iam-ctx 9 | #:region "us-east-1" 10 | #:access-key-id "" 11 | #:secret-access-key "" 12 | #:session-token (and #f "") 13 | #:server-name host)) 14 | 15 | (define c 16 | (make-client 17 | #:bootstrap-host "" 18 | #:bootstrap-port 9198 19 | #:sasl-mechanism&ctx (list 'AWS_MSK_IAM make-auth-ctx) 20 | #:ssl-ctx (ssl-secure-client-context))) 21 | 22 | (get-metadata c) 23 | -------------------------------------------------------------------------------- /example/consumer-offsets.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require kafka 4 | kafka/internal 5 | racket/vector) 6 | 7 | (define c (make-client)) 8 | (define it (make-internal-events c)) 9 | (define (make-deadline-evt) 10 | (alarm-evt (+ (current-inexact-monotonic-milliseconds) 1000) #t)) 11 | (with-handlers ([exn:break? void]) 12 | (let loop ([deadline-evt (make-deadline-evt)]) 13 | (define events 14 | (get-events it)) 15 | (cond 16 | [(vector-empty? events) 17 | (sync deadline-evt) 18 | (loop (make-deadline-evt))] 19 | [else 20 | (for ([e (in-vector events)]) 21 | (println e)) 22 | (loop deadline-evt)]))) 23 | (disconnect-all c) 24 | -------------------------------------------------------------------------------- /example/consumer.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require kafka 4 | kafka/consumer) 5 | 6 | (define c (make-consumer 7 | (make-client) 8 | "example-group" 9 | "example-topic")) 10 | (with-handlers ([exn:break? void] 11 | [exn:fail? (λ (e) ((error-display-handler) (exn-message e) e))]) 12 | (let loop () 13 | (define-values (type data) 14 | (sync (consume-evt c))) 15 | (case type 16 | [(rebalance) 17 | (println `(rebalance ,data))] 18 | [(records) 19 | (println `(records ,(vector-length data))) 20 | (for ([r (in-vector data)]) 21 | (println (list 22 | (record-partition-id r) 23 | (record-offset r) 24 | (record-key r) 25 | (record-value r))))]) 26 | (consumer-commit c) 27 | (loop))) 28 | (consumer-stop c) 29 | -------------------------------------------------------------------------------- /example/producer.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require kafka 4 | kafka/producer 5 | racket/format) 6 | 7 | (define c (make-client)) 8 | (define p (make-producer c)) 9 | (define t "example-topic") 10 | (create-topics 11 | c 12 | (make-CreateTopic 13 | #:name t 14 | #:partitions 2)) 15 | (define evts 16 | (for/list ([i (in-range 128)]) 17 | (define pid (modulo i 2)) 18 | (define k #"a") 19 | (define v (string->bytes/utf-8 (~a i))) 20 | (produce 21 | #:partition pid 22 | #:headers (hash "Content-Type" #"text/plain") 23 | p t k v))) 24 | 25 | (producer-flush p) 26 | (for ([evt (in-list evts)]) 27 | (println (sync evt))) 28 | (disconnect-all c) 29 | -------------------------------------------------------------------------------- /example/proxy.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require kafka 4 | openssl 5 | racket/cmdline) 6 | 7 | (define-values (key-path cert-path) 8 | (let ([key-path #f] 9 | [cert-path #f]) 10 | (command-line 11 | #:once-each 12 | [("-k" "--key") 13 | KEY_PATH 14 | "the SSL private key (in PEM format)" 15 | (set! key-path KEY_PATH)] 16 | [("-c" "--cert") 17 | CERT_PATH 18 | "the SSL certificate chain" 19 | (set! cert-path CERT_PATH)] 20 | #:args [] 21 | (unless (and key-path cert-path) 22 | (error "both the --key and --cert arguments are required")) 23 | (values key-path cert-path)))) 24 | 25 | (define c 26 | (make-client 27 | #:bootstrap-host "broker" 28 | #:bootstrap-port 9092 29 | #:ssl-ctx (ssl-make-client-context 30 | #:private-key `(pem ,key-path) 31 | #:certificate-chain cert-path 32 | 'auto) 33 | #:proxy (make-http-proxy "127.0.0.1" 1080))) 34 | (get-metadata c) 35 | (disconnect-all c) 36 | -------------------------------------------------------------------------------- /example/sasl-ssl.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require kafka 4 | openssl 5 | racket/cmdline 6 | sasl/plain) 7 | 8 | (define-values (key-path cert-path) 9 | (let ([key-path #f] 10 | [cert-path #f]) 11 | (command-line 12 | #:once-each 13 | [("-k" "--key") 14 | KEY_PATH 15 | "the SSL private key (in PEM format)" 16 | (set! key-path KEY_PATH)] 17 | [("-c" "--cert") 18 | CERT_PATH 19 | "the SSL certificate chain" 20 | (set! cert-path CERT_PATH)] 21 | #:args [] 22 | (unless (and key-path cert-path) 23 | (error "both the --key and --cert arguments are required")) 24 | (values key-path cert-path)))) 25 | 26 | (define c 27 | (make-client 28 | #:bootstrap-host "127.0.0.1" 29 | #:bootstrap-port 9092 30 | #:sasl-mechanism&ctx `(plain ,(plain-client-message "client" "client-secret")) 31 | #:ssl-ctx (ssl-make-client-context 32 | #:private-key `(pem ,key-path) 33 | #:certificate-chain cert-path 34 | 'auto))) 35 | 36 | (get-metadata c) 37 | -------------------------------------------------------------------------------- /example/scram-sha256-auth.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require kafka 4 | sasl/scram) 5 | 6 | (define (make-auth-ctx _host _port) 7 | (make-scram-client-ctx 8 | 'sha256 9 | "client" 10 | "client-secret")) 11 | 12 | (define c 13 | (make-client 14 | #:bootstrap-host "127.0.0.1" 15 | #:bootstrap-port 9092 16 | #:sasl-mechanism&ctx (list 'SCRAM-SHA-256 make-auth-ctx))) 17 | 18 | (get-metadata c) 19 | -------------------------------------------------------------------------------- /example/scram-sha512-auth.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require kafka 4 | sasl/scram) 5 | 6 | (define (make-auth-ctx _host _port) 7 | (make-scram-client-ctx 8 | 'sha512 9 | "client" 10 | "client-secret")) 11 | 12 | (define c 13 | (make-client 14 | #:bootstrap-host "127.0.0.1" 15 | #:bootstrap-port 9092 16 | #:sasl-mechanism&ctx (list 'SCRAM-SHA-512 make-auth-ctx))) 17 | 18 | (get-metadata c) 19 | -------------------------------------------------------------------------------- /kafka-lib/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2022 Bogdan Popa 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | 1. Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in the 12 | documentation and/or other materials provided with the distribution. 13 | 14 | 3. Neither the name of the copyright holder nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 24 | TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 25 | PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 26 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 27 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 28 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | -------------------------------------------------------------------------------- /kafka-lib/info.rkt: -------------------------------------------------------------------------------- 1 | #lang info 2 | 3 | (define license 'BSD-3-Clause) 4 | (define version "0.12.1") 5 | (define collection "kafka") 6 | (define deps '("base" 7 | ["binfmt-lib" #:version "0.5"] 8 | ["libzstd" #:version "1.5.5"] 9 | ["lz4-lib" #:version "1.4.1"] 10 | "net-lib" 11 | "sasl-lib" 12 | ["snappy-lib" #:version "1.0"])) 13 | (define build-deps '("rackunit-lib")) 14 | -------------------------------------------------------------------------------- /kafka-lib/internal.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require racket/contract/base 4 | "iterator.rkt" 5 | "private/client.rkt" 6 | "private/serde.rkt") 7 | 8 | (provide 9 | internal-events? 10 | (contract-out 11 | [internal-events-supported? (-> client? boolean?)] 12 | [make-internal-events (-> client? internal-events?)] 13 | [get-events (-> internal-events? vector?)])) 14 | 15 | (define consumer-offsets-topic 16 | "__consumer_offsets") 17 | 18 | (struct internal-events (iter)) 19 | 20 | (define (make-internal-events c) 21 | (internal-events (make-topic-iterator c consumer-offsets-topic 'earliest))) 22 | 23 | (define (get-events ie) 24 | (parse-events (get-records (internal-events-iter ie)))) 25 | 26 | (define (internal-events-supported? c) 27 | (and (get-offsets-topic c) #t)) 28 | 29 | (define (get-offsets-topic c) 30 | (findf 31 | (λ (t) (equal? (TopicMetadata-name t) consumer-offsets-topic)) 32 | (Metadata-topics (client-metadata c)))) 33 | 34 | (define (parse-events records) 35 | (for*/vector ([record (in-vector records)] 36 | #:when (record-value record) 37 | [event (in-value 38 | (parse-Internal 39 | (record-key record) 40 | (record-value record)))] 41 | #:when event) 42 | event)) 43 | -------------------------------------------------------------------------------- /kafka-lib/iterator.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require racket/contract/base 4 | racket/match 5 | racket/promise 6 | "private/batch.rkt" 7 | "private/client.rkt" 8 | "private/common.rkt" 9 | "private/error.rkt" 10 | "private/logger.rkt" 11 | "private/record.rkt" 12 | "private/serde.rkt") 13 | 14 | ;; iterator ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 15 | 16 | (provide 17 | (contract-out 18 | [struct record ([partition-id (or/c #f exact-integer?)] 19 | [offset exact-integer?] 20 | [timestamp exact-integer?] 21 | [key (or/c #f bytes?)] 22 | [value (or/c #f bytes?)] 23 | [headers (hash/c string? bytes?)])] 24 | 25 | [topic-iterator? (-> any/c boolean?)] 26 | [make-topic-iterator (->* (client? string?) 27 | (offset/c) 28 | topic-iterator?)] 29 | [reset-topic-iterator! (-> topic-iterator? offset/c void?)] 30 | [get-records (->* (topic-iterator?) 31 | (#:max-bytes exact-positive-integer?) 32 | (vectorof record?))])) 33 | 34 | (define offset/c 35 | (or/c 'earliest 'latest 36 | (list/c 'timestamp exact-nonnegative-integer?) 37 | (list/c 'exact exact-nonnegative-integer?) 38 | (list/c 'recent exact-positive-integer?))) 39 | 40 | (struct topic-iterator 41 | (client topic [metadata #:mutable] [offsets #:mutable])) 42 | 43 | (define (make-topic-iterator c topic [offset 'latest]) 44 | (define metadata 45 | (get-topic c topic)) 46 | (unless metadata 47 | (error 'make-topic-iterator "topic not found")) 48 | (define offsets 49 | (get-offsets c topic offset)) 50 | (topic-iterator c topic metadata offsets)) 51 | 52 | (define (reset-topic-iterator! it offset) 53 | (match-define (topic-iterator c topic _ _) it) 54 | (define metadata (get-topic c topic)) 55 | (unless metadata 56 | (error 'reset-topic-iterator! "topic not found")) 57 | (define offsets 58 | (get-offsets c topic offset)) 59 | (set-topic-iterator-metadata! it metadata) 60 | (set-topic-iterator-offsets! it offsets)) 61 | 62 | (define (get-records it #:max-bytes [max-bytes (* 1 1024 1024)]) 63 | (match-define (topic-iterator c topic metadata offsets) it) 64 | (define-values (records updated-offsets) 65 | (get-records* c topic metadata offsets max-bytes)) 66 | (begin0 records 67 | (set-topic-iterator-offsets! it updated-offsets))) 68 | 69 | 70 | ;; help ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 71 | 72 | (define (get-topic c name) 73 | (findf 74 | (λ (t) (equal? (TopicMetadata-name t) name)) 75 | (Metadata-topics (reload-metadata c)))) 76 | 77 | (define (get-offsets c topic-name offset) 78 | (match offset 79 | [(or 'earliest 'latest) 80 | (find-offsets c topic-name offset)] 81 | [`(timestamp ,timestamp) 82 | (find-offsets c topic-name timestamp)] 83 | [`(recent ,n) 84 | (for/hasheq ([(pid offset) (in-hash (find-offsets c topic-name 'latest))]) 85 | (values pid (max 0 (- offset n))))] 86 | [`(exact ,offset) 87 | (for/hasheqv ([part (TopicMetadata-partitions (get-topic c topic-name))]) 88 | (define pid (PartitionMetadata-id part)) 89 | (values pid offset))])) 90 | 91 | (define (find-offsets c topic-name timestamp) 92 | (define nodes-by-t&p 93 | (collect-nodes-by-topic&pid 94 | (client-metadata c) 95 | (list topic-name))) 96 | (define pids-by-node 97 | (for/fold ([nodes (hasheqv)]) 98 | ([(t&p node-id) (in-hash nodes-by-t&p)]) 99 | (define pid (cdr t&p)) 100 | (hash-update nodes node-id (λ (pids) (cons pid pids)) null))) 101 | (define promises 102 | (for/list ([(node-id pids) (in-hash pids-by-node)]) 103 | (delay/thread 104 | (define conn (get-node-connection c node-id)) 105 | (define topics 106 | (hash topic-name 107 | (for/hasheqv ([pid (in-list pids)]) 108 | (values pid timestamp)))) 109 | (sync (make-ListOffsets-evt conn topics))))) 110 | (define offsets 111 | (for*/hasheqv ([promise (in-list promises)] 112 | [res (in-value (force promise))] 113 | [parts (in-hash-values res)] 114 | [part (in-list parts)]) 115 | (define pid (PartitionOffset-id part)) 116 | (unless (zero? (PartitionOffset-error-code part)) 117 | (raise-server-error (PartitionOffset-error-code part))) 118 | (values pid (PartitionOffset-offset part)))) 119 | (log-kafka-debug 120 | "found offsets for topic ~s~n timestamp: ~s~n offsets: ~s" 121 | topic-name timestamp offsets) 122 | (cond 123 | ;; When searching by timestamp, if the timestamp exceeds the 124 | ;; timestamp of the latest record, or if the partition has no 125 | ;; data, then -1 is returned. In those cases, perform another set 126 | ;; of requests to get the latest offsets for those partitions and 127 | ;; fill in the gaps. 128 | [(and (exact-integer? timestamp) 129 | (ormap negative? (hash-values offsets))) 130 | (define latest-offsets 131 | (find-offsets c topic-name 'latest)) 132 | (for/hasheq ([(pid offset) (in-hash offsets)]) 133 | (values pid (if (negative? offset) 134 | (hash-ref latest-offsets pid 0) 135 | offset)))] 136 | [else 137 | offsets])) 138 | 139 | (define (get-records* c topic-name metadata offsets max-bytes) 140 | ;; While it is possible to batch requests per node, it's a bad idea 141 | ;; because one partition may starve out multiple smaller partitions 142 | ;; on the same node. Instead, just make one request per (node, 143 | ;; topic, partition) tuple. 144 | (define response-promises 145 | (for/list ([p (in-list (TopicMetadata-partitions metadata))] 146 | #:when (>= (PartitionMetadata-leader-id p) 0)) 147 | (define node-id (PartitionMetadata-leader-id p)) 148 | (define partition-id (PartitionMetadata-id p)) 149 | (define topic-partition 150 | (make-TopicPartition 151 | #:id partition-id 152 | #:offset (hash-ref offsets partition-id 0))) 153 | (define topic-partitions 154 | (hash topic-name (list topic-partition))) 155 | (delay/thread 156 | (sync (make-Fetch-evt (get-node-connection c node-id) topic-partitions 1000 0 max-bytes))))) 157 | (define responses 158 | (for/list ([promise (in-list response-promises)]) 159 | (with-handlers ([exn:fail? (λ (e) 160 | (begin0 #f 161 | ((error-display-handler) 162 | (format "get-internal-events: ~a" (exn-message e)) 163 | e)))]) 164 | (force promise)))) 165 | (define records 166 | (for*/vector ([response (in-list responses)] 167 | #:when response 168 | [parts (in-hash-values (FetchResponse-topics response))] 169 | [part (in-list parts)] 170 | [pid (in-value (FetchResponsePartition-id part))] 171 | [batch (in-list (FetchResponsePartition-batches part))] 172 | [record (in-vector (batch-records batch))] 173 | #:when (>= (record-offset record) 174 | (hash-ref offsets pid 0))) 175 | (begin0 record 176 | (set-record-partition-id! record pid)))) 177 | (define updated-offsets 178 | (for*/fold ([offsets offsets]) 179 | ([response (in-list responses)] 180 | #:when response 181 | [parts (in-hash-values (FetchResponse-topics response))] 182 | [part (in-list parts)] 183 | [b (in-list (FetchResponsePartition-batches part))] 184 | #:unless (zero? (batch-size b))) 185 | (define pid (FetchResponsePartition-id part)) 186 | (define size (batch-size b)) 187 | (define last-record (vector-ref (batch-records b) (sub1 size))) 188 | (hash-set offsets pid (add1 (record-offset last-record))))) 189 | (values records updated-offsets)) 190 | -------------------------------------------------------------------------------- /kafka-lib/logger.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require "private/logger.rkt") 4 | 5 | (provide 6 | kafka-logger 7 | fault? 8 | fault-original-error) 9 | -------------------------------------------------------------------------------- /kafka-lib/main.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require racket/contract/base 4 | racket/lazy-require 5 | racket/promise 6 | racket/string 7 | "private/client.rkt" 8 | "private/error.rkt" 9 | "private/proxy.rkt" 10 | "private/serde.rkt") 11 | 12 | (lazy-require 13 | [openssl (ssl-client-context?)] 14 | [sasl (sasl-ctx?)]) 15 | 16 | (provide 17 | (all-from-out "private/serde.rkt") 18 | (contract-out 19 | [exn:fail:kafka? (-> any/c boolean?)] 20 | [exn:fail:kafka:client? (-> any/c boolean?)] 21 | [exn:fail:kafka:server? (-> any/c boolean?)] 22 | [exn:fail:kafka:server-code (-> exn:fail:kafka:server? exact-integer?)] 23 | [error-code-symbol (-> exact-integer? symbol?)] 24 | 25 | [proxy? (-> any/c boolean?)] 26 | [make-http-proxy (-> string? (integer-in 1 65535) proxy?)] 27 | 28 | [client? (-> any/c boolean?)] 29 | [make-client (->* () 30 | (#:id non-empty-string? 31 | #:bootstrap-host string? 32 | #:bootstrap-port (integer-in 0 65535) 33 | #:sasl-mechanism&ctx (or/c 34 | #f 35 | (list/c 'plain string?) 36 | (list/c symbol? sasl-ctx-proc/c)) 37 | #:ssl-ctx (or/c #f ssl-client-context?) 38 | #:proxy (or/c #f proxy?)) 39 | client?)] 40 | [client-metadata (-> client? Metadata?)] 41 | [reload-metadata (-> client? Metadata?)] 42 | [disconnect-all (-> client? void?)] 43 | 44 | [get-metadata (-> client? string? ... Metadata?)] 45 | [describe-cluster (-> client? Cluster?)] 46 | [describe-configs (-> client? DescribeResource? DescribeResource? ... (listof DescribedResource?))] 47 | [alter-configs (->* (client? AlterResource?) 48 | (#:validate? boolean?) 49 | #:rest (listof AlterResource?) 50 | (listof AlteredResource?))] 51 | [describe-producers (-> client? (hash/c string? (non-empty-listof exact-nonnegative-integer?)) DescribedProducers?)] 52 | [create-topics (-> client? CreateTopic? CreateTopic? ... CreatedTopics?)] 53 | [delete-topics (-> client? string? string? ... DeletedTopics?)] 54 | [find-group-coordinator (-> client? string? Coordinator?)] 55 | [list-groups (-> client? (listof Group?))] 56 | [describe-groups (-> client? string? ... (listof Group?))] 57 | [delete-groups (-> client? string? ... (listof DeletedGroup?))] 58 | [fetch-offsets (->* (client? string?) ((hash/c string? (listof exact-nonnegative-integer?))) GroupOffsets?)] 59 | [reset-offsets (-> client? 60 | string? 61 | (hash/c topic&partition/c exact-nonnegative-integer?) 62 | (hash/c topic&partition/c CommitPartitionResult?))] 63 | [list-offsets (-> client? 64 | (hash/c topic&partition/c (or/c 'earliest 'latest exact-nonnegative-integer?)) 65 | (hash/c topic&partition/c PartitionOffset?))])) 66 | 67 | ;; auth ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 68 | 69 | (provide 70 | sasl-ctx-proc/c) 71 | 72 | (define sasl-ctx-proc/c 73 | (-> string? (integer-in 0 65535) sasl-ctx?)) 74 | 75 | 76 | ;; admin ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 77 | 78 | (define topic&partition/c 79 | (cons/c string? exact-nonnegative-integer?)) 80 | 81 | (define (get-metadata c . topics) 82 | (sync (make-Metadata-evt (get-controller-connection c) topics))) 83 | 84 | (define (describe-cluster c) 85 | (sync (make-DescribeCluster-evt (get-controller-connection c)))) 86 | 87 | (define (describe-configs c . resources) 88 | (define resources-by-node-id 89 | (for/fold ([resources-by-node-id (hasheqv)]) 90 | ([res (in-list resources)]) 91 | (define node-id 92 | (and (eq? (DescribeResource-type res) 'broker) 93 | (string->number (DescribeResource-name res)))) 94 | (hash-update resources-by-node-id node-id (λ (rs) (cons res rs)) null))) 95 | (define described-resourcess 96 | (for/list ([(node-id resources) (in-hash resources-by-node-id)]) 97 | (delay/thread 98 | (define conn 99 | (if node-id 100 | (get-node-connection c node-id) 101 | (get-connection c))) 102 | (define res 103 | (sync (make-DescribeConfigs-evt conn resources))) 104 | (DescribedResources-resources res)))) 105 | (apply append (map force described-resourcess))) 106 | 107 | (define (alter-configs c 108 | #:validate? [validate? #f] 109 | . resources) 110 | (define resources-by-node-id 111 | (for/fold ([resources-by-node-id (hasheqv)]) 112 | ([res (in-list resources)]) 113 | (define node-id 114 | (and (eq? (AlterResource-type res) 'broker) 115 | (string->number (AlterResource-name res)))) 116 | (hash-update resources-by-node-id node-id (λ (rs) (cons res rs)) null))) 117 | (define altered-resources 118 | (for/list ([(node-id resources) (in-hash resources-by-node-id)]) 119 | (delay/thread 120 | (define conn 121 | (if node-id 122 | (get-node-connection c node-id) 123 | (get-connection c))) 124 | (define res 125 | (sync (make-AlterConfigs-evt conn resources validate?))) 126 | (AlteredResources-resources res)))) 127 | (apply append (map force altered-resources))) 128 | 129 | (define (describe-producers c topics) 130 | (when (zero? (hash-count topics)) 131 | (raise-argument-error 'describe-producers "(non-empty-hash/c string? (listof integer?))" topics)) 132 | (sync (make-DescribeProducers-evt (get-controller-connection c) topics))) 133 | 134 | (define (create-topics c topic0 . topics) 135 | (sync (make-CreateTopics-evt (get-controller-connection c) (cons topic0 topics)))) 136 | 137 | (define (delete-topics c topic0 . topics) 138 | (sync (make-DeleteTopics-evt (get-controller-connection c) (cons topic0 topics)))) 139 | 140 | (define (find-group-coordinator c group-id) 141 | (sync (make-FindCoordinator-evt (get-controller-connection c) group-id))) 142 | 143 | (define (list-groups c) 144 | (define groupss 145 | (for/list ([b (in-list (Metadata-brokers (client-metadata c)))]) 146 | (delay/thread 147 | (define node-id (BrokerMetadata-node-id b)) 148 | (sync (make-ListGroups-evt (get-node-connection c node-id)))))) 149 | (apply append (map force groupss))) 150 | 151 | (define (describe-groups c . groups) 152 | (define groupss 153 | (for/list ([(node-id group-ids) (in-hash (get-groups-by-coordinator c groups))]) 154 | (delay/thread 155 | (define res 156 | (sync (make-DescribeGroups-evt (get-node-connection c node-id) group-ids))) 157 | (DescribedGroups-groups res)))) 158 | (apply append (map force groupss))) 159 | 160 | (define (delete-groups c . groups) 161 | (define deleted-groupss 162 | (for/list ([(node-id group-ids) (in-hash (get-groups-by-coordinator c groups))]) 163 | (delay/thread 164 | (define res 165 | (sync (make-DeleteGroups-evt (get-node-connection c node-id) group-ids))) 166 | (DeletedGroups-groups res)))) 167 | (apply append (map force deleted-groupss))) 168 | 169 | (define (fetch-offsets c group [topics (hash)]) 170 | (define node-id 171 | (Coordinator-node-id 172 | (find-group-coordinator c group))) 173 | (sync (make-FetchOffsets-evt (get-node-connection c node-id) group topics))) 174 | 175 | (define (reset-offsets c group offsets) 176 | (define node-id 177 | (Coordinator-node-id 178 | (find-group-coordinator c group))) 179 | (define topics 180 | (for/fold ([topics (hash)]) 181 | ([(t&p offset) (in-hash offsets)]) 182 | (define topic (car t&p)) 183 | (define pid (cdr t&p)) 184 | (define req 185 | (make-CommitPartition 186 | #:id pid 187 | #:offset offset)) 188 | (hash-update topics topic (λ (reqs) (cons req reqs)) null))) 189 | (define res-topics 190 | (sync (make-Commit-evt (get-node-connection c node-id) group -1 "" topics))) 191 | (for*/hash ([(topic partition-ress) (in-hash res-topics)] 192 | [partition-res (in-list partition-ress)]) 193 | (values (cons topic (CommitPartitionResult-id partition-res)) partition-res))) 194 | 195 | (define (list-offsets c topic&partitions) 196 | (define offsetss 197 | (for/list ([(node-id t&ps) (in-hash (get-topic-partitions-by-leader c topic&partitions))]) 198 | (define topics 199 | (for/fold ([topics (hash)]) 200 | ([t&p (in-list t&ps)]) 201 | (define topic (car t&p)) 202 | (define part (cdr t&p)) 203 | (define offset (hash-ref topic&partitions t&p)) 204 | (hash-update 205 | topics 206 | topic 207 | (λ (parts) (hash-set parts part offset)) 208 | hasheqv))) 209 | (delay/thread 210 | (sync (make-ListOffsets-evt (get-node-connection c node-id) topics))))) 211 | (for*/hash ([offsets (in-list (map force offsetss))] 212 | [(topic parts) (in-hash offsets)] 213 | [part (in-list parts)]) 214 | (values (cons topic (PartitionOffset-id part)) part))) 215 | 216 | 217 | ;; help ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 218 | 219 | (define (get-groups-by-coordinator c groups) 220 | (define promises 221 | (for/list ([group-id (in-list groups)]) 222 | (delay/thread 223 | (find-group-coordinator c group-id)))) 224 | (for/fold ([nodes-to-groups (hasheqv)]) 225 | ([group-id (in-list groups)] 226 | [promise (in-list promises)]) 227 | (hash-update 228 | nodes-to-groups 229 | (Coordinator-node-id (force promise)) 230 | (λ (gs) (cons group-id gs)) 231 | null))) 232 | 233 | (define (get-topic-partitions-by-leader c topic&partitions) 234 | (define topic&partitions-to-node-ids 235 | (for*/hash ([topic (in-list (Metadata-topics (client-metadata c)))] 236 | [part (in-list (TopicMetadata-partitions topic))] 237 | #:when (>= (PartitionMetadata-leader-id part) 0)) 238 | (define topic&partition 239 | (cons (TopicMetadata-name topic) 240 | (PartitionMetadata-id part))) 241 | (values topic&partition (PartitionMetadata-leader-id part)))) 242 | (for/fold ([nodes-to-topic&partition (hasheqv)]) 243 | ([(topic&partition node-id) (in-hash topic&partitions-to-node-ids)] 244 | #:when (hash-has-key? topic&partitions topic&partition)) 245 | (hash-update 246 | nodes-to-topic&partition 247 | node-id 248 | (λ (t&ps) (cons topic&partition t&ps)) 249 | null))) 250 | -------------------------------------------------------------------------------- /kafka-lib/private/assignor.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require racket/contract/base 4 | racket/generic 5 | racket/list 6 | "help.rkt" 7 | (prefix-in cproto: "protocol-consumer.bnf")) 8 | 9 | 10 | ;; assignor generics ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 11 | 12 | (provide 13 | assignor? 14 | gen:assignor 15 | (contract-out 16 | [assignor-name (-> assignor? string?)] 17 | [assignor-metadata (-> assignor? (listof string?) bytes?)] 18 | [assignor-assign (-> assignor? 19 | (listof topic-partition?) 20 | (non-empty-listof metadata?) 21 | (hash/c string? (hash/c string? (listof exact-nonnegative-integer?))))])) 22 | 23 | (define-generics assignor 24 | [assignor-name assignor] 25 | [assignor-metadata assignor topics] 26 | [assignor-assign assignor topic-partitions metas]) 27 | 28 | 29 | ;; member-metadata ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 30 | 31 | (provide 32 | (contract-out 33 | [struct metadata ([member-id string?] 34 | [version exact-nonnegative-integer?] 35 | [topics (listof string?)] 36 | [data bytes?])] 37 | [struct topic-partition ([topic string?] 38 | [pid exact-nonnegative-integer?])])) 39 | 40 | (struct metadata (member-id version topics data) 41 | #:transparent) 42 | 43 | (struct topic-partition (topic pid) 44 | #:transparent) 45 | 46 | (define (enc-member-metadata topics [version 0]) 47 | (with-output-bytes 48 | (cproto:un-MemberMetadata 49 | `((Version_1 . ,version) 50 | (ArrayLen_1 . ,(length topics)) 51 | (TopicName_1 . ,topics) 52 | (Data_1 . #""))))) 53 | 54 | 55 | ;; range ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 56 | 57 | (provide 58 | range) 59 | 60 | (define range 61 | (let () 62 | (struct range () 63 | #:methods gen:assignor 64 | [(define (assignor-name _) 65 | "range") 66 | 67 | (define (assignor-metadata _ topics) 68 | (enc-member-metadata topics)) 69 | 70 | (define (assignor-assign _ topic-partitions metas) 71 | (define members-by-topic 72 | (for*/fold ([members-by-topic (hash)] 73 | #:result (for/hash ([(topic mids) (in-hash members-by-topic)]) 74 | (values topic (reverse mids)))) 75 | ([m (in-list metas)] 76 | [t (in-list (metadata-topics m))]) 77 | (define mid (metadata-member-id m)) 78 | (hash-update members-by-topic t (λ (ms) (cons mid ms)) null))) 79 | (for/fold ([assignments (hash)]) 80 | ([(topic member-ids) (in-hash members-by-topic)]) 81 | (define pids 82 | (for/list ([t&p (in-list topic-partitions)] #:when (string=? (topic-partition-topic t&p) topic)) 83 | (topic-partition-pid t&p))) 84 | (define-values (partitions-per-member members-with-extra) 85 | (quotient/remainder 86 | (length pids) 87 | (length member-ids))) 88 | (for/fold ([assignments assignments]) 89 | ([(member-id idx) (in-indexed (in-list member-ids))]) 90 | (define pos 91 | (+ (* idx partitions-per-member) 92 | (min idx members-with-extra))) 93 | (define num 94 | (if (< idx members-with-extra) 95 | (add1 partitions-per-member) 96 | partitions-per-member)) 97 | (define partitions 98 | (take (drop pids pos) num)) 99 | (hash-update assignments member-id (λ (t&ps) (hash-set t&ps topic partitions)) hash))))]) 100 | 101 | (range))) 102 | 103 | (module+ test 104 | (require rackunit) 105 | (check-equal? 106 | (assignor-assign range null (list (metadata "m1" 0 '("t1" "t2") #""))) 107 | (hash 108 | "m1" (hash "t1" null 109 | "t2" null))) 110 | (check-equal? 111 | (assignor-assign 112 | range 113 | (list 114 | (topic-partition "t1" 0)) 115 | (list 116 | (metadata "m1" 0 '("t1") #"") 117 | (metadata "m2" 0 '("t1") #"") 118 | (metadata "m3" 0 '("t1") #""))) 119 | (hash "m1" (hash "t1" '(0)) 120 | "m2" (hash "t1" null) 121 | "m3" (hash "t1" null))) 122 | (check-equal? 123 | (assignor-assign 124 | range 125 | (list 126 | (topic-partition "t1" 0) 127 | (topic-partition "t1" 1) 128 | (topic-partition "t1" 2)) 129 | (list 130 | (metadata "m1" 0 '("t1") #"") 131 | (metadata "m2" 0 '("t1") #""))) 132 | (hash 133 | "m1" (hash "t1" '(0 1)) 134 | "m2" (hash "t1" '(2)))) 135 | (check-equal? 136 | (assignor-assign 137 | range 138 | (list 139 | (topic-partition "t1" 0) 140 | (topic-partition "t1" 1) 141 | (topic-partition "t1" 2) 142 | (topic-partition "t2" 0) 143 | (topic-partition "t2" 1) 144 | (topic-partition "t2" 2) 145 | (topic-partition "t3" 0) 146 | (topic-partition "t3" 1) 147 | (topic-partition "t3" 2) 148 | (topic-partition "t3" 3)) 149 | (list 150 | (metadata "m1" 0 '("t1" "t2") #"") 151 | (metadata "m2" 0 '("t1" "t2" "t3") #""))) 152 | (hash 153 | "m1" (hash 154 | "t1" '(0 1) 155 | "t2" '(0 1)) 156 | "m2" (hash 157 | "t1" '(2) 158 | "t2" '(2) 159 | "t3" '(0 1 2 3))))) 160 | 161 | 162 | ;; round robin ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 163 | 164 | (provide 165 | round-robin) 166 | 167 | (define round-robin 168 | (let () 169 | (struct round-robin () 170 | #:methods gen:assignor 171 | [(define (assignor-name _) 172 | "roundrobin") 173 | 174 | (define (assignor-metadata _ topics) 175 | (enc-member-metadata topics)) 176 | 177 | (define (assignor-assign _ topic-partitions metas) 178 | (define meta-by-id 179 | (for/hash ([m (in-list metas)]) 180 | (values (metadata-member-id m) m))) 181 | (define member-ids 182 | (sort (map metadata-member-id metas) string (batch-len b) (* 20 1024 1024)))) 206 | 207 | (test-case "write nulls" 208 | (define b (make-batch)) 209 | (batch-append! b #f #"1" #:timestamp 5) 210 | (batch-append! b #"b" #f #:timestamp 20) 211 | (define batch-bs 212 | (call-with-output-bytes 213 | (lambda (out) 214 | (write-batch b out)))) 215 | (define batch-bs-in 216 | (open-input-bytes batch-bs)) 217 | (check-equal? 218 | (proto:RecordBatch batch-bs-in) 219 | '((BaseOffset_1 . 0) 220 | (BatchLength_1 . 65) 221 | (PartitionLeaderEpoch_1 . 0) 222 | (Magic_1 . 2) 223 | (CRC_1 . 2331736347) 224 | (BatchAttributes_1 . 0) 225 | (LastOffsetDelta_1 . 1) 226 | (FirstTimestamp_1 . 5) 227 | (MaxTimestamp_1 . 20) 228 | (ProducerID_1 . -1) 229 | (ProducerEpoch_1 . -1) 230 | (BaseSequence_1 . -1) 231 | (RecordCount_1 . 2))) 232 | (check-equal? 233 | (proto:Record batch-bs-in) 234 | '((Length_1 . 7) 235 | (Attributes_1 . 0) 236 | (TimestampDelta_1 . 0) 237 | (OffsetDelta_1 . 0) 238 | (Key_1 . #f) 239 | (Value_1 . #"1") 240 | (Headers_1 (HeadersLen_1 . 0) (Header_1)))) 241 | (check-equal? 242 | (proto:Record batch-bs-in) 243 | '((Length_1 . 7) 244 | (Attributes_1 . 0) 245 | (TimestampDelta_1 . 15) 246 | (OffsetDelta_1 . 1) 247 | (Key_1 . #"b") 248 | (Value_1 . #f) 249 | (Headers_1 (HeadersLen_1 . 0) (Header_1)))))) 250 | 251 | (define header-len 49) 252 | (define (read-batch in) 253 | (define header (proto:RecordBatch in)) 254 | (define data-len 255 | (- (ref 'BatchLength_1 header) header-len)) 256 | (define the-batch 257 | (batch 258 | (ref 'BaseOffset_1 header) 259 | (ref 'PartitionLeaderEpoch_1 header) 260 | (ref 'BatchAttributes_1 header) 261 | (ref 'LastOffsetDelta_1 header) 262 | (ref 'FirstTimestamp_1 header) 263 | (ref 'MaxTimestamp_1 header) 264 | (ref 'ProducerID_1 header) 265 | (ref 'ProducerEpoch_1 header) 266 | (ref 'BaseSequence_1 header) 267 | (ref 'RecordCount_1 header) 268 | #f ;; buf 269 | #f ;; data 270 | #f ;; data-out 271 | #f ;; records 272 | )) 273 | 274 | ;; Pump the data through a pipe so that we can issue writes to the 275 | ;; other side while we parse the data. 276 | (define-values (data-in data-out) 277 | (make-pipe pump-buf-len)) 278 | (thread 279 | (lambda () 280 | (define in* (make-limited-input-port in data-len #f)) 281 | (define buf (make-bytes pump-buf-len)) 282 | (with-handlers ([exn:fail? 283 | (lambda (e) 284 | (log-kafka-fault e "batch pump failed"))]) 285 | (let loop () 286 | (define n-read 287 | (read-bytes-avail! buf in*)) 288 | (unless (eof-object? n-read) 289 | (write-bytes buf data-out 0 n-read) 290 | (loop)))) 291 | (close-output-port data-out))) 292 | 293 | (define compression 294 | (batch-compression the-batch)) 295 | (define records-in 296 | (case compression 297 | [(none) data-in] 298 | [(gzip) 299 | (define out (open-output-bytes)) 300 | (gunzip-through-ports data-in out) 301 | (open-input-bytes (get-output-bytes out))] 302 | [(lz4) 303 | (define out (open-output-bytes)) 304 | (lz4-decompress-through-ports data-in out #:validate-content? #t) 305 | (open-input-bytes (get-output-bytes out))] 306 | [(snappy) 307 | (open-input-bytes (unsnappy (read-bytes data-len data-in)))] 308 | [(zstd) 309 | (open-input-bytes 310 | (zstd-decompress 311 | (read-bytes data-len data-in) 312 | zstd-max-decompressed-len))] 313 | [else 314 | (error 'read-batch "unsupported compression type: ~a" compression)])) 315 | 316 | (define base-offset (batch-base-offset the-batch)) 317 | (define base-timestamp (batch-first-timestamp the-batch)) 318 | (define size (batch-size the-batch)) 319 | (define records 320 | (for/vector #:length size ([_ (in-range size)]) 321 | (define rec (proto:Record records-in)) 322 | (parse-record rec base-offset base-timestamp))) 323 | (begin0 the-batch 324 | (set-batch-records! the-batch records))) 325 | 326 | (module+ test 327 | (test-case "read uncompressed" 328 | (define b0 (make-batch)) 329 | (batch-append! b0 #"a" #"1" #:timestamp 0) 330 | (define in 331 | (open-input-bytes 332 | (call-with-output-bytes 333 | (lambda (out) 334 | (write-batch b0 out))))) 335 | (define b1 (read-batch in)) 336 | (check-equal? (batch-compression b1) 'none) 337 | (check-equal? (batch-size b1) 1) 338 | (check-equal? (record-key (vector-ref (batch-records b1) 0)) #"a") 339 | (check-equal? (record-value (vector-ref (batch-records b1) 0)) #"1")) 340 | 341 | (test-case "read gzipped" 342 | (define b0 (make-batch #:compression 'gzip)) 343 | (batch-append! b0 #"a" #"1" #:timestamp 0) 344 | (define in 345 | (open-input-bytes 346 | (call-with-output-bytes 347 | (lambda (out) 348 | (write-batch b0 out))))) 349 | (define b1 (read-batch in)) 350 | (check-equal? (batch-compression b1) 'gzip) 351 | (check-equal? (batch-size b1) 1) 352 | (check-equal? (record-key (vector-ref (batch-records b1) 0)) #"a") 353 | (check-equal? (record-value (vector-ref (batch-records b1) 0)) #"1"))) 354 | 355 | 356 | ;; record-data ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 357 | 358 | (struct record-data (len bs) 359 | #:mutable) 360 | 361 | (define (make-record-data [cap (* 16 1024)]) 362 | (record-data 0 (make-bytes cap))) 363 | 364 | (define (record-data-set! rd k src) 365 | (define dst (record-data-bs rd)) 366 | (for ([(b idx) (in-indexed (in-bytes src))]) 367 | (bytes-set! dst (+ k idx) b))) 368 | 369 | (define (reset-record-data! rd) 370 | (set-record-data-len! rd 0)) 371 | 372 | (define (append-record-data! rd src [start 0] [end (bytes-length src)]) 373 | (define dst (record-data-bs rd)) 374 | (define len (record-data-len rd)) 375 | (define cap (bytes-length dst)) 376 | (define needed (- end start)) 377 | (define available (- cap len)) 378 | (cond 379 | [(>= available needed) 380 | (bytes-copy! dst len src start end) 381 | (set-record-data-len! rd (+ len needed))] 382 | 383 | [else 384 | (define bs (make-bytes (+ needed (* cap 2)))) 385 | (bytes-copy! bs 0 dst 0 len) 386 | (bytes-copy! bs len src start end) 387 | (set-record-data-bs! rd bs) 388 | (set-record-data-len! rd (+ len needed))])) 389 | 390 | (define (open-output-record-data rd) 391 | (make-output-port 392 | 'record-data 393 | always-evt 394 | (λ (bs start end _flush? _enable-breaks?) 395 | (begin0 (- end start) 396 | (append-record-data! rd bs start end))) 397 | void)) 398 | 399 | (define (open-output-record-data/gzip rd) 400 | (define rd-out (open-output-record-data rd)) 401 | (define-values (in out) 402 | (make-pipe)) 403 | (define thd 404 | (thread 405 | (lambda () 406 | (gzip-through-ports in rd-out #f (current-seconds))))) 407 | (make-output-port 408 | 'record-data/gzip 409 | always-evt 410 | out 411 | (lambda () 412 | (close-output-port out) 413 | (void (sync thd))))) 414 | 415 | (define (copy-record-data rd out) 416 | (write-bytes (record-data-bs rd) out 0 (record-data-len rd))) 417 | -------------------------------------------------------------------------------- /kafka-lib/private/client.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require racket/lazy-require 4 | racket/match 5 | racket/promise 6 | racket/random 7 | "connection.rkt" 8 | "error.rkt" 9 | "serde.rkt") 10 | 11 | (lazy-require 12 | [sasl (sasl-state 13 | sasl-next-message 14 | sasl-receive-message)]) 15 | 16 | ;; API ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 17 | 18 | (provide 19 | client? 20 | make-client 21 | client-metadata 22 | get-connection 23 | get-controller-connection 24 | get-node-connection 25 | reload-metadata 26 | disconnect-all) 27 | 28 | (struct client (manager-ch manager)) 29 | 30 | (define (make-client 31 | #:id [id "racket-kafka"] 32 | #:bootstrap-host [host "127.0.0.1"] 33 | #:bootstrap-port [port 9092] 34 | #:sasl-mechanism&ctx [sasl-mechanism&ctx #f] 35 | #:ssl-ctx [ssl-ctx #f] 36 | #:proxy [proxy #f]) 37 | (define bootstrap-conn #f) 38 | (define metadata 39 | (dynamic-wind 40 | (lambda () 41 | (set! bootstrap-conn (connect id host port proxy ssl-ctx))) 42 | (lambda () 43 | (when sasl-mechanism&ctx 44 | (apply authenticate host port bootstrap-conn sasl-mechanism&ctx)) 45 | (sync (make-Metadata-evt bootstrap-conn null))) 46 | (lambda () 47 | (disconnect bootstrap-conn)))) 48 | (define manager-ch 49 | (make-channel)) 50 | (define manager 51 | (thread/suspend-to-kill 52 | (make-manager manager-ch id sasl-mechanism&ctx ssl-ctx proxy metadata))) 53 | (client manager-ch manager)) 54 | 55 | (define (get-connection c [node-ids #f]) 56 | (force (send c get-best-connection node-ids))) 57 | 58 | (define (get-controller-connection c) 59 | (define metadata 60 | (client-metadata c)) 61 | (get-node-connection c (λ (b) 62 | (= (BrokerMetadata-node-id b) 63 | (Metadata-controller-id metadata))))) 64 | 65 | (define (get-node-connection c node-id) 66 | (define metadata (client-metadata c)) 67 | (define maybe-broker 68 | (findf 69 | (cond 70 | [(procedure? node-id) node-id] 71 | [else (λ (b) (= (BrokerMetadata-node-id b) node-id))]) 72 | (Metadata-brokers metadata))) 73 | (unless maybe-broker 74 | (raise-argument-error 'get-node-connection "node-id/c" node-id)) 75 | (force (send c get-connection maybe-broker))) 76 | 77 | (define (client-metadata c) 78 | (send c get-metadata)) 79 | 80 | (define (reload-metadata c) 81 | (send c reload-metadata (get-controller-connection c))) 82 | 83 | (define (disconnect-all c) 84 | (send c disconnect-all)) 85 | 86 | 87 | ;; manager ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 88 | 89 | (define ((make-manager manager-ch client-id sasl-mechanism&ctx ssl-ctx proxy metadata)) 90 | (define (connect* broker) 91 | (define host (BrokerMetadata-host broker)) 92 | (define port (BrokerMetadata-port broker)) 93 | (define conn (connect client-id host port proxy ssl-ctx)) 94 | (begin0 conn 95 | (when sasl-mechanism&ctx 96 | (apply authenticate host port conn sasl-mechanism&ctx)))) 97 | (define (enliven broker conn) 98 | (if (connected? conn) conn (connect* broker))) 99 | (let loop ([st (make-state metadata)]) 100 | (loop 101 | (apply 102 | sync 103 | (handle-evt 104 | manager-ch 105 | (lambda (msg) 106 | (match msg 107 | [`(get-best-connection ,res-ch ,nack ,node-ids) 108 | (let connect-loop ([st st]) 109 | (with-handlers ([exn:fail? (λ (e) (state-add-req st (req res-ch nack e)))]) 110 | (match-define (state meta conns _) st) 111 | (define brokers (Metadata-brokers meta)) 112 | (define connected-node-ids (hash-keys conns)) 113 | (define unconnected-brokers 114 | (for*/list ([b (in-list brokers)] 115 | [node-id (in-value (BrokerMetadata-node-id b))] 116 | #:unless (memv node-id connected-node-ids) 117 | #:when (if node-ids (memv node-id node-ids) #t)) 118 | b)) 119 | (define filtered-conns 120 | (for/hasheqv ([(node-id conn-promise) (in-hash conns)] 121 | #:when (if node-ids (memv node-id node-ids) #t)) 122 | (values node-id conn-promise))) 123 | (cond 124 | [(null? unconnected-brokers) 125 | (define dead-conns 126 | (for/hasheqv ([(node-id conn-promise) (in-hash filtered-conns)] 127 | #:unless (with-handlers ([exn:fail? (λ (_) #f)]) 128 | (connected? (force conn-promise)))) 129 | (values node-id #t))) 130 | (cond 131 | [(hash-empty? filtered-conns) 132 | (define err (client-error "invalid node subset: ~e" node-ids)) 133 | (state-add-req st (req res-ch nack err))] 134 | [(hash-empty? dead-conns) 135 | (define filtered-live-conns 136 | (for/hasheqv ([(node-id conn-promise) (in-hash filtered-conns)]) 137 | (values node-id (force conn-promise)))) 138 | (define-values (node-id conn) 139 | (find-best-connection filtered-live-conns)) 140 | (define broker 141 | (findf (λ (b) (= (BrokerMetadata-node-id b) node-id)) brokers)) 142 | (define conn* 143 | (delay/thread (enliven broker conn))) 144 | (log-kafka-debug "client: picked node ~e from ~e" node-id (hash-keys filtered-live-conns)) 145 | (state-add-req 146 | (state-set-conn st node-id conn) 147 | (req res-ch nack conn*))] 148 | [else 149 | (log-kafka-debug "client: found stale connections; retrying") 150 | (define all-live-conns 151 | (for/hasheqv ([(node-id conn-promise) (in-hash conns)] 152 | #:unless (hash-has-key? dead-conns node-id)) 153 | (values node-id conn-promise))) 154 | (connect-loop (state-set-conns st all-live-conns))])] 155 | [else 156 | (define broker (random-ref unconnected-brokers)) 157 | (define node-id (BrokerMetadata-node-id broker)) 158 | (define conn (delay/thread (connect* broker))) 159 | (log-kafka-debug "client: picked unconnected node ~e" node-id) 160 | (state-add-req 161 | (state-set-conn st node-id conn) 162 | (req res-ch nack conn))])))] 163 | 164 | [`(get-connection ,res-ch ,nack ,broker) 165 | (define node-id 166 | (BrokerMetadata-node-id broker)) 167 | (define maybe-conn-promise 168 | (hash-ref (state-conns st) node-id #f)) 169 | (define conn-promise 170 | (delay/thread 171 | (cond 172 | [maybe-conn-promise 173 | (define conn 174 | (with-handlers ([exn:fail? (λ (_) (connect* broker))]) 175 | (force maybe-conn-promise))) 176 | (enliven broker conn)] 177 | [else 178 | (connect* broker)]))) 179 | (state-add-req 180 | (state-set-conn st node-id conn-promise) 181 | (req res-ch nack conn-promise))] 182 | 183 | [`(get-metadata ,res-ch ,nack) 184 | (state-add-req st (req res-ch nack (state-meta st)))] 185 | 186 | [`(reload-metadata ,res-ch ,nack ,ctl-conn) 187 | (with-handlers ([exn:fail? (λ (e) (state-add-req st (req res-ch nack e)))]) 188 | (define meta (sync (make-Metadata-evt ctl-conn null))) 189 | (state-add-req 190 | (state-set-meta st meta) 191 | (req res-ch nack meta)))] 192 | 193 | [`(disconnect-all ,res-ch ,nack) 194 | (for ([conn-promise (in-hash-values (state-conns st))]) 195 | (with-handlers ([exn:fail? void]) 196 | (disconnect (force conn-promise)))) 197 | (state-add-req 198 | (state-clear-conns st) 199 | (req res-ch nack (void)))] 200 | 201 | [_ 202 | (begin0 st 203 | (log-kafka-error "client: unexpected message ~e" msg))]))) 204 | (append 205 | (for/list ([r (in-list (state-reqs st))]) 206 | (match-define (req res-ch _ res) r) 207 | (handle-evt 208 | (channel-put-evt res-ch res) 209 | (λ (_) (state-remove-req st r)))) 210 | (for/list ([r (in-list (state-reqs st))]) 211 | (match-define (req _ nack _) r) 212 | (handle-evt nack (λ (_) (state-remove-req st r))))))))) 213 | 214 | (define-syntax-rule (send c id . args) 215 | (sync (make-manager-evt c 'id . args))) 216 | 217 | (define (make-manager-evt c id . args) 218 | (define res-ch 219 | (make-channel)) 220 | (handle-evt 221 | (nack-guard-evt 222 | (lambda (nack) 223 | (thread-resume 224 | (client-manager c) 225 | (current-thread)) 226 | (begin0 res-ch 227 | (channel-put 228 | (client-manager-ch c) 229 | `(,id ,res-ch ,nack ,@args))))) 230 | (lambda (res-or-exn) 231 | (begin0 res-or-exn 232 | (when (exn:fail? res-or-exn) 233 | (raise res-or-exn)))))) 234 | 235 | 236 | ;; manager state ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 237 | 238 | (struct req (res-ch nack res)) 239 | 240 | ;; conns: node id -> promise of Connection 241 | (struct state (meta conns reqs) 242 | #:transparent) 243 | 244 | (define (make-state meta) 245 | (state meta (hasheqv) null)) 246 | 247 | (define (state-set-meta st meta) 248 | (struct-copy state st [meta meta])) 249 | 250 | (define (state-add-req st req) 251 | (struct-copy state st [reqs (cons req (state-reqs st))])) 252 | 253 | (define (state-remove-req st req) 254 | (struct-copy state st [reqs (remq req (state-reqs st))])) 255 | 256 | (define (state-set-conns st conns) 257 | (struct-copy state st [conns conns])) 258 | 259 | (define (state-set-conn st node-id conn) 260 | (struct-copy state st [conns (hash-set (state-conns st) node-id conn)])) 261 | 262 | (define (state-clear-conns st) 263 | (struct-copy state st [conns (hasheqv)])) 264 | 265 | 266 | ;; help ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 267 | 268 | (define (find-best-connection conns) 269 | (for/fold ([best null] 270 | [least-reqs +inf.0] 271 | #:result (apply values (random-ref best))) 272 | ([(node-id conn) (in-hash conns)]) 273 | (define reqs 274 | (get-requests-in-flight conn)) 275 | (cond 276 | [(= reqs least-reqs) 277 | (values `((,node-id ,conn) . ,best) least-reqs)] 278 | [(< reqs least-reqs) 279 | (values `((,node-id ,conn)) reqs)] 280 | [else 281 | (values best least-reqs)]))) 282 | 283 | (define (authenticate host port conn mechanism ctx) 284 | (sync (make-SaslHandshake-evt conn mechanism)) 285 | (case mechanism 286 | [(plain) 287 | (define req 288 | (if (string? ctx) 289 | (string->bytes/utf-8 ctx) 290 | ctx)) 291 | (sync (make-SaslAuthenticate-evt conn req))] 292 | [else 293 | (let ([ctx (ctx host port)]) 294 | (let loop () 295 | (case (sasl-state ctx) 296 | [(done) 297 | (void)] 298 | [(error) 299 | (error 'authenticate "SASL: unexpected error")] 300 | [(receive) 301 | (error 'authenticate "SASL: receive not supported")] 302 | [(send/receive) 303 | (define req (sasl-next-message ctx)) 304 | (define res (sync (make-SaslAuthenticate-evt conn (->bytes req)))) 305 | (sasl-receive-message ctx (SaslAuthenticateResponse-data res)) 306 | (loop)] 307 | [(send/done) 308 | (define req (sasl-next-message ctx)) 309 | (sync (make-SaslAuthenticate-evt conn (->bytes req)))])))]) 310 | (void)) 311 | 312 | (define (->bytes s) 313 | (if (bytes? s) s (string->bytes/utf-8 s))) 314 | -------------------------------------------------------------------------------- /kafka-lib/private/common.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require "serde/metadata.rkt") 4 | 5 | (provide 6 | collect-nodes-by-topic&pid) 7 | 8 | (define (collect-nodes-by-topic&pid metadata topics) 9 | (for*/hash ([t (in-list (Metadata-topics metadata))] 10 | [topic (in-value (TopicMetadata-name t))] 11 | #:when (member topic topics string=?) 12 | [p (in-list (TopicMetadata-partitions t))] 13 | [pid (in-value (PartitionMetadata-id p))] 14 | #:when (>= (PartitionMetadata-leader-id p) 0)) 15 | (values (cons topic pid) (PartitionMetadata-leader-id p)))) 16 | 17 | 18 | ;; evts ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 19 | 20 | (provide 21 | pure-evt) 22 | 23 | (define (pure-evt v0 . vs) 24 | (handle-evt always-evt (λ (_) (apply values v0 vs)))) 25 | -------------------------------------------------------------------------------- /kafka-lib/private/connection.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require racket/lazy-require 4 | racket/match 5 | racket/tcp 6 | "error.rkt" 7 | "help.rkt" 8 | "logger.rkt" 9 | (prefix-in proto: "protocol.bnf") 10 | "proxy.rkt") 11 | 12 | (lazy-require 13 | [openssl (ssl-connect)]) 14 | 15 | ;; connection ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 16 | 17 | (provide 18 | connection? 19 | connect 20 | connected? 21 | disconnect 22 | make-request-evt 23 | get-requests-in-flight) 24 | 25 | (struct connection (ch mgr [versions #:mutable])) 26 | 27 | (define (connect [client-id "racket-kafka"] 28 | [host "127.0.0.1"] 29 | [port 9092] 30 | [proxy #f] 31 | [ssl-ctx #f]) 32 | (define-values (in out) 33 | (cond 34 | [proxy (proxy-connect proxy host port ssl-ctx)] 35 | [ssl-ctx (ssl-connect host port ssl-ctx)] 36 | [else (tcp-connect host port)])) 37 | (define ch (make-channel)) 38 | (define mgr (thread/suspend-to-kill (make-manager client-id in out ch))) 39 | (define conn (connection ch mgr (hasheqv))) 40 | (begin0 conn 41 | (set-connection-versions! conn (get-api-versions conn)))) 42 | 43 | (define (connected? conn) 44 | (sync 45 | (make-message-evt conn `(connected?)) 46 | (handle-evt 47 | (thread-dead-evt (connection-mgr conn)) 48 | (lambda (_) #f)))) 49 | 50 | (define (disconnect conn) 51 | (define ch (connection-ch conn)) 52 | (define mgr (connection-mgr conn)) 53 | (thread-resume mgr (current-thread)) 54 | (void 55 | (sync 56 | (thread-dead-evt mgr) 57 | (handle-evt 58 | (channel-put-evt ch `(disconnect)) 59 | (λ (_) mgr))))) 60 | 61 | (define (get-requests-in-flight conn) 62 | (sync (make-message-evt conn '(requests-in-flight)))) 63 | 64 | 65 | ;; manager ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 66 | 67 | (provide 68 | (all-from-out "logger.rkt")) 69 | 70 | (define (read-port amount in) 71 | (define bs (read-bytes amount in)) 72 | (when (eof-object? bs) 73 | (raise 74 | (exn:fail:network 75 | "unexpected EOF\n the other end closed the connection" 76 | (current-continuation-marks)))) 77 | (open-input-bytes bs)) 78 | 79 | (define ((make-manager client-id in out ch)) 80 | (let loop ([s (make-state)]) 81 | (apply 82 | sync 83 | (handle-evt 84 | (if (state-connected? s) in never-evt) 85 | (lambda (_) 86 | (loop 87 | (with-handlers ([exn:fail:network? 88 | (lambda (e) 89 | (log-kafka-warning "connection failed: ~a" (exn-message e)) 90 | (fail-pending-reqs 91 | (set-state-disconnected s) 92 | (client-error "disconnected")))] 93 | [exn:fail? 94 | (lambda (e) 95 | (begin0 s 96 | (log-kafka-error "failed to process response: ~a" (exn-message e))))]) 97 | (define size-in (read-port 4 in)) 98 | (define size (proto:Size size-in)) 99 | (define resp-in (read-port size in)) 100 | (define resp-id (proto:CorrelationID resp-in)) 101 | (cond 102 | [(find-state-req s resp-id) 103 | => (λ (req) 104 | (define tags 105 | (and (KReq-flexible? req) 106 | (proto:Tags resp-in))) 107 | (define response 108 | ((KReq-parser req) resp-in)) 109 | (define updated-req 110 | (struct-copy KReq req [res #:parent Req (KRes response tags)])) 111 | (update-state-req s resp-id updated-req))] 112 | [else 113 | (begin0 s 114 | (log-kafka-warning "dropped response w/o associated request~n id: ~a" resp-id))]))))) 115 | (handle-evt 116 | ch 117 | (lambda (msg) 118 | (match msg 119 | [`(disconnect) 120 | (close-input-port in) 121 | (close-output-port out) 122 | (log-kafka-debug "client ~a disconnected" client-id)] 123 | 124 | [`(connected? ,nack ,ch) 125 | (define req (Req nack ch (state-connected? s))) 126 | (loop (add-state-req s req))] 127 | 128 | [`(requests-in-flight ,nack ,ch) 129 | (define req (Req nack ch (state-req-count s))) 130 | (loop (add-state-req s req))] 131 | 132 | [`(request ,immed-response ,flexible? ,k ,v ,tags ,request-data ,parser ,nack ,ch) 133 | #:when (state-connected? s) 134 | (loop 135 | (with-handlers ([exn:fail? 136 | (lambda (err) 137 | (define req (Req nack ch err)) 138 | (add-state-req s req))]) 139 | (define res 140 | (if immed-response 141 | (KRes immed-response (hasheqv)) 142 | pending)) 143 | (define req (KReq nack ch res flexible? parser)) 144 | (define header-data 145 | (with-output-bytes 146 | ((if flexible? 147 | proto:un-RequestHeaderV2 148 | proto:un-RequestHeaderV1) 149 | `((APIKey_1 . ,k) 150 | (APIVersion_1 . ,v) 151 | (CorrelationID_1 . ,(state-next-id s)) 152 | (ClientID_1 . ,client-id) 153 | (Tags_1 . ,tags))))) 154 | (define size 155 | (+ (bytes-length header-data) 156 | (bytes-length request-data))) 157 | (proto:un-Size size out) 158 | (write-bytes header-data out) 159 | (write-bytes request-data out) 160 | (flush-output out) 161 | (add-state-req s req)))] 162 | 163 | [`(request ,_ ,_ ,_ ,_ ,_ ,_ ,_ ,nack ,ch) 164 | (define err (client-error "disconnected")) 165 | (define req (Req nack ch err)) 166 | (loop (add-state-req s req))] 167 | 168 | [msg 169 | (log-kafka-error "invalid message: ~e" msg) 170 | (loop s)]))) 171 | (append 172 | (for/list ([(id r) (in-hash (state-reqs s))] 173 | #:unless (pending? (Req-res r))) 174 | (handle-evt 175 | (channel-put-evt (Req-ch r) (Req-res r)) 176 | (lambda (_) 177 | (loop (remove-state-req s id))))) 178 | (for/list ([(id r) (in-hash (state-reqs s))]) 179 | (handle-evt 180 | (Req-nack r) 181 | (lambda (_) 182 | (loop (remove-state-req s id))))))))) 183 | 184 | (define (make-request-evt conn 185 | #:key key 186 | #:version v 187 | #:tags [tags (hasheqv)] 188 | #:parser parser 189 | #:data [data #""] 190 | #:flexible? [flexible? #f] 191 | #:immed-response [immed-response #f]) 192 | (define msg `(request ,immed-response ,flexible? ,key ,v ,tags ,data ,parser)) 193 | (handle-evt (make-message-evt conn msg) KRes-data)) 194 | 195 | (define (make-message-evt conn msg) 196 | (define ch (make-channel)) 197 | (define mgr (connection-mgr conn)) 198 | (handle-evt 199 | (nack-guard-evt 200 | (lambda (nack) 201 | (thread-resume mgr (current-thread)) 202 | (begin0 ch 203 | (sync 204 | (thread-dead-evt mgr) 205 | (channel-put-evt 206 | (connection-ch conn) 207 | (append msg `(,nack ,ch))))))) 208 | (lambda (res-or-exn) 209 | (begin0 res-or-exn 210 | (when (exn:fail? res-or-exn) 211 | (raise res-or-exn)))))) 212 | 213 | 214 | ;; manager state ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 215 | 216 | (struct Req (nack ch res)) 217 | (struct KReq Req (flexible? parser)) 218 | (struct KRes (data tags)) 219 | 220 | (struct state (connected? seq reqs)) 221 | 222 | (define (make-state) 223 | (state #t 0 (hasheqv))) 224 | 225 | (define (state-next-id s) 226 | (state-seq s)) 227 | 228 | (define (state-req-count s) 229 | (hash-count (state-reqs s))) 230 | 231 | (define (add-state-req s req) 232 | (define id (state-next-id s)) 233 | (define reqs (hash-set (state-reqs s) id req)) 234 | (struct-copy state s 235 | [seq (add1 id)] 236 | [reqs reqs])) 237 | 238 | (define (find-state-req s id) 239 | (hash-ref (state-reqs s) id #f)) 240 | 241 | (define (update-state-req s id req) 242 | (struct-copy state s [reqs (hash-set (state-reqs s) id req)])) 243 | 244 | (define (remove-state-req s id) 245 | (struct-copy state s [reqs (hash-remove (state-reqs s) id)])) 246 | 247 | (define (set-state-disconnected s) 248 | (struct-copy state s [connected? #f])) 249 | 250 | (define (fail-pending-reqs s err) 251 | (struct-copy state s [reqs (for/hasheqv ([(id req) (in-hash (state-reqs s))]) 252 | (define updated-req 253 | (if (and (KReq? req) (pending? (Req-res req))) 254 | (struct-copy KReq req [res #:parent Req err]) 255 | req)) 256 | (values id updated-req))])) 257 | 258 | (define pending 259 | (gensym 'pending)) 260 | 261 | (define (pending? v) 262 | (eq? v pending)) 263 | 264 | 265 | ;; version ranges ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 266 | 267 | (provide 268 | (struct-out version-range) 269 | get-api-versions 270 | find-best-version) 271 | 272 | (struct version-range (min max) 273 | #:transparent) 274 | 275 | (define (get-api-versions conn) 276 | (sync 277 | (handle-evt 278 | (make-request-evt 279 | conn 280 | #:key 18 281 | #:version 0 282 | #:parser proto:APIVersionsResponseV0) 283 | (lambda (res) 284 | (define err-code (ref 'ErrorCode_1 res)) 285 | (unless (zero? err-code) 286 | (raise-server-error err-code)) 287 | (for/hasheqv ([rng (in-list (ref 'APIVersionRange_1 res))]) 288 | (values 289 | (ref 'APIKey_1 rng) 290 | (version-range 291 | (ref 'MinVersion_1 rng) 292 | (ref 'MaxVersion_1 rng)))))))) 293 | 294 | (define (find-best-version conn key [supported (version-range 0 +inf.0)]) 295 | (define server-rng 296 | (hash-ref (connection-versions conn) key #f)) 297 | (and server-rng 298 | (<= (version-range-min supported) 299 | (version-range-max server-rng)) 300 | (inexact->exact 301 | (max (version-range-min supported) 302 | (min (version-range-max supported) 303 | (version-range-max server-rng)))))) 304 | -------------------------------------------------------------------------------- /kafka-lib/private/crc.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require (for-syntax racket/base 4 | racket/fixnum) 5 | racket/fixnum) 6 | 7 | (provide 8 | crc 9 | crc-update) 10 | 11 | (begin-for-syntax 12 | (unless (> (most-positive-fixnum) #xFFFFFFFF) 13 | (error 'crc "the CRC implementation requires fixnums to be at least 32bit wide"))) 14 | 15 | (define mask #xFFFFFFFF) 16 | (define table 17 | (fxvector 18 | #x00000000 #xf26b8303 #xe13b70f7 #x1350f3f4 19 | #xc79a971f #x35f1141c #x26a1e7e8 #xd4ca64eb 20 | #x8ad958cf #x78b2dbcc #x6be22838 #x9989ab3b 21 | #x4d43cfd0 #xbf284cd3 #xac78bf27 #x5e133c24 22 | #x105ec76f #xe235446c #xf165b798 #x030e349b 23 | #xd7c45070 #x25afd373 #x36ff2087 #xc494a384 24 | #x9a879fa0 #x68ec1ca3 #x7bbcef57 #x89d76c54 25 | #x5d1d08bf #xaf768bbc #xbc267848 #x4e4dfb4b 26 | #x20bd8ede #xd2d60ddd #xc186fe29 #x33ed7d2a 27 | #xe72719c1 #x154c9ac2 #x061c6936 #xf477ea35 28 | #xaa64d611 #x580f5512 #x4b5fa6e6 #xb93425e5 29 | #x6dfe410e #x9f95c20d #x8cc531f9 #x7eaeb2fa 30 | #x30e349b1 #xc288cab2 #xd1d83946 #x23b3ba45 31 | #xf779deae #x05125dad #x1642ae59 #xe4292d5a 32 | #xba3a117e #x4851927d #x5b016189 #xa96ae28a 33 | #x7da08661 #x8fcb0562 #x9c9bf696 #x6ef07595 34 | #x417b1dbc #xb3109ebf #xa0406d4b #x522bee48 35 | #x86e18aa3 #x748a09a0 #x67dafa54 #x95b17957 36 | #xcba24573 #x39c9c670 #x2a993584 #xd8f2b687 37 | #x0c38d26c #xfe53516f #xed03a29b #x1f682198 38 | #x5125dad3 #xa34e59d0 #xb01eaa24 #x42752927 39 | #x96bf4dcc #x64d4cecf #x77843d3b #x85efbe38 40 | #xdbfc821c #x2997011f #x3ac7f2eb #xc8ac71e8 41 | #x1c661503 #xee0d9600 #xfd5d65f4 #x0f36e6f7 42 | #x61c69362 #x93ad1061 #x80fde395 #x72966096 43 | #xa65c047d #x5437877e #x4767748a #xb50cf789 44 | #xeb1fcbad #x197448ae #x0a24bb5a #xf84f3859 45 | #x2c855cb2 #xdeeedfb1 #xcdbe2c45 #x3fd5af46 46 | #x7198540d #x83f3d70e #x90a324fa #x62c8a7f9 47 | #xb602c312 #x44694011 #x5739b3e5 #xa55230e6 48 | #xfb410cc2 #x092a8fc1 #x1a7a7c35 #xe811ff36 49 | #x3cdb9bdd #xceb018de #xdde0eb2a #x2f8b6829 50 | #x82f63b78 #x709db87b #x63cd4b8f #x91a6c88c 51 | #x456cac67 #xb7072f64 #xa457dc90 #x563c5f93 52 | #x082f63b7 #xfa44e0b4 #xe9141340 #x1b7f9043 53 | #xcfb5f4a8 #x3dde77ab #x2e8e845f #xdce5075c 54 | #x92a8fc17 #x60c37f14 #x73938ce0 #x81f80fe3 55 | #x55326b08 #xa759e80b #xb4091bff #x466298fc 56 | #x1871a4d8 #xea1a27db #xf94ad42f #x0b21572c 57 | #xdfeb33c7 #x2d80b0c4 #x3ed04330 #xccbbc033 58 | #xa24bb5a6 #x502036a5 #x4370c551 #xb11b4652 59 | #x65d122b9 #x97baa1ba #x84ea524e #x7681d14d 60 | #x2892ed69 #xdaf96e6a #xc9a99d9e #x3bc21e9d 61 | #xef087a76 #x1d63f975 #x0e330a81 #xfc588982 62 | #xb21572c9 #x407ef1ca #x532e023e #xa145813d 63 | #x758fe5d6 #x87e466d5 #x94b49521 #x66df1622 64 | #x38cc2a06 #xcaa7a905 #xd9f75af1 #x2b9cd9f2 65 | #xff56bd19 #x0d3d3e1a #x1e6dcdee #xec064eed 66 | #xc38d26c4 #x31e6a5c7 #x22b65633 #xd0ddd530 67 | #x0417b1db #xf67c32d8 #xe52cc12c #x1747422f 68 | #x49547e0b #xbb3ffd08 #xa86f0efc #x5a048dff 69 | #x8ecee914 #x7ca56a17 #x6ff599e3 #x9d9e1ae0 70 | #xd3d3e1ab #x21b862a8 #x32e8915c #xc083125f 71 | #x144976b4 #xe622f5b7 #xf5720643 #x07198540 72 | #x590ab964 #xab613a67 #xb831c993 #x4a5a4a90 73 | #x9e902e7b #x6cfbad78 #x7fab5e8c #x8dc0dd8f 74 | #xe330a81a #x115b2b19 #x020bd8ed #xf0605bee 75 | #x24aa3f05 #xd6c1bc06 #xc5914ff2 #x37faccf1 76 | #x69e9f0d5 #x9b8273d6 #x88d28022 #x7ab90321 77 | #xae7367ca #x5c18e4c9 #x4f48173d #xbd23943e 78 | #xf36e6f75 #x0105ec76 #x12551f82 #xe03e9c81 79 | #x34f4f86a #xc69f7b69 #xd5cf889d #x27a40b9e 80 | #x79b737ba #x8bdcb4b9 #x988c474d #x6ae7c44e 81 | #xbe2da0a5 #x4c4623a6 #x5f16d052 #xad7d5351)) 82 | 83 | (define (crc-update n bs [start 0] [end (bytes-length bs)]) 84 | (for/fold ([n (fxxor n mask)] #:result (fxxor n mask)) 85 | ([b (in-bytes bs start end)]) 86 | (define idx (fxand (fxxor n b) #xFF)) 87 | (define v (fxvector-ref table idx)) 88 | (fxand (fxxor v (fxrshift n 8)) mask))) 89 | 90 | (define (crc bs [start 0] [end (bytes-length bs)]) 91 | (crc-update 0 bs start end)) 92 | 93 | (module+ test 94 | (require rackunit) 95 | (check-equal? (crc #"hello") 2591144780) 96 | (check-equal? (crc #" hello " 2 7) 2591144780) 97 | (check-equal? 98 | (crc 99 | (subbytes 100 | (bytes-append 101 | #"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x3a" 102 | #"\x00\x00\x00\x00\x02\xff\x5e\x0a\x7f\x00\x00\x00" 103 | #"\x00\x00\x00\x00\x00\x01\x7e\xe7\xeb\xbf\xcd\x00" 104 | #"\x00\x01\x7e\xe7\xeb\xbf\xcd\xff\xff\xff\xff\xff" 105 | #"\xff\xff\xff\xff\xff\xff\xff\xff\xff\x00\x00\x00" 106 | #"\x01\x10\x00\x00\x00\x02\x61\x02\x61\x00") 107 | 21)) 108 | 4284353151)) 109 | -------------------------------------------------------------------------------- /kafka-lib/private/error.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require (for-syntax racket/base 4 | racket/string 5 | syntax/parse/pre)) 6 | 7 | (provide 8 | exn:fail:kafka? 9 | exn:fail:kafka:client? 10 | exn:fail:kafka:server? 11 | exn:fail:kafka:server-code 12 | client-error 13 | raise-client-error 14 | server-error 15 | raise-server-error 16 | error-code-symbol 17 | error-code-message 18 | make-error-code?) 19 | 20 | (struct exn:fail:kafka exn:fail ()) 21 | (struct exn:fail:kafka:server exn:fail:kafka (code)) 22 | (struct exn:fail:kafka:client exn:fail:kafka ()) 23 | 24 | (define (client-error message . args) 25 | (exn:fail:kafka:client (apply format message args) (current-continuation-marks))) 26 | 27 | (define (raise-client-error . args) 28 | (raise (apply client-error args))) 29 | 30 | (define server-error 31 | (case-lambda 32 | [(code) 33 | (server-error code (error-code-message code))] 34 | [(code message . args) 35 | (exn:fail:kafka:server (apply format message args) (current-continuation-marks) code)])) 36 | 37 | (define (raise-server-error . args) 38 | (raise (apply server-error args))) 39 | 40 | (define-syntax (define-error-codes stx) 41 | (syntax-parse stx 42 | [(_ sym-id message-id [code:number message:str] ...+) 43 | #:with (sym ...) (for/list ([message-stx (syntax-e #'(message ...))]) 44 | (datum->syntax message-stx (string->symbol (string-replace (syntax->datum message-stx) " " "-")))) 45 | #'(begin 46 | (define (sym-id c) 47 | (case c 48 | [(code) 'sym] ... 49 | [else 'unknown])) 50 | (define (message-id c) 51 | (case c 52 | [(code) message] ... 53 | [else "unknown error code"])))])) 54 | 55 | (define-error-codes error-code-symbol error-code-message 56 | [-1 "unknown server error"] 57 | [0 "no error"] 58 | [1 "offset out of range"] 59 | [2 "corrupt message"] 60 | [3 "unknown topic or partition"] 61 | [4 "invalid fetch size"] 62 | [5 "leader not available"] 63 | [6 "not leader or follower"] 64 | [7 "request timed out"] 65 | [8 "broker not available"] 66 | [9 "replica not available"] 67 | [10 "message too large"] 68 | [11 "stale controller epoch"] 69 | [12 "offset metadata too large"] 70 | [13 "network exception"] 71 | [14 "coordinator load in progress"] 72 | [15 "coordinator not available"] 73 | [16 "not coordinator"] 74 | [17 "invalid topic exception"] 75 | [18 "record list too large"] 76 | [19 "not enough replicas"] 77 | [20 "not enough replicas after append"] 78 | [21 "invalid required acks"] 79 | [22 "illegal generation"] 80 | [23 "inconsistent group protocol"] 81 | [24 "invalid group id"] 82 | [25 "unknown member id"] 83 | [26 "invalid session timeout"] 84 | [27 "rebalance in progress"] 85 | [28 "invalid commit offset size"] 86 | [29 "topic authorization failed"] 87 | [30 "group authorization failed"] 88 | [31 "cluster authorization failed"] 89 | [32 "invalid timestamp"] 90 | [33 "unsupported SASL mechanism"] 91 | [34 "illegal SASL state"] 92 | [35 "unsupported version"] 93 | [36 "topic already exists"] 94 | [37 "invalid partitions"] 95 | [38 "invalid replication factor"] 96 | [39 "invalid replica assignment"] 97 | [40 "invalid config"] 98 | [41 "not controller"] 99 | [42 "invalid request"] 100 | [43 "unsupported for message format"] 101 | [44 "policy violation"] 102 | [45 "out of order sequence number"] 103 | [46 "duplicate sequence number"] 104 | [47 "invalid producer epoch"] 105 | [48 "invalid txn state"] 106 | [49 "invalid producer id mapping"] 107 | [50 "invalid transaction timeout"] 108 | [51 "concurrent transactions"] 109 | [52 "transaction coordinator fenced"] 110 | [53 "transaction id authorization failed"] 111 | [54 "security disabled"] 112 | [55 "operation not attempted"] 113 | [56 "kafka storage error"] 114 | [57 "log dir not found"] 115 | [58 "SASL authentication failed"] 116 | [59 "unknown producer id"] 117 | [60 "reassignment in progress"] 118 | [61 "delegation token authentication disabled"] 119 | [62 "delegation token not found"] 120 | [63 "delegation token owner mismatch"] 121 | [64 "delegation token request not allowed"] 122 | [65 "delegation token authorization failed"] 123 | [66 "delegation token expired"] 124 | [67 "invalid principal type"] 125 | [68 "non-empty group"] 126 | [69 "group id not found"] 127 | [70 "fetch session id not found"] 128 | [71 "invalid fetch session id epoch"] 129 | [72 "listener not found"] 130 | [73 "topic deletion disabled"] 131 | [74 "fenced leader epoch"] 132 | [75 "unknown leader epoch"] 133 | [76 "unsupported compression type"] 134 | [77 "stale broker epoch"] 135 | [78 "offset not available"] 136 | [79 "member id required"] 137 | [80 "preferred leader not available"] 138 | [81 "group max size reached"] 139 | [82 "fenced instance id"] 140 | [83 "eligibile leaders not available"] 141 | [84 "election not needed"] 142 | [85 "no reassignment in progress"] 143 | [86 "group subscribed to topic"] 144 | [87 "invalid record"] 145 | [88 "unstable offset commit"] 146 | [89 "throttling quota exceeded"] 147 | [90 "producer fenced"] 148 | [91 "resource not found"] 149 | [92 "duplicate resource"] 150 | [93 "unacceptable credential"] 151 | [94 "inconsistent voter set"] 152 | [95 "invalid update version"] 153 | [96 "feature update failed"] 154 | [97 "principal deserialization failure"] 155 | [98 "snapshot not found"] 156 | [99 "position out of range"] 157 | [100 "unknown topic id"] 158 | [101 "duplicate broker registration"] 159 | [102 "broker id not registered"] 160 | [103 "inconsistent topic id"] 161 | [104 "inconsistent clusetr id"] 162 | [105 "transaction id not found"] 163 | [106 "fetch session topic id error"]) 164 | 165 | (define ((make-error-code? sym-or-proc) e) 166 | (and (exn:fail:kafka:server? e) 167 | (let ([sym (error-code-symbol (exn:fail:kafka:server-code e))]) 168 | (if (procedure? sym-or-proc) 169 | (sym-or-proc sym) 170 | (eq? sym sym-or-proc))))) 171 | -------------------------------------------------------------------------------- /kafka-lib/private/help.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require racket/port) 4 | 5 | (provide 6 | null8 7 | null16 8 | null32 9 | ref 10 | opt 11 | with-output-bytes) 12 | 13 | (define null8 #"\xFF") 14 | (define null16 #"\xFF\xFF") 15 | (define null32 #"\xFF\xFF\xFF\xFF") 16 | 17 | (define ref 18 | (case-lambda 19 | [(id v) 20 | (define p (assq id v)) 21 | (unless p 22 | (error 'ref "key not found: ~s~n have: ~e" id (map car v))) 23 | (cdr p)] 24 | [(id . args) 25 | (ref id (apply ref args))])) 26 | 27 | (define opt 28 | (case-lambda 29 | [(id v) 30 | (define p (assq id v)) 31 | (and p (cdr p))] 32 | [(id . args) 33 | (opt id (apply ref args))])) 34 | 35 | (define-syntax-rule (with-output-bytes e0 e ...) 36 | (with-output-to-bytes 37 | (lambda () 38 | e0 e ...))) 39 | -------------------------------------------------------------------------------- /kafka-lib/private/logger.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require (for-syntax racket/base)) 4 | 5 | (provide 6 | kafka-logger 7 | log-kafka-debug 8 | log-kafka-info 9 | log-kafka-warning 10 | log-kafka-error 11 | log-kafka-fatal 12 | log-kafka-fault 13 | (struct-out fault)) 14 | 15 | (define-logger kafka) 16 | 17 | (struct fault (original-error)) 18 | 19 | (define (make-fault err) 20 | (fault err)) 21 | 22 | (define-syntax (log-kafka-fault stx) 23 | (syntax-case stx () 24 | [(_ err format-str format-arg ...) 25 | #'(let ([l kafka-logger]) 26 | (when (log-level? l 'debug 'kafka) 27 | (log-message 28 | l 'debug 29 | (format format-str format-arg ...) 30 | (make-fault err))))])) 31 | -------------------------------------------------------------------------------- /kafka-lib/private/protocol-consumer.bnf: -------------------------------------------------------------------------------- 1 | #lang binfmt 2 | 3 | @foreign-parsers "protocol-native.rkt" 4 | {Bytes un-Bytes} 5 | {String un-String} 6 | ; 7 | 8 | MemberMetadata = Version ArrayLen TopicName{ArrayLen_1} Data; 9 | Version = i16be; 10 | ArrayLen = i32be; 11 | TopicName = String; 12 | Data = Bytes; 13 | 14 | MemberAssignment = Version ArrayLen Assignment{ArrayLen_1} Data; 15 | Assignment = TopicName ArrayLen PartitionID{ArrayLen_1}; 16 | PartitionID = i32be; 17 | -------------------------------------------------------------------------------- /kafka-lib/private/protocol-internal.bnf: -------------------------------------------------------------------------------- 1 | #lang binfmt 2 | 3 | @foreign-parsers "protocol-native.rkt" 4 | {Bytes un-Bytes} 5 | {String un-String} 6 | ; 7 | 8 | MemberMetadataV0 = MemberID ClientID ClientHost SessionTimeout Subscription Assignment; 9 | MemberID = String; 10 | ClientID = String; 11 | ClientHost = String; 12 | SessionTimeout = i32be; 13 | Subscription = Bytes; 14 | Assignment = Bytes; 15 | MemberMetadataV1 = MemberID ClientID ClientHost RebalanceTimeout SessionTimeout Subscription Assignment; 16 | RebalanceTimeout = i32be; 17 | MemberMetadataV2 = MemberID ClientID ClientHost RebalanceTimeout SessionTimeout Subscription Assignment; 18 | MemberMetadataV3 = MemberID GroupInstanceID ClientID ClientHost RebalanceTimeout SessionTimeout Subscription Assignment; 19 | GroupInstanceID = String; 20 | 21 | GroupMetadataKeyV2 = Group; 22 | Group = String; 23 | 24 | GroupMetadataValueV0 = ProtocolType Generation Protocol Leader ArrayLen MemberMetadataV0{ArrayLen_1}; 25 | ProtocolType = String; 26 | Generation = i32be; 27 | Protocol = String; 28 | Leader = String; 29 | GroupMetadataValueV1 = ProtocolType Generation Protocol Leader ArrayLen MemberMetadataV1{ArrayLen_1}; 30 | GroupMetadataValueV2 = ProtocolType Generation Protocol Leader CurrentStateTimestamp ArrayLen MemberMetadataV2{ArrayLen_1}; 31 | CurrentStateTimestamp = i64be; 32 | GroupMetadataValueV3 = ProtocolType Generation Protocol Leader CurrentStateTimestamp ArrayLen MemberMetadataV3{ArrayLen_1}; 33 | 34 | OffsetCommitKeyV01 = Group Topic PartitionID; 35 | Topic = String; 36 | PartitionID = i32be; 37 | 38 | OffsetCommitValueV0 = Offset; # Metadata CommitTimestamp; 39 | OffsetCommitValueV1 = Offset; # Metadata CommitTimestamp ExpireTimestamp; 40 | OffsetCommitValueV2 = Offset; # Metadata CommitTimestamp; 41 | OffsetCommitValueV3 = Offset; # LeaderEpoch? Metadata CommitTimestamp; 42 | 43 | Offset = i64be; 44 | LeaderEpoch = i32be; 45 | Metadata = Bytes; 46 | CommitTimestamp = i64be; 47 | ExpireTimestamp = i64be; 48 | 49 | ArrayLen = i32be; 50 | -------------------------------------------------------------------------------- /kafka-lib/private/protocol-native.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require binfmt/runtime/parser 4 | binfmt/runtime/res 5 | binfmt/runtime/unparser) 6 | 7 | (provide 8 | (rename-out 9 | [parse-bool Bool] 10 | [unparse-bool un-Bool] 11 | [parse-uuid UUID] 12 | [unparse-uuid un-UUID] 13 | [parse-bytes Bytes] 14 | [unparse-bytes un-Bytes] 15 | [parse-batch-bytes BatchBytes] 16 | [unparse-batch-bytes un-BatchBytes] 17 | [parse-compact-bytes CompactBytes] 18 | [unparse-compact-bytes un-CompactBytes] 19 | [parse-string String] 20 | [unparse-string un-String] 21 | [parse-batch-string BatchString] 22 | [unparse-batch-string un-BatchString] 23 | [parse-compact-string CompactString] 24 | [unparse-compact-string un-CompactString] 25 | [parse-compact-array-len CompactArrayLen] 26 | [unparse-compact-array-len un-CompactArrayLen] 27 | [parse-tags Tags] 28 | [unparse-tags un-Tags])) 29 | 30 | (define (expect what len in) 31 | (define bs (read-bytes len in)) 32 | (cond 33 | [(eof-object? bs) (make-err in "unexpected EOF while reading ~a" what)] 34 | [(< (bytes-length bs) len) (make-err in "input ended before ~a" what)] 35 | [else (ok bs)])) 36 | 37 | (define (parse-bool in) 38 | (define b (read-byte in)) 39 | (if (eof-object? b) 40 | (make-err in "unexpected EOF while reading Bool") 41 | (ok (not (= b 0))))) 42 | 43 | (define (unparse-bool out v) 44 | (begin0 (ok (and v #t)) 45 | (write-byte (if v 1 0) out))) 46 | 47 | (define (parse-uuid in) 48 | (expect 'UUID 16 in)) 49 | 50 | (define (unparse-uuid out v) 51 | (begin0 (ok v) 52 | (write-bytes v out))) 53 | 54 | (define (parse-bytes in) 55 | (res-bind 56 | (parse-i32be in) 57 | (lambda (len) 58 | (cond 59 | [(= len -1) (ok #f)] 60 | [else (expect 'bytes len in)])))) 61 | 62 | (define (unparse-bytes out bs) 63 | (cond 64 | [(not bs) 65 | (unparse-i32be out -1)] 66 | [else 67 | (res-bind 68 | (unparse-i32be out (bytes-length bs)) 69 | (lambda (_) 70 | (begin0 (ok bs) 71 | (write-bytes bs out))))])) 72 | 73 | (define (parse-batch-bytes in) 74 | (res-bind 75 | (parse-varint32 in) 76 | (lambda (len) 77 | (cond 78 | [(= len -1) (ok #f)] 79 | [else (expect 'batch-bytes len in)])))) 80 | 81 | (define (unparse-batch-bytes out bs) 82 | (cond 83 | [(not bs) 84 | (unparse-varint32 out -1)] 85 | [else 86 | (res-bind 87 | (unparse-varint32 out (bytes-length bs)) 88 | (lambda (_) 89 | (begin0 (ok bs) 90 | (write-bytes bs out))))])) 91 | 92 | (define (parse-compact-bytes in) 93 | (res-bind 94 | (parse-uvarint32 in) 95 | (lambda (len) 96 | (cond 97 | [(zero? len) (ok #f)] 98 | [else (expect 'compact-bytes (sub1 len) in)])))) 99 | 100 | (define (unparse-compact-bytes out bs) 101 | (cond 102 | [(not bs) (unparse-uvarint32 out 0)] 103 | [else 104 | (res-bind 105 | (unparse-uvarint32 out (add1 (bytes-length bs))) 106 | (lambda (_) 107 | (begin0 (ok bs) 108 | (write-bytes bs out))))])) 109 | 110 | (define (parse-string in) 111 | (res-bind 112 | (parse-i16be in) 113 | (lambda (len) 114 | (cond 115 | [(= len -1) (ok #f)] 116 | [else (res-bind 117 | (expect 'string len in) 118 | (compose1 ok bytes->string/utf-8))])))) 119 | 120 | (define (unparse-string out s) 121 | (cond 122 | [(not s) 123 | (unparse-i16be out -1)] 124 | [else 125 | (define bs (string->bytes/utf-8 s)) 126 | (res-bind 127 | (unparse-i16be out (bytes-length bs)) 128 | (lambda (_) 129 | (begin0 (ok s) 130 | (write-bytes bs out))))])) 131 | 132 | (define (parse-batch-string in) 133 | (res-bind 134 | (parse-varint32 in) 135 | (lambda (len) 136 | (cond 137 | [(= len -1) (ok #f)] 138 | [else (res-bind 139 | (expect 'batch-string len in) 140 | (compose1 ok bytes->string/utf-8))])))) 141 | 142 | (define (unparse-batch-string out s) 143 | (cond 144 | [(not s) 145 | (unparse-varint32 out -1)] 146 | [else 147 | (define bs (string->bytes/utf-8 s)) 148 | (res-bind 149 | (unparse-varint32 out (bytes-length bs)) 150 | (lambda (_) 151 | (begin0 (ok s) 152 | (write-bytes bs out))))])) 153 | 154 | (define (parse-compact-string in) 155 | (res-bind 156 | (parse-uvarint32 in) 157 | (lambda (len) 158 | (cond 159 | [(zero? len) (ok #f)] 160 | [else 161 | (res-bind 162 | (expect 'compact-string (sub1 len) in) 163 | (compose1 ok bytes->string/utf-8))])))) 164 | 165 | (define (unparse-compact-string out s) 166 | (cond 167 | [(not s) (unparse-uvarint32 out 0)] 168 | [else 169 | (define bs (string->bytes/utf-8 s)) 170 | (res-bind 171 | (unparse-uvarint32 out (add1 (bytes-length bs))) 172 | (lambda (_) 173 | (begin0 (ok s) 174 | (write-bytes bs out))))])) 175 | 176 | (define (parse-compact-array-len in) 177 | (res-bind 178 | (parse-uvarint32 in) 179 | (lambda (len) 180 | (cond 181 | [(zero? len) (ok 0)] 182 | [else (ok (sub1 len))])))) 183 | 184 | (define (unparse-compact-array-len out len) 185 | (if (zero? len) 186 | (unparse-uvarint32 out 0) 187 | (unparse-uvarint32 out (add1 len)))) 188 | 189 | (define (parse-tags in) 190 | (res-bind 191 | (parse-uvarint32 in) 192 | (lambda (len) 193 | (let loop ([len len] [tags (hasheqv)]) 194 | (cond 195 | [(zero? len) 196 | (ok tags)] 197 | [else 198 | (res-bind 199 | (parse-tag-pair in) 200 | (lambda (tag-pair) 201 | (loop (sub1 len) 202 | (hash-set tags 203 | (car tag-pair) 204 | (cdr tag-pair)))))]))))) 205 | 206 | (define (parse-tag-pair in) 207 | (res-bind 208 | (parse-uvarint32 in) 209 | (lambda (key) 210 | (res-bind 211 | (parse-compact-bytes in) 212 | (lambda (value) 213 | (ok (cons key value))))))) 214 | 215 | (define (unparse-tags out v) 216 | (res-bind 217 | (unparse-uvarint32 out (hash-count v)) 218 | (lambda (_) 219 | (let loop ([tags (sort (hash->list v) #:key car <)]) 220 | (cond 221 | [(null? tags) (ok v)] 222 | [else 223 | (res-bind 224 | (unparse-tag-pair out (car tags)) 225 | (lambda (_) 226 | (loop (cdr tags))))]))))) 227 | 228 | (define (unparse-tag-pair out v) 229 | (res-bind 230 | (unparse-uvarint32 out (car v)) 231 | (lambda (_) 232 | (unparse-compact-bytes out (cdr v))))) 233 | -------------------------------------------------------------------------------- /kafka-lib/private/proxy.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require racket/generic 4 | racket/lazy-require 5 | racket/match) 6 | 7 | (lazy-require 8 | [net/http-client (http-conn-CONNECT-tunnel)]) 9 | 10 | ;; generics ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 11 | 12 | (provide 13 | proxy? 14 | proxy-connect) 15 | 16 | (define-generics proxy 17 | {proxy-connect proxy target-host target-port ssl-ctx}) 18 | 19 | 20 | ;; http proxy ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 21 | 22 | (provide 23 | make-http-proxy) 24 | 25 | (define (make-http-proxy host port) 26 | (http-proxy host port)) 27 | 28 | (struct http-proxy (host port) 29 | #:methods gen:proxy 30 | [(define (proxy-connect p target-host target-port ssl-ctx) 31 | (match-define (http-proxy proxy-host proxy-port) p) 32 | (define-values (_ssl-ctx in out _abandon) 33 | (http-conn-CONNECT-tunnel 34 | #:ssl? ssl-ctx 35 | proxy-host proxy-port 36 | target-host target-port)) 37 | (values in out))]) 38 | -------------------------------------------------------------------------------- /kafka-lib/private/record.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require "help.rkt") 4 | 5 | (provide 6 | (struct-out record) 7 | parse-record) 8 | 9 | (struct record 10 | ([partition-id #:mutable] 11 | offset 12 | timestamp 13 | key 14 | value 15 | headers)) 16 | 17 | (define (parse-record r 18 | [base-offset 0] 19 | [base-timestamp 0]) 20 | (record 21 | #f 22 | (+ base-offset (ref 'OffsetDelta_1 r)) 23 | (+ base-timestamp (ref 'TimestampDelta_1 r)) 24 | (ref 'Key_1 r) 25 | (ref 'Value_1 r) 26 | (for*/hash ([e (in-list (or (opt 'Header_1 'Headers_1 r) null))] 27 | [k (in-value (ref 'HeaderKey_1 e))] 28 | #:when k 29 | [v (in-value (ref 'HeaderValue_1 e))]) 30 | (values k (or v #""))))) 31 | -------------------------------------------------------------------------------- /kafka-lib/private/serde.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (define-syntax-rule (reprovide mod ...) 4 | (begin 5 | (require mod ...) 6 | (provide (all-from-out mod ...)))) 7 | 8 | (reprovide 9 | "serde/authorized-operation.rkt" 10 | "serde/alter-configs.rkt" 11 | "serde/commit.rkt" 12 | "serde/contract.rkt" 13 | "serde/create-topics.rkt" 14 | "serde/delete-groups.rkt" 15 | "serde/delete-topics.rkt" 16 | "serde/describe-cluster.rkt" 17 | "serde/describe-configs.rkt" 18 | "serde/describe-groups.rkt" 19 | "serde/describe-producers.rkt" 20 | "serde/fetch-offsets.rkt" 21 | "serde/fetch.rkt" 22 | "serde/find-coordinator.rkt" 23 | "serde/group.rkt" 24 | "serde/heartbeat.rkt" 25 | "serde/internal.rkt" 26 | "serde/join-group.rkt" 27 | "serde/leave-group.rkt" 28 | "serde/list-groups.rkt" 29 | "serde/list-offsets.rkt" 30 | "serde/metadata.rkt" 31 | "serde/produce.rkt" 32 | "serde/sasl-authenticate.rkt" 33 | "serde/sasl-handshake.rkt" 34 | "serde/sync-group.rkt") 35 | -------------------------------------------------------------------------------- /kafka-lib/private/serde/alter-configs.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require racket/contract/base 4 | "core.rkt" 5 | "resource.rkt") 6 | 7 | (define-record AlterResource 8 | ([type resource-type/c] 9 | [name string?] 10 | [configs (hash/c string? (or/c #f string?))])) 11 | 12 | (define-record AlteredResource 13 | ([error-code error-code/c] 14 | [error-message (or/c #f string?)] 15 | [type resource-type/c] 16 | [name string?])) 17 | 18 | (define-record AlteredResources 19 | ([(throttle-time-ms #f) (or/c #f exact-nonnegative-integer?)] 20 | [resources (listof AlteredResource?)])) 21 | 22 | (define-request AlterConfigs 23 | (resources validate-only?) 24 | #:code 33 25 | #:version 0 26 | #:response proto:AlterConfigsResponseV0 27 | (lambda (resources validate-only?) 28 | (with-output-bytes 29 | (define data 30 | (for/list ([res (in-list resources)]) 31 | (define configs (AlterResource-configs res)) 32 | (define len (hash-count configs)) 33 | (define config-data 34 | (for/list ([(k v) (in-hash configs)]) 35 | `((ConfigName_1 . ,k) 36 | (ConfigValue_1 . ,v)))) 37 | `((ResourceType_1 . ,(->resource-type (AlterResource-type res))) 38 | (ResourceName_1 . ,(AlterResource-name res)) 39 | (Configs_1 . ((ArrayLen_1 . ,len) 40 | (Config_1 . ,config-data)))))) 41 | (proto:un-AlterConfigsRequestV0 42 | `((ArrayLen_1 . ,(length resources)) 43 | (AlterConfigsResourceV0_1 . ,data) 44 | (ValidateOnly_1 . ,validate-only?))))) 45 | (lambda (res) 46 | (make-AlteredResources 47 | #:throttle-time-ms (ref 'ThrottleTimeMs_1 res) 48 | #:resources (for/list ([r (in-list (ref 'AlterConfigsResponseDataV0_1 res))]) 49 | (make-AlteredResource 50 | #:error-code (ref 'ErrorCode_1 r) 51 | #:error-message (ref 'ErrorMessage_1 r) 52 | #:type (resource-type-> (ref 'ResourceType_1 r)) 53 | #:name (ref 'ResourceName_1 r)))))) 54 | -------------------------------------------------------------------------------- /kafka-lib/private/serde/authorized-operation.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require racket/contract/base 4 | racket/list) 5 | 6 | (provide 7 | authorized-operation/c 8 | integer->authorized-operations) 9 | 10 | (define operations 11 | #(unknown any all read write create delete alter describe cluster-action describe-configs alter-configs idempotent-write create-tokens describe-tokens)) 12 | 13 | (define authorized-operation/c 14 | (apply or/c (vector->list operations))) 15 | 16 | (define (integer->authorized-operations n) 17 | (let loop ([i 0] [n n] [ops null]) 18 | (if (<= n 0) 19 | (remove-duplicates (reverse ops)) 20 | (loop (add1 i) 21 | (arithmetic-shift n -1) 22 | (if (= (bitwise-and n 1) 1) 23 | (cons (get-operation i) ops) 24 | ops))))) 25 | 26 | (define (get-operation i) 27 | (if (< i (vector-length operations)) 28 | (vector-ref operations i) 29 | 'unknown)) 30 | -------------------------------------------------------------------------------- /kafka-lib/private/serde/commit.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require "core.rkt") 4 | 5 | (define-record CommitPartition 6 | ([id exact-nonnegative-integer?] 7 | [offset exact-nonnegative-integer?] 8 | [(timestamp (current-milliseconds)) exact-nonnegative-integer?] 9 | [(metadata "") string?])) 10 | 11 | (define-record CommitPartitionResult 12 | ([id exact-nonnegative-integer?] 13 | [error-code error-code/c])) 14 | 15 | (define-request Commit 16 | (group-id 17 | generation-id 18 | member-id 19 | topics) 20 | #:code 8 21 | 22 | #:version 1 23 | #:response proto:OffsetCommitResponseV1 24 | (lambda (group-id generation-id member-id topics) 25 | (with-output-bytes 26 | (proto:un-OffsetCommitRequestV1 27 | `((GroupID_1 . ,group-id) 28 | (GenerationID_1 . ,generation-id) 29 | (MemberID_1 . ,member-id) 30 | (ArrayLen_1 . ,(hash-count topics)) 31 | (OffsetCommitRequestTopicV1_1 . ,(for/list ([(topic partitions) (in-hash topics)]) 32 | (define partition-data 33 | (for/list ([part (in-list partitions)]) 34 | `((PartitionID_1 . ,(CommitPartition-id part)) 35 | (CommittedOffset_1 . ,(CommitPartition-offset part)) 36 | (CommitTimestamp_1 . ,(CommitPartition-timestamp part)) 37 | (CommittedMetadata_1 . ,(CommitPartition-metadata part))))) 38 | 39 | `((TopicName_1 . ,topic) 40 | (ArrayLen_1 . ,(length partitions)) 41 | (OffsetCommitRequestPartitionV1_1 . ,partition-data)))))))) 42 | (lambda (res) 43 | (for/hash ([t (in-list (ref 'OffsetCommitResponseTopicV1_1 res))]) 44 | (define topic (ref 'TopicName_1 t)) 45 | (values topic (for/list ([p (in-list (ref 'OffsetCommitResponsePartitionV1_1 t))]) 46 | (make-CommitPartitionResult 47 | #:id (ref 'PartitionID_1 p) 48 | #:error-code (ref 'ErrorCode_1 p))))))) 49 | -------------------------------------------------------------------------------- /kafka-lib/private/serde/contract.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require racket/contract/base) 4 | 5 | (provide 6 | (all-defined-out)) 7 | 8 | (define error-code/c exact-integer?) 9 | (define port/c (integer-in 0 65535)) 10 | (define tags/c hash?) 11 | -------------------------------------------------------------------------------- /kafka-lib/private/serde/core.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require (for-syntax racket/base 4 | racket/list 5 | racket/syntax 6 | syntax/parse/pre) 7 | racket/contract/base 8 | racket/pretty 9 | "../connection.rkt" 10 | "../error.rkt" 11 | "../help.rkt" 12 | (prefix-in proto: "../protocol.bnf") 13 | "contract.rkt") 14 | 15 | (provide 16 | (all-from-out 17 | "../connection.rkt" 18 | "../help.rkt" 19 | "../protocol.bnf" 20 | "contract.rkt") 21 | define-record 22 | define-request) 23 | 24 | (begin-for-syntax 25 | (define (id-stx->keyword stx) 26 | (datum->syntax stx (string->keyword (symbol->string (syntax-e stx))))) 27 | 28 | (define-syntax-class record-fld 29 | (pattern [id:id ctc:expr] 30 | #:with required? #'#t 31 | #:with kwd (id-stx->keyword #'id) 32 | #:with arg #'id) 33 | (pattern [(id:id def:expr) ctc:expr] 34 | #:with required? #'#f 35 | #:with kwd (id-stx->keyword #'id) 36 | #:with arg #'[id def]))) 37 | 38 | (define-syntax (define-record stx) 39 | (syntax-parse stx 40 | [(_ id:id (fld:record-fld ...)) 41 | #:with id? (format-id #'id "~a?" #'id) 42 | #:with constructor-id (format-id #'id "make-~a" #'id) 43 | #:with (constructor-arg ...) (flatten 44 | (for/list ([kwd (in-list (syntax-e #'(fld.kwd ...)))] 45 | [arg (in-list (syntax-e #'(fld.arg ...)))]) 46 | (list kwd arg))) 47 | #:with (required-ctor-arg-ctc ...) (flatten 48 | (for/list ([req? (in-list (syntax->datum #'(fld.required? ...)))] 49 | [kwd (in-list (syntax-e #'(fld.kwd ...)))] 50 | [ctc (in-list (syntax-e #'(fld.ctc ...)))] 51 | #:when req?) 52 | (list kwd ctc))) 53 | #:with (optional-ctor-arg-ctc ...) (flatten 54 | (for/list ([req? (in-list (syntax->datum #'(fld.required? ...)))] 55 | [kwd (in-list (syntax-e #'(fld.kwd ...)))] 56 | [ctc (in-list (syntax-e #'(fld.ctc ...)))] 57 | #:unless req?) 58 | (list kwd ctc))) 59 | #:with (fld-accessor-id ...) (for/list ([fld-id (in-list (syntax-e #'(fld.id ...)))]) 60 | (format-id fld-id "~a-~a" #'id fld-id)) 61 | #'(begin 62 | (provide 63 | (contract-out 64 | [struct id ([fld.id fld.ctc] ...)] 65 | [constructor-id (->* (required-ctor-arg-ctc ...) 66 | (optional-ctor-arg-ctc ...) 67 | id?)])) 68 | (struct id (fld.id ...) 69 | #:transparent 70 | #:methods gen:custom-write 71 | [(define write-proc 72 | (make-record-constructor-printer 73 | (λ (_) 'constructor-id) 74 | (λ (_) (list 'fld.kwd ...)) 75 | (λ (r) (list (fld-accessor-id r) ...))))]) 76 | (define (constructor-id constructor-arg ...) 77 | (id fld.id ...)))])) 78 | 79 | (define ((make-record-constructor-printer get-id get-kwds get-flds) r out mode) 80 | (define (do-print port [col #f]) 81 | (define indent (if col (make-string (add1 col) #\space) " ")) 82 | (define recur 83 | (case mode 84 | [(#t) write] 85 | [(#f) display] 86 | [(0 1) (λ (d p) (print d p mode))])) 87 | (define write? (not (integer? mode))) 88 | (write-string (if write? "#<" "(") port) 89 | (write (get-id r) port) 90 | (for ([kwd (in-list (get-kwds r))] 91 | [fld (in-list (get-flds r))]) 92 | (when col 93 | (pretty-print-newline port (pretty-print-columns))) 94 | (write-string indent port) 95 | (fprintf port "~a " kwd) 96 | (recur fld port)) 97 | (write-string (if write? ">" ")") port)) 98 | (if (and (pretty-printing) 99 | (integer? (pretty-print-columns))) 100 | ((let/ec esc 101 | (define pretty-port 102 | (make-tentative-pretty-print-output-port 103 | out 104 | (- (pretty-print-columns) 1) 105 | (lambda () 106 | (esc 107 | (lambda () 108 | (tentative-pretty-print-port-cancel pretty-port) 109 | (define-values (_line col _pos) 110 | (port-next-location out)) 111 | (do-print out col)))))) 112 | (do-print pretty-port) 113 | (tentative-pretty-print-port-transfer pretty-port out) 114 | void)) 115 | (do-print out)) 116 | (void)) 117 | 118 | (begin-for-syntax 119 | (define-syntax-class request-arg 120 | (pattern id:id) 121 | (pattern [id:id def:expr]))) 122 | 123 | (define-syntax (define-request stx) 124 | (syntax-parse stx 125 | [(_ id:id 126 | (arg:request-arg ...) 127 | {~seq #:code key:number} 128 | {~seq #:version version-num:number 129 | {~optional {~seq #:flexible flexible}} 130 | #:response parser:expr 131 | {~optional {~seq #:immed-response immed-response-expr}} 132 | encoder:expr 133 | decoder:expr} ...+) 134 | #:with make-evt-id (format-id #'id "make-~a-evt" #'id) 135 | #:with version-rng-id (format-id #'id "supported-~a-versions" #'id) 136 | #:with version-rng (let ([versions (map syntax->datum (syntax-e #'(version-num ...)))]) 137 | (with-syntax ([min-v (apply min versions)] 138 | [max-v (apply max versions)]) 139 | #'(version-range min-v max-v))) 140 | #'(begin 141 | (provide 142 | make-evt-id 143 | version-rng-id) 144 | (define version-rng-id version-rng) 145 | (define (make-evt-id conn arg ...) 146 | (case (find-best-version conn key version-rng-id) 147 | [(version-num) 148 | (request-evt 149 | conn decoder 150 | #:key key 151 | #:version version-num 152 | #:data (encoder arg.id ...) 153 | #:parser parser 154 | #:flexible? {~? flexible #f} 155 | #:immed-response {~? (immed-response-expr arg.id ...) #f})] ... 156 | [else 157 | (error 'make-evt-id "no supported version")])))])) 158 | 159 | (define (request-evt conn proc 160 | #:key key 161 | #:version ver 162 | #:data data 163 | #:parser parser-proc 164 | #:flexible? flexible? 165 | #:immed-response immed-response) 166 | (handle-evt 167 | (make-request-evt 168 | conn 169 | #:key key 170 | #:version ver 171 | #:data data 172 | #:parser parser-proc 173 | #:flexible? flexible? 174 | #:immed-response immed-response) 175 | (lambda (res) 176 | (cond 177 | [immed-response res] 178 | [else 179 | (define err-code (or (opt 'ErrorCode_1 res) 0)) 180 | (unless (zero? err-code) 181 | (raise-server-error 182 | err-code 183 | "~a~n key: ~s~n version: ~s" 184 | (error-code-message err-code) key ver)) 185 | (proc res)])))) 186 | -------------------------------------------------------------------------------- /kafka-lib/private/serde/create-topics.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require racket/contract/base 4 | "core.rkt") 5 | 6 | (define-record CreatedTopic 7 | ([error-code error-code/c] 8 | [error-message (or/c #f string?)] 9 | [name string?])) 10 | 11 | (define-record CreatedTopics 12 | ([topics (listof CreatedTopic?)])) 13 | 14 | (define-record CreateTopic 15 | ([name string?] 16 | [partitions exact-positive-integer?] 17 | [(replication-factor 1) exact-positive-integer?] 18 | [(assignments (hasheqv)) (hash/c exact-nonnegative-integer? (listof exact-nonnegative-integer?))] 19 | [(configs (hash)) (hash/c string? string?)])) 20 | 21 | (define-request CreateTopics 22 | (topics 23 | [timeout-ms 30000] 24 | [validate-only? #f]) 25 | #:code 19 26 | #:version 1 27 | #:response proto:CreateTopicsResponseV1 28 | (lambda (topics timeout-ms validate-only?) 29 | (define topic-requests 30 | (for/list ([t (in-list topics)]) 31 | (define assignments (CreateTopic-assignments t)) 32 | (define configs (CreateTopic-configs t)) 33 | `((TopicName_1 . ,(CreateTopic-name t)) 34 | (NumPartitions_1 . ,(CreateTopic-partitions t)) 35 | (ReplicationFactor_1 . ,(CreateTopic-replication-factor t)) 36 | (Assignments_1 37 | . ((ArrayLen_1 . ,(hash-count assignments)) 38 | (Assignment_1 . ,(for/list ([(pid bids) (in-hash assignments)]) 39 | `((PartitionID_1 . ,pid) 40 | (BrokerIDs_1 . ((ArrayLen_1 . ,(length bids)) 41 | (BrokerID_1 . ,bids)))))))) 42 | (Configs_1 43 | . ((ArrayLen_1 . ,(hash-count configs)) 44 | (Config_1 . ,(for/list ([(name value) (in-hash configs)]) 45 | `((ConfigName_1 . ,name) 46 | (ConfigValue_1 . ,value))))))))) 47 | 48 | (with-output-bytes 49 | (proto:un-CreateTopicsRequestV1 50 | `((CreateTopicsRequestsV1_1 . ((ArrayLen_1 . ,(length topics)) 51 | (CreateTopicRequestV1_1 . ,topic-requests))) 52 | (TimeoutMs_1 . ,timeout-ms) 53 | (ValidateOnly_1 . ,validate-only?))))) 54 | (lambda (res) 55 | (CreatedTopics 56 | (for/list ([t (in-list (ref 'CreateTopicsResponseTopicV1_1 res))]) 57 | (make-CreatedTopic 58 | #:name (ref 'TopicName_1 t) 59 | #:error-code (ref 'ErrorCode_1 t) 60 | #:error-message (ref 'ErrorMessage_1 t)))))) 61 | -------------------------------------------------------------------------------- /kafka-lib/private/serde/delete-groups.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require racket/contract/base 4 | "core.rkt") 5 | 6 | (define-record DeletedGroup 7 | ([id string?] 8 | [(error-code 0) error-code/c])) 9 | 10 | (define-record DeletedGroups 11 | ([(throttle-time-ms #f) (or/c #f exact-integer?)] 12 | [groups (listof DeletedGroup?)])) 13 | 14 | (define-request DeleteGroups 15 | (group-ids) 16 | #:code 42 17 | #:version 0 18 | #:response proto:DeleteGroupsResponseV0 19 | (make-enc-delete-groupsv0 proto:un-DeleteGroupsRequestV0) 20 | dec-delete-groupsv0 21 | 22 | #:version 1 23 | #:response proto:DeleteGroupsResponseV1 24 | (make-enc-delete-groupsv0 proto:un-DeleteGroupsRequestV1) 25 | dec-delete-groupsv0) 26 | 27 | (define ((make-enc-delete-groupsv0 unparser) group-ids) 28 | (with-output-bytes 29 | (unparser 30 | `((ArrayLen_1 . ,(length group-ids)) 31 | (GroupID_1 . ,group-ids))))) 32 | 33 | 34 | (define (dec-delete-groupsv0 res) 35 | (define groups 36 | (for/list ([group (in-list (ref 'DeleteGroupsResponseDataV0_1 res))]) 37 | (make-DeletedGroup 38 | #:id (ref 'GroupID_1 group) 39 | #:error-code (ref 'ErrorCode_1 group)))) 40 | (make-DeletedGroups 41 | #:throttle-time-ms (ref 'ThrottleTimeMs_1 res) 42 | #:groups groups)) 43 | -------------------------------------------------------------------------------- /kafka-lib/private/serde/delete-topics.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require racket/contract/base 4 | "core.rkt") 5 | 6 | (define-record DeletedTopic 7 | ([(error-code 0) error-code/c] 8 | [(error-message #f) (or/c #f string?)] 9 | [name string?] 10 | [(uuid #f) (or/c #f bytes?)] 11 | [(tags #f) (or/c #f tags/c)])) 12 | 13 | (define-record DeletedTopics 14 | ([(throttle-time-ms #f) (or/c #f exact-integer?)] 15 | [topics (listof DeletedTopic?)] 16 | [(tags #f) (or/c #f tags/c)])) 17 | 18 | (define-request DeleteTopics 19 | (topics 20 | [timeout-ms 30000] 21 | [tags (hasheqv)]) 22 | #:code 20 23 | #:version 0 24 | #:response proto:DeleteTopicsResponseV0 25 | enc-delete-topicsv0 26 | (lambda (res) 27 | (define topics 28 | (for/list ([topic (in-list (ref 'DeleteTopicsResponseDataV0_1 res))]) 29 | (make-DeletedTopic 30 | #:error-code (ref 'ErrorCode_1 topic) 31 | #:name (ref 'TopicName_1 topic)))) 32 | (make-DeletedTopics #:topics topics)) 33 | 34 | #:version 1 35 | #:response proto:DeleteTopicsResponseV1 36 | enc-delete-topicsv0 37 | dec-delete-topicsv1 38 | 39 | #:version 2 40 | #:response proto:DeleteTopicsResponseV2 41 | enc-delete-topicsv0 42 | dec-delete-topicsv1 43 | 44 | #:version 3 45 | #:response proto:DeleteTopicsResponseV3 46 | enc-delete-topicsv0 47 | dec-delete-topicsv1 48 | 49 | #:version 4 50 | #:flexible #t 51 | #:response proto:DeleteTopicsResponseV4 52 | enc-delete-topicsv4 53 | dec-delete-topicsv4) 54 | 55 | (define (enc-delete-topicsv0 topics timeout-ms tags) 56 | (unless (zero? (hash-count tags)) 57 | (raise-argument-error 'DeleteTopics "(hasheqv)" tags)) 58 | (with-output-bytes 59 | (proto:un-DeleteTopicsRequestV0 60 | `((TopicNames_1 . ((ArrayLen_1 . ,(length topics)) 61 | (TopicName_1 . ,topics))) 62 | (TimeoutMs_1 . ,timeout-ms))))) 63 | 64 | (define (enc-delete-topicsv4 topics timeout-ms tags) 65 | (with-output-bytes 66 | (proto:un-DeleteTopicsRequestV4 67 | `((CompactTopicNames_1 . ((CompactArrayLen_1 . ,(length topics)) 68 | (CompactTopicName_1 . ,topics))) 69 | (TimeoutMs_1 . ,timeout-ms) 70 | (Tags_1 . ,tags))))) 71 | 72 | (define (dec-delete-topicsv1 res) 73 | (define topics 74 | (for/list ([topic (in-list (ref 'DeleteTopicsResponseDataV0_1 'DeleteTopicsResponsesV0_1 res))]) 75 | (make-DeletedTopic 76 | #:error-code (ref 'ErrorCode_1 topic) 77 | #:name (ref 'TopicName_1 topic)))) 78 | (make-DeletedTopics 79 | #:throttle-time-ms (ref 'ThrottleTimeMs_1 res) 80 | #:topics topics)) 81 | 82 | (define (dec-delete-topicsv4 res) 83 | (define topics 84 | (for/list ([topic (in-list (ref 'DeleteTopicsResponseDataV4_1 'DeleteTopicsResponsesV4_1 res))]) 85 | (make-DeletedTopic 86 | #:error-code (ref 'ErrorCode_1 topic) 87 | #:name (ref 'CompactTopicName_1 topic) 88 | #:tags (ref 'Tags_1 topic)))) 89 | (make-DeletedTopics 90 | #:throttle-time-ms (ref 'ThrottleTimeMs_1 res) 91 | #:topics topics 92 | #:tags (ref 'Tags_1 res))) 93 | -------------------------------------------------------------------------------- /kafka-lib/private/serde/describe-cluster.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require racket/contract/base 4 | "authorized-operation.rkt" 5 | "core.rkt" 6 | "metadata.rkt") 7 | 8 | (define-record Cluster 9 | ([(throttle-time-ms #f) (or/c #f exact-nonnegative-integer?)] 10 | [error-code error-code/c] 11 | [error-message (or/c #f string?)] 12 | [cluster-id string?] 13 | [controller-id exact-nonnegative-integer?] 14 | [brokers (listof BrokerMetadata?)] 15 | [authorized-operations (listof authorized-operation/c)])) 16 | 17 | (define-request DescribeCluster 18 | ([include-authorized-operations? #t]) 19 | #:code 60 20 | #:version 0 21 | #:flexible #t 22 | #:response proto:DescribeClusterResponseV0 23 | (lambda (include-authorized-operations?) 24 | (with-output-bytes 25 | (proto:un-DescribeClusterRequestV0 26 | `((IncludeClusterAuthorizedOperations_1 . ,include-authorized-operations?) 27 | (Tags_1 . ,(hash)))))) 28 | (lambda (res) 29 | (make-Cluster 30 | #:throttle-time-ms (ref 'ThrottleTimeMs_1 res) 31 | #:error-code (ref 'ErrorCode_1 res) 32 | #:error-message (ref 'CompactErrorMessage_1 res) 33 | #:cluster-id (ref 'CompactClusterID_1 res) 34 | #:controller-id (ref 'ControllerID_1 res) 35 | #:authorized-operations (integer->authorized-operations 36 | (ref 'ClusterAuthorizedOperations_1 res)) 37 | #:brokers (for/list ([b (in-list (ref 'CompactBroker_1 res))]) 38 | (make-BrokerMetadata 39 | #:node-id (ref 'BrokerID_1 b) 40 | #:host (ref 'CompactHost_1 b) 41 | #:port (ref 'Port_1 b) 42 | #:rack (ref 'CompactRack_1 b)))))) 43 | -------------------------------------------------------------------------------- /kafka-lib/private/serde/describe-configs.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require racket/contract/base 4 | "core.rkt" 5 | "resource.rkt") 6 | 7 | (define-record DescribeResource 8 | ([type resource-type/c] 9 | [name string?] 10 | [(keys null) (listof string?)])) 11 | 12 | (define-record ResourceConfig 13 | ([name string?] 14 | [value (or/c #f string?)] 15 | [read-only? boolean?] 16 | [default? boolean?] 17 | [sensitive? boolean?])) 18 | 19 | (define-record DescribedResource 20 | ([error-code error-code/c] 21 | [error-message (or/c #f string?)] 22 | [type resource-type/c] 23 | [name string?] 24 | [configs (listof ResourceConfig?)])) 25 | 26 | (define-record DescribedResources 27 | ([(throttle-time-ms #f) (or/c #f exact-nonnegative-integer?)] 28 | [resources (listof DescribedResource?)])) 29 | 30 | (define-request DescribeConfigs 31 | (resources) 32 | #:code 32 33 | #:version 0 34 | #:response proto:DescribeConfigsResponseV0 35 | (lambda (resources) 36 | (with-output-bytes 37 | (proto:un-DescribeConfigsRequestV0 38 | `((ArrayLen_1 . ,(length resources)) 39 | (DescribeConfigsResourceV0_1 . ,(for/list ([res (in-list resources)]) 40 | (define len (length (DescribeResource-keys res))) 41 | `((ResourceType_1 . ,(->resource-type (DescribeResource-type res))) 42 | (ResourceName_1 . ,(DescribeResource-name res)) 43 | (ArrayLen_1 . ,(if (zero? len) -1 len)) 44 | (ConfigName_1 . ,(DescribeResource-keys res))))))))) 45 | (lambda (res) 46 | (make-DescribedResources 47 | #:throttle-time-ms (ref 'ThrottleTimeMs_1 res) 48 | #:resources (for/list ([r (in-list (ref 'DescribeConfigsResultV0_1 res))]) 49 | (make-DescribedResource 50 | #:error-code (ref 'ErrorCode_1 r) 51 | #:error-message (ref 'ErrorMessage_1 r) 52 | #:type (resource-type-> (ref 'ResourceType_1 r)) 53 | #:name (ref 'ResourceName_1 r) 54 | #:configs (for/list ([c (in-list (ref 'ResourceConfigV0_1 r))]) 55 | (make-ResourceConfig 56 | #:name (ref 'ConfigName_1 c) 57 | #:value (ref 'ConfigValue_1 c) 58 | #:read-only? (= (ref 'ReadOnly_1 c) 1) 59 | #:default? (= (ref 'IsDefault_1 c) 1) 60 | #:sensitive? (= (ref 'IsSensitive_1 c) 1)))))))) 61 | -------------------------------------------------------------------------------- /kafka-lib/private/serde/describe-groups.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require racket/contract/base 4 | "authorized-operation.rkt" 5 | "core.rkt" 6 | "group.rkt") 7 | 8 | (define-record DescribedGroups 9 | ([throttle-time-ms (or/c #f exact-nonnegative-integer?)] 10 | [groups (listof Group?)])) 11 | 12 | (define-request DescribeGroups 13 | ([groups null] 14 | [include-authorized-operations? #t]) 15 | #:code 15 16 | 17 | #:version 0 18 | #:response proto:DescribeGroupsResponseV0 19 | enc-describe-groups-v0 20 | dec-describe-groups-v0 21 | 22 | #:version 1 23 | #:response proto:DescribeGroupsResponseV1 24 | enc-describe-groups-v0 25 | dec-describe-groups-v1 26 | 27 | #:version 2 28 | #:response proto:DescribeGroupsResponseV2 29 | enc-describe-groups-v0 30 | dec-describe-groups-v2 31 | 32 | #:version 3 33 | #:response proto:DescribeGroupsResponseV3 34 | enc-describe-groups-v3 35 | dec-describe-groups-v3) 36 | 37 | (define (enc-describe-groups-v0 groups _include-authorized-operations?) 38 | (with-output-bytes 39 | (proto:un-DescribeGroupsRequestV0 40 | `((ArrayLen_1 . ,(length groups)) 41 | (GroupID_1 . ,groups))))) 42 | 43 | (define (enc-describe-groups-v3 groups include-authorized-operations?) 44 | (with-output-bytes 45 | (proto:un-DescribeGroupsRequestV3 46 | `((ArrayLen_1 . ,(length groups)) 47 | (GroupID_1 . ,groups) 48 | (IncludeAuthorizedOperations_1 . ,include-authorized-operations?))))) 49 | 50 | (define ((make-dec-describe-groups [groups-key 'DescribeGroupsGroupV0_1] 51 | [members-key 'DescribeGroupsMemberV0_1]) res) 52 | (make-DescribedGroups 53 | #:throttle-time-ms (opt 'ThrottleTimeMs_1 res) 54 | #:groups (for/list ([g (in-list (ref groups-key res))]) 55 | (make-Group 56 | #:error-code (ref 'ErrorCode_1 g) 57 | #:id (ref 'GroupID_1 g) 58 | #:state (ref 'GroupState_1 g) 59 | #:protocol-type (ref 'ProtocolType_1 g) 60 | #:protocol-data (ref 'ProtocolData_1 g) 61 | #:authorized-operations (integer->authorized-operations 62 | (or (opt 'AuthorizedOperations_1 g) -1)) 63 | #:members (for/list ([m (in-list (ref members-key g))]) 64 | (make-GroupMember 65 | #:id (ref 'MemberID_1 m) 66 | #:client-id (ref 'ClientID_1 m) 67 | #:client-host (ref 'ClientHost_1 m) 68 | #:metadata (ref 'MemberMetadata_1 m) 69 | #:assignment (ref 'MemberAssignmentData_1 m))))))) 70 | 71 | (define dec-describe-groups-v0 (make-dec-describe-groups)) 72 | (define dec-describe-groups-v1 (make-dec-describe-groups)) 73 | (define dec-describe-groups-v2 (make-dec-describe-groups)) 74 | (define dec-describe-groups-v3 (make-dec-describe-groups 'DescribeGroupsGroupV3_1)) 75 | -------------------------------------------------------------------------------- /kafka-lib/private/serde/describe-producers.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require racket/contract/base 4 | "core.rkt") 5 | 6 | (define-record Producer 7 | ([id exact-nonnegative-integer?] 8 | [epoch exact-integer?] 9 | [last-sequence exact-integer?] 10 | [last-timestamp exact-integer?] 11 | [coordinator-epoch exact-integer?] 12 | [current-txn-start-offset exact-integer?])) 13 | 14 | (define-record PartitionProducers 15 | ([id exact-nonnegative-integer?] 16 | [error-code error-code/c] 17 | [error-message (or/c #f string?)] 18 | [producers (listof Producer?)])) 19 | 20 | (define-record TopicProducers 21 | ([name string?] 22 | [partitions (listof PartitionProducers?)])) 23 | 24 | (define-record DescribedProducers 25 | ([throttle-time-ms (or/c #f exact-nonnegative-integer?)] 26 | [topics (listof TopicProducers?)])) 27 | 28 | (define-request DescribeProducers 29 | (topics) 30 | #:code 61 31 | #:version 0 32 | #:flexible #t 33 | #:response proto:DescribeProducersResponseV0 34 | (lambda (topics) 35 | (define topics-data 36 | (for/list ([(topic partitions) (in-hash topics)]) 37 | `((CompactTopicName_1 . ,topic) 38 | (CompactArrayLen_1 . ,(length partitions)) 39 | (PartitionID_1 . ,partitions) 40 | (Tags_1 . ,(hash))))) 41 | 42 | (with-output-bytes 43 | (proto:un-DescribeProducersRequestV0 44 | `((CompactArrayLen_1 . ,(hash-count topics)) 45 | (DescribeProducersRequestTopicV0_1 . ,topics-data) 46 | (Tags_1 . ,(hash)))))) 47 | (lambda (res) 48 | (make-DescribedProducers 49 | #:throttle-time-ms (ref 'ThrottleTimeMs_1 res) 50 | #:topics (for/list ([topic (in-list (ref 'DescribeProducersResponseTopicV0_1 res))]) 51 | (make-TopicProducers 52 | #:name (ref 'CompactTopicName_1 topic) 53 | #:partitions (for/list ([part (in-list (ref 'DescribeProducersResponseTopicPartitionV0_1 topic))]) 54 | (make-PartitionProducers 55 | #:id (ref 'PartitionID_1 part) 56 | #:error-code (ref 'ErrorCode_1 part) 57 | #:error-message (ref 'CompactErrorMessage_1 part) 58 | #:producers (for/list ([prod (in-list (ref 'DescribeProducersResponseProducerV0_1 part))]) 59 | (make-Producer 60 | #:id (ref 'ProducerID_1 prod) 61 | #:epoch (ref 'ProducerEpoch_1 prod) 62 | #:last-sequence (ref 'LastSequence_1 prod) 63 | #:last-timestamp (ref 'LastTimestamp_1 prod) 64 | #:coordinator-epoch (ref 'CoordinatorEpoch_1 prod) 65 | #:current-txn-start-offset (ref 'CurrentTxnStartOffset_1 prod)))))))))) 66 | -------------------------------------------------------------------------------- /kafka-lib/private/serde/fetch-offsets.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require racket/contract/base 4 | "core.rkt") 5 | 6 | (define-record GroupPartitionOffset 7 | ([id exact-nonnegative-integer?] 8 | [error-code error-code/c] 9 | [offset (or/c -1 exact-nonnegative-integer?)] 10 | [metadata string?])) 11 | 12 | (define-record GroupOffsets 13 | ([(throttle-time-ms #f) (or/c #f exact-nonnegative-integer?)] 14 | [(error-code 0) error-code/c] 15 | [topics (hash/c string? (listof GroupPartitionOffset?))])) 16 | 17 | (define-request FetchOffsets 18 | (group-id 19 | [topics (hash)]) 20 | #:code 9 21 | 22 | #:version 1 23 | #:response proto:OffsetFetchResponseV1 24 | (make-enc-offset-fetch-v1 #f) 25 | dec-offset-fetch-v1 26 | 27 | #:version 2 28 | #:response proto:OffsetFetchResponseV2 29 | (make-enc-offset-fetch-v1) 30 | dec-offset-fetch-v2 31 | 32 | #:version 3 33 | #:response proto:OffsetFetchResponseV3 34 | (make-enc-offset-fetch-v1) 35 | dec-offset-fetch-v3 36 | 37 | #:version 4 38 | #:response proto:OffsetFetchResponseV4 39 | (make-enc-offset-fetch-v1) 40 | dec-offset-fetch-v3) 41 | 42 | (define ((make-enc-offset-fetch-v1 [supports-null? #t]) group-id partitions-by-topic) 43 | (define topics 44 | (for/list ([(topic partitions) (in-hash partitions-by-topic)]) 45 | `((TopicName_1 . ,topic) 46 | (ArrayLen_1 . ,(length partitions)) 47 | (OffsetFetchRequestPartitionV1_1 . ,partitions)))) 48 | (with-output-bytes 49 | (proto:un-OffsetFetchRequestV1 50 | `((GroupID_1 . ,group-id) 51 | (ArrayLen_1 . ,(if (and supports-null? (null? topics)) 52 | -1 53 | (hash-count partitions-by-topic))) 54 | (OffsetFetchRequestTopicV1_1 . ,topics))))) 55 | 56 | (define (dec-offset-fetch-v1 res) 57 | (make-GroupOffsets 58 | #:topics (dec-offset-fetch-v1-topics res))) 59 | 60 | (define (dec-offset-fetch-v2 res) 61 | (make-GroupOffsets 62 | #:error-code (ref 'ErrorCode_1 res) 63 | #:topics (dec-offset-fetch-v1-topics res))) 64 | 65 | (define (dec-offset-fetch-v3 res) 66 | (make-GroupOffsets 67 | #:throttle-time-ms (ref 'ThrottleTimeMs_1 res) 68 | #:error-code (ref 'ErrorCode_1 res) 69 | #:topics (dec-offset-fetch-v1-topics res))) 70 | 71 | (define (dec-offset-fetch-v1-topics res) 72 | (for/hash ([t (in-list (ref 'OffsetFetchResponseTopicV1_1 res))]) 73 | (define topic (ref 'TopicName_1 t)) 74 | (values topic (for/list ([p (in-list (ref 'OffsetFetchResponsePartitionV1_1 t))]) 75 | (make-GroupPartitionOffset 76 | #:id (ref 'PartitionID_1 p) 77 | #:error-code (ref 'ErrorCode_1 p) 78 | #:offset (ref 'CommittedOffset_1 p) 79 | #:metadata (ref 'OffsetFetchMetadata_1 p)))))) 80 | -------------------------------------------------------------------------------- /kafka-lib/private/serde/fetch.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require binfmt/runtime/parser 4 | binfmt/runtime/res 5 | racket/contract/base 6 | "core.rkt") 7 | 8 | (define-record FetchResponsePartition 9 | ([id exact-nonnegative-integer?] 10 | [(error-code 0) error-code/c] 11 | [high-watermark exact-nonnegative-integer?] 12 | [last-stable-offset exact-nonnegative-integer?] 13 | [batches list?])) 14 | 15 | (define-record FetchResponse 16 | ([throttle-time-ms exact-nonnegative-integer?] 17 | [topics (hash/c string? (listof FetchResponsePartition?))])) 18 | 19 | (define-record TopicPartition 20 | ([id exact-nonnegative-integer?] 21 | [(offset 0) exact-nonnegative-integer?] 22 | [(max-bytes (* 100 1024 1024)) exact-nonnegative-integer?])) 23 | 24 | (define-request Fetch 25 | (topic-partitions 26 | max-wait-ms 27 | [min-bytes 0] 28 | [max-bytes (* 100 1024 1024)] 29 | [isolation-level 'read-committed]) 30 | #:code 1 31 | 32 | #:version 4 33 | #:response read-FetchResponseV4 34 | (lambda (topic-partitions max-wait-ms min-bytes max-bytes isolation-level) 35 | (with-output-bytes 36 | (proto:un-FetchRequestV4 37 | `((ReplicaID_1 . -1) 38 | (MaxWaitMs_1 . ,max-wait-ms) 39 | (MinBytes_1 . ,min-bytes) 40 | (MaxBytes_1 . ,max-bytes) 41 | (IsolationLevel_1 . ,(case isolation-level 42 | [(read-uncommitted) 0] 43 | [(read-committed) 1])) 44 | (ArrayLen_1 . ,(hash-count topic-partitions)) 45 | (FetchTopicV4_1 . ,(for/list ([(topic partitions) (in-hash topic-partitions)]) 46 | `((TopicName_1 . ,topic) 47 | (ArrayLen_1 . ,(length partitions)) 48 | (FetchPartitionV4_1 . ,(for/list ([p (in-list partitions)]) 49 | `((PartitionID_1 . ,(TopicPartition-id p)) 50 | (FetchOffset_1 . ,(TopicPartition-offset p)) 51 | (MaxBytes_1 . ,(TopicPartition-max-bytes p)))))))))))) 52 | (lambda (res) 53 | (make-FetchResponse 54 | #:throttle-time-ms (ref 'ThrottleTimeMs_1 res) 55 | #:topics (for/hash ([t (in-list (ref 'FetchResponseDataV4_1 res))]) 56 | (define topic (ref 'TopicName_1 t)) 57 | (define parts 58 | (for/list ([p (in-list (ref 'FetchResponsePartitionV4_1 t))]) 59 | (make-FetchResponsePartition 60 | #:id (ref 'PartitionID_1 p) 61 | #:error-code (ref 'ErrorCode_1 p) 62 | #:high-watermark (ref 'HighWatermark_1 p) 63 | #:last-stable-offset (ref 'LastStableOffset_1 p) 64 | #:batches (ref 'Records_1 p)))) 65 | (values topic parts))))) 66 | 67 | ;; Kafka simply lifts batches off disk, so it's likely that responses 68 | ;; will be truncated. This procedure has to treat parse failures as 69 | ;; signals that all the data that could have been read, has been. 70 | ;; 71 | ;; xref: https://kafka.apache.org/documentation/#impl_reads 72 | (define (read-FetchResponseV4 in) 73 | ;; ThrottleTimeMs 74 | (res-bind 75 | (parse-i32be in) 76 | (lambda (throttle-time-ms) 77 | ;; ArrayLen 78 | (res-bind 79 | (parse-i32be in) 80 | (lambda (len) 81 | ;; FetchResponseDataV4{ArrayLen_1} 82 | (define topics 83 | (let loop ([remaining len] [topics null]) 84 | (cond 85 | [(> remaining 0) 86 | (define topic 87 | (with-handlers ([exn:fail? (λ (_) #f)]) 88 | (proto:FetchResponseDataV4 in))) 89 | (if topic 90 | (loop 91 | (sub1 remaining) 92 | (cons topic topics)) 93 | (reverse topics))] 94 | [else 95 | (reverse topics)]))) 96 | `((ThrottleTimeMs_1 . ,throttle-time-ms) 97 | (ArrayLen_1 . ,len) 98 | (FetchResponseDataV4_1 . ,topics))))))) 99 | -------------------------------------------------------------------------------- /kafka-lib/private/serde/find-coordinator.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require racket/contract/base 4 | "core.rkt") 5 | 6 | (define-record Coordinator 7 | ([(throttle-time-ms #f) (or/c #f exact-nonnegative-integer?)] 8 | [error-code error-code/c] 9 | [(error-message #f) (or/c #f string?)] 10 | [node-id exact-nonnegative-integer?] 11 | [host string?] 12 | [port (integer-in 0 65535)])) 13 | 14 | (define-request FindCoordinator 15 | (key [key-type 0]) 16 | #:code 10 17 | 18 | #:version 0 19 | #:response proto:FindCoordinatorResponseV0 20 | (lambda (key _key-type) 21 | (with-output-bytes 22 | (proto:un-FindCoordinatorRequestV0 key))) 23 | dec-find-coordinator-res 24 | 25 | #:version 1 26 | #:response proto:FindCoordinatorResponseV1 27 | enc-find-coordinator-v1 28 | dec-find-coordinator-res 29 | 30 | #:version 2 31 | #:response proto:FindCoordinatorResponseV2 32 | enc-find-coordinator-v1 33 | dec-find-coordinator-res) 34 | 35 | (define (enc-find-coordinator-v1 key type) 36 | (with-output-bytes 37 | (proto:un-FindCoordinatorRequestV1 38 | `((CoordinatorKey_1 . ,key) 39 | (CoordinatorKeyType_1 . ,type))))) 40 | 41 | (define (dec-find-coordinator-res res) 42 | (make-Coordinator 43 | #:throttle-time-ms (opt 'ThrottleTimeMs_1 res) 44 | #:error-code (ref 'ErrorCode_1 res) 45 | #:error-message (or (opt 'ErrorMessage_1 res) 46 | (opt 'CompactErrorMessage_1 res)) 47 | #:node-id (ref 'NodeID_1 res) 48 | #:host (or (opt 'Host_1 res) 49 | (ref 'CompactHost_1 res)) 50 | #:port (ref 'Port_1 res))) 51 | -------------------------------------------------------------------------------- /kafka-lib/private/serde/group.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require racket/contract/base 4 | racket/port 5 | (prefix-in consumer: "../protocol-consumer.bnf") 6 | "authorized-operation.rkt" 7 | "core.rkt") 8 | 9 | (provide 10 | (contract-out 11 | [Group-topics (-> Group? (listof string?))] 12 | [GroupMember-topics (-> GroupMember? (listof string?))] 13 | [GroupMember-assignments (-> GroupMember? (hash/c string? (listof exact-integer?)))])) 14 | 15 | (define-record GroupMember 16 | ([id string?] 17 | [client-id string?] 18 | [client-host string?] 19 | [metadata bytes?] 20 | [assignment bytes?])) 21 | 22 | (define-record Group 23 | ([(error-code 0) error-code/c] 24 | [id string?] 25 | [(state #f) (or/c #f string?)] 26 | [protocol-type string?] 27 | [(protocol-data #f) (or/c #f string?)] 28 | [(members null) (listof GroupMember?)] 29 | [(authorized-operations null) (listof authorized-operation/c)])) 30 | 31 | (define (Group-topics g) 32 | (case (Group-protocol-type g) 33 | [("consumer") 34 | (for*/fold ([topics (hash)] #:result (sort (hash-keys topics) string bytes? bytes? (or/c #f Internal/c))] 13 | [InternalGroupMember-topics (-> InternalGroupMember? (listof string?))] 14 | [InternalGroupMember-assignments (-> InternalGroupMember? (hash/c string? (listof exact-nonnegative-integer?)))])) 15 | 16 | (define-record InternalOffsetCommit 17 | ([group string?] 18 | [topic string?] 19 | [partition-id exact-nonnegative-integer?] 20 | [offset exact-integer?])) 21 | 22 | (define-record InternalGroupMember 23 | ([id string?] 24 | [client-id string?] 25 | [client-host string?] 26 | [(rebalance-timeout #f) (or/c #f exact-integer?)] 27 | [session-timeout exact-integer?] 28 | [(subscription #f) (or/c #f bytes?)] 29 | [(assignment #f) (or/c #f bytes?)])) 30 | 31 | (define (InternalGroupMember-topics m) 32 | (with-handlers ([exn:fail? (λ (_) null)]) 33 | (define res 34 | (call-with-input-bytes (InternalGroupMember-subscription m) 35 | consumer:MemberMetadata)) 36 | (ref 'TopicName_1 res))) 37 | 38 | (define (InternalGroupMember-assignments m) 39 | (with-handlers ([exn:fail? (λ (_) (hash))]) 40 | (define res 41 | (call-with-input-bytes (InternalGroupMember-assignment m) 42 | consumer:MemberAssignment)) 43 | (for/hash ([topic (in-list (ref 'Assignment_1 res))]) 44 | (values 45 | (ref 'TopicName_1 topic) 46 | (ref 'PartitionID_1 topic))))) 47 | 48 | (define-record InternalGroupMetadata 49 | ([group string?] 50 | [generation exact-integer?] 51 | [protocol-type (or/c #f string?)] 52 | [protocol-data (or/c #f string?)] 53 | [leader (or/c #f string?)] 54 | [members (listof InternalGroupMember?)])) 55 | 56 | (define Internal/c 57 | (or/c InternalOffsetCommit? InternalGroupMetadata?)) 58 | 59 | (define (parse-Internal key-bs value-bs) 60 | (define-values (type key) 61 | (call-with-input-bytes key-bs 62 | (lambda (in) 63 | (case (read-bytes 2 in) 64 | [(#"\0\0" #"\0\1") 65 | (values 'commit (internal:OffsetCommitKeyV01 in))] 66 | [(#"\0\2") 67 | (values 'group (internal:GroupMetadataKeyV2 in))] 68 | [else 69 | (values #f #f)])))) 70 | (case type 71 | [(commit) 72 | (define value 73 | (call-with-input-bytes value-bs 74 | (lambda (in) 75 | (case (read-bytes 2 in) 76 | [(#"\0\0" #"\0\1" #"\0\2" #"\0\3") 77 | (internal:OffsetCommitValueV0 in)] 78 | [else #f])))) 79 | (and value 80 | (make-InternalOffsetCommit 81 | #:group (ref 'Group_1 key) 82 | #:topic (ref 'Topic_1 key) 83 | #:partition-id (ref 'PartitionID_1 key) 84 | #:offset value))] 85 | [(group) 86 | (define-values (members-key value) 87 | (call-with-input-bytes value-bs 88 | (lambda (in) 89 | (case (read-bytes 2 in) 90 | [(#"\0\0") (values 'MemberMetadataV0_1 (internal:GroupMetadataValueV0 in))] 91 | [(#"\0\1") (values 'MemberMetadataV1_1 (internal:GroupMetadataValueV1 in))] 92 | [(#"\0\2") (values 'MemberMetadataV2_1 (internal:GroupMetadataValueV2 in))] 93 | [(#"\0\3") (values 'MemberMetadataV3_1 (internal:GroupMetadataValueV3 in))] 94 | [else (values #f #f)])))) 95 | (and value 96 | (make-InternalGroupMetadata 97 | #:group key 98 | #:generation (ref 'Generation_1 value) 99 | #:protocol-type (ref 'ProtocolType_1 value) 100 | #:protocol-data (ref 'Protocol_1 value) 101 | #:leader (ref 'Leader_1 value) 102 | #:members (for/list ([m (in-list (ref members-key value))]) 103 | (make-InternalGroupMember 104 | #:id (ref 'MemberID_1 m) 105 | #:client-id (ref 'ClientID_1 m) 106 | #:client-host (ref 'ClientHost_1 m) 107 | #:rebalance-timeout (ref 'RebalanceTimeout_1 m) 108 | #:session-timeout (ref 'SessionTimeout_1 m) 109 | #:subscription (ref 'Subscription_1 m) 110 | #:assignment (ref 'Assignment_1 m)))))] 111 | [else 112 | #f])) 113 | 114 | (module+ test 115 | (require rackunit) 116 | 117 | (test-case "invalid key & value" 118 | (check-false (parse-Internal #"" #""))) 119 | 120 | (test-case "valid key, bad value" 121 | (check-false (parse-Internal #"\0\2\0\rexample-group" #""))) 122 | 123 | (test-case "valid offset commit" 124 | (check-equal? 125 | (parse-Internal 126 | #"\0\1\0\rexample-group\0\rexample-topic\0\0\0\1" 127 | #"\0\3\0\0\0\0\0\0\b\0\377\377\377\377\0\0\0\0\1\203\340\243\341\f") 128 | (make-InternalOffsetCommit 129 | #:group "example-group" 130 | #:topic "example-topic" 131 | #:partition-id 1 132 | #:offset 2048))) 133 | 134 | (test-case "valid group metadata" 135 | (check-equal? 136 | (parse-Internal 137 | #"\0\2\0\rexample-group" 138 | #"\0\3\0\bconsumer\0\0\0\27\0\5range\0001racket-kafka-35d27953-bcfa-4be9-a5b0-58ae5cebe04a\0\0\1\203\340\235'\21\0\0\0\2\0001racket-kafka-35d27953-bcfa-4be9-a5b0-58ae5cebe04a\377\377\0\fracket-kafka\0\v/172.18.0.1\0\0u0\0\0u0\0\0\0\31\0\0\0\0\0\1\0\rexample-topic\0\0\0\0\0\0\0!\0\0\0\0\0\1\0\rexample-topic\0\0\0\1\0\0\0\0\0\0\0\0\0001racket-kafka-1f34c0ba-75b8-4d68-a4f5-63afcaa16522\377\377\0\fracket-kafka\0\v/172.18.0.1\0\0u0\0\0u0\0\0\0\31\0\0\0\0\0\1\0\rexample-topic\0\0\0\0\0\0\0!\0\0\0\0\0\1\0\rexample-topic\0\0\0\1\0\0\0\1\0\0\0\0") 139 | (make-InternalGroupMetadata 140 | #:group "example-group" 141 | #:generation 23 142 | #:protocol-type "consumer" 143 | #:protocol-data "range" 144 | #:leader "racket-kafka-35d27953-bcfa-4be9-a5b0-58ae5cebe04a" 145 | #:members (list 146 | (make-InternalGroupMember 147 | #:id "racket-kafka-35d27953-bcfa-4be9-a5b0-58ae5cebe04a" 148 | #:client-id "racket-kafka" 149 | #:client-host "/172.18.0.1" 150 | #:rebalance-timeout 30000 151 | #:session-timeout 30000 152 | #:subscription #"\0\0\0\0\0\1\0\rexample-topic\0\0\0\0" 153 | #:assignment #"\0\0\0\0\0\1\0\rexample-topic\0\0\0\1\0\0\0\0\0\0\0\0") 154 | (make-InternalGroupMember 155 | #:id "racket-kafka-1f34c0ba-75b8-4d68-a4f5-63afcaa16522" 156 | #:client-id "racket-kafka" 157 | #:client-host "/172.18.0.1" 158 | #:rebalance-timeout 30000 159 | #:session-timeout 30000 160 | #:subscription #"\0\0\0\0\0\1\0\rexample-topic\0\0\0\0" 161 | #:assignment #"\0\0\0\0\0\1\0\rexample-topic\0\0\0\1\0\0\0\1\0\0\0\0"))))) 162 | 163 | (test-case "group member accessors" 164 | (define m 165 | (make-InternalGroupMember 166 | #:id "racket-kafka-35d27953-bcfa-4be9-a5b0-58ae5cebe04a" 167 | #:client-id "racket-kafka" 168 | #:client-host "/172.18.0.1" 169 | #:rebalance-timeout 30000 170 | #:session-timeout 30000 171 | #:subscription #"\0\0\0\0\0\1\0\rexample-topic\0\0\0\0" 172 | #:assignment #"\0\0\0\0\0\1\0\rexample-topic\0\0\0\1\0\0\0\0\0\0\0\0")) 173 | (check-equal? 174 | (InternalGroupMember-topics m) 175 | '("example-topic")) 176 | (check-equal? 177 | (InternalGroupMember-assignments m) 178 | (hash "example-topic" '(0))))) 179 | -------------------------------------------------------------------------------- /kafka-lib/private/serde/join-group.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require racket/contract/base 4 | "core.rkt") 5 | 6 | (define-record Protocol 7 | ([name string?] 8 | [metadata bytes?])) 9 | 10 | (define-record Member 11 | ([id string?] 12 | [metadata bytes?])) 13 | 14 | (define-record JoinGroupResponse 15 | ([(throttle-time-ms #f) (or/c #f exact-nonnegative-integer?)] 16 | [error-code error-code/c] 17 | [generation-id exact-nonnegative-integer?] 18 | [protocol-name string?] 19 | [leader string?] 20 | [member-id string?] 21 | [members (listof Member?)])) 22 | 23 | (define-request JoinGroup 24 | (group-id 25 | [session-timeout-ms 30000] 26 | [member-id ""] 27 | [protocol-type "consumer"] 28 | [protocols null]) 29 | #:code 11 30 | 31 | #:version 0 32 | #:response proto:JoinGroupResponseV0 33 | (lambda (group-id session-timeout-ms member-id protocol-type protocols) 34 | (with-output-bytes 35 | (proto:un-JoinGroupRequestV0 36 | `((GroupID_1 . ,group-id) 37 | (SessionTimeoutMs_1 . ,session-timeout-ms) 38 | (MemberID_1 . ,member-id) 39 | (ProtocolType_1 . ,protocol-type) 40 | (ArrayLen_1 . ,(length protocols)) 41 | (Protocol_1 ,@(for/list ([p (in-list protocols)]) 42 | `((ProtocolName_1 . ,(Protocol-name p)) 43 | (ProtocolMetadata_1 . ,(Protocol-metadata p))))))))) 44 | (lambda (res) 45 | (make-JoinGroupResponse 46 | #:throttle-time-ms (opt 'ThrottleTimeMs_1 res) 47 | #:error-code (ref 'ErrorCode_1 res) 48 | #:generation-id (ref 'GenerationID_1 res) 49 | #:protocol-name (ref 'ProtocolName_1 res) 50 | #:leader (ref 'LeaderID_1 res) 51 | #:member-id (ref 'MemberID_1 res) 52 | #:members (for/list ([m (ref 'Member_1 res)]) 53 | (make-Member 54 | #:id (ref 'MemberID_1 m) 55 | #:metadata (ref 'MemberMetadata_1 m)))))) 56 | -------------------------------------------------------------------------------- /kafka-lib/private/serde/leave-group.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require racket/contract/base 4 | "core.rkt") 5 | 6 | (define-record LeaveGroupResponse 7 | ([(throttle-time-ms #f) (or/c #f exact-nonnegative-integer?)] 8 | [error-code error-code/c])) 9 | 10 | (define-request LeaveGroup 11 | (group-id member-id) 12 | #:code 13 13 | 14 | #:version 0 15 | #:response proto:LeaveGroupResponseV0 16 | (make-enc-leave-group proto:un-LeaveGroupRequestV0) 17 | dec-leave-group 18 | 19 | #:version 1 20 | #:response proto:LeaveGroupRequestV1 21 | (make-enc-leave-group proto:un-LeaveGroupRequestV1) 22 | dec-leave-group 23 | 24 | #:version 2 25 | #:response proto:LeaveGroupRequestV2 26 | (make-enc-leave-group proto:un-LeaveGroupRequestV2) 27 | dec-leave-group) 28 | 29 | (define ((make-enc-leave-group unparse) group-id member-id) 30 | (with-output-bytes 31 | (unparse 32 | `((GroupID_1 . ,group-id) 33 | (MemberID_1 . ,member-id))))) 34 | 35 | (define (dec-leave-group res) 36 | (make-LeaveGroupResponse 37 | #:throttle-time-ms (opt 'ThrottleTimeMs_1 res) 38 | #:error-code (or (opt 'ErrorCode_1 res) 0))) 39 | -------------------------------------------------------------------------------- /kafka-lib/private/serde/list-groups.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require "core.rkt" 4 | "group.rkt") 5 | 6 | (define-request ListGroups () 7 | #:code 16 8 | #:version 0 9 | #:response proto:ListGroupsResponseV0 10 | (lambda () #"") 11 | (dec-list-groups-response-v012 'ListGroupsGroupV0_1) 12 | 13 | #:version 1 14 | #:response proto:ListGroupsResponseV1 15 | (lambda () #"") 16 | (dec-list-groups-response-v012 'ListGroupsGroupV1_1) 17 | 18 | #:version 2 19 | #:response proto:ListGroupsResponseV2 20 | (lambda () #"") 21 | (dec-list-groups-response-v012 'ListGroupsGroupV2_1) 22 | 23 | #:version 3 24 | #:flexible #t 25 | #:response proto:ListGroupsResponseV3 26 | (lambda () 27 | (with-output-bytes 28 | (proto:un-ListGroupsRequestV3 (hash)))) 29 | (lambda (res) 30 | (for/list ([g (in-list (ref 'ListGroupsGroupV3_1 res))]) 31 | (make-Group 32 | #:id (ref 'CompactGroupID_1 g) 33 | #:protocol-type (ref 'CompactProtocolType_1 g))))) 34 | 35 | (define ((dec-list-groups-response-v012 list-id) res) 36 | (for/list ([g (in-list (ref list-id res))]) 37 | (make-Group 38 | #:id (ref 'GroupID_1 g) 39 | #:protocol-type (ref 'ProtocolType_1 g)))) 40 | -------------------------------------------------------------------------------- /kafka-lib/private/serde/list-offsets.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require "core.rkt") 4 | 5 | (define-record PartitionOffset 6 | ([id exact-nonnegative-integer?] 7 | [error-code error-code/c] 8 | [timestamp exact-nonnegative-integer?] 9 | [offset exact-integer?])) 10 | 11 | (define-request ListOffsets 12 | (topics [isolation-level 'read-committed]) 13 | #:code 2 14 | 15 | #:version 2 16 | #:response proto:ListOffsetsResponseV2 17 | (lambda (topics isolation-level) 18 | (with-output-bytes 19 | (proto:un-ListOffsetsRequestV2 20 | `((ReplicaID_1 . -1) 21 | (IsolationLevel_1 . ,(case isolation-level 22 | [(read-committed) 1] 23 | [(read-uncommitted) 0] 24 | [else (raise-argument-error 'ListOffsets "(or/c 'read-committed 'read-uncommitted)" isolation-level)])) 25 | (ArrayLen_1 . ,(hash-count topics)) 26 | (ListOffsetsRequestTopicV2_1 . ,(for/list ([(topic partitions) (in-hash topics)]) 27 | (define parts 28 | (for/list ([(pid timestamp) (in-hash partitions)]) 29 | `((PartitionID_1 . ,pid) 30 | (Timestamp_1 . ,(case timestamp 31 | [(latest) -1] 32 | [(earliest) -2] 33 | [else timestamp]))))) 34 | 35 | `((TopicName_1 . ,topic) 36 | (ArrayLen_1 . ,(hash-count partitions)) 37 | (ListOffsetsRequestPartitionV2_1 . ,parts)))))))) 38 | (lambda (res) 39 | (for/hash ([t (in-list (ref 'ListOffsetsResponseTopicV2_1 res))]) 40 | (define topic (ref 'TopicName_1 t)) 41 | (values topic (for/list ([p (in-list (ref 'ListOffsetsResponsePartitionV2_1 t))]) 42 | (make-PartitionOffset 43 | #:id (ref 'PartitionID_1 p) 44 | #:error-code (ref 'ErrorCode_1 p) 45 | #:timestamp (ref 'Timestamp_1 p) 46 | #:offset (ref 'Offset_1 p))))))) 47 | -------------------------------------------------------------------------------- /kafka-lib/private/serde/metadata.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require racket/contract/base 4 | "core.rkt") 5 | 6 | (define-record BrokerMetadata 7 | ([node-id exact-nonnegative-integer?] 8 | [host string?] 9 | [port port/c] 10 | [rack (or/c #f string?)])) 11 | 12 | (define-record PartitionMetadata 13 | ([error-code error-code/c] 14 | [id exact-nonnegative-integer?] 15 | [leader-id exact-integer?] 16 | [replica-node-ids (listof exact-nonnegative-integer?)] 17 | [in-sync-replica-node-ids (listof exact-nonnegative-integer?)])) 18 | 19 | (define-record TopicMetadata 20 | ([error-code error-code/c] 21 | [name string?] 22 | [internal? boolean?] 23 | [partitions (listof PartitionMetadata?)])) 24 | 25 | (define-record Metadata 26 | ([brokers (listof BrokerMetadata?)] 27 | [topics (listof TopicMetadata?)] 28 | [controller-id exact-integer?] 29 | [(cluster-id #f) (or/c #f string?)])) 30 | 31 | (define-request Metadata (topics) 32 | #:code 3 33 | #:version 1 34 | #:response proto:MetadataResponseV1 35 | enc-metadata-request 36 | dec-metadata-response 37 | 38 | #:version 2 39 | #:response proto:MetadataResponseV2 40 | enc-metadata-request 41 | dec-metadata-response) 42 | 43 | (define (enc-metadata-request topics) 44 | (if (null? topics) 45 | null32 46 | (with-output-bytes 47 | (proto:un-MetadataRequestV1 48 | `((ArrayLen_1 . ,(length topics)) 49 | (TopicName_1 . ,topics)))))) 50 | 51 | (define (dec-metadata-response res) 52 | (Metadata 53 | (for/list ([broker (in-list (ref 'Broker_1 'Brokers_1 res))]) 54 | (BrokerMetadata 55 | (ref 'NodeID_1 broker) 56 | (ref 'Host_1 broker) 57 | (ref 'Port_1 broker) 58 | (ref 'Rack_1 broker))) 59 | (for/list ([topic (in-list (ref 'TopicMetadata_1 'TopicMetadatas_1 res))]) 60 | (TopicMetadata 61 | (ref 'TopicErrorCode_1 topic) 62 | (ref 'TopicName_1 topic) 63 | (not (zero? (ref 'IsInternal_1 topic))) 64 | (for/list ([part (in-list (ref 'PartitionMetadata_1 'PartitionMetadatas_1 topic))]) 65 | (PartitionMetadata 66 | (ref 'PartitionErrorCode_1 part) 67 | (ref 'PartitionID_1 part) 68 | (ref 'Leader_1 part) 69 | (ref 'Replica_1 part) 70 | (ref 'ISR_1 part))))) 71 | (ref 'ControllerID_1 res) 72 | (opt 'ClusterID_1 res))) 73 | -------------------------------------------------------------------------------- /kafka-lib/private/serde/produce.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require racket/contract/base 4 | "core.rkt") 5 | 6 | (define-record ProduceResponsePartition 7 | ([id exact-nonnegative-integer?] 8 | [error-code error-code/c] 9 | [offset exact-integer?])) 10 | 11 | (define-record RecordResult 12 | ([topic string?] 13 | [partition ProduceResponsePartition?])) 14 | 15 | (define-record ProduceResponseTopic 16 | ([name string?] 17 | [partitions (listof ProduceResponsePartition?)])) 18 | 19 | (define-record ProduceResponse 20 | ([topics (listof ProduceResponseTopic?)])) 21 | 22 | (define-record PartitionData 23 | ([id exact-integer?] 24 | [batch bytes?])) 25 | 26 | (define-record TopicData 27 | ([name string?] 28 | [partitions (listof PartitionData?)])) 29 | 30 | (define-request Produce 31 | (data 32 | [acks 'none] 33 | [timeout-ms 30000]) 34 | #:code 0 35 | #:version 3 36 | #:response proto:ProduceResponseV3 37 | #:immed-response (λ (_data acks _timeout-ms) 38 | (and (eq? acks 'none) 39 | (ProduceResponse null))) 40 | enc-producev3 41 | dec-producev2) 42 | 43 | (define (acks->integer acks) 44 | (case acks 45 | [(none) 0] 46 | [(leader) 1] 47 | [(all) -1] 48 | [else (raise-argument-error 'acks->integer "(or/c -1 0 1)" acks)])) 49 | 50 | (define (enc-producev3 data acks timeout-ms) 51 | (with-output-bytes 52 | (proto:un-ProduceRequestV3 53 | `((TransactionalID_1 . #f) 54 | (Acks_1 . ,(acks->integer acks)) 55 | (TimeoutMs_1 . ,timeout-ms) 56 | (ArrayLen_1 . ,(length data)) 57 | (TopicData_1 . ,(for/list ([d (in-list data)]) 58 | (define parts (TopicData-partitions d)) 59 | `((TopicName_1 . ,(TopicData-name d)) 60 | (ArrayLen_1 . ,(length parts)) 61 | (PartitionData_1 . ,(for/list ([p (in-list parts)]) 62 | `((PartitionID_1 . ,(PartitionData-id p)) 63 | (Records_1 . ,(PartitionData-batch p)))))))))))) 64 | 65 | (define (dec-producev2 res) 66 | (ProduceResponse 67 | (for/list ([t (in-list (ref 'ProduceResponseDataV2_1 res))]) 68 | (ProduceResponseTopic 69 | (ref 'TopicName_1 t) 70 | (for/list ([p (in-list (ref 'PartitionResponseV2_1 t))]) 71 | (make-ProduceResponsePartition 72 | #:id (ref 'PartitionID_1 p) 73 | #:error-code (ref 'ErrorCode_1 p) 74 | #:offset (ref 'Offset_1 p))))))) 75 | -------------------------------------------------------------------------------- /kafka-lib/private/serde/resource.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require racket/contract/base) 4 | 5 | (provide 6 | resource-type/c 7 | resource-type-> 8 | ->resource-type) 9 | 10 | (define resource-type/c 11 | (or/c 'unknown 'any 'topic 'group 'broker 'cluster 'transaction-id 'delegation-token 'user)) 12 | 13 | (define (->resource-type type) 14 | (case type 15 | [(unknown) 0] 16 | [(any) 1] 17 | [(topic) 2] 18 | [(group) 3] 19 | [(broker) 4] 20 | [(cluster) 4] 21 | [(transactional-id) 5] 22 | [(delegation-token) 6] 23 | [(user) 7] 24 | [else (raise-argument-error '->resource-type "resource-type/c" type)])) 25 | 26 | (define (resource-type-> type) 27 | (case type 28 | [(0) 'unknown] 29 | [(1) 'any] 30 | [(2) 'topic] 31 | [(3) 'group] 32 | [(4) 'broker] 33 | [(4) 'cluster] 34 | [(5) 'transactional-id] 35 | [(6) 'delegation-token] 36 | [(7) 'user] 37 | [else (raise-argument-error 'resource-type-> "(integer-in 0 7)" type)])) 38 | -------------------------------------------------------------------------------- /kafka-lib/private/serde/sasl-authenticate.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require "core.rkt") 4 | 5 | (define-record SaslAuthenticateResponse 6 | ([data bytes?] 7 | [session-lifetime-ms exact-integer?])) 8 | 9 | (define-request SaslAuthenticate 10 | ([data bytes?]) 11 | #:code 36 12 | #:version 1 13 | #:response proto:SaslAuthenticateResponseV1 14 | (lambda (data) 15 | (with-output-bytes 16 | (proto:un-SaslAuthenticateRequestV1 data))) 17 | (lambda (res) 18 | (make-SaslAuthenticateResponse 19 | #:data (ref 'Bytes_1 res) 20 | #:session-lifetime-ms (ref 'SessionLifetimeMs_1 res)))) 21 | -------------------------------------------------------------------------------- /kafka-lib/private/serde/sasl-handshake.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require racket/contract/base 4 | "core.rkt") 5 | 6 | (define-record SaslHandshakeResponse 7 | ([(mechanisms null) (listof symbol?)])) 8 | 9 | (define-request SaslHandshake 10 | ([mechanism 'plain]) 11 | #:code 17 12 | #:version 1 13 | #:response proto:SaslHandshakeResponseV1 14 | (lambda (mechanism) 15 | (with-output-bytes 16 | (proto:un-SaslHandshakeRequestV1 17 | (string-upcase (symbol->string mechanism))))) 18 | (lambda (res) 19 | (make-SaslHandshakeResponse 20 | #:mechanisms (map (compose1 string->symbol string-downcase) 21 | (ref 'SaslHandshakeMechanism_1 res))))) 22 | -------------------------------------------------------------------------------- /kafka-lib/private/serde/sync-group.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require "core.rkt") 4 | 5 | (define-record Assignment 6 | ([member-id string?] 7 | [data bytes?])) 8 | 9 | (define-request SyncGroup 10 | (group-id 11 | generation-id 12 | member-id 13 | member-assignments) 14 | #:code 14 15 | 16 | #:version 0 17 | #:response proto:SyncGroupResponseV0 18 | (lambda (group-id generation-id member-id member-assignments) 19 | (with-output-bytes 20 | (proto:un-SyncGroupRequestV0 21 | `((GroupID_1 . ,group-id) 22 | (GenerationID_1 . ,generation-id) 23 | (MemberID_1 . ,member-id) 24 | (ArrayLen_1 . ,(length member-assignments)) 25 | (MemberAssignment_1 ,@(for/list ([a (in-list member-assignments)]) 26 | `((MemberID_1 . ,(Assignment-member-id a)) 27 | (MemberAssignmentData_1 . ,(Assignment-data a))))))))) 28 | (lambda (res) 29 | (ref 'MemberAssignmentData_1 res))) 30 | -------------------------------------------------------------------------------- /kafka-test/info.rkt: -------------------------------------------------------------------------------- 1 | #lang info 2 | 3 | (define license 'BSD-3-Clause) 4 | (define collection "test") 5 | (define deps '("base" 6 | "binfmt" 7 | "kafka-lib" 8 | "rackunit-lib")) 9 | (define implies '("kafka-lib")) 10 | -------------------------------------------------------------------------------- /kafka-test/kafka/fixtures/truncated-records.bin: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:7145c3e0b14b4dddc729684dce5bed22a0e86565baa7362527146547d3a19ef7 3 | size 1048580 4 | -------------------------------------------------------------------------------- /kafka-test/kafka/fixtures/zstd-frame-no-length.bin: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:7d78375d7fbf814761f8e6499782c9127ee4cf163bba7f48aa0a3b7ae4227a28 3 | size 1125 4 | -------------------------------------------------------------------------------- /kafka-test/kafka/records.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require binfmt/runtime/res 4 | kafka/private/batch 5 | kafka/private/batch-native 6 | racket/port 7 | racket/runtime-path 8 | rackunit) 9 | 10 | (define-runtime-path truncated-records.bin 11 | "fixtures/truncated-records.bin") 12 | (define-runtime-path zstd-frame-no-length.bin 13 | "fixtures/zstd-frame-no-length.bin") 14 | 15 | (define (check-read-truncated-records in) 16 | (define res (Records in)) 17 | (check-true (ok? res)) 18 | (check-equal? (length (ok-v res)) 6158) 19 | (check-equal? 20 | (for/sum ([b (in-list (ok-v res))]) 21 | (batch-size b)) 22 | 6230)) 23 | 24 | (test-case "read truncated records" 25 | (test-case "from file" 26 | (call-with-input-file truncated-records.bin 27 | (lambda (in) 28 | (check-read-truncated-records in)))) 29 | (test-case "from pipe" 30 | (call-with-input-file truncated-records.bin 31 | (lambda (file-in) 32 | (define-values (in out) 33 | (make-pipe)) 34 | (thread 35 | (lambda () 36 | (copy-port file-in out) 37 | (close-output-port out))) 38 | (check-read-truncated-records in))))) 39 | 40 | (test-case "read zstd-compressed data where the decompressed length is unknown" 41 | (call-with-input-file zstd-frame-no-length.bin 42 | (lambda (in) 43 | (define res (read-batch in)) 44 | (check-not-false res) 45 | (check-equal? (vector-length (batch-records res)) 1000)))) 46 | -------------------------------------------------------------------------------- /kafka/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2022 Bogdan Popa 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | 1. Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in the 12 | documentation and/or other materials provided with the distribution. 13 | 14 | 3. Neither the name of the copyright holder nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 24 | TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 25 | PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 26 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 27 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 28 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | -------------------------------------------------------------------------------- /kafka/info.rkt: -------------------------------------------------------------------------------- 1 | #lang info 2 | 3 | (define license 'BSD-3-Clause) 4 | (define collection "kafka") 5 | (define deps '(("base" #:version "8.4") 6 | "kafka-lib")) 7 | (define build-deps '("racket-doc" 8 | "sasl-doc" 9 | "sasl-lib" 10 | "scribble-lib")) 11 | (define implies '("kafka-lib")) 12 | -------------------------------------------------------------------------------- /kafka/scribblings/info.rkt: -------------------------------------------------------------------------------- 1 | #lang info 2 | 3 | (define scribblings '(("kafka.scrbl" () ("Databases")))) 4 | -------------------------------------------------------------------------------- /tests/00-wait-for-kafka.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require kafka) 4 | 5 | (define deadline (+ (current-seconds) 60)) 6 | (let loop () 7 | (cond 8 | [(> (current-seconds) deadline) 9 | (error 'wait-for-kafka "kafka not available")] 10 | [else 11 | (with-handlers ([exn:fail? 12 | (lambda (e) 13 | ((error-display-handler) (exn-message e) e) 14 | (sleep 1) 15 | (loop))]) 16 | (define k (make-client)) 17 | (void (find-group-coordinator k "example")))])) 18 | -------------------------------------------------------------------------------- /tests/01-publish-consume.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require kafka 4 | kafka/consumer 5 | kafka/producer 6 | racket/match 7 | rackunit 8 | rackunit/text-ui) 9 | 10 | (define t "01-publish-consume") 11 | (define k (make-client)) 12 | 13 | (run-tests 14 | (test-suite 15 | "publish-consume" 16 | #:before 17 | (lambda () 18 | (create-topics k (make-CreateTopic 19 | #:name t 20 | #:partitions 2))) 21 | #:after 22 | (lambda () 23 | (delete-topics k t) 24 | (disconnect-all k)) 25 | 26 | (let () 27 | (define p (make-producer k)) 28 | (define evts 29 | (list 30 | (produce p t #"k-1" #"v-1" #:partition 0) 31 | (produce p t #"k-2" #"v-2" #:partition 1))) 32 | (producer-flush p) 33 | (for ([res (in-list (map sync evts))]) 34 | (match-define (RecordResult _ (ProduceResponsePartition _ err _)) res) 35 | (check-true (zero? err))) 36 | (producer-stop p) 37 | 38 | (define c (make-consumer k "00-consume" t)) 39 | (test-case "consume before end" 40 | (define-values (type data) 41 | (sync (consume-evt c))) 42 | (check-equal? type 'records) 43 | (check-equal? (vector-length data) 2) 44 | (check-equal? (record-key (vector-ref data 0)) #"k-1") 45 | (check-equal? (record-key (vector-ref data 1)) #"k-2") 46 | (check-equal? (record-value (vector-ref data 0)) #"v-1") 47 | (check-equal? (record-value (vector-ref data 1)) #"v-2") 48 | (consumer-commit c)) 49 | 50 | (test-case "consume after end" 51 | (define-values (type data) 52 | (sync (consume-evt c 100))) 53 | (check-equal? type 'records) 54 | (check-equal? (vector-length data) 0) 55 | (consumer-commit c)) 56 | (consumer-stop c)))) 57 | -------------------------------------------------------------------------------- /tests/02-commit-after-rebalance.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require kafka 4 | kafka/consumer 5 | kafka/producer 6 | rackunit 7 | rackunit/text-ui) 8 | 9 | (define t "02-commit-after-rebalance") 10 | (define k (make-client)) 11 | 12 | (run-tests 13 | (test-suite 14 | "commit-after-rebalance" 15 | #:before 16 | (lambda () 17 | (create-topics k (make-CreateTopic 18 | #:name t 19 | #:partitions 2)) 20 | 21 | (define p (make-producer k)) 22 | (produce p t #"a" #"a" #:partition 0) 23 | (produce p t #"b" #"b" #:partition 1) 24 | (producer-flush p) 25 | (producer-stop p)) 26 | #:after 27 | (lambda () 28 | (delete-topics k t) 29 | (disconnect-all k)) 30 | 31 | (let () 32 | (define g "commit-after-rebalance-group") 33 | (define k0 (make-client)) 34 | (define c0 (make-consumer k0 g t)) 35 | (sync (consume-evt c0)) 36 | (define k1 (make-client)) 37 | (define c1 (make-consumer k1 g t)) 38 | (consumer-stop c1) 39 | (disconnect-all k1) 40 | (consumer-commit c0) 41 | (let () 42 | (define-values (type _data) 43 | (sync (consume-evt c0))) 44 | (check-equal? type 'rebalance)) 45 | (let () 46 | (define-values (type data) 47 | (sync (consume-evt c0))) 48 | (check-equal? type 'records) 49 | (check-equal? (vector-length data) 2)) 50 | (consumer-stop c0) 51 | (disconnect-all k0)))) 52 | -------------------------------------------------------------------------------- /tests/03-delete-groups.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require kafka 4 | kafka/consumer 5 | kafka/producer 6 | rackunit 7 | rackunit/text-ui) 8 | 9 | (define t "03-delete-groups") 10 | (define k (make-client)) 11 | 12 | (run-tests 13 | (test-suite 14 | "delete-groups" 15 | #:before 16 | (lambda () 17 | (create-topics k (make-CreateTopic 18 | #:name t 19 | #:partitions 2)) 20 | 21 | (define p (make-producer k)) 22 | (produce p t #"a" #"a" #:partition 0) 23 | (produce p t #"b" #"b" #:partition 1) 24 | (producer-flush p) 25 | (producer-stop p)) 26 | #:after 27 | (lambda () 28 | (delete-topics k t) 29 | (disconnect-all k)) 30 | 31 | (let () 32 | (define g "03-delete-groups-group") 33 | (define c (make-consumer k g t)) 34 | (sync (consume-evt c)) 35 | (consumer-stop c) 36 | (disconnect-all k) 37 | (delete-groups k g) 38 | (check-false (member g (map Group-id (list-groups k))))))) 39 | -------------------------------------------------------------------------------- /tests/99-publish-consume-for-a-while.rkt: -------------------------------------------------------------------------------- 1 | #lang racket/base 2 | 3 | (require kafka 4 | kafka/consumer 5 | kafka/producer 6 | racket/format 7 | rackunit 8 | rackunit/text-ui) 9 | 10 | (define N 10000) 11 | (define P 8) 12 | (define t "99-publish-consume-for-a-while") 13 | 14 | (run-tests 15 | (test-suite 16 | "publish-consume-for-a-while" 17 | #:before 18 | (lambda () 19 | (define k (make-client)) 20 | (delete-topics k t) 21 | (create-topics k (make-CreateTopic 22 | #:name t 23 | #:partitions P)) 24 | (disconnect-all k)) 25 | 26 | #:after 27 | (lambda () 28 | (define k (make-client)) 29 | (delete-topics k t) 30 | (disconnect-all k)) 31 | 32 | (let () 33 | (define msgs null) 34 | (define msg-ch (make-channel)) 35 | 36 | (define g "publish-consume-for-a-while-group") 37 | (define consumer-ch (make-channel)) 38 | (define consumer-thds 39 | (for/list ([i (in-range P)]) 40 | (thread 41 | (lambda () 42 | (let join-loop () 43 | (with-handlers ([exn:fail:kafka:server? 44 | (lambda (e) 45 | (define code (exn:fail:kafka:server-code e)) 46 | (case (error-code-symbol code) 47 | [(coordinator-not-available rebalance-in-progress) 48 | (join-loop)] 49 | [else 50 | (raise e)]))]) 51 | (define k (make-client #:id (~a "consumer-" i))) 52 | (define c (make-consumer k g t)) 53 | (let loop () 54 | (sync 55 | (handle-evt 56 | consumer-ch 57 | (lambda (_) 58 | (consumer-stop c) 59 | (disconnect-all k))) 60 | (handle-evt 61 | (consume-evt c) 62 | (lambda (type data) 63 | (case type 64 | [(records) 65 | (consumer-commit c) 66 | (for ([r (in-vector data)]) 67 | (channel-put msg-ch (record-value r))) 68 | (loop)] 69 | 70 | [else 71 | (loop)]))))))))))) 72 | 73 | (define producer-ch (make-channel)) 74 | (define producer-thd 75 | (thread 76 | (lambda () 77 | (define k (make-client #:id "producer")) 78 | (define p (make-producer k #:flush-interval 500)) 79 | (let loop ([n 0] [evts null]) 80 | (cond 81 | [(= (length evts) 500) 82 | (time (for-each sync evts)) 83 | (printf "published 500~n") 84 | (loop n null)] 85 | [(< n N) 86 | (define produce-evt 87 | (produce p t #"k" #"v" #:partition (modulo n 8))) 88 | (sleep (/ (random 5) 1000.0)) 89 | (loop (add1 n) (cons produce-evt evts))] 90 | [else 91 | (time (for-each sync evts)) 92 | (printf "publish done~n") 93 | (sync producer-ch) 94 | (disconnect-all k)]))))) 95 | 96 | (let loop ([n 0]) 97 | (when (zero? (modulo n 100)) 98 | (printf "received ~s~n" n)) 99 | (unless (= n N) 100 | (set! msgs (cons (channel-get msg-ch) msgs)) 101 | (loop (add1 n)))) 102 | 103 | (channel-put producer-ch '(stop)) 104 | (thread-wait producer-thd) 105 | (for ([_ (in-range P)]) 106 | (channel-put consumer-ch '(stop))) 107 | (for-each thread-wait consumer-thds)))) 108 | --------------------------------------------------------------------------------