├── .github └── workflows │ └── deploy.yml ├── .gitignore ├── LICENSE ├── README.md ├── dev └── bytegeist │ └── dev.clj ├── project.clj ├── src-java └── bytegeist │ └── protobuf │ └── Util.java ├── src └── bytegeist │ └── bytegeist.clj └── test └── bytegeist └── bytegeist_test.clj /.github/workflows/deploy.yml: -------------------------------------------------------------------------------- 1 | name: Clojars Deploy 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | 9 | jobs: 10 | deploy: 11 | name: Deploy 12 | runs-on: ubuntu-18.04 13 | steps: 14 | - name: Checkout 15 | uses: actions/checkout@v2 16 | - name: Lint 17 | uses: DeLaGuardo/clojure-lint-action@v1 18 | with: 19 | clj-kondo-args: --lint src test 20 | github_token: ${{ secrets.GITHUB_TOKEN }} 21 | - name: Test 22 | run: lein test 23 | - name: Deploy 24 | if: github.event_name == 'push' && github.ref == 'refs/heads/master' 25 | env: 26 | CLOJARS_USER: ${{ secrets.CLOJARS_USER }} 27 | CLOJARS_PASS: ${{ secrets.CLOJARS_PASS }} 28 | run: lein deploy 29 | 30 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /classes 3 | /checkouts 4 | pom.xml 5 | pom.xml.asc 6 | *.jar 7 | *.class 8 | /.lein-* 9 | /.nrepl-port 10 | .hgignore 11 | .hg/ 12 | *.iml 13 | .idea/ 14 | /dev/bytegeist/ignore.clj 15 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Yonatan Elhanan 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # bytegeist 2 | 3 | [![Build Status](https://img.shields.io/github/workflow/status/yonatane/bytegeist/Clojars%20Deploy?event=push&branch=master&label=Build)](https://github.com/yonatane/bytegeist/actions) 4 | 5 | Binary protocol specs for clojure 6 | 7 | WIP 8 | 9 | ```clojure 10 | [bytegeist "0.1.0-SNAPSHOT"] 11 | ``` 12 | 13 | ### Current goals 14 | 15 | 1. Define the binary encoding of any protocol using plain data 16 | 2. Simply define multiple versions of a protocol without duplication or extra programming 17 | 3. Protobuf without code generation 18 | 4. Support NIO ByteBuffer, Netty ByteBuf and InputStream/OutputStream 19 | 5. Enable extensions for custom types and I/O 20 | 6. Fast 21 | 22 | ### Examples 23 | 24 | [The tests](test/bytegeist/bytegeist_test.clj) contain the most updated examples. 25 | 26 | [Skazka (Kafka proxy)](https://github.com/yonatane/skazka/blob/851873f7a75b9c37f3313d041c4caeddfafa9db0/src/skazka/protocol.clj#L1) 27 | implements some of the kafka protocol. 28 | 29 | Reading and writing simple data: 30 | 31 | ```clojure 32 | (require '[bytegeist.bytegeist :as g]) 33 | (import '(io.netty.buffer ByteBuf Unpooled)) 34 | (def b (Unpooled/buffer)) 35 | 36 | ;; Just an integer 37 | (g/write :int32 b 2020) 38 | (g/read :int32 b) 39 | ;=> 2020 40 | 41 | ;; A length-delimited string, first 2 bytes hold the length 42 | (def username [:string {:length :short}]) 43 | 44 | (g/write username b "byteme72") 45 | (g/read username b) 46 | ;=> "byteme72" 47 | ``` 48 | 49 | Maps: 50 | 51 | ```clojure 52 | ;; A map with predefined fields 53 | (def user [:map 54 | [:username [:string {:length :short}]] 55 | [:year :int32]]) 56 | ;; Or reuse previously defined types and also compile the spec once 57 | (def user (g/spec [:map 58 | [:username username] 59 | [:year :int32]])) 60 | 61 | (g/write user b {:username "byteme72" 62 | :year 2020}) 63 | (g/read user b) 64 | ;=> {:username "byteme72", :year 2020} 65 | ``` 66 | 67 | Inline map inside its parent: 68 | 69 | ```clojure 70 | ;; A message where the header has multiple versions 71 | (def message 72 | [:map 73 | [:header {:inline true} 74 | [:multi {:dispatch :version} 75 | [0 [:map [:version :short] [:h1 :int32]]] 76 | [1 [:map [:version :short] [:h1 :uvarint32] [:h2 :bool]]]]] 77 | [:data [:string {:length :int32}]]]) 78 | 79 | (g/write message b {:version 0 :h1 123 :data "old"}) 80 | (g/write message b {:version 1 :h1 123 :h2 true :data "new"}) 81 | 82 | (g/read message b) 83 | => {:version 0, :h1 123, :data "old"} 84 | (g/read message b) 85 | => {:version 1, :h1 123, :h2 true, :data "new"} 86 | ``` 87 | 88 | Map-of: 89 | 90 | ```clojure 91 | ;; Map-of int to bytes, prefixed by the number of fields 92 | (def tagged-fields 93 | (let [tag :uvarint32 94 | data [:bytes {:length :uvarint32}]] 95 | [:map-of {:length :uvarint32} tag data])) 96 | 97 | (g/write tagged-fields b {0 (.getBytes "hello") 98 | 1 (byte-array 10000)}) 99 | (g/read tagged-fields b) 100 | ;=> {0 #object["[B" 0x667e83eb "[B@667e83eb"], 1 #object["[B" 0xed01c80 "[B@ed01c80"]} 101 | ``` 102 | 103 | ### Multi spec 104 | 105 | You can select a spec at runtime according to any field, 106 | as long as all fields up to, and including that field, are the same. 107 | 108 | ```clojure 109 | (def person 110 | [:multi {:dispatch :type} 111 | ["student" 112 | [:map 113 | [:type [:string {:length :short}]] 114 | [:grade :short]]] 115 | ["employee" 116 | [:map 117 | [:type [:string {:length :short}]] 118 | [:salary :int]]]]) 119 | 120 | (g/write person buf {:type "employee" 121 | :salary 99999}) 122 | 123 | (g/write person buf {:type "student" 124 | :grade 100}) 125 | 126 | (g/read person buf) 127 | ; => {:type "employee", :salary 99999} 128 | (g/read person buf) 129 | ; => {:type "student", :grade 100} 130 | ``` 131 | 132 | Dispatch on multiple fields, with a function of those fields to determine the dispatch value: 133 | 134 | ```clojure 135 | (def message 136 | [:multi {:dispatch [:type :version] 137 | :dispatch-fn (fn [[t v]] [(keyword t) (if (> v 2) :new :old)])} 138 | [[:produce :old] 139 | [:map 140 | [:type [:string {:length :short}]] 141 | [:version :short] 142 | [:data [:string {:length :int}]]]] 143 | [[:produce :new] 144 | [:map 145 | [:type [:string {:length :short}]] 146 | [:version :short] 147 | [:client-id [:string {:length :uvarint32}]] ; A new field in versions 3 and up 148 | [:data [:string {:length :uvarint32}]]]] 149 | [[:fetch :old] 150 | [:map 151 | [:type [:string {:length :short}]] 152 | [:version :short] 153 | [:partitions [:vector {:length :uvarint32} :uvarint32]]]]]) 154 | 155 | (g/write message buf {:type "produce" 156 | :version 3 157 | :client-id "test client" 158 | :data "test data"}) 159 | 160 | (g/read message buf) 161 | ; => {:type "produce", :version 3, :client-id "test client", :data "test data"} 162 | ``` 163 | 164 | ### Built-in types 165 | 166 | `:bool` 167 | `:boolean` 168 | `:byte` 169 | `:int16` 170 | `:short` 171 | `:int24` 172 | `:int32` 173 | `:int` 174 | `:int64` 175 | `:long` 176 | `:double` 177 | `:float` 178 | `:ubyte` 179 | `:uint32` 180 | `:uvarint32` 181 | `:map` 182 | `:map-of` 183 | `:vector` 184 | `:tuple` 185 | `:string` 186 | `:bytes` 187 | `:multi` 188 | 189 | ### Acknowledgements 190 | 191 | [Metosin/malli](https://github.com/metosin/malli) defines the schema notation bytegeist adopted. 192 | 193 | [Funcool/octet](https://github.com/funcool/octet) was used before bytegeist. 194 | 195 | ![YourKit](https://www.yourkit.com/images/yklogo.png)
196 | YourKit supports open source projects with innovative and intelligent tools for monitoring and profiling Java applications. 197 | YourKit is the creator of YourKit Java Profiler. 198 | 199 | ## License 200 | 201 | Copyright © 2020 Yonatan Elhanan 202 | 203 | Distributed under the MIT License 204 | -------------------------------------------------------------------------------- /dev/bytegeist/dev.clj: -------------------------------------------------------------------------------- 1 | (ns bytegeist.dev) 2 | -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject bytegeist "0.1.0-SNAPSHOT" 2 | :description "Binary protocol specs for clojure" 3 | :url "https://github.com/yonatane/bytegeist" 4 | :license {:name "MIT License"} 5 | :deploy-repositories [["releases" {:url "https://repo.clojars.org" 6 | :sign-releases false 7 | :username :env/clojars_user 8 | :password :env/clojars_pass}] 9 | ["snapshots" {:url "https://repo.clojars.org" 10 | :username :env/clojars_user 11 | :password :env/clojars_pass}]] 12 | :dependencies [[org.clojure/clojure "1.10.1"] 13 | [io.netty/netty-buffer "4.1.49.Final"]] 14 | :profiles {:dev {:source-paths ["dev"] 15 | :dependencies [[org.clojure/tools.namespace "1.0.0"] 16 | [criterium "0.4.5"] 17 | [com.clojure-goes-fast/clj-async-profiler "0.4.1"]]}} 18 | :java-source-paths ["src-java"] 19 | :repl-options {:init-ns bytegeist.dev} 20 | :global-vars {*warn-on-reflection* true} 21 | :pedantic? :abort) 22 | -------------------------------------------------------------------------------- /src-java/bytegeist/protobuf/Util.java: -------------------------------------------------------------------------------- 1 | package bytegeist.protobuf; 2 | 3 | import io.netty.buffer.ByteBuf; 4 | 5 | public class Util { 6 | // Mostly copied from com.google.protobuf.CodedInputStream.readRawVarint32 7 | public static int readUnsignedVarint32(ByteBuf buf) { 8 | int firstByte = buf.readByte(); 9 | if ((firstByte & 0x80) == 0) { 10 | return firstByte; 11 | } 12 | int result = firstByte & 0x7f; 13 | int offset = 7; 14 | for (; offset < 32; offset += 7) { 15 | final int b = buf.readByte(); 16 | result |= (b & 0x7f) << offset; 17 | if ((b & 0x80) == 0) { 18 | return result; 19 | } 20 | } 21 | // Keep reading up to 64 bits. 22 | for (; offset < 64; offset += 7) { 23 | final int b = buf.readByte(); 24 | if ((b & 0x80) == 0) { 25 | return result; 26 | } 27 | } 28 | throw new RuntimeException("Varint32 too long"); 29 | } 30 | 31 | // Mostly copied from com.google.protobuf.CodedOutputStream.SafeDirectNioEncoder.writeUInt32NoTag 32 | public static void writeUnsignedVarint32(ByteBuf buf, int value) { 33 | while (true) { 34 | if ((value & ~0x7F) == 0) { 35 | buf.writeByte((byte) value); 36 | return; 37 | } else { 38 | buf.writeByte((byte) ((value & 0x7F) | 0x80)); 39 | value >>>= 7; 40 | } 41 | } 42 | } 43 | 44 | // Copied from com.google.protobuf.CodedOutputStream.computeUInt32SizeNoTag 45 | public static int sizeUnsignedVarint32(final int value) { 46 | if ((value & (~0 << 7)) == 0) { 47 | return 1; 48 | } 49 | if ((value & (~0 << 14)) == 0) { 50 | return 2; 51 | } 52 | if ((value & (~0 << 21)) == 0) { 53 | return 3; 54 | } 55 | if ((value & (~0 << 28)) == 0) { 56 | return 4; 57 | } 58 | return 5; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/bytegeist/bytegeist.clj: -------------------------------------------------------------------------------- 1 | (ns bytegeist.bytegeist 2 | (:refer-clojure :exclude [byte double float read read-string]) 3 | (:import (io.netty.buffer ByteBuf) 4 | (java.nio.charset StandardCharsets))) 5 | 6 | (declare spec read write) 7 | 8 | #_(defn- append 9 | [s fields] 10 | (conj s fields)) 11 | 12 | (defn- override? 13 | [field] 14 | (= :override (first field))) 15 | 16 | (defn- remove-override-tag 17 | [field] 18 | (vec (rest field))) 19 | 20 | (defn- normalize-field 21 | [field] 22 | (cond-> field (override? field) remove-override-tag)) 23 | 24 | (defn- map-props 25 | [s] 26 | (let [props (nth s 1)] 27 | (when (map? props) 28 | props))) 29 | 30 | (defn- map-fields 31 | [s] 32 | (if (map-props s) 33 | (nthrest s 2) 34 | (rest s))) 35 | 36 | (defn- field-name 37 | [field] 38 | (first field)) 39 | 40 | (defn- contains-field? 41 | [coll field] 42 | (some #{(field-name field)} (map field-name coll))) 43 | 44 | (defn- map-add-fields 45 | [s fields] 46 | (let [; Ordered normalized fields 47 | all-ordered 48 | (mapv normalize-field (concat (map-fields s) fields)) 49 | 50 | ; Map field-name to its overridden type 51 | final-types 52 | (into {} all-ordered) 53 | 54 | ; Reconstruct the map fields, old fields retain their position but their type might be overridden. 55 | final-fields 56 | (reduce 57 | (fn [r field] 58 | (let [fname (field-name field)] 59 | (if (contains-field? r field) 60 | r 61 | (conj r [fname (get final-types fname)])))) 62 | [] 63 | all-ordered) 64 | 65 | root (if-let [props (map-props s)] 66 | [:map props] 67 | [:map])] 68 | 69 | (into root final-fields))) 70 | 71 | (defn add-fields 72 | [s fields] 73 | (cond 74 | (= :map (nth s 0)) 75 | (map-add-fields s fields) 76 | 77 | :else 78 | (throw (IllegalArgumentException. "Unsupported spec for add-fields")))) 79 | 80 | (defprotocol Spec 81 | (-read [_ b] "Relative read and increment the index") 82 | (-write [_ b v] "Relative write and increment the index")) 83 | 84 | (defprotocol MapSpec 85 | (-properties [_]) 86 | (-fields [_])) 87 | 88 | (def bool 89 | (reify 90 | Spec 91 | (-read [_ b] 92 | (.readBoolean ^ByteBuf b)) 93 | (-write [_ b v] 94 | (.writeBoolean ^ByteBuf b (boolean v))))) 95 | 96 | (def byte 97 | (reify 98 | Spec 99 | (-read [_ b] 100 | (.readByte ^ByteBuf b)) 101 | (-write [_ b v] 102 | (.writeByte ^ByteBuf b (unchecked-int v))))) 103 | 104 | (def int16 105 | (reify 106 | Spec 107 | (-read [_ b] 108 | (.readShort ^ByteBuf b)) 109 | (-write [_ b v] 110 | (.writeShort ^ByteBuf b (unchecked-int v))))) 111 | 112 | (def int24 113 | (reify 114 | Spec 115 | (-read [_ b] 116 | (.readMedium ^ByteBuf b)) 117 | (-write [_ b v] 118 | (.writeMedium ^ByteBuf b (unchecked-int v))))) 119 | 120 | (def int32 121 | (reify 122 | Spec 123 | (-read [_ b] 124 | (.readInt ^ByteBuf b)) 125 | (-write [_ b v] 126 | (.writeInt ^ByteBuf b (unchecked-int v))))) 127 | 128 | (def int64 129 | (reify 130 | Spec 131 | (-read [_ b] 132 | (.readLong ^ByteBuf b)) 133 | (-write [_ b v] 134 | (.writeLong ^ByteBuf b (unchecked-long v))))) 135 | 136 | (def double 137 | (reify 138 | Spec 139 | (-read [_ b] 140 | (.readDouble ^ByteBuf b)) 141 | (-write [_ b v] 142 | (.writeDouble ^ByteBuf b (unchecked-double v))))) 143 | 144 | (def float 145 | (reify 146 | Spec 147 | (-read [_ b] 148 | (.readFloat ^ByteBuf b)) 149 | (-write [_ b v] 150 | (.writeFloat ^ByteBuf b (unchecked-float v))))) 151 | 152 | (def ubyte 153 | (reify 154 | Spec 155 | (-read [_ b] 156 | (.readUnsignedByte ^ByteBuf b)) 157 | (-write [_ b v] 158 | (.writeByte ^ByteBuf b (unchecked-int v))))) 159 | 160 | (def uint32 161 | (reify 162 | Spec 163 | (-read [_ b] 164 | (.readUnsignedInt ^ByteBuf b)) 165 | (-write [_ b v] 166 | (.writeInt ^ByteBuf b (unchecked-int v))))) 167 | 168 | (def uvarint32 169 | (reify 170 | Spec 171 | (-read [_ b] 172 | (bytegeist.protobuf.Util/readUnsignedVarint32 ^ByteBuf b)) 173 | (-write [_ b v] 174 | (bytegeist.protobuf.Util/writeUnsignedVarint32 ^ByteBuf b (unchecked-int v))))) 175 | 176 | (declare spec) 177 | 178 | (defn- length-based-frame-spec [length-spec data-spec] 179 | (reify 180 | Spec 181 | (-read [_ b] 182 | (some-> length-spec (read b)) 183 | (read data-spec b)) 184 | (-write [_ b v] 185 | (let [frame-index (.writerIndex ^ByteBuf b) 186 | _ (write length-spec b 0) 187 | length-length (- (.writerIndex ^ByteBuf b) frame-index)] 188 | (write data-spec b v) 189 | (let [data-length (- (.writerIndex ^ByteBuf b) frame-index length-length)] 190 | (.writerIndex ^ByteBuf b frame-index) 191 | (write length-spec b data-length) 192 | (.writerIndex ^ByteBuf b (+ frame-index length-length data-length))))))) 193 | 194 | (defn- compile-field 195 | [field] 196 | (if (= 2 (count field)) 197 | (let [[field-name field-spec] field] 198 | [field-name nil (spec field-spec)]) 199 | (let [[field-name field-props field-spec] field] 200 | [field-name field-props (spec field-spec)]))) 201 | 202 | (defn map-spec 203 | [s] 204 | (let [props (map-props s) 205 | length-spec (some-> props :length spec) 206 | fields-data (map-fields s) 207 | compiled-fields (mapv compile-field fields-data) 208 | data-spec (reify 209 | MapSpec 210 | (-properties [_] props) 211 | (-fields [_] compiled-fields) 212 | Spec 213 | (-read [_ b] 214 | (into {} 215 | (mapcat (fn [[field-name field-props field-spec]] 216 | (if (:inline field-props) 217 | (read field-spec b) 218 | [[field-name (read field-spec b)]]))) 219 | compiled-fields)) 220 | (-write [_ b v] 221 | (run! (fn [[field-name field-props field-spec :as field]] 222 | (try 223 | (if (:inline field-props) 224 | (write field-spec b v) 225 | (write field-spec b (get v field-name))) 226 | (catch Exception e 227 | (throw (ex-info "Failed write" {:field field :v v} e))))) 228 | compiled-fields)))] 229 | (if length-spec 230 | (length-based-frame-spec length-spec data-spec) 231 | data-spec))) 232 | 233 | (defn map-of-spec 234 | [[_ {:keys [length adjust]} k-spec v-spec]] 235 | (let [length-spec (when (not (int? length)) (spec length)) 236 | adjust (or adjust 0) 237 | k-spec (spec k-spec) 238 | v-spec (spec v-spec)] 239 | (if length-spec 240 | (reify 241 | Spec 242 | (-read [_ b] 243 | (let [len (- (read length-spec b) adjust)] 244 | (if (< len 0) 245 | nil 246 | (into {} (repeatedly len #(vector (read k-spec b) (read v-spec b))))))) 247 | (-write [_ b m] 248 | (if (nil? m) 249 | (write length-spec b (dec adjust)) 250 | (do (write length-spec b (+ (count m) adjust)) 251 | (run! (fn [[k v]] 252 | (write k-spec b k) 253 | (write v-spec b v)) 254 | m))))) 255 | (reify 256 | Spec 257 | (-read [_ b] 258 | (let [len length] 259 | (into {} (repeatedly len #(vector (read k-spec b) (read v-spec b)))))) 260 | (-write [_ b m] 261 | (run! (fn [[k v]] 262 | (write k-spec b k) 263 | (write v-spec b v)) 264 | m)))))) 265 | 266 | (defn tuple-spec 267 | [s] 268 | (let [specs (mapv spec (rest s))] 269 | (reify 270 | Spec 271 | (-read [_ b] 272 | (mapv (fn [item-spec] (read item-spec b)) specs)) 273 | (-write [_ b v] 274 | (loop [i 0] 275 | (when (< i (count specs)) 276 | (let [item-spec (nth specs i) item (nth v i)] 277 | (write item-spec b item) 278 | (recur (inc i))))))))) 279 | 280 | (defn fixed-length-vector-spec 281 | [length item-spec] 282 | (let [item-spec (spec item-spec)] 283 | (reify 284 | Spec 285 | (-read [_ b] 286 | (vec (repeatedly length #(read item-spec ^ByteBuf b)))) 287 | (-write [_ b v] 288 | (run! #(write item-spec b %) v))))) 289 | 290 | (defn length-delimited-vector-spec 291 | [delimiter-spec item-spec adjust] 292 | (let [adjust (or adjust 0)] 293 | (reify 294 | Spec 295 | (-read [_ b] 296 | (let [len (- (read delimiter-spec b) adjust)] 297 | (if (< len 0) 298 | nil 299 | (vec (repeatedly len #(read item-spec ^ByteBuf b)))))) 300 | (-write [_ b v] 301 | (if (nil? v) 302 | (write delimiter-spec b (dec adjust)) 303 | (do (write delimiter-spec b (+ (count v) adjust)) 304 | (run! #(write item-spec b %) v))))))) 305 | 306 | (defn vector-spec 307 | [[_ {:keys [length adjust]} item-spec]] 308 | (cond 309 | (int? length) 310 | (fixed-length-vector-spec length item-spec) 311 | 312 | :else 313 | (length-delimited-vector-spec (spec length) (spec item-spec) adjust))) 314 | 315 | (defn- read-bytes 316 | [^ByteBuf b length] 317 | (let [byts (byte-array length)] 318 | (.readBytes ^ByteBuf b byts) 319 | byts)) 320 | 321 | (defn- write-bytes 322 | [^ByteBuf b byts] 323 | (.writeBytes ^ByteBuf b (bytes byts))) 324 | 325 | (defn- read-string 326 | [^ByteBuf b length] 327 | (.readCharSequence ^ByteBuf b length StandardCharsets/UTF_8)) 328 | 329 | (defn- write-string 330 | [^ByteBuf b str] 331 | (.writeBytes ^ByteBuf b (.getBytes ^String str StandardCharsets/UTF_8))) 332 | 333 | (defn- fixed-length-spec 334 | [length read write] 335 | (reify 336 | Spec 337 | (-read [_ b] 338 | (read b length)) 339 | (-write [_ b v] 340 | (write b v)))) 341 | 342 | (defn- fixed-length-bytes-spec 343 | [length] 344 | (fixed-length-spec length read-bytes write-bytes)) 345 | 346 | (defn- fixed-length-string-spec 347 | [length] 348 | (fixed-length-spec length read-string write-string)) 349 | 350 | (defn- length-delimited-bytes-spec 351 | [delimiter-spec adjust] 352 | (let [adjust (or adjust 0)] 353 | (reify 354 | Spec 355 | (-read [_ b] 356 | (let [length (- (read delimiter-spec b) adjust)] 357 | (if (< length 0) 358 | nil 359 | (read-bytes b length)))) 360 | (-write [_ b byts] 361 | (if (nil? byts) 362 | (write delimiter-spec ^ByteBuf b (dec adjust)) 363 | (do (write delimiter-spec ^ByteBuf b (+ (alength (bytes byts)) adjust)) 364 | (write-bytes b byts))))))) 365 | 366 | (defn- length-delimited-string-spec 367 | [delimiter-spec adjust] 368 | (let [adjust (or adjust 0)] 369 | (reify 370 | Spec 371 | (-read [_ b] 372 | (let [length (- (read delimiter-spec b) adjust)] 373 | (if (< length 0) 374 | nil 375 | (read-string b length)))) 376 | (-write [_ b v] 377 | (if (nil? v) 378 | (write delimiter-spec ^ByteBuf b (dec adjust)) 379 | (let [byts (.getBytes ^String v StandardCharsets/UTF_8)] 380 | (write delimiter-spec ^ByteBuf b (+ (alength byts) adjust)) 381 | (write-bytes b byts))))))) 382 | 383 | (defn- length-spec-compiler 384 | [fixed-length-spec-fn length-delimited-spec-fn] 385 | (fn length-delimited-spec 386 | [[_ {:keys [length adjust]}]] 387 | (cond 388 | (int? length) 389 | (fixed-length-spec-fn length) 390 | 391 | :else 392 | (length-delimited-spec-fn (spec length) adjust)))) 393 | 394 | (def string-spec (length-spec-compiler fixed-length-string-spec length-delimited-string-spec)) 395 | 396 | (def bytes-spec (length-spec-compiler fixed-length-bytes-spec length-delimited-bytes-spec)) 397 | 398 | (defn- positions 399 | [pred coll] 400 | (keep-indexed (fn [idx x] 401 | (when (pred x) 402 | idx)) 403 | coll)) 404 | 405 | (defn- multi-spec 406 | [[_ {:keys [dispatch dispatch-fn] :or {dispatch-fn identity}} & children]] 407 | (let [cases (into {} (map (fn [[k v]] [k (spec v)])) children) 408 | first-spec (-> cases vals first) 409 | first-spec-fields (-fields first-spec) 410 | first-spec-props (or (-properties first-spec) {}) 411 | field-pred (if (keyword? dispatch) #{dispatch} (set dispatch)) 412 | last-dispatch-pos (last (positions field-pred (map first first-spec-fields))) 413 | fields-upto-dispatch (subvec first-spec-fields 0 (inc last-dispatch-pos)) 414 | initial-reader (spec (into [:map first-spec-props] fields-upto-dispatch)) 415 | dispatch-f (if (keyword? dispatch) (comp dispatch-fn dispatch) #(dispatch-fn (mapv % dispatch)))] 416 | (reify 417 | Spec 418 | (-read [_ b] 419 | (let [mark (.readerIndex ^ByteBuf b) 420 | initial (read initial-reader b) 421 | _ (.readerIndex ^ByteBuf b mark) 422 | dispatch-val (dispatch-f initial)] 423 | (if-some [matched (get cases dispatch-val)] 424 | (read matched b) 425 | (throw (ex-info "Invalid dispatch value" {:dispatch-options (vals cases) 426 | :dispatch-value dispatch-val}))))) 427 | 428 | (-write [_ b v] 429 | (let [dispatch-val (dispatch-f v)] 430 | (if-some [matched (get cases dispatch-val)] 431 | (write matched b v) 432 | (throw (ex-info "Invalid dispatch value" {:dispatch-options (vals cases) 433 | :dispatch-value dispatch-val})))))))) 434 | 435 | ;; Registry 436 | 437 | (def registry 438 | {:bool bool 439 | :boolean bool 440 | :byte byte 441 | :int16 int16 442 | :short int16 443 | :int24 int24 444 | :int32 int32 445 | :int int32 446 | :int64 int64 447 | :long int64 448 | :double double 449 | :float float 450 | :ubyte ubyte 451 | :uint32 uint32 452 | :uvarint32 uvarint32}) 453 | 454 | (def f-registry 455 | {:map map-spec 456 | :map-of map-of-spec 457 | :vector vector-spec 458 | :tuple tuple-spec 459 | :string string-spec 460 | :bytes bytes-spec 461 | :multi multi-spec}) 462 | 463 | (defn compile-spec-vector 464 | [s] 465 | (let [shape (nth s 0) 466 | f (or (get f-registry shape) (throw (ex-info "Unknown spec" {:input s})))] 467 | (f s))) 468 | 469 | (defn spec [s] 470 | (cond 471 | (satisfies? Spec s) 472 | s 473 | 474 | (keyword? s) 475 | (or (get registry s) (throw (ex-info "Unknown spec" {:input s}))) 476 | 477 | (vector? s) 478 | (compile-spec-vector s) 479 | 480 | :else 481 | (throw (ex-info "Unsupported spec input type" {:input s})))) 482 | 483 | (defn read [s b] 484 | (-read (spec s) b)) 485 | 486 | (defn write [s b v] 487 | (-write (spec s) b v)) 488 | -------------------------------------------------------------------------------- /test/bytegeist/bytegeist_test.clj: -------------------------------------------------------------------------------- 1 | (ns bytegeist.bytegeist-test 2 | (:require [clojure.test :refer [deftest testing is are assert-expr do-report]] 3 | [clojure.walk] 4 | [bytegeist.bytegeist :as g]) 5 | (:import (io.netty.buffer Unpooled) 6 | (clojure.lang ExceptionInfo))) 7 | 8 | (def max-ubyte (-> Byte/MAX_VALUE inc (* 2) dec)) 9 | (def max-int24 8388607) 10 | (def min-int24 -8388608) 11 | (def max-uint (-> Integer/MAX_VALUE inc (* 2) dec)) 12 | (defn max-uvarint [num-bytes] (long (dec (Math/pow 2 (* 7 num-bytes))))) 13 | 14 | (defn write-read [s v] 15 | (let [b (Unpooled/buffer)] 16 | (g/write s b v) 17 | (g/read s b))) 18 | 19 | (defn seq-bytes [v] 20 | (clojure.walk/postwalk (fn [x] (if (bytes? x) (seq x) x)) v)) 21 | 22 | (defn preserved? 23 | [s v] 24 | (= (seq-bytes v) (seq-bytes (write-read s v)))) 25 | 26 | (defmethod assert-expr 'preserved? [msg form] 27 | (let [s (nth form 1) 28 | v (nth form 2)] 29 | `(let [read-back# (write-read ~s ~v) 30 | report-type# (if (= (seq-bytes ~v) (seq-bytes read-back#)) :pass :fail)] 31 | (do-report {:type report-type#, :message ~msg, 32 | :expected '~v, :actual read-back#})))) 33 | 34 | (def message 35 | [:map 36 | [:size :int32]]) 37 | 38 | (def request-header-v0 39 | [:map 40 | [:api-key :int16] 41 | [:api-version :int16] 42 | [:correlation-id :int32]]) 43 | 44 | (def request-header-v1 45 | (-> request-header-v0 46 | (g/add-fields [[:client-id [:string {:length :short}]]]))) 47 | 48 | (deftest fail-unsupported-input 49 | (is (thrown-with-msg? Exception #"Unsupported" (g/spec 123)))) 50 | 51 | (deftest fail-unknown-spec 52 | (is (thrown-with-msg? Exception #"Unknown" (g/spec :not-here)))) 53 | 54 | (deftest spec 55 | (testing "Spec new version from previous" 56 | (let [v0 [:map [:a :int32]] 57 | v1 (g/add-fields v0 [[:b :int16]]) 58 | literal-v1 [:map 59 | [:a :int32] 60 | [:b :int16]]] 61 | (is (= literal-v1 v1))))) 62 | 63 | (deftest primitive-write-read 64 | (testing "boolean" 65 | (are [v] (let [b (Unpooled/buffer 1 1)] 66 | (g/write g/bool b v) 67 | (= v (g/read g/bool b))) 68 | true 69 | false)) 70 | 71 | (testing "signed" 72 | (are [num-bytes s v] (let [b (Unpooled/buffer num-bytes num-bytes)] 73 | (g/write s b v) 74 | (= v (g/read s b))) 75 | 1 g/byte Byte/MAX_VALUE 76 | 1 g/byte Byte/MIN_VALUE 77 | 2 g/int16 Short/MAX_VALUE 78 | 2 g/int16 Short/MIN_VALUE 79 | 3 g/int24 max-int24 80 | 3 g/int24 min-int24 81 | 4 g/int32 Integer/MAX_VALUE 82 | 4 g/int32 Integer/MIN_VALUE 83 | 4 g/float Float/MAX_VALUE 84 | 4 g/float Float/MIN_VALUE 85 | 8 g/int64 Long/MAX_VALUE 86 | 8 g/int64 Long/MIN_VALUE 87 | 8 g/double Double/MIN_VALUE 88 | 8 g/double Double/MAX_VALUE)) 89 | 90 | (testing "unsigned within range" 91 | (are [num-bytes s v] (let [b (Unpooled/buffer num-bytes num-bytes)] 92 | (g/write s b v) 93 | (= v (g/read s b))) 94 | 1 g/ubyte max-ubyte 95 | 4 g/uint32 max-uint 96 | 1 g/uvarint32 0 97 | 1 g/uvarint32 (max-uvarint 1) 98 | 2 g/uvarint32 (max-uvarint 2) 99 | 3 g/uvarint32 (max-uvarint 3) 100 | 4 g/uvarint32 (max-uvarint 4))) 101 | (testing "unsigned wrap" 102 | (are [num-bytes s w r] (let [b (Unpooled/buffer num-bytes num-bytes)] 103 | (g/write s b w) 104 | (= r (g/read s b))) 105 | 1 g/ubyte (inc max-ubyte) 0 106 | 1 g/ubyte -1 max-ubyte 107 | 4 g/uint32 (inc max-uint) 0 108 | 4 g/uint32 -1 max-uint)) 109 | (testing "uvarint out of bounds" 110 | (is (thrown? Exception 111 | (let [b (Unpooled/buffer 1 1)] 112 | (g/write g/uvarint32 b (inc (max-uvarint 1)))))) 113 | (is (thrown? Exception 114 | (let [b (Unpooled/buffer 2 2)] 115 | (g/write g/uvarint32 b (inc (max-uvarint 2)))))) 116 | (is (thrown? Exception 117 | (let [b (Unpooled/buffer 3 3)] 118 | (g/write g/uvarint32 b (inc (max-uvarint 3)))))) 119 | (is (thrown? Exception 120 | (let [b (Unpooled/buffer 4 4)] 121 | (g/write g/uvarint32 b (inc (max-uvarint 4)))))))) 122 | 123 | (deftest string-test 124 | (testing "No offset" 125 | (are [length v] (let [s [:string {:length length}] 126 | b (Unpooled/buffer)] 127 | (g/write s b v) 128 | (= v (g/read s b))) 129 | 0 "" 130 | 3 "abc" 131 | :short nil 132 | :short "" 133 | :short "short-delimited" 134 | :int "int-delimited" 135 | :uvarint32 "" 136 | :uvarint32 "uvarint-delimited")) 137 | 138 | (testing "Adjustment 1" 139 | (are [length v] (let [s [:string {:length length :adjust 1}] 140 | b (Unpooled/buffer)] 141 | (g/write s b v) 142 | (= v (g/read s b))) 143 | :short nil 144 | :short "" 145 | :short "short-delimited" 146 | :int "int-delimited" 147 | :uvarint32 nil 148 | :uvarint32 "" 149 | :uvarint32 "uvarint-delimited"))) 150 | 151 | (deftest bytes-spec 152 | (testing "No adjustment" 153 | (are [length v] (let [s [:bytes {:length length}]] 154 | (preserved? s v)) 155 | 0 (byte-array 0) 156 | 3 (byte-array 3 (byte 1)) 157 | :short nil 158 | :short (byte-array 0) 159 | :short (byte-array 3 (byte 1)) 160 | :uvarint32 (byte-array 0) 161 | :uvarint32 (byte-array (max-uvarint 4) (byte 1)))) 162 | 163 | (testing "Adjustment 1" 164 | (are [length v] (let [s [:bytes {:length length :adjust 1}]] 165 | (preserved? s v)) 166 | :short nil 167 | :short (byte-array 0) 168 | :short (byte-array 3 (byte 1)) 169 | :uvarint32 nil 170 | :uvarint32 (byte-array 0) 171 | :uvarint32 (byte-array (max-uvarint 4) (byte 1))))) 172 | 173 | (deftest tuple-spec 174 | (are [s v] (preserved? s v) 175 | [:tuple :bool] 176 | [true] 177 | 178 | [:tuple :uvarint32 :bool :int24 [:map [:a [:tuple :int32 :uint32]]]] 179 | [(max-uvarint 2) false max-int24 {:a [0 max-uint]}])) 180 | 181 | (deftest vector-spec 182 | (are [s v] (preserved? s v) 183 | [:vector {:length 3} :bool] 184 | [true false true] 185 | 186 | [:vector {:length :short} :bool] 187 | nil 188 | 189 | [:vector {:length :short} :bool] 190 | [] 191 | 192 | [:vector {:length :short} :bool] 193 | [true false true] 194 | 195 | [:vector {:length :uvarint32, :adjust 1} :uvarint32] 196 | nil 197 | 198 | [:vector {:length :uvarint32, :adjust 1} :uvarint32] 199 | [] 200 | 201 | [:vector {:length :uvarint32, :adjust 1} :uvarint32] 202 | [0 (max-uvarint 1) (max-uvarint 2) (max-uvarint 3) (max-uvarint 4)] 203 | 204 | [:vector {:length :uvarint32 205 | :adjust 1} 206 | [:map 207 | [:a :int32] 208 | [:b [:string {:length :uvarint32, :adjust 1}]] 209 | [:c [:vector {:length :uvarint32, :adjust 1} [:tuple :bool :short]]]]] 210 | [{:a 1 :b "test-string" :c [[true 1] [false 2] [true 3]]}])) 211 | 212 | (deftest map-spec 213 | (are [s v] (preserved? s v) 214 | [:map [:a :int32]] 215 | {:a 1} 216 | 217 | [:map 218 | [:a :int32] 219 | [:m [:map [:b :int16]]]] 220 | {:a 1 221 | :m {:b 2}} 222 | 223 | [:map 224 | [:a :int32] 225 | [:b :uint32] 226 | [:m [:map 227 | [:b :uint32] 228 | [:c :int24]]]] 229 | {:a 1 230 | :b 2 231 | :m {:b (max-uvarint 3) 232 | :c max-int24}} 233 | 234 | [:map {:length :int32} 235 | [:a :int32] 236 | [:b :uint32] 237 | [:m [:map 238 | [:b :uint32] 239 | [:c :int24]]]] 240 | {:a 1 241 | :b 2 242 | :m {:b (max-uvarint 3) 243 | :c max-int24}})) 244 | 245 | (deftest multi-spec-single-field 246 | (let [role [:multi {:dispatch :type} 247 | ["student" 248 | [:map 249 | [:type [:string {:length :short}]] 250 | [:grade :short]]] 251 | ["employee" 252 | [:map 253 | [:type [:string {:length :short}]] 254 | [:salary :int]]]]] 255 | (testing "Successful dispatch" 256 | (is (preserved? role {:type "student" 257 | :grade 100})) 258 | (is (preserved? role {:type "employee" 259 | :salary 99999}))) 260 | (testing "Unmatched dispatch throws on write" 261 | (let [b (Unpooled/buffer)] 262 | (is (thrown? ExceptionInfo #"Invalid dispatch" 263 | (g/write role b {:type "lizard"}))))) 264 | (testing "Unmatched dispatch throws on read" 265 | (let [lizard [:map [:type [:string {:length :short}]]] 266 | b (Unpooled/buffer)] 267 | (g/write lizard b {:type "lizard"}) 268 | (is (thrown? ExceptionInfo #"Invalid dispatch" 269 | (g/read role b))))))) 270 | 271 | (deftest multi-spec-multiple-fields 272 | (let [message 273 | [:multi {:dispatch [:type :version]} 274 | [["produce" 1] 275 | [:map 276 | [:type [:string {:length :short}]] 277 | [:version :short] 278 | [:data [:string {:length :int}]]]] 279 | [["produce" 2] 280 | [:map 281 | [:type [:string {:length :short}]] 282 | [:version :short] 283 | [:client-id [:string {:length :uvarint32}]] 284 | [:data [:string {:length :uvarint32}]]]] 285 | [["fetch" 1] 286 | [:map 287 | [:type [:string {:length :short}]] 288 | [:version :short] 289 | [:partitions [:vector {:length :uvarint32} :uvarint32]]]]] 290 | 291 | produce-v1 {:type "produce", :version 1 292 | :data "test data"} 293 | produce-v2 {:type "produce", :version 2 294 | :client-id "test client" 295 | :data "test data"} 296 | fetch-v1 {:type "fetch", :version 1 297 | :partitions [0 1 2]}] 298 | 299 | (is (preserved? message produce-v1)) 300 | (is (preserved? message produce-v2)) 301 | (is (preserved? message fetch-v1)))) 302 | 303 | (deftest multi-spec-fn 304 | (let [message 305 | [:multi {:dispatch [:type :version] 306 | :dispatch-fn (fn [[t v]] [(keyword t) (if (> v 2) :new :old)])} 307 | [[:produce :old] 308 | [:map 309 | [:type [:string {:length :short}]] 310 | [:version :short] 311 | [:data [:string {:length :int}]]]] 312 | [[:produce :new] 313 | [:map 314 | [:type [:string {:length :short}]] 315 | [:version :short] 316 | [:client-id [:string {:length :uvarint32}]] 317 | [:data [:string {:length :uvarint32}]]]] 318 | [[:fetch :old] 319 | [:map 320 | [:type [:string {:length :short}]] 321 | [:version :short] 322 | [:partitions [:vector {:length :uvarint32} :uvarint32]]]]]] 323 | (is (preserved? message {:type "produce", :version 3 324 | :client-id "test client" 325 | :data "test data"})))) 326 | 327 | (deftest map-of-spec 328 | (let [tag :uvarint32 329 | data [:bytes {:length :uvarint32}] 330 | tagged-fields [:map-of {:length :uvarint32} tag data]] 331 | (is (preserved? tagged-fields {0 (.getBytes "hello") 332 | 1 (byte-array 10000)})))) 333 | 334 | (deftest inline 335 | (testing "A map inside another" 336 | (let [message 337 | [:map 338 | [:leaf :int32] 339 | [:nested [:map [:leaf :int32]]] 340 | [:inlined {:inline true} [:map [:inline-leaf :int32]]]]] 341 | (is (preserved? message {:leaf 1 342 | :nested {:leaf 2} 343 | :inline-leaf 3})))) 344 | (testing "Two multi-specs inlined in a map" 345 | (let [message 346 | [:map 347 | [:header {:inline true} 348 | [:multi {:dispatch :header-version} 349 | [0 [:map 350 | [:header-version :short] 351 | [:header-val [:string {:length :short}]]]] 352 | [1 [:map 353 | [:header-version :short] 354 | [:header-val [:string {:length :uvarint32}]] 355 | [:extra-val [:string {:length :uvarint32}]]]]]] 356 | [:data [:string {:length :int32}]] 357 | [:footer {:inline true} 358 | [:multi {:dispatch :footer-version} 359 | [0 [:map 360 | [:footer-version :short] 361 | [:footer-val [:string {:length :short}]]]] 362 | [1 [:map 363 | [:footer-version :short] 364 | [:footer-val :uvarint32]]]]]] 365 | old-header {:header-version 0 366 | :header-val "old"} 367 | new-header {:header-version 1 368 | :header-val "new" 369 | :extra-val "extra"} 370 | data {:data "same"} 371 | old-footer {:footer-version 0 372 | :footer-val "str"} 373 | new-footer {:footer-version 1 374 | :footer-val 123}] 375 | (doall (for [header [old-header new-header] 376 | footer [old-footer new-footer]] 377 | (is (preserved? message (merge header data footer)))))))) 378 | --------------------------------------------------------------------------------