├── .gitignore ├── README.md ├── cljs-src └── msgpack_cljs │ └── core.cljs └── project.clj /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /classes 3 | /checkouts 4 | pom.xml 5 | pom.xml.asc 6 | *.jar 7 | *.class 8 | /.lein-* 9 | /.nrepl-port 10 | .hgignore 11 | .hg/ 12 | 13 | /static 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # msgpack-cljs 2 | 3 | A Clojurescript implementation of msgpack. This library's structure is *heavily* adapted from the great work done on [clojure-msgpack](https://github.com/edma2/clojure-msgpack). Support for Clojure `keyword`, `symbol`, `char`, `ratio`, and `set` are all included by default as Extension types for convenience and to be compatible with `clojure-msgpack`. 4 | 5 | ## Usage 6 | 7 | `pack` takes an arbitrary object and converts it into a `Uint8Array`. 8 | 9 | `unpack` takes a `Uint8Array` and converts it back into an arbitrary object. 10 | 11 | ## Improvements 12 | 13 | This library could stand to be refactored. We just use this internally at [PKC](https://pkc.io), so, if there is demand (or interest) in making this more feature-rich, we are open to making improvements. Just be vocal, otherwise we'll keep it simple. 14 | 15 | ## Copyright 16 | Copyright 2018 PKC 17 | 18 | Licensed under the Apache License, Version 2.0 (the "License"); 19 | you may not use this file except in compliance with the License. 20 | You may obtain a copy of the License at: 21 | 22 | [http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0) 23 | 24 | Unless required by applicable law or agreed to in writing, software 25 | distributed under the License is distributed on an "AS IS" BASIS, 26 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 27 | See the License for the specific language governing permissions and 28 | limitations under the License. 29 | -------------------------------------------------------------------------------- /cljs-src/msgpack_cljs/core.cljs: -------------------------------------------------------------------------------- 1 | (ns msgpack-cljs.core 2 | (:require [goog.array] 3 | [goog.crypt] 4 | [goog.math.Long])) 5 | 6 | (def msgpack-stream-default-size 64) 7 | 8 | (defn string->bytes [s] 9 | (js/Uint8Array. (.stringToUtf8ByteArray goog.crypt s))) 10 | 11 | (defn bytes->string [bs] 12 | (.utf8ByteArrayToString goog.crypt bs)) 13 | 14 | (defn array= [a b] 15 | (.equals goog.array a b)) 16 | 17 | (defrecord Extended [type data]) 18 | 19 | (defprotocol IExtendable 20 | (extension [this])) 21 | 22 | (defprotocol IStream 23 | (inc-offset! [this new-offset]) 24 | (resize-on-demand! [this n]) 25 | (stream->uint8array [this])) 26 | 27 | (defprotocol IInputStream 28 | (write [this buffer]) 29 | (write-u8 [this u8]) 30 | (write-i8 [this i8]) 31 | (write-u16 [this u16]) 32 | (write-i16 [this i16]) 33 | (write-u32 [this u32]) 34 | (write-i32 [this i32]) 35 | (write-i64 [this i64]) 36 | (write-f64 [this f64])) 37 | 38 | (deftype MsgpackInputStream [^:unsynchronized-mutable bytes ^:unsynchronized-mutable offset] 39 | IStream 40 | (resize-on-demand! [this n] 41 | (let [base (+ offset n)] 42 | (when (> base (.-byteLength bytes)) 43 | (let [new-bytes (js/Uint8Array. (bit-or 0 (* 1.5 base))) 44 | old-bytes (js/Uint8Array. (.-buffer bytes))] 45 | (set! bytes (js/DataView. (.-buffer new-bytes))) 46 | (.set new-bytes old-bytes 0))))) 47 | (inc-offset! [this n] (set! offset (+ offset n))) 48 | (stream->uint8array [this] 49 | (js/Uint8Array. (.-buffer bytes) 0 offset)) 50 | 51 | IInputStream 52 | (write [this buffer] 53 | (resize-on-demand! this (.-byteLength buffer)) 54 | (.set (js/Uint8Array. (.-buffer bytes)) buffer offset) 55 | (inc-offset! this (.-byteLength buffer))) 56 | (write-u8 [this u8] 57 | (resize-on-demand! this 1) 58 | (.setUint8 bytes offset u8 false) 59 | (inc-offset! this 1)) 60 | (write-i8 [this i8] 61 | (resize-on-demand! this 1) 62 | (.setInt8 bytes offset i8 false) 63 | (inc-offset! this 1)) 64 | (write-u16 [this u16] 65 | (resize-on-demand! this 2) 66 | (.setUint16 bytes offset u16 false) 67 | (inc-offset! this 2)) 68 | (write-i16 [this i16] 69 | (resize-on-demand! this 2) 70 | (.setInt16 bytes offset i16 false) 71 | (inc-offset! this 2)) 72 | (write-u32 [this u32] 73 | (resize-on-demand! this 4) 74 | (.setUint32 bytes offset u32 false) 75 | (inc-offset! this 4)) 76 | (write-i32 [this i32] 77 | (resize-on-demand! this 4) 78 | (.setInt32 bytes offset i32 false) 79 | (inc-offset! this 4)) 80 | 81 | ; msgpack stores integers in big-endian 82 | (write-i64 [this u64] 83 | (let [glong (goog.math.Long/fromNumber u64)] 84 | (write-i32 this (.getHighBits glong)) 85 | (write-i32 this (.getLowBits glong)))) 86 | 87 | (write-f64 [this f64] 88 | (resize-on-demand! this 8) 89 | (.setFloat64 bytes offset f64 false) 90 | (inc-offset! this 8))) 91 | 92 | (defn input-stream [] 93 | (MsgpackInputStream. (js/DataView. (js/ArrayBuffer. msgpack-stream-default-size)) 0)) 94 | 95 | (defn pack-bytes [stream bytes] 96 | (let [n (.-byteLength bytes)] 97 | (cond 98 | (<= n 0xff) (doto stream (write-u8 0xc4) (write-u8 n) (write bytes)) 99 | (<= n 0xffff) (doto stream (write-u8 0xc5) (write-u16 n) (write bytes)) 100 | (<= n 0xffffffff) (doto stream (write-u8 0xc6) (write-u32 n) (write bytes)) 101 | :else (throw (js/Error. "bytes too large to pack"))))) 102 | 103 | ; we will support doubles only 104 | (defn pack-float [stream f] 105 | (doto stream (write-u8 0xcb) (write-f64 f))) 106 | 107 | (defn pack-int [stream i] 108 | (cond 109 | ; +fixnum 110 | (<= 0 i 127) (write-u8 stream i) 111 | ; -fixnum 112 | (<= -32 i -1) (write-i8 stream i) 113 | 114 | ; uint 8 115 | (<= 0 i 0xff) (doto stream (write-u8 0xcc) (write-u8 i)) 116 | ; uint 16 117 | (<= 0 i 0xffff) (doto stream (write-u8 0xcd) (write-u16 i)) 118 | ; uint 32 119 | (<= 0 i 0xffffffff) (doto stream (write-u8 0xce) (write-u32 i)) 120 | ; uint 64 121 | (<= 0 i 0xffffffffffffffff) (doto stream (write-u8 0xcf) (write-i64 i)) 122 | 123 | ; int 8 124 | (<= -0x80 i -1) (doto stream (write-u8 0xd0) (write-i8 i)) 125 | ; int 16 126 | (<= -0x8000 i -1) (doto stream (write-u8 0xd1) (write-i16 i)) 127 | ; int 32 128 | (<= -0x80000000 i -1) (doto stream (write-u8 0xd2) (write-i32 i)) 129 | ; int 64 130 | (<= -0x8000000000000000 i -1) (doto stream (write-u8 0xd3) (write-i64 i)) 131 | 132 | :else (throw (js/Error. (str "Integer value out of bounds: " i))))) 133 | 134 | (defn pack-number [stream n] 135 | (if-not (integer? n) 136 | (pack-float stream n) 137 | (pack-int stream n))) 138 | 139 | (defn pack-string [stream s] 140 | (let [bytes (string->bytes s) 141 | len (.-byteLength bytes)] 142 | (cond 143 | (<= len 0x1f) (doto stream (write-u8 (bit-or 2r10100000 len)) (write bytes)) 144 | (<= len 0xff) (doto stream (write-u8 0xd9) (write-u8 len) (write bytes)) 145 | (<= len 0xffff) (doto stream (write-u8 0xda) (write-u16 len) (write bytes)) 146 | (<= len 0xffffffff) (doto stream (write-u8 0xdb) (write-u32 len) (write bytes)) 147 | :else (throw (js/Error. "string too large to pack"))))) 148 | 149 | (declare pack) 150 | 151 | (defprotocol Packable 152 | "Objects that can be serialized as MessagePack types" 153 | (pack-stream [this stream])) 154 | 155 | (defn pack-coll [stream coll] 156 | (doseq [x coll] 157 | (pack-stream x stream))) 158 | 159 | (extend-protocol IExtendable 160 | PersistentHashSet 161 | (extension [this] 162 | (Extended. 0x07 (pack (vec this)))) 163 | Keyword 164 | (extension [this] 165 | (Extended. 0x03 (pack (.substring (str this) 1) (input-stream)))) 166 | cljs.core.Symbol 167 | (extension [this] 168 | (Extended. 0x04 (pack (str this))))) 169 | 170 | (defn pack-extended [s {:keys [type data]}] 171 | (let [len (.-byteLength data)] 172 | (case len 173 | 1 (write-u8 s 0xd4) 174 | 2 (write-u8 s 0xd5) 175 | 4 (write-u8 s 0xd6) 176 | 8 (write-u8 s 0xd7) 177 | 16 (write-u8 s 0xd8) 178 | (cond 179 | (<= len 0xff) (doto s (write-u8 0xc7) (write-u8 len)) 180 | (<= len 0xffff) (doto s (write-u8 0xc8) (write-u16 len)) 181 | (<= len 0xffffffff) (doto s (write-u8 0xc9) (write-u32 len)) 182 | :else (throw (js/Error. "extended type too large to pack")))) 183 | (write-u8 s type) 184 | (write s data))) 185 | 186 | (defn pack-seq [s seq] 187 | (let [len (count seq)] 188 | (cond 189 | (<= len 0xf) (doto s (write-u8 (bit-or 2r10010000 len)) (pack-coll seq)) 190 | (<= len 0xffff) (doto s (write-u8 0xdc) (write-u16 len) (pack-coll seq)) 191 | (<= len 0xffffffff) (doto s (write-u8 0xdd) (write-u32 len) (pack-coll seq)) 192 | :else (throw (js/Error. "seq type too large to pack"))))) 193 | 194 | (defn pack-map [s map] 195 | (let [len (count map) 196 | pairs (interleave (keys map) (vals map))] 197 | (cond 198 | (<= len 0xf) (doto s (write-u8 (bit-or 2r10000000 len)) (pack-coll pairs)) 199 | (<= len 0xffff) (doto s (write-u8 0xde) (write-u16 len) (pack-coll pairs)) 200 | (<= len 0xffffffff) (doto s (write-u8 0xdf) (write-u32 len) (pack-coll pairs)) 201 | :else (throw (js/Error. "map type too large to pack"))))) 202 | 203 | (extend-protocol Packable 204 | nil 205 | (pack-stream [_ s] (write-u8 s 0xc0)) 206 | 207 | boolean 208 | (pack-stream [bool s] (write-u8 s (if bool 0xc3 0xc2))) 209 | 210 | number 211 | (pack-stream [n s] (pack-number s n)) 212 | 213 | string 214 | (pack-stream [str s] (pack-string s str)) 215 | 216 | Extended 217 | (pack-stream [ext s] (pack-extended s ext)) 218 | 219 | PersistentVector 220 | (pack-stream [seq s] (pack-seq s seq)) 221 | 222 | EmptyList 223 | (pack-stream [seq s] (pack-seq s seq)) 224 | 225 | List 226 | (pack-stream [seq s] (pack-seq s seq)) 227 | 228 | LazySeq 229 | (pack-stream [seq s] (pack-seq s (vec seq))) 230 | 231 | js/Uint8Array 232 | (pack-stream [u8 s] (pack-bytes s u8)) 233 | 234 | PersistentArrayMap 235 | (pack-stream [array-map s] (pack-map s array-map)) 236 | 237 | PersistentHashMap 238 | (pack-stream [hmap s] (pack-map s hmap)) 239 | 240 | PersistentHashSet 241 | (pack-stream [hset s] (pack-stream (extension hset) s)) 242 | 243 | Keyword 244 | (pack-stream [kw s] (pack-stream (extension kw) s)) 245 | 246 | Symbol 247 | (pack-stream [sym s] (pack-stream (extension sym) s))) 248 | 249 | (declare unpack-stream) 250 | 251 | (defprotocol IOutputStream 252 | (read [this n]) 253 | (read-bytes [this n]) 254 | (read-u8 [this]) 255 | (read-i8 [this]) 256 | (read-u16 [this]) 257 | (read-i16 [this]) 258 | (read-u32 [this]) 259 | (read-i32 [this]) 260 | (read-i64 [this]) 261 | (read-f32 [this]) 262 | (read-f64 [this])) 263 | 264 | (deftype MsgpackOutputStream [bytes ^:unsynchronized-mutable offset] 265 | IStream 266 | (inc-offset! [this n] (set! offset (+ offset n))) 267 | (resize-on-demand! [this n] nil) 268 | (stream->uint8array [this] 269 | (js/Uint8Array. (.-buffer bytes))) 270 | 271 | IOutputStream 272 | (read [this n] 273 | (let [old-offset offset] 274 | (inc-offset! this n) 275 | (.slice (.-buffer bytes) old-offset offset))) 276 | (read-bytes [this n] 277 | (js/Uint8Array. (read this n))) 278 | (read-u8 [this] 279 | (let [u8 (.getUint8 bytes offset)] 280 | (inc-offset! this 1) 281 | u8)) 282 | (read-i8 [this] 283 | (let [i8 (.getInt8 bytes offset)] 284 | (inc-offset! this 1) 285 | i8)) 286 | (read-u16 [this] 287 | (let [u16 (.getUint16 bytes offset)] 288 | (inc-offset! this 2) 289 | u16)) 290 | (read-i16 [this] 291 | (let [i16 (.getInt16 bytes offset false)] 292 | (inc-offset! this 2) 293 | i16)) 294 | (read-u32 [this] 295 | (let [u32 (.getUint32 bytes offset false)] 296 | (inc-offset! this 4) 297 | u32)) 298 | (read-i32 [this] 299 | (let [i32 (.getInt32 bytes offset false)] 300 | (inc-offset! this 4) 301 | i32)) 302 | (read-i64 [this] 303 | (let [high-bits (.getInt32 bytes offset false) 304 | low-bits (.getInt32 bytes (+ offset 4) false)] 305 | (inc-offset! this 8) 306 | (.toNumber (goog.math.Long. low-bits high-bits)))) 307 | (read-f32 [this] 308 | (let [f32 (.getFloat32 bytes offset false)] 309 | (inc-offset! this 4) 310 | f32)) 311 | (read-f64 [this] 312 | (let [f64 (.getFloat64 bytes offset false)] 313 | (inc-offset! this 8) 314 | f64))) 315 | 316 | (defn output-stream [bytes] 317 | (MsgpackOutputStream. (js/DataView. bytes) 0)) 318 | 319 | (defn read-str [stream n] 320 | (bytes->string (read-bytes stream n))) 321 | 322 | (defn unpack-n [stream n] 323 | (let [v (transient [])] 324 | (dotimes [_ n] 325 | (conj! v (unpack-stream stream))) 326 | (persistent! v))) 327 | 328 | (defn unpack-map [stream n] 329 | (apply hash-map (unpack-n stream (* 2 n)))) 330 | 331 | (declare unpack-ext) 332 | 333 | (defn unpack-stream [stream] 334 | (let [byte (read-u8 stream)] 335 | (case byte 336 | 0xc0 nil 337 | 0xc2 false 338 | 0xc3 true 339 | 0xc4 (read-bytes stream (read-u8 stream)) 340 | 0xc5 (read-bytes stream (read-u16 stream)) 341 | 0xc6 (read-bytes stream (read-u32 stream)) 342 | 0xc7 (unpack-ext stream (read-u8 stream)) 343 | 0xc8 (unpack-ext stream (read-u16 stream)) 344 | 0xc9 (unpack-ext stream (read-u32 stream)) 345 | 0xca (read-f32 stream) 346 | 0xcb (read-f64 stream) 347 | 0xcc (read-u8 stream) 348 | 0xcd (read-u16 stream) 349 | 0xce (read-u32 stream) 350 | 0xcf (read-i64 stream) 351 | 0xd0 (read-i8 stream) 352 | 0xd1 (read-i16 stream) 353 | 0xd2 (read-i32 stream) 354 | 0xd3 (read-i64 stream) 355 | 0xd4 (unpack-ext stream 1) 356 | 0xd5 (unpack-ext stream 2) 357 | 0xd6 (unpack-ext stream 4) 358 | 0xd7 (unpack-ext stream 8) 359 | 0xd8 (unpack-ext stream 16) 360 | 0xd9 (read-str stream (read-u8 stream)) 361 | 0xda (read-str stream (read-u16 stream)) 362 | 0xdb (read-str stream (read-u32 stream)) 363 | 0xdc (unpack-n stream (read-u16 stream)) 364 | 0xdd (unpack-n stream (read-u32 stream)) 365 | 0xde (unpack-map stream (read-u16 stream)) 366 | 0xdf (unpack-map stream (read-u32 stream)) 367 | (cond 368 | (= (bit-and 2r11100000 byte) 2r11100000) byte 369 | (= (bit-and 2r10000000 byte) 0) byte 370 | (= (bit-and 2r11100000 byte) 2r10100000) (read-str stream (bit-and 2r11111 byte)) 371 | (= (bit-and 2r11110000 byte) 2r10010000) (unpack-n stream (bit-and 2r1111 byte)) 372 | (= (bit-and 2r11110000 byte) 2r10000000) (unpack-map stream (bit-and 2r1111 byte)) 373 | :else (throw (js/Error. "invalid msgpack stream")))))) 374 | 375 | (defn keyword-deserializer [bytes] 376 | (keyword 377 | (unpack-stream 378 | (output-stream bytes)))) 379 | 380 | (defn symbol-deserializer [bytes] 381 | (symbol 382 | (unpack-stream 383 | (output-stream bytes)))) 384 | 385 | (defn char-deserializer [bytes] 386 | (unpack-stream 387 | (output-stream bytes))) 388 | 389 | (defn ratio-deserializer [bytes] 390 | (let [[n d] (unpack-stream (output-stream bytes))] 391 | (/ n d))) 392 | 393 | (defn set-deserializer [bytes] 394 | (set (unpack-stream (output-stream bytes)))) 395 | 396 | (defn unpack-ext [stream n] 397 | (let [type (read-u8 stream)] 398 | (case type 399 | 3 (keyword-deserializer (read stream n)) 400 | 4 (symbol-deserializer (read stream n)) 401 | 5 (char-deserializer (read stream n)) 402 | 6 (ratio-deserializer (read stream n)) 403 | 7 (set-deserializer (read stream n))))) 404 | 405 | (defn unpack [bytes] 406 | (unpack-stream 407 | (output-stream (.-buffer bytes)))) 408 | 409 | (defn pack [obj] 410 | (let [stream (input-stream)] 411 | (pack-stream obj stream) 412 | (stream->uint8array stream))) 413 | -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject msgpack-cljs "0.1.0-SNAPSHOT" 2 | :description "A ClojureScript library for serializing and deserializing to msgpack." 3 | :plugins [[lein-cljsbuild "LATEST"]] 4 | :license {:name "APACHE LICENSE, VERSION 2.0 (CURRENT)" 5 | :url "https://www.apache.org/licenses/LICENSE-2.0"} 6 | :dependencies [[org.clojure/clojure "LATEST"] 7 | [org.clojure/clojurescript "LATEST"]] 8 | :cljsbuild {:builds 9 | [{:id "dev" 10 | :source-paths ["cljs-src"] 11 | :compiler 12 | {:output-to "static/development/index.js" 13 | :source-map true 14 | :output-dir "static/development/js" 15 | :optimizations :none 16 | :main msgpack-cljs.core 17 | :asset-path "/development/js" 18 | :cache-analysis true 19 | :pretty-print true}} 20 | {:id "release" 21 | :source-paths ["cljs-src"] 22 | :compiler 23 | {:output-to "static/release/index.js" 24 | :source-map "static/release/index.js.map" 25 | :main msgpack-cljs.core 26 | :output-dir "static/release/js" 27 | :optimizations :advanced 28 | :pseudo-names false}}]}) 29 | --------------------------------------------------------------------------------