├── README.md ├── project.clj ├── src └── byte_spec.clj └── test └── bytes_test.clj /README.md: -------------------------------------------------------------------------------- 1 | byte-spec 2 | ============== 3 | 4 | #### A declarative DSL for reading and writing binary file formats 5 | 6 | This library implements a small DSL that is used to describe binary 7 | serialization formats. It was written to support reading and writing 8 | SuperCollider synthesizer definition files for Project Overtone, but it can be 9 | used for many binary file formats, network packets, etc... 10 | 11 | First you describe your binary format like this: 12 | 13 | (defspec basic-spec 14 | :a :int8 15 | :b :int16 16 | :c :int32 17 | :d :float32 18 | :e :float64 19 | :f :string) 20 | 21 | ;; An object to serialize 22 | (def foo {:a 10 :b 20 :c 40 :d 23.2 :e 23.2 :f "asddf"}) 23 | 24 | ;; And serialize it to a byte array like this: 25 | (spec-write-bytes basic-spec foo) ;; => [...bytes...] 26 | 27 | ;; reading in a byte array with the basic-spec format works like this: 28 | (spec-read-bytes basic-spec bytes) 29 | 30 | There is also support for arrays of the basic types and arrays of nested spec 31 | types. Take a look at the tests for some more examples, and of course browsing 32 | the source is always a good idea. Not a lot of work has gone into making this 33 | especially general purpose beyond what we've needed for Overtone, but if people 34 | are interested in using it I'm happy to make it more useful for other projects 35 | as well. 36 | 37 | For a more full example of usage checkout overtone.core.synthdef in the project 38 | Overtone source. Feel free to email me with questions, comments, or patches. 39 | 40 | ### Project Info: 41 | 42 | Include in your project.clj like so: 43 | 44 | [overtone/byte-spec "0.0.2-SNAPSHOT"] 45 | 46 | #### Source Repository 47 | Downloads and the source repository can be found on GitHub: 48 | 49 | http://github.com/rosejn/byte-spec 50 | 51 | Eventually there will be more documentation for this library, but in the 52 | meantime you can see it in use within the context of Project Overtone, located 53 | here: 54 | 55 | http://github.com/rosejn/overtone 56 | 57 | #### Mailing List 58 | 59 | For any questions, comments or patches, use the Overtone google group here: 60 | 61 | http://groups.google.com/group/overtone 62 | 63 | ### Authors 64 | 65 | * Jeff Rose 66 | -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject overtone/byte-spec "0.2.0-SNAPSHOT" 2 | :description "A DSL for reading and writing binary file formats in Clojure." 3 | :dependencies [[org.clojure/clojure "1.2.0"] 4 | [org.clojure/clojure-contrib "1.2.0"]]) 5 | -------------------------------------------------------------------------------- /src/byte_spec.clj: -------------------------------------------------------------------------------- 1 | (ns byte-spec 2 | (:import (java.net URL) 3 | (java.io FileInputStream FileOutputStream 4 | DataInputStream DataOutputStream 5 | BufferedInputStream BufferedOutputStream 6 | ByteArrayOutputStream ByteArrayInputStream))) 7 | 8 | ; This file implements a DSL for specifying the layout of binary data formats. 9 | ; Look at synthdef.clj that defines the format for SuperCollider 10 | ; synthesizer definition (.scsyndef) files for an example of usage. 11 | 12 | (def *spec-out* nil) 13 | (def *spec-in* nil) 14 | 15 | (defn- bytes-to-int [bytes] 16 | (-> bytes (ByteArrayInputStream.) (DataInputStream.) (.readInt))) 17 | 18 | (defn- read-pstring [] 19 | (let [len (.readByte *spec-in*) 20 | bytes (byte-array len)] 21 | (.readFully *spec-in* bytes) 22 | (String. bytes))) 23 | 24 | (defn- write-pstring [s] 25 | (.writeByte *spec-out* (count s)) 26 | (.write *spec-out* (.getBytes s))) 27 | 28 | ;; Standard numeric types + Pascal style strings. 29 | ;; pstring => a byte giving the string length followed by the ascii bytes 30 | (def READERS { 31 | :int8 #(.readByte *spec-in*) 32 | :int16 #(.readShort *spec-in*) 33 | :int32 #(.readInt *spec-in*) 34 | :int64 #(.readLong *spec-in*) 35 | :float32 #(.readFloat *spec-in*) 36 | :float64 #(.readDouble *spec-in*) 37 | 38 | :byte #(.readByte *spec-in*) 39 | :short #(.readShort *spec-in*) 40 | :int #(.readInt *spec-in*) 41 | :long #(.readLong *spec-in*) 42 | :float #(.readFloat *spec-in*) 43 | :double #(.readDouble *spec-in*) 44 | 45 | :string read-pstring 46 | }) 47 | 48 | (def WRITERS { 49 | :int8 #(.writeByte *spec-out* %1) 50 | :int16 #(.writeShort *spec-out* %1) 51 | :int32 #(.writeInt *spec-out* %1) 52 | :int64 #(.writeLong *spec-out* %1) 53 | :float32 #(.writeFloat *spec-out* %1) 54 | :float64 #(.writeDouble *spec-out* %1) 55 | 56 | :byte #(.writeByte *spec-out* %1) 57 | :short #(.writeShort *spec-out* %1) 58 | :int #(.writeInt *spec-out* %1) 59 | :long #(.writeLong *spec-out* %1) 60 | :float #(.writeFloat *spec-out* %1) 61 | :double #(.writeDouble *spec-out* %1) 62 | 63 | :string write-pstring 64 | }) 65 | 66 | ; TODO: Make this complete 67 | ; For now it just does enough to handle SuperCollider oddity 68 | (defn- coerce-default [value ftype] 69 | (if (and (string? value) (= :int32 ftype)) 70 | (bytes-to-int (.getBytes value)) 71 | value)) 72 | 73 | (defn make-spec [spec-name field-specs] 74 | (loop [specs field-specs 75 | fields []] 76 | (if specs 77 | (let [fname (first specs) 78 | ftype (second specs) 79 | fdefault (if (and (> (count specs) 2) 80 | (not (keyword? (nth specs 2)))) 81 | (nth specs 2) 82 | nil) 83 | fdefault (coerce-default fdefault ftype) 84 | spec {:fname fname 85 | :ftype ftype 86 | :fdefault fdefault} 87 | specs (if (nil? fdefault) 88 | (next (next specs)) 89 | (next (next (next specs)))) 90 | fields (conj fields spec)] 91 | ;(println (str "field: " spec)) 92 | (recur specs fields)) 93 | {:name (str spec-name) 94 | :specs fields}))) 95 | 96 | ;; A spec is just a hash-map containing a named vector of field specs 97 | (defmacro defspec [spec-name & field-specs] 98 | `(def ~spec-name (make-spec ~(str spec-name) [~@field-specs]))) 99 | 100 | (defn spec [s & data] 101 | (let [field-names (map #(:fname %1) (:specs s))] 102 | (apply hash-map (interleave field-names data)))) 103 | 104 | (declare spec-read) 105 | 106 | (defn- spec-read-array [spec size] 107 | ;(println (str "[" (if (map? spec) (:name spec) spec) "] size = " size)) 108 | (loop [i size 109 | ary []] 110 | (if (pos? i) 111 | (let [next-val (if (contains? READERS spec) 112 | ((spec READERS)) 113 | (spec-read spec))] 114 | (recur (dec i) (conj ary next-val))) 115 | ary))) 116 | 117 | (defn spec-read 118 | "Returns an instantiation of the provided spec, with data read from 119 | a DataInputStream bound to *spec-in*." 120 | [spec] 121 | (loop [specs (:specs spec) 122 | data {}] 123 | ;(println (str "spec-read - " (:name spec))) 124 | (if specs 125 | (let [{:keys [fname ftype fdefault]} (first specs) 126 | ;_ (println (str ftype ": " fname " default: -" fdefault "-" )) 127 | fval (cond 128 | ; basic type 129 | (contains? READERS ftype) ((ftype READERS)) 130 | 131 | ; sub-spec 132 | (map? ftype) (spec-read ftype) 133 | 134 | ; array 135 | (vector? ftype) (spec-read-array (first ftype) 136 | ((keyword (str "n-" (name fname))) data)))] 137 | ;(println (str ftype ": " fname " <- " (if (vector? fval) (str "[" (:name (first ftype)) "]") fval))) 138 | (recur (next specs) (assoc data fname fval))) 139 | data))) 140 | 141 | (defn spec-read-bytes [spec bytes] 142 | (binding [*spec-in* (-> bytes (ByteArrayInputStream.) (BufferedInputStream.) (DataInputStream.))] 143 | (spec-read spec))) 144 | 145 | (defn spec-read-url [spec url] 146 | (with-open [ins (.openStream url)] 147 | (binding [*spec-in* (-> ins (BufferedInputStream.) (DataInputStream.))] 148 | (spec-read spec)))) 149 | 150 | (declare spec-write) 151 | 152 | (defn- spec-write-array [spec ary] 153 | ;(println (str "WRITE-A: [ " 154 | ; (if (map? spec) (:name spec) spec) " ](" (count ary) ")")) 155 | ;(println "ary: " ary) 156 | (let [nxt-writer (cond 157 | (contains? WRITERS spec) (spec WRITERS) 158 | (map? spec) (partial spec-write spec) 159 | true (throw (IllegalArgumentException. 160 | (str "Invalid spec: " spec))))] 161 | (doseq [item ary] 162 | (nxt-writer item)))) 163 | 164 | (defn spec-write-basic [ftype fname fval fdefault] 165 | (if-let [val (or fval fdefault)] 166 | ((ftype WRITERS) val) 167 | (throw (Exception. (str "No value was given for '" fname "' field and it has no default."))))) 168 | 169 | (defn count-for [fname] 170 | (keyword (.substring (name fname) 2))) 171 | 172 | (defn spec-write 173 | "Serializes the data according to spec, writing bytes onto *spec-out*." 174 | [spec data] 175 | (doseq [{:keys [fname ftype fdefault]} (:specs spec)] 176 | (cond 177 | ; count of another field starting with n- 178 | (.startsWith (name fname) "n-") 179 | (let [wrt (ftype WRITERS) 180 | c-field (get data (count-for fname)) 181 | cnt (count c-field)] 182 | (wrt cnt)) 183 | 184 | ; an array of sub-specs 185 | (vector? ftype) (spec-write-array (first ftype) (fname data)) 186 | 187 | ; a single sub-spec 188 | (map? ftype) (spec-write ftype (fname data)) 189 | 190 | ; a basic type 191 | (contains? WRITERS ftype) (spec-write-basic ftype fname (fname data) fdefault) 192 | 193 | true (throw (IllegalArgumentException. 194 | (str "Invalid field spec: " fname " " ftype)))))) 195 | 196 | (defn spec-write-file [spec data path] 197 | (with-open [spec-out (-> path (FileOutputStream.) (BufferedOutputStream.) (DataOutputStream.))] 198 | (binding [*spec-out* spec-out] 199 | (spec-write spec data)))) 200 | 201 | (defn spec-write-bytes [spec data] 202 | (let [bos (ByteArrayOutputStream.)] 203 | (with-open [out (DataOutputStream. bos)] 204 | (binding [*spec-out* out] 205 | (spec-write spec data))) 206 | (.toByteArray bos))) 207 | 208 | (defn bytes-and-back [spec obj] 209 | (spec-read-bytes spec (spec-write-bytes spec obj))) 210 | 211 | -------------------------------------------------------------------------------- /test/bytes_test.clj: -------------------------------------------------------------------------------- 1 | (ns bytes-test 2 | (:use 3 | clojure.test 4 | byte-spec)) 5 | 6 | (defspec basic-type-spec 7 | :a :int8 8 | :b :int16 9 | :c :int32 10 | :d :float32 11 | :e :float64 12 | :f :string) 13 | 14 | (deftest basic-type-test [] 15 | (let [a {:a 1 :b 2 :c 3 :d 4 :e 5 :f "six"} 16 | b (bytes-and-back basic-type-spec a) 17 | c (spec basic-type-spec 1 2 3 4 5 "six")] 18 | (is (= a b)) 19 | (is (= a c)) 20 | (is (= b c)) 21 | (is (thrown? Exception (spec-write-bytes basic-type-spec {:a 123}))))) 22 | 23 | (defspec array-spec 24 | :n-a :int8 25 | :a [:int16] 26 | :n-b :int32 27 | :b [:float32] 28 | :n-c :int64 29 | :c [:string]) 30 | 31 | (defn- floatify 32 | "Convert all numbers in col to floats." 33 | [col] 34 | (map #(if (number? %1) (float %1) %1) col)) 35 | 36 | (deftest array-test [] 37 | (let [a {:n-a (byte 4) 38 | :a [1 2 3 4] 39 | :n-b (int 6) 40 | :b (floatify [3.23 4.3223 53.32 253.2 53.2 656.5]) 41 | :n-c (long 3) 42 | :c ["foo" "bar" "baz"]} 43 | b (bytes-and-back array-spec a)] 44 | (is (= a b)) 45 | (is (= 4 (:n-a b))) 46 | (is (= 6 (:n-b b))) 47 | (is (= 3 (:n-c b))))) 48 | 49 | (defspec rhythm-spec 50 | :name :string 51 | :length :int16 52 | :n-triggers :int32 53 | :triggers [:int8]) 54 | 55 | (defspec melody-spec 56 | :name :string 57 | :n-notes :int32 58 | :notes [:int16]) 59 | 60 | (defspec song-spec 61 | :name :string 62 | :bpm :int8 63 | :rhythm rhythm-spec 64 | :melody melody-spec) 65 | 66 | (deftest nested-spec-test [] 67 | (let [r (spec rhythm-spec "test rhythm" 100 (short 5) [1 2 3 4 5]) 68 | m (spec melody-spec "test melody" (int 12) [2 3 4 54 23 43 98 23 98 54 87 23]) 69 | s (spec song-spec "test song" 234 r m) 70 | s2 (bytes-and-back song-spec s) 71 | m2 (:melody s2) 72 | r2 (:rhythm s2)] 73 | (is (= 5 (:n-triggers r2))) 74 | (is (= 12 (:n-notes m2))) 75 | (is (= r2)) 76 | (is (= m2)) 77 | (is (= s s)))) 78 | 79 | (defn bytes-tests [] 80 | (binding [*test-out* *out*] 81 | (run-tests 'bytes-test))) 82 | 83 | --------------------------------------------------------------------------------