├── .luacheckrc ├── LICENSE.txt ├── Makefile ├── README.md ├── luajit-msgpack-pure.lua └── tests ├── bench.lua ├── cases.mpac ├── cases_compact.mpac ├── pr-11.lua └── test.lua /.luacheckrc: -------------------------------------------------------------------------------- 1 | std = "luajit" 2 | 3 | ignore = { 4 | -- unused argument self 5 | "212/self", 6 | } 7 | 8 | exclude_files = {"tests/pr-11.lua"} 9 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (C) 2011-2019 by Pierre Chapuis 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PREFIX=/usr/local 2 | LMODNAME=luajit-msgpack-pure 3 | 4 | LUA=luajit 5 | LMODFILE=$(LMODNAME).lua 6 | 7 | ABIVER=5.1 8 | INSTALL_SHARE=$(PREFIX)/share 9 | INSTALL_LMOD=$(INSTALL_SHARE)/lua/$(ABIVER) 10 | 11 | BENCH_NLOOP=50000 12 | 13 | all: 14 | @echo "This is a pure module. Nothing to make :)" 15 | 16 | test: 17 | $(LUA) tests/test.lua 18 | 19 | bench: 20 | $(LUA) tests/bench.lua $(BENCH_NLOOP) 21 | 22 | install: 23 | install -m0644 $(LMODFILE) $(INSTALL_LMOD)/$(LMODFILE) 24 | 25 | uninstall: 26 | rm -f $(INSTALL_LMOD)/$(LMODFILE) 27 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # luajit-msgpack-pure 2 | 3 | ## Warning 4 | 5 | This library implements MessagePack [spec v4](https://github.com/msgpack/msgpack/blob/814275fb2760d794f9f899b0a5b85db0f80c3332/spec-old.md) with some extensions compatible with [msgpack-js](https://github.com/creationix/msgpack-js). *It does not implement [spec v5](https://github.com/msgpack/msgpack/blob/814275fb2760d794f9f899b0a5b85db0f80c3332/spec.md).* If you need support for spec v5 (including STR8, BIN types, etc), see [alternatives](#alternatives). 6 | 7 | ## Presentation 8 | 9 | This is yet another implementation of MessagePack for LuaJIT. 10 | However, unlike [luajit-msgpack](https://github.com/catwell/cw-lua/tree/master/luajit-msgpack), 11 | luajit-msgpack-pure does not depend on the MessagePack C library. 12 | Everything is re-implemented in LuaJIT code (using the FFI but only to 13 | manipulate data structures). 14 | 15 | ## Alternatives 16 | 17 | If I had to pick a single MessagePack implementation, it would be [lua-MessagePack](https://fperrad.frama.io/lua-MessagePack/). It is pure Lua, its performance is very close to luajit-msgpack-pure and it supports the latest revision of the standard. If it had existed earlier, I would not have written this one. *If you start a new project, use it.* 18 | 19 | Another interesting implementation is [lua-cmsgpack](https://github.com/antirez/lua-cmsgpack), written in C specifically for use in Redis. 20 | 21 | Other implementations: 22 | 23 | - [lua-msgpack](https://github.com/kengonakajima/lua-msgpack) (pure Lua) 24 | - [lua-msgpack-native](https://github.com/kengonakajima/lua-msgpack-native) 25 | (Lua-specific C implementation targeting luvit) 26 | - [MPLua](https://github.com/nobu-k/mplua) (binding) 27 | 28 | Before luajit-msgpack-pure, I had written [luajit-msgpack](https://github.com/catwell/cw-lua/tree/master/luajit-msgpack), a FFI binding which is now deprecated. 29 | 30 | ## TODO 31 | 32 | - Missing datatype tests 33 | - Comparison tests vs. other implementations 34 | 35 | ## Usage 36 | 37 | ### Basics 38 | 39 | ```lua 40 | local mp = require "luajit-msgpack-pure" 41 | local my_data = {this = {"is",4,"test"}} 42 | local encoded = mp.pack(my_data) 43 | local offset,decoded = mp.unpack(encoded) 44 | assert(offset == #encoded) 45 | ``` 46 | 47 | ### Concatenating encoded data 48 | 49 | ```lua 50 | local mp = require "luajit-msgpack-pure" 51 | local my_data_1 = 42 52 | local my_data_2 = "foo" 53 | local encoded = mp.pack(my_data_1) .. mp.pack(my_data_2) 54 | local offset_1,decoded_1 = mp.unpack(encoded) 55 | assert(decoded_1 == 42) 56 | local offset_2,decoded_2 = mp.unpack(encoded,offset_1) 57 | assert(decoded_2 == "foo") 58 | local offset_3,decoded_3 = mp.unpack(encoded,offset_2) 59 | assert((not offset_3) and (decoded_3 == nil)) 60 | ``` 61 | 62 | ### Setting floating point precision 63 | 64 | ```lua 65 | local mp = require "luajit-msgpack-pure" 66 | local my_data = math.pi 67 | local encoded_1 = mp.pack(my_data) -- default is double 68 | local offset_1,decoded_1 = mp.unpack(encoded_1) 69 | assert(offset_1 == 9) -- 1 byte overhead + 8 bytes double 70 | assert(decoded_1 == math.pi) 71 | mp.set_fp_type("float") 72 | local encoded_2 = mp.pack(my_data) 73 | local offset_2,decoded_2 = mp.unpack(encoded_2) 74 | assert(offset_2 == 5) -- 1 byte overhead + 5 bytes float 75 | assert(decoded_2 ~= math.pi) -- loss of precision 76 | mp.set_fp_type("double") -- back to double precision 77 | local encoded_3 = mp.pack(my_data) 78 | local offset_3,decoded_3 = mp.unpack(encoded_3) 79 | assert((offset_3 == 9) and (decoded_3 == math.pi)) 80 | ``` 81 | ## Copyright 82 | 83 | Copyright (c) 2011-2019 Pierre Chapuis 84 | -------------------------------------------------------------------------------- /luajit-msgpack-pure.lua: -------------------------------------------------------------------------------- 1 | local ffi = require "ffi" 2 | local bit = require "bit" 3 | local math = require "math" 4 | 5 | local IS_LUAFFI = not rawget(_G,"jit") 6 | 7 | -- standard cdefs 8 | 9 | ffi.cdef[[ 10 | void free(void *ptr); 11 | void *realloc(void *ptr, size_t size); 12 | void *malloc(size_t size); 13 | ]] 14 | 15 | -- cache bitops 16 | local bor,band,rshift = bit.bor,bit.band,bit.rshift 17 | 18 | -- shared ffi data 19 | local t_buf = ffi.new("unsigned char[8]") 20 | local t_buf2 = ffi.new("unsigned char[8]") 21 | 22 | -- VLA ctype constructor 23 | local uchar_vla = ffi.typeof("unsigned char[?]") 24 | 25 | -- endianness 26 | 27 | local LITTLE_ENDIAN = ffi.abi("le") 28 | local rcopy = function(dst,src,len) 29 | local n = len-1 30 | for i=0,n do dst[i] = src[n-i] end 31 | end 32 | 33 | -- buffer 34 | 35 | local MSGPACK_SBUFFER_INIT_SIZE = 8192 36 | 37 | local buffer = {} 38 | 39 | local sbuffer_init = function(self) 40 | self.size = 0 41 | self.alloc = MSGPACK_SBUFFER_INIT_SIZE 42 | self.data = ffi.cast("unsigned char *",ffi.C.malloc(MSGPACK_SBUFFER_INIT_SIZE)) 43 | end 44 | 45 | local sbuffer_destroy = function(self) 46 | ffi.C.free(buffer.data) 47 | end 48 | 49 | local sbuffer_realloc = function(self,len) 50 | if self.alloc - self.size < len then 51 | local nsize = self.alloc * 2 52 | while nsize < self.alloc + len do nsize = nsize * 2 end 53 | self.data = ffi.cast("unsigned char *",ffi.C.realloc(self.data,nsize)) 54 | self.alloc = nsize 55 | end 56 | end 57 | 58 | local sbuffer_append_str = function(self,buf,len) 59 | sbuffer_realloc(self,len) 60 | ffi.copy(self.data+self.size,buf,len) 61 | self.size = self.size + len 62 | end 63 | 64 | local sbuffer_append_byte = function(self,b) 65 | sbuffer_realloc(self,1) 66 | self.data[self.size] = b 67 | self.size = self.size + 1 68 | end 69 | 70 | local sbuffer_append_tbl = function(self,t) 71 | local len = #t 72 | sbuffer_realloc(self,len) 73 | local p = self.data + self.size - 1 74 | for i=1,len do p[i] = t[i] end 75 | self.size = self.size + len 76 | end 77 | 78 | local sbuffer_append_intx, sbuffer_append_int64 79 | if LITTLE_ENDIAN then 80 | sbuffer_append_intx = function(self,n,x,h) 81 | local t = {h} 82 | for i=x-8,8,-8 do t[#t+1] = band(rshift(n,i),0xff) end 83 | t[#t+1] = band(n,0xff) 84 | sbuffer_append_tbl(self,t) 85 | end 86 | sbuffer_append_int64 = function(self,n,h) 87 | local q,r = math.floor(n/2^32),n%(2^32) 88 | local t = {h} 89 | for i=24,8,-8 do t[#t+1] = band(rshift(q,i),0xff) end 90 | t[5] = band(q,0xff) 91 | for i=24,8,-8 do t[#t+1] = band(rshift(r,i),0xff) end 92 | t[9] = band(r,0xff) 93 | sbuffer_append_tbl(self,t) 94 | end 95 | else 96 | sbuffer_append_intx = function(self,n,x,h) 97 | local t = {h, band(n,0xff)} 98 | for i=8,x-8,8 do t[#t+1] = band(rshift(n,i),0xff) end 99 | sbuffer_append_tbl(self,t) 100 | end 101 | sbuffer_append_int64 = function(self,n,h) 102 | local q,r = math.floor(n/2^32),n%(2^32) 103 | local t = {h, band(r,0xff)} 104 | for i=8,24,8 do t[#t+1] = band(rshift(r,i),0xff) end 105 | t[6] = band(q,0xff) 106 | for i=8,24,8 do t[#t+1] = band(rshift(q,i),0xff) end 107 | sbuffer_append_tbl(self,t) 108 | end 109 | end 110 | 111 | --- packers 112 | 113 | local packers = {} 114 | 115 | packers.dynamic = function(data) 116 | return packers[type(data)](data) 117 | end 118 | 119 | packers["nil"] = function() 120 | sbuffer_append_byte(buffer,0xc0) 121 | end 122 | 123 | packers.boolean = function(data) 124 | if data then -- pack true 125 | sbuffer_append_byte(buffer,0xc3) 126 | else -- pack false 127 | sbuffer_append_byte(buffer,0xc2) 128 | end 129 | end 130 | 131 | local set_fp_type = function(t) 132 | local ptype,typebyte,_posinf,_neginf,_nan 133 | if t == "double" then 134 | typebyte,ptype = 0xcb,ffi.typeof("double *") 135 | _posinf = {typebyte,0x7f,0xf0,0x00,0x00,0x00,0x00,0x00,0x00} 136 | _neginf = {typebyte,0xff,0xf0,0x00,0x00,0x00,0x00,0x00,0x00} 137 | _nan = {typebyte,0xff,0xf8,0x00,0x00,0x00,0x00,0x00,0x00} 138 | elseif t == "float" then 139 | typebyte,ptype = 0xca,ffi.typeof("float *") 140 | _posinf = {typebyte,0x7f,0x80,0x00,0x00} 141 | _neginf = {typebyte,0xff,0x80,0x00,0x00} 142 | _nan = {typebyte,0xff,0x88,0x00,0x00} 143 | else return nil end 144 | local len = ffi.sizeof(t) 145 | if LITTLE_ENDIAN then 146 | packers.fpnum = function(n) 147 | ffi.cast(ptype,t_buf2)[0] = n 148 | rcopy(t_buf,t_buf2,len) 149 | sbuffer_append_byte(buffer,typebyte) 150 | sbuffer_append_str(buffer,t_buf,len) 151 | end 152 | else 153 | packers.fpnum = function(n) 154 | ffi.cast(ptype,t_buf)[0] = n 155 | sbuffer_append_byte(buffer,typebyte) 156 | sbuffer_append_str(buffer,t_buf,len) 157 | end 158 | end 159 | packers.posinf = function() 160 | sbuffer_append_tbl(buffer,_posinf) 161 | end 162 | packers.neginf = function() 163 | sbuffer_append_tbl(buffer,_neginf) 164 | end 165 | packers.nan = function() 166 | sbuffer_append_tbl(buffer,_nan) 167 | end 168 | return true 169 | end 170 | 171 | set_fp_type("double") -- default 172 | 173 | packers.number = function(n) 174 | if math.floor(n) == n then -- integer 175 | if n >= 0 then -- positive integer 176 | if n < 128 then -- positive fixnum 177 | sbuffer_append_byte(buffer,n) 178 | elseif n < 256 then -- uint8 179 | sbuffer_append_tbl(buffer,{0xcc,n}) 180 | elseif n < 2^16 then -- uint16 181 | sbuffer_append_intx(buffer,n,16,0xcd) 182 | elseif n < 2^32 then -- uint32 183 | sbuffer_append_intx(buffer,n,32,0xce) 184 | elseif n == math.huge then -- +inf 185 | packers.posinf() 186 | else -- uint64 187 | sbuffer_append_int64(buffer,n,0xcf) 188 | end 189 | else -- negative integer 190 | if n >= -32 then -- negative fixnum 191 | sbuffer_append_byte(buffer,bor(0xe0,n)) 192 | elseif n >= -128 then -- int8 193 | sbuffer_append_tbl(buffer,{0xd0,n}) 194 | elseif n >= -2^15 then -- int16 195 | sbuffer_append_intx(buffer,n,16,0xd1) 196 | elseif n >= -2^31 then -- int32 197 | sbuffer_append_intx(buffer,n,32,0xd2) 198 | elseif n == -math.huge then -- -inf 199 | packers.neginf() 200 | else -- int64 201 | sbuffer_append_int64(buffer,n,0xd3) 202 | end 203 | end 204 | elseif n ~= n then -- nan 205 | packers.nan() 206 | else -- floating point 207 | packers.fpnum(n) 208 | end 209 | end 210 | 211 | packers.string = function(data) 212 | local n = #data 213 | if n < 32 then 214 | sbuffer_append_byte(buffer,bor(0xa0,n)) 215 | elseif n < 2^16 then 216 | sbuffer_append_intx(buffer,n,16,0xda) 217 | elseif n < 2^32 then 218 | sbuffer_append_intx(buffer,n,32,0xdb) 219 | else 220 | error("overflow") 221 | end 222 | sbuffer_append_str(buffer,data,n) 223 | end 224 | 225 | packers["function"] = function(data) 226 | error("unimplemented", data) 227 | end 228 | 229 | packers.userdata = function(data) 230 | if IS_LUAFFI then 231 | return packers.cdata(data) 232 | else 233 | error("unimplemented") 234 | end 235 | end 236 | 237 | packers.thread = function(data) 238 | error("unimplemented", data) 239 | end 240 | 241 | packers.array = function(data,ndata) 242 | ndata = ndata or #data 243 | if ndata < 16 then 244 | sbuffer_append_byte(buffer,bor(0x90,ndata)) 245 | elseif ndata < 2^16 then 246 | sbuffer_append_intx(buffer,ndata,16,0xdc) 247 | elseif ndata < 2^32 then 248 | sbuffer_append_intx(buffer,ndata,32,0xdd) 249 | else 250 | error("overflow") 251 | end 252 | for i=1,ndata do packers[type(data[i])](data[i]) end 253 | end 254 | 255 | packers.map = function(data,ndata) 256 | if not ndata then 257 | ndata = 0 258 | for _ in pairs(data) do ndata = ndata+1 end 259 | end 260 | if ndata < 16 then 261 | sbuffer_append_byte(buffer,bor(0x80,ndata)) 262 | elseif ndata < 2^16 then 263 | sbuffer_append_intx(buffer,ndata,16,0xde) 264 | elseif ndata < 2^32 then 265 | sbuffer_append_intx(buffer,ndata,32,0xdf) 266 | else 267 | error("overflow") 268 | end 269 | for k,v in pairs(data) do 270 | packers[type(k)](k) 271 | packers[type(v)](v) 272 | end 273 | end 274 | 275 | local set_table_classifier = function(f) 276 | packers.table = function(data) 277 | local obj_type,ndata = f(data) 278 | packers[obj_type](data,ndata) 279 | end 280 | end 281 | 282 | local table_classifier_keys = function(data) 283 | -- slightly slower, does not access values at all 284 | local is_map,ndata,nmax = false,0,0 285 | for k,_ in pairs(data) do 286 | if (type(k) == "number") and (k > 0) and (math.floor(k) == k) then 287 | if k > nmax then nmax = k end 288 | else is_map = true end 289 | ndata = ndata+1 290 | end 291 | if (nmax ~= ndata) then -- there are holes 292 | is_map = true 293 | end -- else nmax == ndata == #data 294 | return (is_map and "map" or "array"),ndata 295 | end 296 | 297 | local table_classifier_values = function(data) 298 | -- slightly faster, accesses values 299 | local is_map,ndata = false,0 300 | for _ in pairs(data) do 301 | ndata = ndata + 1 302 | if rawget(data,ndata) == nil then is_map = true end 303 | end 304 | return (is_map and "map" or "array"),ndata 305 | end 306 | 307 | set_table_classifier(table_classifier_keys) 308 | 309 | packers.cdata = function(data) -- msgpack-js 310 | local n = ffi.sizeof(data) 311 | if not n then 312 | error("cannot pack cdata of unknown size") 313 | elseif n < 2^16 then 314 | sbuffer_append_intx(buffer,n,16,0xd8) 315 | elseif n < 2^32 then 316 | sbuffer_append_intx(buffer,n,32,0xd9) 317 | else 318 | error("overflow") 319 | end 320 | sbuffer_append_str(buffer,data,n) 321 | end 322 | 323 | -- types decoding 324 | 325 | local types_map = { 326 | [0xc0] = "nil", 327 | [0xc2] = "false", 328 | [0xc3] = "true", 329 | [0xc4] = "nil", -- msgpack-js 330 | [0xca] = "float", 331 | [0xcb] = "double", 332 | [0xcc] = "uint8", 333 | [0xcd] = "uint16", 334 | [0xce] = "uint32", 335 | [0xcf] = "uint64", 336 | [0xd0] = "int8", 337 | [0xd1] = "int16", 338 | [0xd2] = "int32", 339 | [0xd3] = "int64", 340 | [0xd8] = "buf16", -- msgpack-js 341 | [0xd9] = "buf32", -- msgpack-js 342 | [0xda] = "raw16", 343 | [0xdb] = "raw32", 344 | [0xdc] = "array16", 345 | [0xdd] = "array32", 346 | [0xde] = "map16", 347 | [0xdf] = "map32", 348 | } 349 | 350 | local type_for = function(n) 351 | if types_map[n] then return types_map[n] 352 | elseif n < 0xc0 then 353 | if n < 0x80 then return "fixnum_pos" 354 | elseif n < 0x90 then return "fixmap" 355 | elseif n < 0xa0 then return "fixarray" 356 | else return "fixraw" end 357 | elseif n > 0xdf then return "fixnum_neg" 358 | else return "undefined" end 359 | end 360 | 361 | local types_len_map = { 362 | uint16 = 2, uint32 = 4, uint64 = 8, 363 | int16 = 2, int32 = 4, int64 = 8, 364 | float = 4, double = 8, 365 | } 366 | 367 | --- unpackers 368 | 369 | local unpackers = {} 370 | 371 | local unpack_number 372 | if LITTLE_ENDIAN then 373 | unpack_number = function(buf,offset,ntype,nlen) 374 | rcopy(t_buf,buf.data+offset+1,nlen) 375 | return tonumber(ffi.cast(ntype,t_buf)[0]) 376 | end 377 | else 378 | unpack_number = function(buf,offset,ntype) 379 | return tonumber(ffi.cast(ntype,buf.data+offset+1)[0]) 380 | end 381 | end 382 | 383 | local unpacker_number = function(buf,offset) 384 | local obj_type = type_for(buf.data[offset]) 385 | local nlen = types_len_map[obj_type] 386 | local ntype 387 | if (obj_type == "float") or (obj_type == "double") then 388 | ntype = obj_type .. " *" 389 | else ntype = obj_type .. "_t *" end 390 | if offset + nlen >= buf.size then return nil,nil end 391 | return offset+nlen+1,unpack_number(buf,offset,ntype,nlen) 392 | end 393 | 394 | local unpack_map = function(buf,offset,n) 395 | local r = {} 396 | local k,v 397 | for _=1,n do 398 | offset,k = unpackers.dynamic(buf,offset) 399 | if not offset then 400 | -- Whole map is not available in the buffer 401 | return nil, r 402 | end 403 | offset,v = unpackers.dynamic(buf,offset) 404 | if not offset then 405 | -- Whole map is not available in the buffer 406 | return nil, r 407 | end 408 | r[k] = v 409 | end 410 | return offset,r 411 | end 412 | 413 | local unpack_array = function(buf,offset,n) 414 | local r = {} 415 | for i=1,n do 416 | offset,r[i] = unpackers.dynamic(buf,offset) 417 | if not offset then 418 | -- Whole array is not available in the buffer 419 | return nil, r 420 | end 421 | end 422 | return offset,r 423 | end 424 | 425 | unpackers.dynamic = function(buf,offset) 426 | assert(offset, "non nil offset is expected") 427 | if offset >= buf.size then return nil,nil end 428 | local obj_type = type_for(buf.data[offset]) 429 | return unpackers[obj_type](buf,offset) 430 | end 431 | 432 | unpackers.undefined = function(buf,offset) 433 | error("unimplemented", buf, offset) 434 | end 435 | 436 | unpackers["nil"] = function(_,offset) 437 | return offset+1,nil 438 | end 439 | 440 | unpackers["false"] = function(_,offset) 441 | return offset+1,false 442 | end 443 | 444 | unpackers["true"] = function(_,offset) 445 | return offset+1,true 446 | end 447 | 448 | unpackers.fixnum_pos = function(buf,offset) 449 | return offset+1,buf.data[offset] 450 | end 451 | 452 | unpackers.uint8 = function(buf,offset) 453 | if offset + 1 >= buf.size then return nil,nil end 454 | return offset+2,buf.data[offset+1] 455 | end 456 | 457 | unpackers.uint16 = unpacker_number 458 | unpackers.uint32 = unpacker_number 459 | unpackers.uint64 = unpacker_number 460 | 461 | unpackers.fixnum_neg = function(buf,offset) 462 | -- alternative to cast below: 463 | -- return offset+1,-band(bxor(buf.data[offset],0x1f),0x1f)-1 464 | return offset+1,ffi.cast("int8_t *",buf.data)[offset] 465 | end 466 | 467 | unpackers.int8 = function(buf,offset) 468 | if offset + 1 >= buf.size then return nil,nil end 469 | return offset+2,ffi.cast("int8_t *",buf.data+offset+1)[0] 470 | end 471 | 472 | unpackers.int16 = unpacker_number 473 | unpackers.int32 = unpacker_number 474 | unpackers.int64 = unpacker_number 475 | 476 | unpackers.float = unpacker_number 477 | unpackers.double = unpacker_number 478 | 479 | unpackers.fixraw = function(buf,offset) 480 | local n = band(buf.data[offset],0x1f) 481 | if offset + n >= buf.size then return nil,nil end 482 | return offset+n+1,ffi.string(buf.data+offset+1,n) 483 | end 484 | 485 | unpackers.buf16 = function(buf,offset) 486 | if offset + 2 >= buf.size then return nil,nil end 487 | local n = unpack_number(buf,offset,"uint16_t *",2) 488 | if offset + n + 2 >= buf.size then return nil,nil end 489 | local r = uchar_vla(n) 490 | ffi.copy(r,buf.data+offset+3,n) 491 | return offset+n+3,r 492 | end 493 | 494 | unpackers.buf32 = function(buf,offset) 495 | if offset + 4 >= buf.size then return nil,nil end 496 | local n = unpack_number(buf,offset,"uint32_t *",4) 497 | if offset + n + 4 >= buf.size then return nil,nil end 498 | local r = uchar_vla(n) 499 | ffi.copy(r,buf.data+offset+5,n) 500 | return offset+n+5,r 501 | end 502 | 503 | unpackers.raw16 = function(buf,offset) 504 | if offset + 2 >= buf.size then return nil,nil end 505 | local n = unpack_number(buf,offset,"uint16_t *",2) 506 | if offset + n + 2 >= buf.size then return nil,nil end 507 | return offset+n+3,ffi.string(buf.data+offset+3,n) 508 | end 509 | 510 | unpackers.raw32 = function(buf,offset) 511 | if offset + 4 >= buf.size then return nil,nil end 512 | local n = unpack_number(buf,offset,"uint32_t *",4) 513 | if offset + n + 4 >= buf.size then return nil,nil end 514 | return offset+n+5,ffi.string(buf.data+offset+5,n) 515 | end 516 | 517 | unpackers.fixarray = function(buf,offset) 518 | local n = band(buf.data[offset],0x0f) 519 | return unpack_array(buf,offset+1,n) 520 | end 521 | 522 | unpackers.array16 = function(buf,offset) 523 | if offset + 2 >= buf.size then return nil,nil end 524 | local n = unpack_number(buf,offset,"uint16_t *",2) 525 | return unpack_array(buf,offset+3,n) 526 | end 527 | 528 | unpackers.array32 = function(buf,offset) 529 | if offset + 4 >= buf.size then return nil,nil end 530 | local n = unpack_number(buf,offset,"uint32_t *",4) 531 | return unpack_array(buf,offset+5,n) 532 | end 533 | 534 | unpackers.fixmap = function(buf,offset) 535 | local n = band(buf.data[offset],0x0f) 536 | return unpack_map(buf,offset+1,n) 537 | end 538 | 539 | unpackers.map16 = function(buf,offset) 540 | if offset + 2 >= buf.size then return nil,nil end 541 | local n = unpack_number(buf,offset,"uint16_t *",2) 542 | return unpack_map(buf,offset+3,n) 543 | end 544 | 545 | unpackers.map32 = function(buf,offset) 546 | if offset + 4 >= buf.size then return nil,nil end 547 | local n = unpack_number(buf,offset,"uint32_t *",4) 548 | return unpack_map(buf,offset+5,n) 549 | end 550 | 551 | -- Main functions 552 | 553 | local ljp_pack = function(data) 554 | sbuffer_init(buffer) 555 | packers.dynamic(data) 556 | local s = ffi.string(buffer.data,buffer.size) 557 | sbuffer_destroy(buffer) 558 | return s 559 | end 560 | 561 | local ljp_unpack = function(s,offset) 562 | if offset == nil then offset = 0 end 563 | if type(s) ~= "string" then return false,"invalid argument" end 564 | sbuffer_init(buffer) 565 | sbuffer_append_str(buffer,s,#s) 566 | local data 567 | offset,data = unpackers.dynamic(buffer,offset) 568 | sbuffer_destroy(buffer) 569 | return offset,data 570 | end 571 | 572 | local ljp_pack_raw = function(raw) 573 | sbuffer_init(buffer) 574 | packers.dynamic(raw) 575 | local data, size = buffer.data, buffer.size 576 | buffer.data = nil 577 | return data, size 578 | end 579 | 580 | local ljp_unpack_raw = function(ptr, size, offset) 581 | if offset == nil then offset = 0 end 582 | local buf = { 583 | data = ffi.cast("unsigned char *", ptr), 584 | size = size 585 | } 586 | local data 587 | offset, data = unpackers.dynamic(buf, offset) 588 | return offset, data 589 | end 590 | 591 | return { 592 | pack = ljp_pack, 593 | unpack = ljp_unpack, 594 | pack_raw = ljp_pack_raw, 595 | unpack_raw = ljp_unpack_raw, 596 | set_fp_type = set_fp_type, 597 | table_classifiers = { 598 | keys = table_classifier_keys, 599 | values = table_classifier_values, 600 | }, 601 | set_table_classifier = set_table_classifier, 602 | packers = packers, 603 | unpackers = unpackers, 604 | } 605 | -------------------------------------------------------------------------------- /tests/bench.lua: -------------------------------------------------------------------------------- 1 | local mp = require "luajit-msgpack-pure" 2 | local ok,socket = pcall(require,"socket") 3 | local gettime = ok and socket.gettime or os.clock 4 | 5 | if #arg ~= 1 then 6 | error("Usage: luajit bench.lua 1000000") 7 | end 8 | local nloop = arg[1] 9 | 10 | local printf = function(p,...) 11 | io.stdout:write(string.format(p,...)); io.stdout:flush() 12 | end 13 | 14 | local makeiary = function(n) 15 | local out = {} 16 | for i=1,n do table.insert(out,math.floor(i-n/2)) end 17 | return out 18 | end 19 | 20 | local makedary = function(n) 21 | local out = {} 22 | for i=1,n do table.insert(out,1.5e+35*i) end 23 | return out 24 | end 25 | 26 | local makestr = function(n) 27 | local out = "" 28 | for _=1,n-1 do out = out .. "a" end 29 | out = out .. "b" 30 | return out 31 | end 32 | 33 | local datasets = { 34 | { "empty", {}, nloop }, 35 | { "iary1", {1}, nloop }, 36 | { "iary10", {-5,-4,-3,-2,-1,0,1,2,3,4}, nloop }, 37 | { "iary100", makeiary(100), nloop/10 }, 38 | { "iary1000", makeiary(1000), nloop/100 }, 39 | { "iary10000", makeiary(10000), nloop/1000 }, 40 | { "dary1", {1.5e+35}, nloop/50 }, 41 | { "dary10", makedary(10), nloop/100 }, 42 | { "dary100", makedary(100), nloop/1000 }, 43 | { "dary1000", makedary(1000), nloop/1000 }, 44 | { "str1", { "a"}, nloop }, 45 | { "str10", { makestr(10) }, nloop }, 46 | { "str100", { makestr(100) }, nloop }, 47 | { "str1000", { makestr(1000) }, nloop }, 48 | { "str10000", { makestr(10000) }, nloop/10 }, 49 | { "str20000", { makestr(20000) }, nloop/10 }, 50 | { "str30000", { makestr(30000) }, nloop/10 }, 51 | { "str40000", { makestr(40000) }, nloop/100 }, 52 | { "str80000", { makestr(80000) }, nloop/100 }, 53 | } 54 | 55 | local n,st,et 56 | for _,v in ipairs(datasets) do 57 | st,n = gettime(),v[3] 58 | local offset 59 | local x = v[2] 60 | for _=1,n do 61 | offset = mp.unpack(mp.pack(x)) 62 | end 63 | assert(offset == #mp.pack(x)) 64 | et = gettime() 65 | printf( 66 | "%9s\t %f sec\t%d times/sec\n", 67 | v[1],(et-st),n/(et-st) 68 | ) 69 | end 70 | -------------------------------------------------------------------------------- /tests/cases.mpac: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/catwell/luajit-msgpack-pure/94b9f4a0b3acccb9eb7a66d259b4b7b0ceba974e/tests/cases.mpac -------------------------------------------------------------------------------- /tests/cases_compact.mpac: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/catwell/luajit-msgpack-pure/94b9f4a0b3acccb9eb7a66d259b4b7b0ceba974e/tests/cases_compact.mpac -------------------------------------------------------------------------------- /tests/pr-11.lua: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env luajit 2 | 3 | local ffi = require "ffi" 4 | 5 | local display = function(m,x) 6 | local _t = type(x) 7 | io.stdout:write(string.format("\n%s: %s ",m,_t)) 8 | if _t == "table" then pretty.dump(x) else print(x) end 9 | end 10 | 11 | local printf = function(p,...) 12 | io.stdout:write(string.format(p,...)); io.stdout:flush() 13 | end 14 | 15 | local mp = require "luajit-msgpack-pure" 16 | 17 | local offset,res 18 | local float_val = 0xffff000000000 19 | 20 | local nb_test2 = function(n,sz) 21 | offset,res = mp.unpack(mp.pack(n)) 22 | assert(offset,"decoding failed") 23 | if res ~= n then 24 | assert(false,string.format("wrong value %g, expected %g",res,n)) 25 | end 26 | assert(offset == sz,string.format( 27 | "wrong size %d for number %g (expected %d)", 28 | offset,n,sz 29 | )) 30 | end 31 | 32 | printf(".") 33 | for _=0,1000 do 34 | for n=0,50 do 35 | local v = float_val + n 36 | nb_test2(float_val + n, 9) 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /tests/test.lua: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env luajit 2 | 3 | local RUN_LARGE_TESTS = false 4 | local IS_LUAFFI = not rawget(_G,"jit") 5 | 6 | local pathx = require "pl.path" 7 | local pretty = require "pl.pretty" 8 | local tablex = require "pl.tablex" 9 | require "pl.strict" 10 | 11 | local mp = require "luajit-msgpack-pure" 12 | local ffi = require "ffi" 13 | 14 | local display = function(m,x) 15 | local _t = type(x) 16 | io.stdout:write(string.format("\n%s: %s ",m,_t)) 17 | if _t == "table" then pretty.dump(x) else print(x) end 18 | end 19 | 20 | local printf = function(p,...) 21 | io.stdout:write(string.format(p,...)); io.stdout:flush() 22 | end 23 | 24 | local msgpack_cases = { 25 | false,true,nil,0,0,0,0,0,0,0,0,0,-1,-1,-1,-1,-1,127,127,255,2^16-1, 26 | 2^32-1,-32,-32,-128,-2^15,-2^31,0.0,-0.0,1.0,-1.0, 27 | "a","a","a","","","", 28 | {0},{0},{0},{},{},{},{},{},{},{a=97},{a=97},{a=97},{{}},{{"a"}}, 29 | } 30 | 31 | local data = { 32 | true, 33 | false, 34 | 42, 35 | -42, 36 | 0.79, 37 | "Hello world!", 38 | {}, 39 | {true,false,42,-42,0.79,"Hello","World!"}, 40 | {[0]=17,21,27}, 41 | {[-6]=17,21,27}, 42 | {[-1]=4,1,nil,3}, 43 | {[1]=17,[99999999]=21}, 44 | {[1.2]=5, [2]=7}, 45 | {{"multi","level",{"lists","used",45,{{"trees"}}},"work",{}},"too"}, 46 | {foo="bar",spam="eggs"}, 47 | {nested={maps={"work","too"}}}, 48 | {"we","can",{"mix","integer"},{keys="and"},{2,{maps="as well"}}}, 49 | msgpack_cases, 50 | } 51 | 52 | local offset,res 53 | 54 | -- Custom tests 55 | printf("Custom tests ") 56 | for i=0,#data do -- 0 tests nil! 57 | printf(".") 58 | offset,res = mp.unpack(mp.pack(data[i])) 59 | assert(offset,"decoding failed") 60 | if not tablex.deepcompare(res,data[i]) then 61 | display("expected",data[i]) 62 | display("found",res) 63 | assert(false,string.format("wrong value %d",i)) 64 | end 65 | end 66 | print(" OK") 67 | 68 | -- Integer tests 69 | 70 | printf("Integer tests ") 71 | 72 | local nb_test = function(n,sz,epsilon) 73 | offset,res = mp.unpack(mp.pack(n)) 74 | assert(offset,"decoding failed") 75 | if epsilon then 76 | local diff = math.abs(res - n) 77 | if diff > epsilon then 78 | assert(false,string.format( 79 | "wrong value %g, expected %g, difference %g > epsilon %g", 80 | res,n,diff,epsilon 81 | )) 82 | end 83 | else 84 | if res ~= n then 85 | assert(false,string.format("wrong value %g, expected %g",res,n)) 86 | end 87 | end 88 | assert(offset == sz,string.format( 89 | "wrong size %d for number %g (expected %d)", 90 | offset,n,sz 91 | )) 92 | end 93 | 94 | printf(".") 95 | for n=0,127 do -- positive fixnum 96 | nb_test(n,1) 97 | end 98 | 99 | printf(".") 100 | for n=128,255 do -- uint8 101 | nb_test(n,2) 102 | end 103 | 104 | printf(".") 105 | for n=256,2^16-1 do -- uint16 106 | nb_test(n,3) 107 | end 108 | 109 | -- uint32 110 | printf(".") 111 | for n=2^16,2^16+100 do 112 | nb_test(n,5) 113 | end 114 | for n=2^32-101,2^32-1 do 115 | nb_test(n,5) 116 | end 117 | 118 | printf(".") 119 | for n=2^32,2^32+100 do -- uint64 120 | nb_test(n,9) 121 | end 122 | 123 | printf(".") 124 | for n=-1,-32,-1 do -- negative fixnum 125 | nb_test(n,1) 126 | end 127 | 128 | printf(".") 129 | for n=-33,-128,-1 do -- int8 130 | nb_test(n,2) 131 | end 132 | 133 | printf(".") 134 | for n=-129,-2^15,-1 do -- int16 135 | nb_test(n,3) 136 | end 137 | 138 | -- int32 139 | printf(".") 140 | for n=-2^15-1,-2^15-101,-1 do 141 | nb_test(n,5) 142 | end 143 | for n=-2^31+100,-2^31,-1 do 144 | nb_test(n,5) 145 | end 146 | 147 | printf(".") 148 | for n=-2^31-1,-2^31-101,-1 do -- int64 149 | nb_test(n,9) 150 | end 151 | 152 | print(" OK") 153 | 154 | -- Floating point tests 155 | printf("Floating point tests ") 156 | 157 | printf(".") -- default is double 158 | for _=1,100 do 159 | local n = math.random()*200-100 160 | nb_test(n,9) 161 | end 162 | 163 | printf(".") 164 | 165 | mp.set_fp_type("float") 166 | for _=1,100 do 167 | local n = math.random()*200-100 168 | nb_test(n,5,1e-5) 169 | end 170 | 171 | printf(".") 172 | mp.set_fp_type("double") 173 | for _=1,100 do 174 | local n = math.random()*200-100 175 | nb_test(n,9) 176 | end 177 | 178 | print(" OK") 179 | 180 | -- Quasi-numbers tests 181 | printf("Quasi-numbers tests ") 182 | 183 | -- Notes and conventions: 184 | 185 | -- IEEE 754 format remainder: 186 | -- - Float: [1] sign | [8] exp | [23] frac 187 | -- - Double: [1] sign | [11] exp | [52] frac 188 | 189 | -- Inifinities have an exponent of all ones and a fraction of zero: 190 | -- - +inf: 0x7f800000 / 0x7ff0000000000000 191 | -- - -inf: 0xff800000 / 0xfff0000000000000 192 | 193 | -- NaNs have any sign, an exponent of all ones and any fraction except 0. 194 | -- If the MSB of the fraction is set then it is a QNaN, otherwise it is a SNaN. 195 | 196 | -- NaNs are *packed* as QNaNs (non-signaling), specifically 197 | -- 0xff880000 / 0xfff8000000000000. 198 | -- All QNaNs and SNaNs should be decoded as `nan`, but this might not 199 | -- always be the case currently because of specificities of LuaJIT. 200 | 201 | local _pos_inf,_neg_inf = 1/0,-1/0 202 | 203 | local nan_test = function(sz) 204 | local n = 0/0 205 | offset,res = mp.unpack(mp.pack(n)) 206 | assert(offset,"decoding failed") 207 | if not ((type(res) == "number") and (res ~= res)) then 208 | print(type(res)) 209 | assert(false,string.format("wrong value %g, expected %g",res,n)) 210 | end 211 | assert(offset == sz,string.format( 212 | "wrong size %d for number %g (expected %d)", 213 | offset,n,sz 214 | )) 215 | end 216 | 217 | mp.set_fp_type("float") 218 | 219 | printf(".") 220 | nb_test(_pos_inf,5) 221 | 222 | printf(".") 223 | nb_test(_neg_inf,5) 224 | 225 | printf(".") 226 | nan_test(5) 227 | 228 | mp.set_fp_type("double") 229 | 230 | printf(".") 231 | nb_test(_pos_inf,9) 232 | 233 | printf(".") 234 | nb_test(_neg_inf,9) 235 | 236 | printf(".") 237 | nan_test(9) 238 | 239 | print(" OK") 240 | 241 | -- Raw tests 242 | 243 | printf("Raw tests ") 244 | 245 | local rand_raw = function(len) 246 | local t = {} 247 | for i=1,len do t[i] = string.char(math.random(0,255)) end 248 | return table.concat(t) 249 | end 250 | 251 | local raw_test = function(raw,overhead) 252 | offset,res = mp.unpack(mp.pack(raw)) 253 | assert(offset,"decoding failed") 254 | if res ~= raw then 255 | assert(false,string.format("wrong raw (len %d - %d)",#res,#raw)) 256 | end 257 | assert(offset-#raw == overhead,string.format( 258 | "wrong overhead %d for #raw %d (expected %d)", 259 | offset-#raw,#raw,overhead 260 | )) 261 | end 262 | 263 | printf(".") 264 | for n=0,31 do -- fixraw 265 | raw_test(rand_raw(n),1) 266 | end 267 | 268 | -- raw16 269 | printf(".") 270 | for n=32,32+100 do 271 | raw_test(rand_raw(n),3) 272 | end 273 | for n=2^16-101,2^16-1 do 274 | raw_test(rand_raw(n),3) 275 | end 276 | 277 | -- raw32 278 | printf(".") 279 | for n=2^16,2^16+100 do 280 | raw_test(rand_raw(n),5) 281 | end 282 | if RUN_LARGE_TESTS then 283 | for n=2^32-101,2^32-1 do 284 | raw_test(rand_raw(n),5) 285 | end 286 | end 287 | 288 | print(" OK") 289 | 290 | -- Table tests 291 | printf("Table tests ") 292 | 293 | local rand_array = function(len) -- of positive fixnum-s 294 | local t = {} 295 | for i=1,len do t[i] = math.random(0,127) end 296 | return t 297 | end 298 | 299 | local array_test = function(t,expected_size,overhead) 300 | offset,res = mp.unpack(mp.pack(t)) 301 | assert(offset,"decoding failed") 302 | assert((offset-expected_size) == overhead,string.format( 303 | "wrong overhead %d (expected %d)", 304 | (offset-expected_size),overhead 305 | )) 306 | assert(type(res) == "table",string.format("wrong type %s",type(res))) 307 | local n = #res 308 | assert(n == expected_size,string.format( 309 | "wrong size %d (expected %d)", 310 | n,expected_size 311 | )) 312 | for i=0,n-1 do 313 | assert(t[i] == res[i],"wrong value") 314 | end 315 | end 316 | 317 | -- fix array 318 | printf(".") 319 | for n=0,15 do 320 | array_test(rand_array(n),n,1) 321 | end 322 | 323 | -- array16 324 | printf(".") 325 | for n=16,16+100 do 326 | array_test(rand_array(n),n,3) 327 | end 328 | for n=2^16-101,2^16-1 do 329 | array_test(rand_array(n),n,3) 330 | end 331 | 332 | -- array32 333 | printf(".") 334 | for n=2^16,2^16+100 do 335 | array_test(rand_array(n),n,5) 336 | end 337 | if RUN_LARGE_TESTS then 338 | for n=2^32-101,2^32-1 do 339 | array_test(rand_array(n),n,5) 340 | end 341 | end 342 | 343 | print(" OK") 344 | 345 | -- TODO Map tests 346 | printf("Map tests ") 347 | print(" TODO") 348 | 349 | -- From MessagePack test suite 350 | do 351 | local cases_dir = pathx.abspath(pathx.dirname(arg[0])) 352 | local case_files = { 353 | standard = pathx.join(cases_dir,"cases.mpac"), 354 | compact = pathx.join(cases_dir,"cases_compact.mpac"), 355 | } 356 | local i,f,bindata 357 | local ncases = #msgpack_cases 358 | for case_name,case_file in pairs(case_files) do 359 | printf("MsgPack %s tests ",case_name) 360 | f = assert(io.open(case_file,'rb')) 361 | bindata = f:read("*all") 362 | f:close() 363 | offset,i = 0,0 364 | while true do 365 | i = i+1 366 | printf(".") 367 | offset,res = mp.unpack(bindata,offset) 368 | if not offset then break end 369 | if not tablex.deepcompare(res,msgpack_cases[i]) then 370 | display("expected",msgpack_cases[i]) 371 | display("found",res) 372 | assert(false,string.format("wrong value %d",i)) 373 | end 374 | end 375 | assert( 376 | i-1 == ncases, 377 | string.format("decoded %d values instead of %d",i-1,ncases) 378 | ) 379 | print(" OK") 380 | end 381 | end 382 | 383 | -- msgpack-js compatibility 384 | printf("msgpack-js compatibility tests ") 385 | 386 | local rand_buf = function(len) 387 | local buf = ffi.new("unsigned char[?]",len) 388 | for i=0,len-1 do buf[i] = math.random(0,255) end 389 | return buf 390 | end 391 | 392 | local buf_type = IS_LUAFFI and "userdata" or "cdata" 393 | local buf_test = function(buf,expected_size,overhead) 394 | offset,res = mp.unpack(mp.pack(buf)) 395 | assert(offset,"decoding failed") 396 | assert((offset-expected_size) == overhead,string.format( 397 | "wrong overhead %d (expected %d)", 398 | (offset-expected_size),overhead 399 | )) 400 | assert(type(res) == buf_type,string.format("wrong type %s",type(res))) 401 | local n = ffi.sizeof(res) 402 | assert(n == expected_size,string.format( 403 | "wrong size %d (expected %d)", 404 | n,expected_size 405 | )) 406 | for i=0,n-1 do 407 | assert(buf[i] == res[i],"wrong value") 408 | end 409 | end 410 | 411 | -- undefined 412 | printf(".") 413 | offset,res = mp.unpack(string.char(0xc4)) 414 | assert(offset,"decoding failed") 415 | assert(offset == 1,"wrong size") 416 | assert(res == nil,"wrong value") 417 | 418 | -- buf16 419 | printf(".") 420 | for n=32,32+100 do 421 | buf_test(rand_buf(n),n,3) 422 | end 423 | for n=2^16-101,2^16-1 do 424 | buf_test(rand_buf(n),n,3) 425 | end 426 | 427 | -- buf32 428 | printf(".") 429 | for n=2^16,2^16+100 do 430 | buf_test(rand_buf(n),n,5) 431 | end 432 | if RUN_LARGE_TESTS then 433 | for n=2^32-101,2^32-1 do 434 | buf_test(rand_buf(n),n,5) 435 | end 436 | end 437 | 438 | print(" OK") 439 | --------------------------------------------------------------------------------