├── .github └── workflows │ ├── doc.yml │ └── test.yml ├── .luacheckrc ├── README.md ├── bin-dev-1.rockspec ├── bin-scm-1.rockspec ├── bin.lua ├── bin ├── base.lua ├── basebuf.lua ├── buf.lua ├── fixbuf.lua ├── rbuf.lua └── saferbuf.lua ├── config.ld ├── ex.lua ├── libluabin.c ├── portable_endian.h ├── rockspecs ├── bin-scm-1.rockspec ├── bin-scm-2.rockspec ├── bin-scm-3.rockspec └── bin-scm-4.rockspec ├── rpm.spec ├── test ├── sample.lua ├── scribe.lua └── t.lua ├── tmp └── leftovers.lua └── xd.h /.github/workflows/doc.yml: -------------------------------------------------------------------------------- 1 | name: doc 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | 8 | jobs: 9 | doc: 10 | runs-on: ubuntu-latest 11 | 12 | steps: 13 | - uses: actions/checkout@master 14 | - uses: leafo/gh-actions-lua@v8.0.0 15 | with: 16 | luaVersion: "5.3.5" 17 | 18 | - uses: leafo/gh-actions-luarocks@v4.0.0 19 | 20 | - name: install dependencies 21 | run: | 22 | luarocks install ldoc 23 | 24 | - name: build documentation 25 | run: | 26 | ldoc . 27 | 28 | - name: publish documentation 29 | uses: JamesIves/github-pages-deploy-action@4.1.3 30 | with: 31 | branch: gh-pages 32 | folder: doc 33 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: test 2 | 3 | on: 4 | - push 5 | - pull_request 6 | 7 | jobs: 8 | linter: 9 | runs-on: "ubuntu-latest" 10 | steps: 11 | - uses: actions/checkout@master 12 | - uses: leafo/gh-actions-lua@v8.0.0 13 | with: 14 | luaVersion: luajit-openresty 15 | 16 | - uses: leafo/gh-actions-luarocks@v4.0.0 17 | - name: install and run luacheck 18 | run: | 19 | luarocks install luacheck && luacheck . 20 | 21 | test: 22 | strategy: 23 | matrix: 24 | luaVersion: ["luajit-2.0.5", "luajit-2.1.0-beta3", "luajit-openresty"] 25 | machineTag: ["ubuntu-latest", "macos-latest"] 26 | 27 | runs-on: ${{ matrix.machineTag }} 28 | 29 | steps: 30 | - uses: actions/checkout@master 31 | - uses: leafo/gh-actions-lua@v8.0.0 32 | with: 33 | luaVersion: ${{ matrix.luaVersion }} 34 | 35 | - uses: leafo/gh-actions-luarocks@v4.0.0 36 | 37 | - name: build and fetch dependencies 38 | run: | 39 | luarocks --server https://moonlibs.github.io/rocks/ make rockspecs/bin-scm-4.rockspec 40 | 41 | - name: test-scribe 42 | run: | 43 | lua test/scribe.lua 44 | 45 | - name: test-reb 46 | run: | 47 | lua test/t.lua 48 | -------------------------------------------------------------------------------- /.luacheckrc: -------------------------------------------------------------------------------- 1 | std = "luajit" 2 | include_files = { "bin/", "bin.lua" } 3 | codes = true 4 | ignore = { "631" --[[ line is too long ]] } 5 | read_globals = { 'package.search' } 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SYNOPSIS 2 | 3 | ```lua 4 | 5 | local bin = require 'bin' 6 | local ffi = require 'ffi' 7 | 8 | --[[ For the first: nice hexdump ]]-- 9 | 10 | -- print hexdump of lua string 11 | print(bin.xd("some binary string")) 12 | 13 | --[[ Gives the following output: 14 | [0000] 73 6F 6D 65 20 62 69 6E 61 72 79 20 73 74 72 69 some bin ary stri 15 | [0010] 6E 67 ng 16 | ]] 17 | 18 | -- Accepts arbitrary pointer and could take length 19 | print(bin.xd(ffi.cast("void *","string"),4)) 20 | --[[ Gives the following output: 21 | [0000] 73 74 72 69 stri 22 | ]] 23 | 24 | -- For compact output there is `hex` 25 | print(bin.hex("some binary string")) 26 | -- 736F6D652062696E61727920737472696E67 27 | 28 | --[[ There is also endian conversion functions ]]-- 29 | 30 | print(bin.htobe16(0x1234)) -- 13330 31 | print(bin.htole16(0x1234)) -- 4660 32 | print(bin.be16toh(0x1234)) -- 13330 33 | print(bin.le16toh(0x1234)) -- 4660 34 | print(bin.htobe32(0x1234)) -- 873594880 35 | print(bin.htole32(0x1234)) -- 4660 36 | print(bin.be32toh(0x1234)) -- 873594880 37 | print(bin.le32toh(0x1234)) -- 4660 38 | print(bin.htobe64(0x1234)) -- 3752061439553044480ULL 39 | print(bin.htole64(0x1234)) -- 4660ULL 40 | print(bin.be64toh(0x1234)) -- 3752061439553044480ULL 41 | print(bin.le64toh(0x1234)) -- 4660ULL 42 | 43 | --[[ And buffer objects. Write buffer ]]-- 44 | 45 | local desired_size = 4096 46 | local buf = bin.buf(desired_size) 47 | print(buf) 48 | -- binbuf<0x31004200>[0/4096] 49 | -- address used/total 50 | 51 | -- accepts encoding of integers: 52 | 53 | buf:uint8(0xff) 54 | print(buf:hex()) -- FF 55 | buf:uint32(0x12345678) 56 | print(buf:hex()) -- FF78563412 57 | buf:uint32(0x12345678) 58 | 59 | -- Also support endianness 60 | 61 | buf:uint32be(0x12345678) -- be for big endian 62 | print(buf:hex()) -- FF785634127856341212345678 63 | 64 | buf:uint32le(0x12345678) -- le for little endian 65 | print(buf:hex()) -- FF78563412785634121234567878563412 66 | 67 | --[[ 68 | Complete list of numeric methods: 69 | uint8 int8 uint8be int8le 70 | uint16 int16 uint16be int16le 71 | uint32 int32 uint32be int32le 72 | uint64 int64 uint64be int64le 73 | V = int32le 74 | N = int32be 75 | ber - for BER packed integer (perl pack w) 76 | reb - for reverse BER (big endian) 77 | f float floatbe floatle 78 | d double doublebe doublele <- this one looks so weird ;) 79 | ]] 80 | 81 | -- buffer may be cleared (reset pos to zero or other pos within buf) 82 | buf:clear(8) 83 | print(buf:hex()) -- FF78563412785634 84 | print(buf) -- binbuf<0x31004200>[8/4096] 85 | buf:clear() 86 | print(buf) -- binbuf<0x31004200>[0/4096] 87 | 88 | buf:double(0.123456) 89 | print(buf:hex()) -- FF78563412785634 90 | buf:doublebe(0.123456) 91 | print(buf:hex()) -- BFB67EFACF9ABF3F3FBF9ACFFA7EB6BF 92 | 93 | buf:clear() 94 | 95 | buf:float(0.123456) 96 | print(buf:hex()) -- 80D6FC3D 97 | buf:floatbe(0.123456) 98 | print(buf:hex()) -- 80D6FC3D3DFCD680 99 | 100 | buf:clear() 101 | 102 | --[[ Also there is simple :char ]] 103 | buf:char('x') -- !only one char 104 | buf:char(0xfe) -- but also accepts number as byte 105 | 106 | print(buf:hex()) -- 78FE 107 | 108 | --[[ for strings or buffers there is copy ]] 109 | 110 | buf:copy("putme") 111 | buf:copy("notlong",3) 112 | 113 | print(buf:dump()) 114 | -- [0000] 78 FE 70 75 74 6D 65 6E 6F 74 x.pu tmen ot 115 | 116 | buf:clear() 117 | 118 | --[[ For manual operation there is :alloc ]] 119 | local nbytes = 5 120 | local p = buf:alloc(nbytes) 121 | ffi.copy(p,"12345",5) 122 | print(buf) 123 | print(buf:dump()) 124 | -- binbuf<0x3a801400>[5/4096] 125 | -- [0000] 31 32 33 34 35 1234 5 126 | 127 | --[[ If you need raw char data fro buffer, call :pv() ]] 128 | 129 | local ptr,len = buf:pv() 130 | local str = ffi.string(ptr,len) 131 | 132 | --- 133 | --- Inside object real buffer is allocated internally and will be freed with buf destroy 134 | --- If you need to use buffer (for ex for iovec), you must keep reference to buf 135 | --- But since buf is not a pointer to the beginning of char data there is an :export method. 136 | --- It allows to take out character data itself with correct gc-guarded poiner 137 | --- 138 | 139 | local pv 140 | do 141 | local buf = bin.buf(4096) 142 | buf:copy("test") 143 | local len 144 | pv,len = buf:export() 145 | -- now buffer can't be used anymore for writing data 146 | end 147 | collectgarbage() 148 | -- but pv still points to original data and memory will be freed only when pv will be freed 149 | pv = nil 150 | collectgarbage() 151 | -- now memory was freed. no one should use data under buf or pv 152 | 153 | --[[ 154 | bin.buf requires 2 allocations: for struct and for buffer, but allows reallocation, when needed. If the amount of data required is known, bin.fixbuf may be used as a drop-in replacement. It uses only one malloc, but can't be extended over it's size (error will be thrown) 155 | ]] 156 | 157 | --- 158 | --- There is also read buffer, for reading data, but I will document it 159 | --- a bit later ;) 160 | --- 161 | ``` 162 | -------------------------------------------------------------------------------- /bin-dev-1.rockspec: -------------------------------------------------------------------------------- 1 | package = "bin" 2 | version = "dev-1" 3 | 4 | source = { 5 | url = 'git+https://github.com/moonlibs/bin.git'; 6 | branch = 'master'; 7 | } 8 | 9 | description = { 10 | summary = "Binary tools"; 11 | detailed = "Binary tools"; 12 | homepage = 'https://github.com/moonlibs/bin.git'; 13 | license = 'Artistic'; 14 | maintainer = "Mons Anderson "; 15 | } 16 | 17 | dependencies = { 18 | 'lua >= 5.1'; 19 | 'ffi-reloadable >= 0'; 20 | } 21 | 22 | build = { 23 | type = "builtin", 24 | modules = { 25 | bin = "bin.lua", 26 | ["bin.base"] = "bin/base.lua", 27 | ["bin.basebuf"] = "bin/basebuf.lua", 28 | ["bin.buf"] = "bin/buf.lua", 29 | ["bin.fixbuf"] = "bin/fixbuf.lua", 30 | ["bin.rbuf"] = "bin/rbuf.lua", 31 | ["bin.saferbuf"] = "bin/saferbuf.lua", 32 | ['libluabin-scm-4'] = { 33 | sources = "libluabin.c" 34 | }, 35 | }, 36 | } 37 | -------------------------------------------------------------------------------- /bin-scm-1.rockspec: -------------------------------------------------------------------------------- 1 | package = 'bin' 2 | version = 'scm-1' 3 | 4 | source = { 5 | url = 'git+https://github.com/moonlibs/bin.git'; 6 | branch = 'v1'; 7 | } 8 | 9 | description = { 10 | summary = "Binary tools"; 11 | detailed = "Binary tools"; 12 | homepage = 'https://github.com/moonlibs/bin.git'; 13 | license = 'Artistic'; 14 | maintainer = "Mons Anderson "; 15 | } 16 | 17 | dependencies = { 18 | 'lua >= 5.1'; 19 | 'ffi-reloadable >= 0'; 20 | } 21 | 22 | build = { 23 | type = 'builtin', 24 | modules = { 25 | ['bin'] = 'bin.lua'; 26 | ['libluabin'] = { 27 | sources = { 28 | "libluabin.c", 29 | }; 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /bin.lua: -------------------------------------------------------------------------------- 1 | local M = {} 2 | for method, func in pairs(require 'bin.base') do 3 | M[method] = func 4 | end 5 | 6 | M.fixbuf = require 'bin.fixbuf'.new 7 | M.buf = require 'bin.buf'.new 8 | M.rbuf = require 'bin.rbuf'.new 9 | M.saferbuf = require 'bin.saferbuf'.new 10 | 11 | return M 12 | -------------------------------------------------------------------------------- /bin/base.lua: -------------------------------------------------------------------------------- 1 | --- 2 | -- This module contains basic binary functions. 3 | -- @release v4 4 | -- @module bin.base 5 | local M = { V = 4 } 6 | 7 | local ffi = require 'ffi.reloadable' 8 | local so_lib_path do 9 | local so_lib_name = 'libluabin-scm-'..M.V 10 | if package.search then 11 | so_lib_path = package.search(so_lib_name) 12 | else 13 | so_lib_path = package.searchpath(so_lib_name, package.cpath) 14 | end 15 | assert(so_lib_path, "bin: failed to find "..so_lib_name) 16 | end 17 | local lib = ffi.load(so_lib_path, true) 18 | 19 | ffi.fundef('calloc', [[ void *calloc(size_t count, size_t size); ]]) 20 | ffi.fundef('malloc', [[ void * malloc(size_t size); ]]) 21 | ffi.fundef('realloc', [[ void * realloc(void *ptr, size_t size); ]]) 22 | ffi.fundef('free', [[ void free(void *ptr); ]]) 23 | ffi.fundef('memmove', [[ void * memmove(void *dst, const void *src, size_t len); ]]) 24 | 25 | ffi.fundef('reb_decode',[[ 26 | uint8_t reb_decode(const char *p, size_t size, uint64_t * result); 27 | ]]) 28 | ffi.fundef('reb_encode',[[ 29 | uint8_t reb_encode(uint64_t n, char *buf, size_t size); 30 | ]]) 31 | ffi.fundef('bin_hex',[[ 32 | char * bin_hex(unsigned char *p, size_t size); 33 | ]]) 34 | ffi.typedef('xd_conf',[[ 35 | typedef struct { 36 | uint8_t row; 37 | uint8_t hpad; 38 | uint8_t cpad; 39 | uint8_t hsp; 40 | uint8_t csp; 41 | uint8_t cols; 42 | } xd_conf; 43 | ]]); 44 | ffi.fundef('bin_xd',[[ 45 | char * bin_xd(const char *data, size_t size, xd_conf *cf); 46 | ]]) 47 | 48 | ffi.fundef('bin_htobe16',[[uint16_t bin_htobe16 (uint16_t x);]]) 49 | --- 50 | -- Converts host byte order to big endian 16 bits. 51 | -- @param x number 52 | function M.htobe16(x) return lib.bin_htobe16(x) end 53 | ffi.fundef('bin_htole16',[[uint16_t bin_htole16 (uint16_t x);]]) 54 | --- 55 | -- Converts host byte order to little endian 16 bits. 56 | -- @param x number 57 | function M.htole16(x) return lib.bin_htole16(x) end 58 | ffi.fundef('bin_be16toh',[[uint16_t bin_be16toh (uint16_t x);]]) 59 | --- 60 | -- Converts big endian 16 bits to host byte order. 61 | -- @param x number 62 | function M.be16toh(x) return lib.bin_be16toh(x) end 63 | ffi.fundef('bin_le16toh',[[uint16_t bin_le16toh (uint16_t x);]]) 64 | --- 65 | -- Converts little endian 16 bits to host byte 66 | -- @param x number 67 | function M.le16toh(x) return lib.bin_le16toh(x) end 68 | ffi.fundef('bin_htobe32',[[uint32_t bin_htobe32 (uint32_t x);]]) 69 | --- 70 | -- Converts host byte order to big endian 32 bits. 71 | -- @param x number 72 | function M.htobe32(x) return lib.bin_htobe32(x) end 73 | ffi.fundef('bin_htole32',[[uint32_t bin_htole32 (uint32_t x);]]) 74 | --- 75 | -- Converts host byte order to little endian 32 bits. 76 | -- @param x number 77 | function M.htole32(x) return lib.bin_htole32(x) end 78 | ffi.fundef('bin_be32toh',[[uint32_t bin_be32toh (uint32_t x);]]) 79 | --- 80 | -- Converts big endian 32 bits to host byte order. 81 | -- @param x number 82 | function M.be32toh(x) return lib.bin_be32toh(x) end 83 | ffi.fundef('bin_le32toh',[[uint32_t bin_le32toh (uint32_t x);]]) 84 | --- 85 | -- Converts little endian 32 bits to host byte 86 | -- @param x number 87 | function M.le32toh(x) return lib.bin_le32toh(x) end 88 | ffi.fundef('bin_htobe64',[[uint64_t bin_htobe64 (uint64_t x);]]) 89 | --- 90 | -- Converts host byte order to big endian 64 bits. 91 | -- @param x number 92 | function M.htobe64(x) return lib.bin_htobe64(x) end 93 | ffi.fundef('bin_htole64',[[uint64_t bin_htole64 (uint64_t x);]]) 94 | --- 95 | -- Converts host byte order to little endian 64 bits. 96 | -- @param x number 97 | function M.htole64(x) return lib.bin_htole64(x) end 98 | ffi.fundef('bin_be64toh',[[uint64_t bin_be64toh (uint64_t x);]]) 99 | --- 100 | -- Converts big endian 64 bits to host byte order. 101 | -- @param x number 102 | function M.be64toh(x) return lib.bin_be64toh(x) end 103 | ffi.fundef('bin_le64toh',[[uint64_t bin_le64toh (uint64_t x);]]) 104 | --- 105 | -- Converts little endian 64 bits to host byte 106 | -- @param x number 107 | function M.le64toh(x) return lib.bin_le64toh(x) end 108 | 109 | M.jit_major = require 'jit'.version_num 110 | 111 | if not rawget(_G, 'dostring') then 112 | if not rawget(_G, 'loadstring') then 113 | rawset(_G, 'loadstring', load) 114 | end 115 | local loadstring = rawget(_G, 'loadstring') 116 | rawset(_G, 'dostring', function(str) return assert(loadstring(str))() end) 117 | end 118 | 119 | --- 120 | -- Returns hex representation of the binary data 121 | -- @param data accepts string, allmost any pointer - start of the data. 122 | -- @param len length of the data in bytes 123 | -- @treturn string `hex` representation of the data 124 | -- @usage 125 | -- print(bin.hex "some binary string") 126 | -- 736F6D652062696E61727920737472696E67 127 | function M.hex(data, len) 128 | len = len or #data 129 | local buf = lib.bin_hex(ffi.cast("char*",data),len); 130 | local rv 131 | if buf then 132 | rv = ffi.string(buf) 133 | lib.free(buf) 134 | else 135 | error("Failed") 136 | end 137 | return rv 138 | end 139 | 140 | --- 141 | -- Returns hexdump (hexdump -C) of the binary data 142 | -- @param data accepts string, allmost any pointer - start of the data. 143 | -- @param len length of the data in bytes 144 | -- @treturn string `hexdump` representation of the data 145 | -- @usage 146 | -- print(bin.xd "some binary string") 147 | -- [0000] 73 6F 6D 65 20 62 69 6E 61 72 79 20 73 74 72 69 some bin ary stri 148 | -- [0010] 6E 67 ng 149 | function M.xd(data, len) 150 | len = len or #data 151 | local buf = lib.bin_xd(ffi.cast("char*",data),len,nil); 152 | local rv 153 | if buf then 154 | rv = ffi.string(buf) 155 | lib.free(buf) 156 | else 157 | error("Failed") 158 | end 159 | return rv 160 | end 161 | 162 | return M 163 | -------------------------------------------------------------------------------- /bin/basebuf.lua: -------------------------------------------------------------------------------- 1 | --- 2 | -- This module implements common write buffer operations. 3 | -- 4 | -- basebuf supports `{u,}int{8,16,32,64}{be,le,}` methods. 5 | -- 6 | -- All of them accepts single argument n and pushes it with corresponding casting 7 | -- @classmod bin.basebuf 8 | local buf = {} 9 | 10 | local base = require 'bin.base' 11 | local rbuf = require 'bin.rbuf' 12 | 13 | local ffi = require 'ffi.reloadable' 14 | local C = ffi.C 15 | 16 | ffi.typedef('double_union',[[ 17 | typedef union double_union { 18 | double d; 19 | uint64_t u; 20 | } double_union; 21 | ]]); 22 | ffi.typedef('float_union',[[ 23 | typedef union float_union { 24 | float d; 25 | uint32_t u; 26 | } float_union; 27 | ]]); 28 | 29 | for _,ix in pairs({'','u'}) do 30 | for _,ti in pairs({'8', '16', '32', '64'}) do 31 | local t = ti 32 | local sz = math.floor(tonumber(ti)/8) 33 | local t_t = ffi.typeof(ix..'int'..t..'_t *') 34 | buf[ ix..'int'..t ] = function(self,n) 35 | local p = self:alloc(sz) 36 | ffi.cast(t_t, p)[0] = n 37 | end 38 | if sz > 1 then 39 | local htobe = 'bin_htobe' .. t 40 | local htole = 'bin_htole' .. t 41 | buf[ ix..'int'..t..'le' ] = function(self,n) 42 | local p = self:alloc(sz) 43 | ffi.cast(t_t, p)[0] = C[htole](n) 44 | end 45 | buf[ ix..'int'..t..'be' ] = function(self,n) 46 | local p = self:alloc(sz) 47 | ffi.cast(t_t, p)[0] = C[htobe](n) 48 | end 49 | end 50 | end 51 | end 52 | 53 | buf.V = buf.int32le; 54 | buf.N = buf.int32be; 55 | 56 | local double_t = ffi.typeof('double *') 57 | local float_t = ffi.typeof('float *') 58 | local uint8_t = ffi.typeof('uint8_t *') 59 | 60 | --- 61 | -- Pushes `n` as double (8 bytes) into buffer 62 | -- @param n number 63 | function buf:double(n) 64 | local p = self:alloc(8) 65 | ffi.cast(double_t, p)[0] = n 66 | end 67 | --- 68 | -- Pushes `n` as double (8 bytes) casting to LE into buffer 69 | -- @param n number 70 | function buf:doublele(n) 71 | local p = ffi.cast('double_union *', self:alloc(8)) 72 | p[0].d = n 73 | p[0].u = C.bin_htole64(p[0].u) 74 | end 75 | --- 76 | -- Pushes `n` as double (8 bytes) casting to BE into buffer 77 | -- @param n number 78 | function buf:doublebe(n) 79 | local p = ffi.cast('double_union *', self:alloc(8)) 80 | p[0].d = n 81 | p[0].u = C.bin_htobe64(p[0].u) 82 | end 83 | buf.d = buf.double; 84 | 85 | --- 86 | -- Pushes `n` as float (4 bytes) into buffer 87 | -- @number n 88 | function buf:float(n) 89 | local p = self:alloc(4) 90 | ffi.cast(float_t, p)[0] = n 91 | end 92 | --- 93 | -- Pushes `n` as float (4 bytes) casting to LE into buffer 94 | -- @number n 95 | function buf:floatle(n) 96 | local p = ffi.cast('float_union *', self:alloc(4)) 97 | p[0].d = n 98 | p[0].u = C.bin_htole32(p[0].u) 99 | end 100 | --- 101 | -- Pushes `n` as float (4 bytes) casting to BE into buffer 102 | -- @number n 103 | function buf:floatbe(n) 104 | local p = ffi.cast('float_union *', self:alloc(4)) 105 | p[0].d = n 106 | p[0].u = C.bin_htobe32(p[0].u) 107 | end 108 | buf.f = buf.float; 109 | 110 | -- compatitbility with lower version of LuaJIT 111 | local lims = { 112 | 2ULL^7 , 2ULL^14, 2ULL^21, 113 | 2ULL^28, 2ULL^35, 2ULL^42, 114 | 2ULL^49, 2ULL^56, 2ULL^63, 115 | } 116 | function buf.reb (self, n) 117 | local size = 1 118 | while size <= #lims and n >= lims[size] do 119 | size = size + 1 120 | end 121 | 122 | local p = ffi.cast(uint8_t, self:alloc(size)) 123 | if C.reb_encode(n, p, self.len) == 1 then 124 | error("REB encoding failed for " .. n) 125 | end 126 | end 127 | 128 | --- 129 | -- Pushes number performing variable length encoding 130 | -- 131 | -- [wikipedia](https://en.wikipedia.org/wiki/Variable-length_quantity#General_structure) 132 | -- @function buf:ber 133 | -- @param n number 134 | if base.jit_major >= 20100 then 135 | local func = [[ 136 | local ffi = require 'ffi' 137 | local uint8_t = ffi.typeof('uint8_t *') 138 | return function(self,n) 139 | ]] 140 | for i = 1,9 do 141 | local lim = bit.lshift(1ULL,7*i) 142 | if i == 1 then 143 | func = func .. "if n < "..string.format("0x%xULL",tonumber(lim)).." then\n" 144 | elseif i == 9 then 145 | func = func .. "else --- < "..string.format("0x%xULL",tonumber(lim)).."\n" 146 | else 147 | func = func .. "elseif n < "..string.format("0x%xULL",tonumber(lim)).." then\n" 148 | end 149 | func = func .. "\tlocal p = ffi.cast(uint8_t,self:alloc("..i.."))\n" 150 | if i > 1 then 151 | for ptr = 0,i-2 do 152 | func = func .. "\tp["..ptr.."] = bit.bor(0x80, bit.rshift(n,"..( 7*(i-ptr-1) ).."))\n" 153 | end 154 | end 155 | func = func .. "\tp["..(i-1).."] = bit.band(0x7f, n)\n" 156 | end 157 | func = func .. "end\nend\n" 158 | buf.ber = _G.dostring(func) 159 | end 160 | 161 | --- 162 | -- Pushes first byte of the string 163 | -- @string x 164 | function buf:char(x) 165 | local p = self:alloc(1) 166 | if type(x) == 'string' then 167 | -- assert(#x == 1, "String should be 1 byte length") 168 | ffi.cast(uint8_t, p)[0] = x:byte(1) 169 | else 170 | ffi.cast(uint8_t, p)[0] = tonumber(x) 171 | end 172 | end 173 | 174 | --- 175 | -- Control methods 176 | -- @section controls 177 | 178 | --- 179 | -- Copies data from given string or pointer 180 | -- @param src string or pointer 181 | -- @param[opt=#src] len bytes of src to copy 182 | function buf:copy(data,len) 183 | if not len then len = #data end 184 | 185 | local p = self:alloc(len) 186 | ffi.copy(p,ffi.cast('char *',data),len) 187 | end 188 | 189 | --- 190 | -- Rollbacks cursor to given position 191 | -- @number[opt=0] n 192 | function buf:clear(n) 193 | n = n or 0 194 | assert(n <= self.cur, "position must be less than current length") 195 | self.cur = n 196 | end 197 | 198 | --- 199 | -- Returns hexdump of the buffer 200 | -- @see bin.base.xd 201 | -- @treturn string hexdump 202 | function buf:dump() 203 | local p,len = self:pv() 204 | return base.xd(p,len,nil) 205 | end 206 | buf.xd = buf.dump 207 | 208 | --- 209 | -- Returns hex representation (analog of string.tohex) 210 | -- @see bin.base.hex 211 | -- @treturn string hex 212 | function buf:hex() 213 | local p,len = self:pv() 214 | return base.hex(p,len) 215 | end 216 | 217 | --- 218 | -- Creates new unsafe `bin.rbuf` from the buffer 219 | -- @treturn @{bin.rbuf} rbuf 220 | function buf:reader() 221 | local p,l = self:pv() 222 | return rbuf.new(p, l) 223 | end 224 | 225 | return buf -------------------------------------------------------------------------------- /bin/buf.lua: -------------------------------------------------------------------------------- 1 | --- 2 | -- This module implements write buffer. 3 | -- You should use it if you want to collect some data for write. 4 | -- 5 | -- Example of usage [moonlibs/connection-scribe](https://github.com/moonlibs/connection-scribe/blob/7f7177fcd34dd016e83962dea08327a042be8849/connection/scribe.lua#L362) 6 | -- @module bin.buf 7 | local ffi = require 'ffi.reloadable' 8 | local C = ffi.C 9 | 10 | local basebuf = require 'bin.basebuf' 11 | 12 | local function capacity(sz) 13 | -- grow by 64b blocks or by 1/4 of data aligned by 64b 14 | sz = tonumber(sz) 15 | local alg = math.ceil(sz / 4 / 64) * 64 16 | return math.ceil(sz / alg) * alg 17 | end 18 | 19 | --- 20 | -- Implements write buffer. 21 | -- Has following structure inside 22 | -- 23 | -- struct bin_buf { 24 | -- char *buf; 25 | -- size_t cur; 26 | -- size_t len; 27 | -- }; 28 | -- 29 | -- @type buf 30 | local buf = {} 31 | function buf:alloc( sz ) 32 | if self.len - self.cur < sz then 33 | self.len = capacity(self.cur + sz); 34 | self.buf = C.realloc(self.buf,self.len) 35 | end 36 | if self.buf == nil then 37 | error("Access to exported buffer",2) 38 | end 39 | self.cur = self.cur + sz 40 | return self.buf + self.cur - sz 41 | end 42 | 43 | --- 44 | -- Returns raw pointer *Doesn't grant ownership*. 45 | -- 46 | -- **You can't continue to use pointer after buffer modification or after free of original object** 47 | -- 48 | -- Mind GC! 49 | -- @return `char *` start of the buffer 50 | -- @return `size` of written bytes 51 | function buf:pv() 52 | return self.buf,self.cur 53 | end 54 | 55 | --- 56 | -- Exports data from buffer for external use. 57 | -- 58 | -- Grants ownership of struct to the returned value (including gc) 59 | -- use object after export is **prohibited** 60 | -- @return `char *` start of the buffer 61 | -- @return `size` of written data inside buffer 62 | -- @return `capacity` - allocated bytes of the buffer 63 | function buf:export() 64 | local p,len = self:pv() 65 | self.buf = nil 66 | return ffi.gc(p, C.free), len, self.len 67 | end 68 | 69 | local function bin_buf_str( self ) 70 | return string.format( 71 | 'binbuf<0x%x>[%s/%s]', 72 | tonumber(ffi.cast('int',self.buf)), 73 | tonumber(self.cur),tonumber(self.len)) 74 | end 75 | local function bin_buf_free(self) 76 | -- print("freeing buf ", self.buf) 77 | if self.buf ~= nil then 78 | C.free(self.buf) 79 | end 80 | end 81 | 82 | for k,v in pairs(basebuf) do buf[k] = v end 83 | 84 | ffi.typedef('bin_buf',[[ 85 | typedef struct bin_buf { 86 | char *buf; 87 | size_t cur; 88 | size_t len; 89 | } bin_buf; 90 | ]], { 91 | __gc = bin_buf_free; 92 | __index = buf; 93 | __tostring = bin_buf_str; 94 | 95 | }) 96 | 97 | local M = {} 98 | --- 99 | -- Creates new buffer 100 | -- @constructor 101 | -- @param size initial capacity of the buffer aligned to 64bytes 102 | -- @return @{buf} buffer new empty buffer 103 | function M.new(sz) 104 | sz = sz or 4096; 105 | -- sz = sz or 16; 106 | local b = ffi.new( 'bin_buf'); 107 | b.len = capacity(sz); 108 | -- b.buf = ffi.new('char[?]',b.len) -- memory corruption after free (( 109 | b.buf = C.calloc(b.len,1) -- memory corruption after free (( 110 | return b 111 | end 112 | 113 | return M -------------------------------------------------------------------------------- /bin/fixbuf.lua: -------------------------------------------------------------------------------- 1 | --- 2 | -- This module provides buffer with constant capacity. 3 | -- Module implements every method from @{bin.basebuf} 4 | -- 5 | -- Example of usage [moonlibs/connection-scribe](https://github.com/moonlibs/connection-scribe/blob/7f7177fcd34dd016e83962dea08327a042be8849/connection/scribe.lua#L362) 6 | -- @module bin.fixbuf 7 | 8 | --- 9 | -- Implements write buffer. 10 | -- Has following structure inside 11 | -- 12 | -- struct bin_buf_#capasity { 13 | -- char buf[#capasity]; 14 | -- size_t cur; 15 | -- }; 16 | -- 17 | -- @type fixbuf 18 | local buf = {} 19 | 20 | local ffi = require 'ffi.reloadable' 21 | local C = ffi.C 22 | local basebuf = require 'bin.basebuf' 23 | 24 | 25 | function buf:alloc( sz ) 26 | -- print("alloc for buf ",self, sz, ffi.sizeof(self[0])) 27 | if ffi.sizeof(self[0]) - self.cur < sz then 28 | error("No more space in fixed buffer",2) 29 | end 30 | self.cur = self.cur + sz 31 | return ffi.cast('char *',self) + self.cur - sz 32 | end 33 | 34 | --- 35 | -- Returns raw pointer *Doesn't grant ownership*. 36 | -- 37 | -- **You can't continue to use pointer after buffer modification or after free of original object** 38 | -- 39 | -- Mind GC! 40 | -- @return `char *` ptr 41 | -- @return `size` of written data 42 | function buf:pv() 43 | return ffi.cast('char *',self),self.cur 44 | end 45 | 46 | --- 47 | -- Returns raw pointer, size and free of the buffer and destroys buffer. 48 | -- 49 | -- Grants ownership of struct to the returned value (including gc) 50 | -- use object after export is **prohibited** 51 | -- @return `char *` start of the buffer 52 | -- @return `size` of the buffer 53 | -- @return `free` of the buffer 54 | function buf:export() 55 | --[[ 56 | local newptr = ffi.cast('char *',ffi.gc(self,nil)) 57 | local sz = ffi.sizeof(self[0]) - ffi.sizeof(self.cur) 58 | return ffi.gc(newptr, C.free), self.cur, sz 59 | ]] 60 | return ffi.gc(ffi.cast('char *',ffi.gc(self,nil)),C.free), 61 | self.cur, 62 | ffi.sizeof(self[0]) - ffi.sizeof(self.cur) 63 | end 64 | 65 | local function bin_buf_str( self ) 66 | return string.format( 67 | 'binbuf<0x%x>[%s/%s]', 68 | tonumber(ffi.cast('int',ffi.cast('char *',self))), 69 | tonumber(self.cur),tonumber(ffi.sizeof(self[0]) - ffi.sizeof(self.cur))) 70 | end 71 | 72 | for k,v in pairs(basebuf) do buf[k] = v end 73 | 74 | local sizes = {} 75 | -- 4k to 1Mb 76 | for i = 12,20 do 77 | local sz = 2^i 78 | local cap = sz - ffi.sizeof('size_t') 79 | ffi.typedef('bin_buf_'..cap, 80 | 'typedef struct bin_buf_'..cap..' { char buf['..cap..']; size_t cur; } bin_buf_'..cap..';' 81 | ,{ 82 | __index = buf; 83 | __tostring = bin_buf_str; 84 | } 85 | ) 86 | table.insert(sizes,cap) 87 | end 88 | 89 | local M = {} 90 | 91 | --- 92 | -- Creates buffer from range 4K - 1M. 93 | -- @constructor 94 | -- @param sz requested size 95 | -- @return `fixbuf` 96 | -- @raise if `sz` is bigger than 1M 97 | function M.new(sz) 98 | sz = sz or 4088; 99 | for _,cap in ipairs(sizes) do 100 | if cap >= sz then 101 | -- print("choose ",cap, " for ",sz) 102 | local typename = 'bin_buf_'..cap 103 | -- local b = ffi.new( 'bin_buf_'..cap ); 104 | local b = 105 | ffi.gc( 106 | ffi.cast( typename..'*', C.calloc( 1, ffi.sizeof(typename) ) ), 107 | C.free 108 | ) 109 | b.cur = 0; 110 | return b 111 | end 112 | end 113 | error("Too big size requested "..tostring(sz)) 114 | end 115 | 116 | return M -------------------------------------------------------------------------------- /bin/rbuf.lua: -------------------------------------------------------------------------------- 1 | --- 2 | -- Implements unsafe binary Reader. 3 | -- 4 | -- Same as `bin.basebuf` supports decoding numbers 5 | -- into any endianess. 6 | -- 7 | -- `{u,i}{8,16,32,64}{be,le,}` are supported opposite to defined at @{bin.basebuf}. 8 | -- @module bin.rbuf 9 | 10 | --- 11 | -- Implements unsafe binary Reader. 12 | -- Has following structure. 13 | -- 14 | -- struct bin_rbuf { 15 | -- const char * buf; 16 | -- union { 17 | -- const char *c; 18 | -- const int8_t *i8; 19 | -- const uint8_t *u8; 20 | -- const int16_t *i16; 21 | -- const uint16_t *u16; 22 | -- const int32_t *i32; 23 | -- const uint32_t *u32; 24 | -- const int64_t *i64; 25 | -- const uint64_t *u64; 26 | -- } p; 27 | -- size_t len; 28 | -- }; 29 | -- @type rbuf 30 | local rbuf = {} 31 | local ffi = require 'ffi.reloadable' 32 | local C = ffi.C 33 | 34 | local base = require 'bin.base' 35 | 36 | local function bin_rbuf_str( self ) 37 | return string.format( 38 | 'binrbuf<0x%x>[%s/%s]', 39 | tonumber(ffi.cast('int',ffi.cast('char *',self))), 40 | tonumber(self.p.c - self.buf), 41 | tonumber(self.len) 42 | ) 43 | end 44 | 45 | local rbuf_t = ffi.typedef('bin_rbuf',[[ 46 | typedef struct bin_rbuf { 47 | const char * buf; 48 | union { 49 | const char *c; 50 | const int8_t *i8; 51 | const uint8_t *u8; 52 | const int16_t *i16; 53 | const uint16_t *u16; 54 | const int32_t *i32; 55 | const uint32_t *u32; 56 | const int64_t *i64; 57 | const uint64_t *u64; 58 | } p; 59 | 60 | size_t len; 61 | } bin_rbuf; 62 | ]],{ 63 | __index = rbuf; 64 | __tostring = bin_rbuf_str; 65 | }) 66 | 67 | for _,ix in pairs({'i','u'}) do 68 | for _,t in pairs({'8', '16', '32', '64'}) do 69 | local sz = math.floor(tonumber(t)/8) 70 | local typename = ix..t 71 | 72 | rbuf[ typename ] = function(self) 73 | local n = self.p[typename][0] 74 | self.p[typename] = self.p[typename]+1 75 | return n 76 | end 77 | if sz > 1 then 78 | local htobe = 'bin_htobe' .. t 79 | local htole = 'bin_htole' .. t 80 | rbuf[ typename..'le' ] = function(self) 81 | local n = self.p[typename][0] 82 | self.p[typename] = self.p[typename]+1 83 | return C[htole]( n ) 84 | end 85 | rbuf[ typename..'be' ] = function(self) 86 | local n = self.p[typename][0] 87 | self.p[typename] = self.p[typename]+1 88 | return C[htobe]( n ) 89 | end 90 | end 91 | end 92 | end 93 | 94 | -- TODO: float, double 95 | 96 | rbuf.V = rbuf.i32le; 97 | rbuf.N = rbuf.i32be; 98 | 99 | function rbuf:reb() 100 | local n = ffi.new("uint64_t [1]", 0) 101 | local shift = C.reb_decode(self.p.u8, self:avail(), ffi.cast('uint64_t *', n)) 102 | if shift == 0 then 103 | error("Decoding REB failed", 2) 104 | else 105 | self.p.u8 = self.p.u8 + shift 106 | return n[0] 107 | end 108 | end 109 | 110 | --- 111 | -- Decodes VLQ number Opposite to @{bin.basebuf.ber} (LuaJIT >= 2.1). 112 | -- @function rbuf:ber 113 | -- @return `number` decoded value 114 | if base.jit_major >= 20100 then 115 | function rbuf:ber() 116 | local n = 0ULL 117 | for i = 0,8 do 118 | n = bit.bor( bit.lshift(n,7), bit.band( 0x7f, self.p.u8[i] ) ) 119 | if self.p.u8[i] < 0x80 then 120 | self.p.u8 = self.p.u8 + i + 1 121 | return n 122 | end 123 | end 124 | error("Bad BER sequence",2) 125 | end 126 | end 127 | 128 | --- 129 | -- [unsafe] Extracts `len` bytes from buffer as `string` 130 | -- @number len size of resulting string 131 | -- @return `string` str 132 | function rbuf:str(len) 133 | local str = ffi.string( self.p.c, len ) 134 | self.p.c = self.p.c + len 135 | return str 136 | end 137 | 138 | --- 139 | -- [safe] Returns nice hexdump of the remaining buffer 140 | -- @see bin.base.xd 141 | -- @return `string` hexdump 142 | function rbuf:dump() 143 | return base.xd(self.p.c,self.len - (self.p.c-self.buf),nil) 144 | end 145 | 146 | --- 147 | -- [safe] Returns hex representation of the remaining buffer 148 | -- @see bin.base.hex 149 | -- @return `string` hex 150 | function rbuf:hex() 151 | return base.hex(self.p.c,self.len - (self.p.c-self.buf)) 152 | end 153 | 154 | --- 155 | -- [safe] memmoves remaining buffer to the beginning. 156 | -- shrinks `len` of the buffer. 157 | function rbuf:move() 158 | local size = self.len - (self.p.c - self.buf) 159 | C.memmove(self.buf, self.p.c, size) 160 | self.p.c = self.buf 161 | self.len = size 162 | end 163 | 164 | --- 165 | -- [unsafe] Forwards cursor for `len` bytes (accepts negatives). 166 | -- @number `len` 167 | function rbuf:skip(len) 168 | self.p.c = self.p.c + len 169 | end 170 | 171 | --- 172 | -- [safe] Returns remaining part of the buffer 173 | -- @return `left` bytes 174 | function rbuf:avail() 175 | return self.len - (self.p.c-self.buf) 176 | end 177 | 178 | local M = { buf = rbuf } 179 | 180 | --- 181 | -- Creates rbuf from pointer 182 | -- @constructor 183 | -- @tparam char* p pointer or string 184 | -- @param[opt=#p] l length of the pointer 185 | -- @return `rbuf` 186 | function M.new(p, l) 187 | if not l then l = #p end 188 | local self = rbuf_t{ buf = p; len = l; } 189 | self.p.c = p 190 | return self 191 | end 192 | 193 | return M -------------------------------------------------------------------------------- /bin/saferbuf.lua: -------------------------------------------------------------------------------- 1 | --- 2 | -- Implements Safe binary Reader. 3 | -- 4 | -- Example of usage [ochaton/migrate](https://github.com/ochaton/migrate/blob/29663cf96b0103963a1f78dbbbc91df9bf1fec98/migrate/init.lua#L31) 5 | -- 6 | -- Same as @{bin.rbuf} but safer. 7 | -- Always checks bounds of the buffer. 8 | -- @see bin.rbuf.dump 9 | -- @see bin.rbuf.hex 10 | -- @see bin.rbuf.move 11 | -- @see bin.rbuf.avail 12 | -- @module bin.saferbuf 13 | 14 | --- 15 | -- Safe rbuf implements everything `bin.rbuf` does but adds bounds checks. 16 | -- Has following structure. 17 | -- 18 | -- struct bin_safe_rbuf { 19 | -- const char * buf; 20 | -- union { 21 | -- const char *c; 22 | -- const int8_t *i8; 23 | -- const uint8_t *u8; 24 | -- const int16_t *i16; 25 | -- const uint16_t *u16; 26 | -- const int32_t *i32; 27 | -- const uint32_t *u32; 28 | -- const int64_t *i64; 29 | -- const uint64_t *u64; 30 | -- } p; 31 | -- size_t len; 32 | -- }; 33 | -- @see bin.rbuf.dump 34 | -- @see bin.rbuf.hex 35 | -- @see bin.rbuf.move 36 | -- @see bin.rbuf.avail 37 | -- @type saferbuf 38 | local srbuf = {} 39 | local ffi = require 'ffi.reloadable' 40 | local C = ffi.C 41 | 42 | local base = require 'bin.base' 43 | local rbuf = require 'bin.rbuf'.buf 44 | 45 | srbuf.dump = rbuf.dump 46 | srbuf.hex = rbuf.hex 47 | srbuf.move = rbuf.move 48 | srbuf.avail = rbuf.avail 49 | 50 | local function bin_safe_rbuf_str(self) 51 | return string.format( 52 | 'binsrbuf<0x%x>[%s/%s]', 53 | tonumber(ffi.cast('int',ffi.cast('char *',self))), 54 | tonumber(self.p.c - self.buf), 55 | tonumber(self.len) 56 | ) 57 | end 58 | 59 | local saferbuf_t = ffi.typedef('bin_safe_rbuf',[[ 60 | typedef struct bin_safe_rbuf { 61 | const char * buf; 62 | union { 63 | const char *c; 64 | const int8_t *i8; 65 | const uint8_t *u8; 66 | const int16_t *i16; 67 | const uint16_t *u16; 68 | const int32_t *i32; 69 | const uint32_t *u32; 70 | const int64_t *i64; 71 | const uint64_t *u64; 72 | } p; 73 | 74 | size_t len; 75 | } bin_safe_rbuf; 76 | ]],{ 77 | __index = srbuf; 78 | __tostring = bin_safe_rbuf_str; 79 | }) 80 | 81 | for _,ix in pairs({'i','u'}) do 82 | for _,t in pairs({ 83 | '8', 84 | '16', 85 | '32', 86 | '64', 87 | }) do 88 | local sz = math.floor(tonumber(t)/8) 89 | local typename = ix..t 90 | 91 | srbuf[ typename ] = function(self) 92 | self:have(sz, 3) 93 | local n = self.p[typename][0] 94 | self.p[typename] = self.p[typename]+1 95 | return n 96 | end 97 | if sz > 1 then 98 | local htobe = 'bin_htobe' .. t 99 | local htole = 'bin_htole' .. t 100 | srbuf[ typename..'le' ] = function(self) 101 | self:have(sz, 3) 102 | local n = self.p[typename][0] 103 | self.p[typename] = self.p[typename]+1 104 | return C[htole]( n ) 105 | end 106 | srbuf[ typename..'be' ] = function(self) 107 | self:have(sz, 3) 108 | local n = self.p[typename][0] 109 | self.p[typename] = self.p[typename]+1 110 | return C[htobe]( n ) 111 | end 112 | end 113 | end 114 | end 115 | 116 | -- TODO: float, double 117 | 118 | srbuf.V = srbuf.i32le; 119 | srbuf.N = srbuf.i32be; 120 | 121 | function srbuf:reb() 122 | local n = ffi.new("uint64_t [1]", 0) 123 | local shift = C.reb_decode(self.p.u8, self:avail(), ffi.cast('uint64_t *', n)) 124 | if shift == 0 then 125 | error("Decoding REB failed", 2) 126 | else 127 | self:have(shift, 3) 128 | self.p.u8 = self.p.u8 + shift 129 | return n[0] 130 | end 131 | end 132 | 133 | 134 | --- 135 | -- Decodes VLQ number Opposite to @{bin.basebuf.ber} (LuaJIT >= 2.1). 136 | -- @function srbuf:ber 137 | -- @return `number` decoded value 138 | if base.jit_major >= 20100 then 139 | function srbuf:ber() 140 | local n = 0ULL 141 | for i = 0,8 do 142 | n = bit.bor( bit.lshift(n,7), bit.band( 0x7f, self.p.u8[i] ) ) 143 | self:have(i+1, 3) 144 | if self.p.u8[i] < 0x80 then 145 | self.p.u8 = self.p.u8 + i + 1 146 | return n 147 | end 148 | end 149 | error("Bad BER sequence",2) 150 | end 151 | end 152 | 153 | --- 154 | -- Reads `len` bytes from buffer and returns `string`. 155 | -- @number len 156 | -- @return string 157 | function srbuf:str(len) 158 | self:have(len, 3) 159 | local str = ffi.string(self.p.c, len) 160 | self.p.c = self.p.c + len 161 | return str 162 | end 163 | 164 | --- 165 | -- Forwards cursor for `len` bytes (bounds checks inside) 166 | -- @number len number of bytes 167 | function srbuf:skip(len) 168 | self:have(len, 3) 169 | self.p.c = self.p.c + len 170 | end 171 | 172 | --- 173 | -- Checks that buffer has at least `bytes` till the end 174 | -- @number[opt=0] bytes 175 | -- @number[opt=2] lvl 176 | -- @treturn number bytes to the end of the buffer 177 | function srbuf:have(bytes, lvl) 178 | local avail = self:avail() 179 | bytes = bytes or 0 180 | if bytes > self:avail() then 181 | lvl = lvl or 2 182 | error("not enough bytes", lvl) 183 | end 184 | return avail 185 | end 186 | 187 | local M = {} 188 | --- 189 | -- Creates reader from given pointer 190 | -- @constructor 191 | -- @tparam `char *` ptr string or pointer 192 | -- @number[opt=#str] size of the buffer 193 | -- @return `saferbuf` 194 | function M.new(p, l) 195 | if not l then l = #p end 196 | local self = saferbuf_t{ buf = p; len = l; } 197 | self.p.c = p 198 | return self 199 | end 200 | 201 | return M -------------------------------------------------------------------------------- /config.ld: -------------------------------------------------------------------------------- 1 | project = 'Bin' 2 | title = 'Bin' 3 | description = "Eases work with binary data in LuaJIT" 4 | full_description = [[ 5 | Module bin provides many functions to perfom binary transformations. 6 | 7 | bin implements 2 write buffers (bin.buf and bin.fixbuf) 8 | to ease construct binary data. 9 | 10 | Also bin implements 2 read buffer (bin.saferbuf and bin.rbuf) 11 | to ease organize parsing of binary data. 12 | ]] 13 | file = {'bin'} 14 | readme = 'README.md' 15 | format = 'discount' 16 | style = "!pale" 17 | wrap = true 18 | not_luadoc = true 19 | no_space_before_args = true -------------------------------------------------------------------------------- /ex.lua: -------------------------------------------------------------------------------- 1 | local bin = require 'bin' 2 | local ffi = require 'ffi' 3 | 4 | --[[ For the first: nice hexdump ]]-- 5 | 6 | -- print hexdump of lua string 7 | print(bin.xd("some binary string")) 8 | 9 | --[[ Gives the following output: 10 | [0000] 73 6F 6D 65 20 62 69 6E 61 72 79 20 73 74 72 69 some bin ary stri 11 | [0010] 6E 67 ng 12 | ]] 13 | 14 | -- Accepts arbitrary pointer and could take length 15 | print(bin.xd(ffi.cast("void *","string"),4)) 16 | --[[ Gives the following output: 17 | [0000] 73 74 72 69 stri 18 | ]] 19 | 20 | -- For compact output there is `hex` 21 | print(bin.hex("some binary string")) 22 | -- 736F6D652062696E61727920737472696E67 23 | 24 | --[[ There is also endian conversion functions ]]-- 25 | 26 | print(bin.htobe16(0x1234)) -- 13330 27 | print(bin.htole16(0x1234)) -- 4660 28 | print(bin.be16toh(0x1234)) -- 13330 29 | print(bin.le16toh(0x1234)) -- 4660 30 | print(bin.htobe32(0x1234)) -- 873594880 31 | print(bin.htole32(0x1234)) -- 4660 32 | print(bin.be32toh(0x1234)) -- 873594880 33 | print(bin.le32toh(0x1234)) -- 4660 34 | print(bin.htobe64(0x1234)) -- 3752061439553044480ULL 35 | print(bin.htole64(0x1234)) -- 4660ULL 36 | print(bin.be64toh(0x1234)) -- 3752061439553044480ULL 37 | print(bin.le64toh(0x1234)) -- 4660ULL 38 | 39 | --[[ And buffer objects. Write buffer ]]-- 40 | 41 | local desired_size = 4096 42 | local buf = bin.buf(desired_size) 43 | print(buf) 44 | -- binbuf<0x31004200>[0/4096] 45 | -- address used/total 46 | 47 | -- accepts encoding of integers: 48 | 49 | buf:uint8(0xff) 50 | print(buf:hex()) -- FF 51 | buf:uint32(0x12345678) 52 | print(buf:hex()) -- FF78563412 53 | buf:uint32(0x12345678) 54 | 55 | -- Also support endianness 56 | 57 | buf:uint32be(0x12345678) -- be for big endian 58 | print(buf:hex()) -- FF785634127856341212345678 59 | 60 | buf:uint32le(0x12345678) -- le for little endian 61 | print(buf:hex()) -- FF78563412785634121234567878563412 62 | 63 | --[[ 64 | Complete list of numeric methods: 65 | uint8 int8 uint8be int8le 66 | uint16 int16 uint16be int16le 67 | uint32 int32 uint32be int32le 68 | uint64 int64 uint64be int64le 69 | V = int32le 70 | N = int32be 71 | ber - for BER packed integer (perl pack w) 72 | reb - for reverse BER (big endian) 73 | f float floatbe floatle 74 | d double doublebe doublele <- this one looks so weird ;) 75 | ]] 76 | 77 | -- buffer may be cleared (reset pos to zero or other pos within buf) 78 | buf:clear(8) 79 | print(buf:hex()) -- FF78563412785634 80 | print(buf) -- binbuf<0x31004200>[8/4096] 81 | buf:clear() 82 | print(buf) -- binbuf<0x31004200>[0/4096] 83 | 84 | buf:double(0.123456) 85 | print(buf:hex()) -- FF78563412785634 86 | buf:doublebe(0.123456) 87 | print(buf:hex()) -- BFB67EFACF9ABF3F3FBF9ACFFA7EB6BF 88 | 89 | buf:clear() 90 | 91 | buf:float(0.123456) 92 | print(buf:hex()) -- 80D6FC3D 93 | buf:floatbe(0.123456) 94 | print(buf:hex()) -- 80D6FC3D3DFCD680 95 | 96 | buf:clear() 97 | 98 | --[[ Also there is simple :char ]] 99 | buf:char('x') -- !only one char 100 | buf:char(0xfe) -- but also accepts number as byte 101 | 102 | print(buf:hex()) -- 78FE 103 | 104 | --[[ for strings or buffers there is copy ]] 105 | 106 | buf:copy("putme") 107 | buf:copy("notlong",3) 108 | 109 | print(buf:dump()) 110 | -- [0000] 78 FE 70 75 74 6D 65 6E 6F 74 x.pu tmen ot 111 | 112 | buf:clear() 113 | 114 | --[[ For manual operation there is :alloc ]] 115 | local nbytes = 5 116 | local p = buf:alloc(nbytes) 117 | ffi.copy(p,"12345",5) 118 | print(buf) 119 | print(buf:dump()) 120 | -- binbuf<0x3a801400>[5/4096] 121 | -- [0000] 31 32 33 34 35 1234 5 122 | 123 | --[[ If you need raw char data fro buffer, call :pv() ]] 124 | 125 | local ptr,len = buf:pv() 126 | local str = ffi.string(ptr,len) 127 | 128 | --- 129 | --- Inside object real buffer is allocated internally and will be freed with buf destroy 130 | --- If you need to use buffer (for ex for iovec), you must keep reference to buf 131 | --- But since buf is not a pointer to the beginning of char data there is an :export method. 132 | --- It allows to take out character data itself with correct gc-guarded poiner 133 | --- 134 | 135 | local pv 136 | do 137 | local buf = bin.buf(4096) 138 | buf:copy("test") 139 | local len 140 | pv,len = buf:export() 141 | -- now buffer can't be used anymore for writing data 142 | end 143 | collectgarbage() 144 | -- but pv still points to original data and memory will be freed only when pv will be freed 145 | pv = nil 146 | collectgarbage() 147 | -- now memory was freed. no one should use data under buf or pv 148 | 149 | -------------------------------------------------------------------------------- /libluabin.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "portable_endian.h" 10 | #include "xd.h" 11 | 12 | char * bin_xd(char *data, size_t size, xd_conf *cf) { 13 | return xd(data, size, cf); 14 | } 15 | 16 | uint16_t bin_htobe16 (uint16_t x) { return htobe16(x); } 17 | uint16_t bin_htole16 (uint16_t x) { return htole16(x); } 18 | uint16_t bin_be16toh (uint16_t x) { return be16toh(x); } 19 | uint16_t bin_le16toh (uint16_t x) { return le16toh(x); } 20 | 21 | uint32_t bin_htobe32 (uint32_t x) { return htobe32(x); } 22 | uint32_t bin_htole32 (uint32_t x) { return htole32(x); } 23 | uint32_t bin_be32toh (uint32_t x) { return be32toh(x); } 24 | uint32_t bin_le32toh (uint32_t x) { return le32toh(x); } 25 | 26 | uint64_t bin_htobe64 (uint64_t x) { return htobe64(x); } 27 | uint64_t bin_htole64 (uint64_t x) { return htole64(x); } 28 | uint64_t bin_be64toh (uint64_t x) { return be64toh(x); } 29 | uint64_t bin_le64toh (uint64_t x) { return le64toh(x); } 30 | 31 | char * bin_hex(unsigned char *p, size_t size) { 32 | char *rv = malloc(size*2+1); 33 | char *r = rv; 34 | if (!rv) { 35 | fprintf(stderr,"Can't allocate memory\n"); 36 | return NULL; 37 | } 38 | unsigned char * e = p + size; 39 | for (; p>= 7, ptr++) { 76 | *ptr = (n & 0x7f) | ( n > 0x7f ? 0x80 : 0 ); 77 | } 78 | if (ptr > pend) { 79 | return 1; // fail 80 | } 81 | if (n > 0) { 82 | return 1; // fail, number wasn't packed 83 | } 84 | } 85 | return 0; 86 | } 87 | -------------------------------------------------------------------------------- /portable_endian.h: -------------------------------------------------------------------------------- 1 | // "License": Public Domain 2 | // I, Mathias Panzenböck, place this file hereby into the public domain. Use it at your own risk for whatever you like. 3 | // In case there are jurisdictions that don't support putting things in the public domain you can also consider it to 4 | // be "dual licensed" under the BSD, MIT and Apache licenses, if you want to. This code is trivial anyway. Consider it 5 | // an example on how to get the endian conversion functions on different platforms. 6 | 7 | #ifndef PORTABLE_ENDIAN_H__ 8 | #define PORTABLE_ENDIAN_H__ 9 | 10 | #if (defined(_WIN16) || defined(_WIN32) || defined(_WIN64)) && !defined(__WINDOWS__) 11 | 12 | # define __WINDOWS__ 13 | 14 | #endif 15 | 16 | #if defined(__linux__) || defined(__CYGWIN__) 17 | 18 | # include 19 | 20 | #elif defined(__APPLE__) 21 | 22 | # include 23 | 24 | # define htobe16(x) OSSwapHostToBigInt16(x) 25 | # define htole16(x) OSSwapHostToLittleInt16(x) 26 | # define be16toh(x) OSSwapBigToHostInt16(x) 27 | # define le16toh(x) OSSwapLittleToHostInt16(x) 28 | 29 | # define htobe32(x) OSSwapHostToBigInt32(x) 30 | # define htole32(x) OSSwapHostToLittleInt32(x) 31 | # define be32toh(x) OSSwapBigToHostInt32(x) 32 | # define le32toh(x) OSSwapLittleToHostInt32(x) 33 | 34 | # define htobe64(x) OSSwapHostToBigInt64(x) 35 | # define htole64(x) OSSwapHostToLittleInt64(x) 36 | # define be64toh(x) OSSwapBigToHostInt64(x) 37 | # define le64toh(x) OSSwapLittleToHostInt64(x) 38 | 39 | # define __BYTE_ORDER BYTE_ORDER 40 | # define __BIG_ENDIAN BIG_ENDIAN 41 | # define __LITTLE_ENDIAN LITTLE_ENDIAN 42 | # define __PDP_ENDIAN PDP_ENDIAN 43 | 44 | #elif defined(__OpenBSD__) 45 | 46 | # include 47 | 48 | #elif defined(__NetBSD__) || defined(__FreeBSD__) || defined(__DragonFly__) 49 | 50 | # include 51 | 52 | # define be16toh(x) betoh16(x) 53 | # define le16toh(x) letoh16(x) 54 | 55 | # define be32toh(x) betoh32(x) 56 | # define le32toh(x) letoh32(x) 57 | 58 | # define be64toh(x) betoh64(x) 59 | # define le64toh(x) letoh64(x) 60 | 61 | #elif defined(__WINDOWS__) 62 | 63 | # include 64 | # include 65 | 66 | # if BYTE_ORDER == LITTLE_ENDIAN 67 | 68 | # define htobe16(x) htons(x) 69 | # define htole16(x) (x) 70 | # define be16toh(x) ntohs(x) 71 | # define le16toh(x) (x) 72 | 73 | # define htobe32(x) htonl(x) 74 | # define htole32(x) (x) 75 | # define be32toh(x) ntohl(x) 76 | # define le32toh(x) (x) 77 | 78 | # define htobe64(x) htonll(x) 79 | # define htole64(x) (x) 80 | # define be64toh(x) ntohll(x) 81 | # define le64toh(x) (x) 82 | 83 | # elif BYTE_ORDER == BIG_ENDIAN 84 | 85 | /* that would be xbox 360 */ 86 | # define htobe16(x) (x) 87 | # define htole16(x) __builtin_bswap16(x) 88 | # define be16toh(x) (x) 89 | # define le16toh(x) __builtin_bswap16(x) 90 | 91 | # define htobe32(x) (x) 92 | # define htole32(x) __builtin_bswap32(x) 93 | # define be32toh(x) (x) 94 | # define le32toh(x) __builtin_bswap32(x) 95 | 96 | # define htobe64(x) (x) 97 | # define htole64(x) __builtin_bswap64(x) 98 | # define be64toh(x) (x) 99 | # define le64toh(x) __builtin_bswap64(x) 100 | 101 | # else 102 | 103 | # error byte order not supported 104 | 105 | # endif 106 | 107 | # define __BYTE_ORDER BYTE_ORDER 108 | # define __BIG_ENDIAN BIG_ENDIAN 109 | # define __LITTLE_ENDIAN LITTLE_ENDIAN 110 | # define __PDP_ENDIAN PDP_ENDIAN 111 | 112 | #else 113 | 114 | # error platform not supported 115 | 116 | #endif 117 | 118 | #endif -------------------------------------------------------------------------------- /rockspecs/bin-scm-1.rockspec: -------------------------------------------------------------------------------- 1 | package = 'bin' 2 | version = 'scm-1' 3 | 4 | source = { 5 | url = 'git+https://github.com/moonlibs/bin.git'; 6 | branch = 'v1'; 7 | } 8 | 9 | description = { 10 | summary = "Binary tools"; 11 | detailed = "Binary tools"; 12 | homepage = 'https://github.com/moonlibs/bin.git'; 13 | license = 'Artistic'; 14 | maintainer = "Mons Anderson "; 15 | } 16 | 17 | dependencies = { 18 | 'lua >= 5.1'; 19 | 'ffi-reloadable >= 0'; 20 | } 21 | 22 | build = { 23 | type = 'builtin', 24 | modules = { 25 | ['bin'] = 'bin.lua'; 26 | ['libluabin'] = { 27 | sources = { 28 | "libluabin.c", 29 | }; 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /rockspecs/bin-scm-2.rockspec: -------------------------------------------------------------------------------- 1 | package = 'bin' 2 | version = 'scm-2' 3 | 4 | source = { 5 | url = 'git+https://github.com/moonlibs/bin.git'; 6 | branch = 'v2'; 7 | } 8 | 9 | description = { 10 | summary = "Binary tools"; 11 | detailed = "Binary tools"; 12 | homepage = 'https://github.com/moonlibs/bin.git'; 13 | license = 'Artistic'; 14 | maintainer = "Mons Anderson "; 15 | } 16 | 17 | dependencies = { 18 | 'lua >= 5.1'; 19 | 'ffi-reloadable >= 0'; 20 | } 21 | 22 | build = { 23 | type = 'builtin', 24 | modules = { 25 | ['bin'] = 'bin.lua'; 26 | ['libluabin'] = { 27 | sources = { 28 | "libluabin.c", 29 | }; 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /rockspecs/bin-scm-3.rockspec: -------------------------------------------------------------------------------- 1 | package = 'bin' 2 | version = 'scm-3' 3 | 4 | source = { 5 | url = 'git+https://github.com/moonlibs/bin.git'; 6 | branch = 'v3'; 7 | } 8 | 9 | description = { 10 | summary = "Binary tools"; 11 | detailed = "Binary tools"; 12 | homepage = 'https://github.com/moonlibs/bin.git'; 13 | license = 'Artistic'; 14 | maintainer = "Mons Anderson "; 15 | } 16 | 17 | dependencies = { 18 | 'lua >= 5.1'; 19 | 'ffi-reloadable >= 0'; 20 | } 21 | 22 | build = { 23 | type = 'builtin', 24 | modules = { 25 | ['bin'] = 'bin.lua'; 26 | ['libluabin-'..version] = { 27 | sources = { 28 | "libluabin.c", 29 | }; 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /rockspecs/bin-scm-4.rockspec: -------------------------------------------------------------------------------- 1 | package = "bin" 2 | version = "scm-4" 3 | 4 | source = { 5 | url = 'git+https://github.com/moonlibs/bin.git'; 6 | branch = 'v4'; 7 | } 8 | 9 | description = { 10 | summary = "Binary tools"; 11 | detailed = "Binary tools"; 12 | homepage = 'https://github.com/moonlibs/bin.git'; 13 | license = 'Artistic'; 14 | maintainer = "Mons Anderson "; 15 | } 16 | 17 | dependencies = { 18 | 'lua ~> 5.1'; 19 | 'ffi-reloadable >= 0'; 20 | } 21 | 22 | build = { 23 | type = "builtin", 24 | modules = { 25 | bin = "bin.lua", 26 | ["bin.base"] = "bin/base.lua", 27 | ["bin.basebuf"] = "bin/basebuf.lua", 28 | ["bin.buf"] = "bin/buf.lua", 29 | ["bin.fixbuf"] = "bin/fixbuf.lua", 30 | ["bin.rbuf"] = "bin/rbuf.lua", 31 | ["bin.saferbuf"] = "bin/saferbuf.lua", 32 | ['libluabin-scm-4'] = { 33 | sources = "libluabin.c" 34 | }, 35 | }, 36 | } 37 | -------------------------------------------------------------------------------- /rpm.spec: -------------------------------------------------------------------------------- 1 | %{!?lua_version: %global lua_version %{lua: print(string.sub(_VERSION, 5))}} 2 | %define lua_version 5.1 3 | 4 | %{!?lua_libdir: %global lua_libdir %{_libdir}/lua/%{lua_version}} 5 | %{!?lua_pkgdir: %global lua_pkgdir %{_datadir}/lua/%{lua_version}} 6 | 7 | %{!?luarocks_libdir: %global luarocks_libdir %(luarocks config --lua-libdir)} 8 | 9 | %define __repo bin 10 | 11 | Name: lua-%{__repo} 12 | Version: 3.1.1 13 | Release: 1 14 | License: BSD 15 | Group: Development/Libraries 16 | Url: https://github.com/moonlibs/bin 17 | Summary: FFI-library 18 | BuildArch: x86_64 19 | BuildRoot: %{_tmppath}/%{name} 20 | BuildRequires: lua-ffi-reloadable >= 1 21 | BuildRequires: gcc 22 | Requires: lua-ffi-reloadable >= 1 23 | 24 | %description 25 | Binary tools for LuaJIT 26 | 27 | %prep 28 | rm -rf %{__repo} 29 | 30 | %if %{?SRC_DIR:1}%{!?SRC_DIR:0} 31 | rm -rf %{__repo} 32 | mkdir -p %{__repo} 33 | cp -rai %{SRC_DIR}/* %{__repo} 34 | cd %{__repo} 35 | %else 36 | mkdir %{__repo} 37 | %endif 38 | 39 | %build 40 | 41 | %install 42 | cd %{__repo} 43 | gcc -O2 -fPIC -I/usr/include/lua5.1 -c libluabin.c -o libluabin.o 44 | gcc -shared -o libluabin-scm-3.so -L/usr/lib libluabin.o 45 | 46 | install -d -m 0755 %{buildroot}/usr/share/lua/5.1/ 47 | install -d -m 0755 %{buildroot}/usr/lib64/lua/5.1/ 48 | 49 | install -m 0644 bin.lua %{buildroot}/usr/share/lua/5.1/ 50 | install -m 0644 libluabin-scm-3.so %{buildroot}/usr/lib64/lua/5.1/ 51 | 52 | %clean 53 | rm -rf %{buildroot} 54 | rm -rf %{__repo} 55 | 56 | %files 57 | %defattr(-,root,root) 58 | %{_datadir}/lua/%{lua_version}/* 59 | %{_prefix}/lib64/lua/5.1/ 60 | 61 | %changelog 62 | 63 | -------------------------------------------------------------------------------- /test/sample.lua: -------------------------------------------------------------------------------- 1 | local ffi = require 'ffi' 2 | 3 | -- box.cfg { 4 | -- background = false; 5 | -- logger_nonblock = true; 6 | -- read_only = true; 7 | -- wal_mode = 'none'; 8 | -- } 9 | 10 | local fiber = require 'fiber' 11 | 12 | local function dump(x) 13 | local j = require'json'.new() 14 | j.cfg{ 15 | encode_use_tostring = true; 16 | } 17 | return j.encode(x) 18 | end 19 | 20 | package.path = "./?.lua;"..package.path 21 | print(package.searchpath('bin', package.path)) 22 | 23 | local bin = require 'bin' 24 | 25 | if false then -- bench 26 | -- jit.off() 27 | local clock = require 'clock' 28 | 29 | local N = 1e7 30 | 31 | local st = clock.proc() 32 | local buf = bin.buf(64) 33 | for i = 1,N do 34 | buf.cur = 0 35 | buf:reb(0x12345678LL) 36 | end 37 | local r = clock.proc() - st 38 | print(string.format("%d/%0.6fs = %0.4fs", N,r, N/r)) 39 | 40 | local st = clock.proc() 41 | local buf = bin.buf(64) 42 | for i = 1,N do 43 | buf.cur = 0 44 | buf:reb2(0x12345678LL) 45 | end 46 | local r = clock.proc() - st 47 | print(string.format("%d/%0.6fs = %0.4fs", N,r, N/r)) 48 | 49 | 50 | do return end 51 | end -- bench 52 | 53 | do 54 | local buf = bin.buf(64) 55 | -- buf:copy("abcdef1234567890"); 56 | -- buf:ber(0x12345678LL) 57 | -- buf:ber2(0x12345678LL) 58 | buf:reb(0x1234567890LL) 59 | -- buf:char(0xff) 60 | -- buf:char(0xff) 61 | buf:ber(0x1234567890LL) 62 | print("encoded = ",0x1234567890LL) 63 | 64 | print(buf) 65 | print(buf:dump()) 66 | local rbuf= buf:reader() 67 | print(rbuf) 68 | print("decoded reb = ",rbuf:reb()) 69 | -- rbuf.p.u16 = rbuf.p.u16+1 70 | print("decoded ber = ",rbuf:ber()) 71 | -- print(rbuf:u32()) 72 | -- print(rbuf:N()) 73 | print(rbuf) 74 | end 75 | 76 | do return end 77 | 78 | do 79 | local p,l 80 | do 81 | local buf = bin.buf(64) 82 | print(buf) 83 | for i = 0,127 do buf:char('A') end 84 | print(buf:dump()) 85 | fiber.sleep(0.1) 86 | collectgarbage() 87 | print(buf:dump()) 88 | p,l = buf:export() 89 | end 90 | collectgarbage() 91 | print(bin.xd(p,l)) 92 | end 93 | 94 | print(bin.xd("test")); 95 | 96 | print(string.format("%04x",bin.htole16(0x1234))) 97 | print(string.format("%04x",bin.htobe16(0x1234))) 98 | print(string.format("%08x",bin.htole32(0x12345678))) 99 | print(string.format("%08x",bin.htobe32(0x12345678))) 100 | 101 | print(bin.hex("some 1234")) 102 | 103 | 104 | -- local buf = bin.fixbuf() 105 | local buf = bin.buf() 106 | 107 | print(buf) 108 | print(bin.xd(buf.buf,16)) 109 | local p = buf:alloc(8) 110 | ffi.copy(p,"12345678") 111 | print(bin.xd(buf.buf,16)) 112 | print(buf) 113 | 114 | buf:uint8(0xff) 115 | buf:int8(-300) 116 | buf:int8(0) 117 | buf:uint64be(0x1234567890ABCDEFULL) 118 | 119 | buf:V(0x12345678ULL) 120 | 121 | buf:int8(0xfe) 122 | 123 | buf:ber(0x12345678) 124 | 125 | print(buf) 126 | print(bin.xd(buf.buf,32)) 127 | 128 | print(buf:dump()) 129 | 130 | print(buf:hex()) 131 | 132 | 133 | print("new buf") 134 | local buf = bin.fixbuf() 135 | local p = buf:alloc(8) 136 | ffi.copy(p,"12345678") 137 | buf:ber(0x12345678) 138 | buf:V(0x12345678ULL) 139 | print(buf:dump()) 140 | print(buf:hex()) 141 | 142 | 143 | 144 | -- print(string.format("%016s",tostring(bin.htole64(0x1234567890ABCDEFULL)))) 145 | -- print(string.format("%016x",bin.htobe64(0x12345678))) 146 | 147 | -- local cn = scr("localhost",1463,{}) -------------------------------------------------------------------------------- /test/scribe.lua: -------------------------------------------------------------------------------- 1 | package.path = "./?.lua;"..package.path 2 | print(package.searchpath('bin', package.path)) 3 | 4 | local bin = require 'bin' 5 | local ffi = require 'ffi' 6 | 7 | local function typedef(t,def) 8 | if not pcall(ffi.typeof,t) then 9 | local r,e = pcall(ffi.cdef,def) 10 | if not r then error(e,2) end 11 | end 12 | return ffi.typeof(t) 13 | end 14 | local function fundef(n,def,src) 15 | src = src or ffi.C 16 | local f = function(src,n) return src[n] end 17 | if not pcall(f,src,n) then 18 | local r,e = pcall(ffi.cdef,def) 19 | if not r then error(e,2) end 20 | end 21 | local r,e = pcall(f,src,n) 22 | if not r then 23 | error(e,2) 24 | end 25 | return r 26 | end 27 | 28 | local VERSION_MASK = ffi.cast('uint32_t',0xffff0000) 29 | local VERSION_1 = ffi.cast('uint32_t',0x80010000); 30 | 31 | local M_CALL = 1 32 | local M_REPLY = 2 33 | local M_EXCEPTION = 3 34 | local M_ONEWAY = 4 35 | 36 | local T_STOP = 0 37 | local T_VOID = 1 38 | local T_BOOL = 2 39 | local T_BYTE = 3 40 | -- local T_I08 = 3 41 | local T_I16 = 6 42 | local T_I32 = 8 43 | local T_U64 = 9 44 | local T_I64 = 10 45 | local T_DOUBLE = 4 46 | local T_STRING = 11 47 | -- local T_UTF7 = 11 48 | local T_STRUCT = 12 49 | local T_MAP = 13 50 | local T_SET = 14 51 | local T_LIST = 15 52 | local T_UTF8 = 16 53 | local T_UTF16 = 1 54 | 55 | 56 | typedef('sc_hdr_t',[[ 57 | #pragma pack (push, 1) 58 | typedef struct { 59 | unsigned size : 32; 60 | char v0 : 8; 61 | char v1 : 8; 62 | char t0 : 8; 63 | char t1 : 8; 64 | char len[4]; 65 | char proc[3]; 66 | 67 | unsigned seq : 32; 68 | struct { 69 | unsigned char type; 70 | unsigned char id[2]; 71 | } field; 72 | struct { 73 | unsigned char type; 74 | unsigned int size : 32; 75 | } list; 76 | } sc_hdr_t; 77 | #pragma pack (pop) 78 | ]]) 79 | 80 | local def_hdr = ffi.new('sc_hdr_t',{ 81 | size = 0; 82 | 83 | v0 = 0x80; v1 = 1; 84 | t0 = 0; t1 = 1; 85 | 86 | len = {0,0,0,3}; 87 | proc = "Log"; 88 | 89 | field = { type = T_LIST,id={0,1} }; 90 | list = { T_STRUCT,0}; 91 | }) 92 | local HDR_SZ = ffi.sizeof('sc_hdr_t') 93 | 94 | local _seq = 1233 95 | function seq() 96 | _seq = _seq < 0xffffffff and _seq + 1 or 1 97 | return _seq 98 | end 99 | 100 | 101 | function send(...) 102 | local messages 103 | if type(...) == 'table' then 104 | local t = ... 105 | if #t == 0 then 106 | messages = {t} 107 | else 108 | messages = t 109 | end 110 | else 111 | local cat,msg = ... 112 | messages = {{cat=cat,msg=msg}} 113 | end 114 | 115 | local sz = HDR_SZ 116 | -- print(sz) 117 | for _,rec in pairs(messages) do 118 | if rec.cat then 119 | sz = sz+1+2+4+#rec.cat 120 | end 121 | if rec.msg then 122 | sz = sz+1+2+4+#rec.msg 123 | end 124 | sz = sz+1 125 | end 126 | sz = sz+1 127 | 128 | local buf = bin.fixbuf(sz) 129 | local hdr = ffi.cast( 'sc_hdr_t *', buf:alloc(HDR_SZ) ) 130 | ffi.copy(hdr,def_hdr,HDR_SZ) 131 | 132 | local seq = seq() 133 | 134 | hdr.seq = bin.htobe32( seq ) 135 | hdr.list.size = bin.htobe32( #messages ) 136 | 137 | for _,rec in pairs(messages) do 138 | if rec.cat then 139 | buf:uint8(T_STRING) 140 | buf:uint16be(1) 141 | buf:uint32be(#rec.cat) 142 | buf:copy(rec.cat) 143 | end 144 | if rec.msg then 145 | buf:uint8(T_STRING) 146 | buf:uint16be(2) 147 | buf:uint32be(#rec.msg) 148 | buf:copy(rec.msg) 149 | end 150 | buf:uint8(0) 151 | end 152 | buf:uint8(0) 153 | 154 | local p,len = buf:pv() 155 | hdr = ffi.cast( 'sc_hdr_t *', p ) 156 | hdr.size = bin.htobe32( len - 4 ) 157 | 158 | local p,len,sz = buf:export() 159 | p = ffi.cast('char *',p) 160 | end 161 | 162 | for i=1,1000 do 163 | send({ cat = "category"; msg = "message" }) 164 | end 165 | for i=1,1000 do 166 | send({ cat = "category"; msg = "message" }) 167 | collectgarbage('collect') 168 | end 169 | -------------------------------------------------------------------------------- /test/t.lua: -------------------------------------------------------------------------------- 1 | --[[ 2 | You can run this tests using typing this in the shell: 3 | # LuaJIT (tested on 2.0.5) 4 | luajit test/t.lua 5 | 6 | # Tarantool (tested on 1.10.2-26-gb2ddd18) 7 | tarantool test/t.lua 8 | ]] 9 | 10 | package.path = "./?.lua;"..package.path 11 | print(package.searchpath('bin', package.path)) 12 | -- TODO: import tap based tests 13 | local bin = require 'bin' 14 | 15 | -- test cases was taken from https://rosettacode.org/wiki/Variable-length_quantity#C 16 | local numbers = { 17 | 0x7f, 0x4000, 0, 0x3ffffe, 0x1fffff, 0, 0x200000, 0x3311a1234df31413ULL, 18 | -1ULL, 18446744062075611165ULL 19 | } 20 | 21 | local buf = bin.buf() 22 | for _, num in ipairs(numbers) do 23 | buf:reb(num) 24 | end 25 | 26 | local rbuf = buf:reader() 27 | for _, num in ipairs(numbers) do 28 | local got = rbuf:reb() 29 | assert(got == num, ("Expected %s got %s"):format(num, got)) 30 | print("ok - ", num) 31 | end 32 | 33 | -- memory leak tests: 34 | local buf = bin.rbuf("") -- empty buffer :) 35 | local ok = pcall(buf.reb, buf) 36 | assert(not ok, "REB decoding should fail for empty buf") 37 | print("ok - ", "failed REB decoding for empty buf") 38 | 39 | -- corrupted REB: 40 | local buf = bin.buf(1) -- 1 byte buffer 41 | buf:char(0xff) -- transmit 0xff into it 42 | local rbuf = buf:reader() 43 | 44 | local ok = pcall(rbuf.reb, rbuf) 45 | assert(not ok, "REB decoding should fail") 46 | print("ok - ", "failed REB decoding for corrupted buf") 47 | 48 | -- \xFF \xFF is invalid REB: 49 | local buf = bin.buf(1) 50 | buf:copy(("\xFF"):rep(2)) 51 | 52 | local rbuf = buf:reader() 53 | local ok = pcall(rbuf.reb, rbuf) 54 | assert(not ok, "REB decoding must fail for \xFF\xFF") 55 | print("ok - ", "REB decode fails for \\xFF\\xFF") 56 | 57 | assert(rbuf.len == 2, "rbuf.len is 2") 58 | print("ok - ", "rbuf.len is 2") 59 | 60 | assert(rbuf:avail() == 2, "rbuf:avail() is 2") 61 | print("ok - ", "rbuf:avail() is 2 - no bytes were consumed after malformed REB") 62 | 63 | local buf = bin.buf() 64 | for i = 0, 64 do 65 | local left, center, right = 2ULL^i-1, 2ULL^i, 2ULL^i+1 66 | 67 | buf:reb(left) 68 | buf:reb(center) 69 | buf:reb(right) 70 | end 71 | 72 | local rbuf = buf:reader() 73 | for i = 0, 64 do 74 | local center = 2ULL^i 75 | local left, right = center-1, center+1 76 | 77 | print("ok -", select(2, assert(rbuf:reb() == left, ("reb_decode(reb_encode(%s)) == %s"):format(left, left)))) 78 | print("ok -", select(2, assert(rbuf:reb() == center, ("reb_decode(reb_encode(%s)) == %s"):format(center, center)))) 79 | print("ok -", select(2, assert(rbuf:reb() == right, ("reb_decode(reb_encode(%s)) == %s"):format(right, right)))) 80 | end 81 | -------------------------------------------------------------------------------- /tmp/leftovers.lua: -------------------------------------------------------------------------------- 1 | local bin = require 'bin' 2 | -- local buf = bin 3 | 4 | function buf:ber(n) 5 | n = ffi.cast('uint32_t',n) 6 | -- n = ffi.cast('uint64_t',n) 7 | if n < 0x80 then 8 | local p = ffi.cast(uint8_t,self:alloc(1)) 9 | p[0] = n 10 | elseif n < 0x4000 then 11 | local p = ffi.cast(uint8_t,self:alloc(2)) 12 | -- print("2>",bit.rshift(n,7)) 13 | ffi.cast(uint8_t, p)[0] = bit.bor(0x80, bit.rshift(n,7)) 14 | ffi.cast(uint8_t, p)[1] = bit.band(n,0x7f) 15 | elseif n < 0x200000 then 16 | local p = ffi.cast(uint8_t,self:alloc(3)) 17 | ffi.cast(uint8_t, p)[0] = bit.bor(0x80, bit.rshift(n,14)) 18 | ffi.cast(uint8_t, p)[1] = bit.bor(0x80, bit.rshift(n,7)) 19 | ffi.cast(uint8_t, p)[2] = bit.band(n,0x7f) 20 | elseif n < 0x10000000 then 21 | local p = ffi.cast(uint8_t,self:alloc(4)) 22 | ffi.cast(uint8_t, p)[0] = bit.bor(0x80, bit.rshift(n,21)) 23 | ffi.cast(uint8_t, p)[1] = bit.bor(0x80, bit.rshift(n,14)) 24 | ffi.cast(uint8_t, p)[2] = bit.bor(0x80, bit.rshift(n,7)) 25 | ffi.cast(uint8_t, p)[3] = bit.band(n,0x7f) 26 | else 27 | local p = ffi.cast(uint8_t,self:alloc(5)) 28 | ffi.cast(uint8_t, p)[0] = bit.bor(0x80, bit.rshift(n,28)) 29 | ffi.cast(uint8_t, p)[1] = bit.bor(0x80, bit.rshift(n,21)) 30 | ffi.cast(uint8_t, p)[2] = bit.bor(0x80, bit.rshift(n,14)) 31 | ffi.cast(uint8_t, p)[3] = bit.bor(0x80, bit.rshift(n,7)) 32 | ffi.cast(uint8_t, p)[4] = bit.band(n,0x7f) 33 | end 34 | end 35 | 36 | 37 | function buf:ber2(x) 38 | x = ffi.cast('uint64_t',x) 39 | local p 40 | if x < 0x80 then 41 | p = self:alloc(1) 42 | ffi.cast(uint8_t, p)[0] = x 43 | else 44 | local len = 45 | -- x < 0x80ULL and 1 or 46 | x < 0x4000ULL and 2 or 47 | x < 0x200000ULL and 3 or 48 | x < 0x10000000ULL and 4 or 49 | x < 0x800000000ULL and 5 or 50 | x < 0x40000000000ULL and 6 or 51 | x < 0x2000000000000ULL and 7 or 52 | x < 0x100000000000000ULL and 8 or 9 53 | -- x < 0x8000000000000000 and 9 or 54 | 55 | p = ffi.cast(uint8_t,self:alloc(len)) 56 | for i = 0,len-2 do 57 | p[i] = bit.bor(0x80, bit.rshift(x, 7*(len-i-1) )) 58 | end 59 | p[len-1] = bit.band(x,0x7f) 60 | end 61 | end 62 | 63 | 64 | local clock = require 'clock' 65 | 66 | local N = 1e7 67 | 68 | -- local st = clock.proc() 69 | -- local buf = bin.buf(64) 70 | -- for i = 1,N do 71 | -- buf.cur = 0 72 | -- buf:ber3(0x12345678LL) 73 | -- end 74 | -- local r = clock.proc() - st 75 | -- print(string.format("%d/%0.6fs = %0.4fs", N,r, N/r)) 76 | 77 | local st = clock.proc() 78 | local buf = bin.buf(64) 79 | for i = 1,N do 80 | buf.cur = 0 81 | buf:ber(0x12345678LL) 82 | end 83 | local r = clock.proc() - st 84 | print(string.format("%d/%0.6fs = %0.4fs", N,r, N/r)) 85 | 86 | local st = clock.proc() 87 | local buf = bin.buf(64) 88 | for i = 1,N do 89 | buf.cur = 0 90 | buf:ber2(0x12345678LL) 91 | end 92 | local r = clock.proc() - st 93 | print(string.format("%d/%0.6fs = %0.4fs", N,r, N/r)) 94 | 95 | 96 | 97 | 98 | 99 | function buf:reb(x) 100 | x = ffi.cast('uint64_t',x) 101 | if x < 0x80 then 102 | p = self:alloc(1) 103 | ffi.cast(uint8_t, p)[0] = x 104 | else 105 | local p 106 | while x > 0 do 107 | p = self:alloc(1) 108 | if x > 0x7f then 109 | ffi.cast(uint8_t, p)[0] = bit.bor(bit.band(x,0x7f),0x80) 110 | else 111 | ffi.cast(uint8_t, p)[0] = bit.band(x,0x7f) 112 | end 113 | x = bit.rshift(x,7) 114 | end 115 | end 116 | end 117 | 118 | -- if true then -- bench 119 | -- jit.off() 120 | local clock = require 'clock' 121 | 122 | local N = 1e7 123 | 124 | local st = clock.proc() 125 | local buf = bin.buf(64) 126 | for i = 1,N do 127 | buf.cur = 0 128 | buf:reb(0x12345678LL) 129 | end 130 | local r = clock.proc() - st 131 | print(string.format("%d/%0.6fs = %0.4fs", N,r, N/r)) 132 | 133 | local st = clock.proc() 134 | local buf = bin.buf(64) 135 | for i = 1,N do 136 | buf.cur = 0 137 | buf:reb2(0x12345678LL) 138 | end 139 | local r = clock.proc() - st 140 | print(string.format("%d/%0.6fs = %0.4fs", N,r, N/r)) 141 | 142 | 143 | do return end -------------------------------------------------------------------------------- /xd.h: -------------------------------------------------------------------------------- 1 | #ifndef XD_H__ 2 | #define XD_H__ 3 | 4 | #include 5 | #include 6 | 7 | #define HEX_SZ (16*10 + 1) 8 | #define CHR_SZ (16*8 + 1) 9 | 10 | typedef struct { 11 | uint8_t row; 12 | uint8_t hpad; 13 | uint8_t cpad; 14 | uint8_t hsp; 15 | uint8_t csp; 16 | uint8_t cols; 17 | } xd_conf; 18 | 19 | static xd_conf default_xd_conf = { 16,1,0,1,1,4 }; 20 | 21 | static char * xd(char *data, size_t size, xd_conf *cf) 22 | { 23 | /* dumps size bytes of *data to stdout. Looks like: 24 | * [0000] 75 6E 6B 6E 6F 77 6E 20 30 FF 00 00 00 00 39 00 unknown 0.....9. 25 | * src = 16 bytes. 26 | * dst = 6 + 16 * 3 + 4*2 + 16 + 1 27 | * prefix byte+pad sp between col visual newline 28 | */ 29 | if (!cf) cf = &default_xd_conf; 30 | uint8_t row = cf->row; 31 | uint8_t hpad = cf->hpad; 32 | uint8_t cpad = cf->cpad; 33 | uint8_t hsp = cf->hsp; 34 | uint8_t csp = cf->csp; 35 | uint8_t sp = cf->cols; 36 | 37 | uint8_t every = (uint8_t)row / sp; 38 | 39 | char *p = data; 40 | unsigned char c; 41 | size_t n; 42 | // unsigned addr; 43 | // char bytestr[4] = {0}; 44 | char addrstr[10] = {0}; 45 | char hexstr[ HEX_SZ ] = {0}; 46 | char chrstr[ CHR_SZ ] = {0}; 47 | unsigned hex_sz = row*(2+hpad) + hsp * sp + 1; /* size = bytes<16*2> + 16* + col */ 48 | unsigned chr_sz = row*(2+cpad) + csp * sp + 1; /* size = bytes<16> + 16*cpad + col */ 49 | 50 | if ( hex_sz > HEX_SZ ) { 51 | fprintf(stderr,"Parameters too big: estimated hex size will be %u, but have only %u\n", hex_sz, HEX_SZ); 52 | return NULL; 53 | } 54 | if ( chr_sz > CHR_SZ ) { 55 | fprintf(stderr,"Parameters too big: estimated chr size will be %u, but have only %u\n", chr_sz, CHR_SZ); 56 | return NULL; 57 | } 58 | 59 | size_t sv_sz = ( size + row-1 ) * ( (uint8_t)( 6 + 3 + hex_sz + 2 + chr_sz + 1 + row-1 ) / row ); 60 | /* ^ reserve for incomplete string \n ^ emulation of ceil */ 61 | char *rv = malloc(sv_sz); 62 | if (!rv) { 63 | fprintf(stderr,"Can't allocate memory\n"); 64 | return NULL; 65 | } 66 | char *rvptr = rv; 67 | 68 | char *curhex = hexstr; 69 | char *curchr = chrstr; 70 | for(n=1; n<=size; n++) { 71 | if (n % row == 1) 72 | snprintf(addrstr, sizeof(addrstr), "%04x", ( (int)(p-data) ) & 0xffff ); 73 | 74 | c = *p; 75 | if (c < 0x20 || c > 0x7f) { 76 | c = '.'; 77 | } 78 | 79 | /* store hex str (for left side) */ 80 | snprintf(curhex, 3+hpad, "%02X%-*s", (unsigned char)*p, hpad,""); curhex += 2+hpad; 81 | 82 | /* store char str (for right side) */ 83 | snprintf(curchr, 2+cpad, "%c%-*s", c, cpad, ""); curchr += 1+cpad; 84 | 85 | //warn("n=%d, row=%d, every=%d\n",n,row,every); 86 | if( n % row == 0 ) { 87 | /* line completed */ 88 | //printf("[%-4.4s] %s %s\n", addrstr, hexstr, chrstr); 89 | rvptr += snprintf(rvptr, (p-rvptr+sv_sz) ,"[%-4.4s] %s %s\n", addrstr, hexstr, chrstr); 90 | //sv_catpvf(rv,"[%-4.4s] %-*s %-*s\n", addrstr, hex_sz-1, hexstr, chr_sz-1, chrstr); 91 | hexstr[0] = 0; curhex = hexstr; 92 | chrstr[0] = 0; curchr = chrstr; 93 | } else if( every && ( n % every == 0 ) ) { 94 | /* half line: add whitespaces */ 95 | snprintf(curhex, 1+hsp, "%-*s", hsp, ""); curhex += hsp; 96 | snprintf(curchr, 1+csp, "%-*s", csp, ""); curchr += csp; 97 | } 98 | p++; /* next byte */ 99 | } 100 | 101 | if (curhex > hexstr) { 102 | /* print rest of buffer if not empty */ 103 | //printf("[%4.4s] %s %s\n", addrstr, hexstr, chrstr); 104 | rvptr += snprintf(rvptr, (p-rvptr+sv_sz),"[%-4.4s] %-*s %-*s\n", addrstr, hex_sz-1, hexstr, chr_sz-1, chrstr); 105 | } 106 | //warn("String len: %d, sv_sz=%d",SvCUR(rv),sv_sz); 107 | return rv; 108 | } 109 | 110 | #endif 111 | --------------------------------------------------------------------------------