├── .travis.yml ├── COPYRIGHT ├── README.md ├── lds.lua ├── lds ├── Array.lua ├── HashMap.lua ├── Vector.lua ├── allocator.lua ├── hash.lua ├── init.lua └── jemalloc.lua ├── rockspecs └── lds-scm-1.rockspec └── tests ├── perf_Insert.lua ├── perf_unordered_map_insert.cpp ├── run_tests.sh ├── test_Array.lua ├── test_HashMap.lua └── test_Vector.lua /.travis.yml: -------------------------------------------------------------------------------- 1 | # lds .travis.yml 2 | # Copyright (c) 2020 Evan Wies 3 | # 4 | # Testing on Travis CI 5 | # 6 | # https://travis-ci.org/neomantra/lds 7 | # 8 | 9 | sudo: required 10 | 11 | dist: bionic 12 | 13 | addons: 14 | apt: 15 | packages: 16 | - luajit 17 | - luarocks 18 | - g++ 19 | - libboost1.62-dev 20 | 21 | jobs: 22 | include: 23 | - stage: run unit tests 24 | script: ./tests/run_tests.sh 25 | 26 | - stage: test building the rockspec 27 | script: sudo luarocks build rockspecs/lds-scm-1.rockspec 28 | 29 | - stage: run performance tests 30 | script: 31 | - ./tests/perf_Insert.lua 32 | - g++ -o perf_unordered_map_insert -O3 tests/perf_unordered_map_insert.cpp && ./perf_unordered_map_insert 33 | -------------------------------------------------------------------------------- /COPYRIGHT: -------------------------------------------------------------------------------- 1 | 2 | lds - LuaJIT Data Structures 3 | 4 | Copyright (c) 2012-2020 Evan Wies. All rights reserved. 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | 8 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 9 | 10 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 11 | 12 | [ MIT license: http://www.opensource.org/licenses/mit-license.php ] 13 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # lds - LuaJIT Data Structures 3 | 4 | [![Travis Status](https://travis-ci.org/neomantra/lds.svg?branch=master)](https://travis-ci.org/neomantra/lds) 5 | 6 | **lds** provides data structures which hold [LuaJIT *cdata*](http://luajit.org/ext_ffi_api.html). 7 | 8 | These containers cover the common use cases of Lua *tables*: 9 | 10 | * Array (fixed-size array) 11 | * Vector (dynamically-sized array) 12 | * HashMap (key-value store) 13 | 14 | After installing the library, simply `require` the module and use the returned value: 15 | ``` 16 | local lds = require 'lds' 17 | ``` 18 | 19 | ## Manual Installation 20 | 21 | This library is currently only available as a [GitHub repo](https://github.com/neomantra/lds). To install, clone the repo and copy `lds.lua` and the **lds** directory (the one *inside* the repo) to somewhere on your `LUA_PATH`. 22 | 23 | 24 | ``` 25 | # The path /usr/local/share/lua/5.1 is just an example, it is common on Ubuntu platforms 26 | 27 | git clone https://github.com/neomantra/lds.git 28 | cp -r lds/lds.lua lds/lds /usr/local/share/lua/5.1 29 | ``` 30 | 31 | 32 | ## LuaRocks Installation 33 | 34 | To install from the [LuaRocks](https://luarocks.org/) [rockspec](https://github.com/neomantra/lds/blob/master/rockspecs/lds-scm-1.rockspec): 35 | 36 | ``` 37 | git clone https://github.com/neomantra/lds.git 38 | luarocks build rockspecs/lds-scm-1.rockspec 39 | ``` 40 | 41 | This library is *NOT* officially loaded to the LuaRocks repository. 42 | 43 | ## Example 44 | 45 | The following example shows a *cdata* being stored in a Vector and then retrieved from there are stored in a HashMap. 46 | 47 | ```lua 48 | local ffi = require 'ffi' 49 | local lds = require 'lds' 50 | 51 | ffi.cdef([[ 52 | struct Item { 53 | int id; 54 | char name[16]; 55 | int inventory; 56 | }; 57 | ]]) 58 | 59 | local Item_t = ffi.typeof('struct Item') 60 | 61 | -- Create a Vector and add some items 62 | local v = lds.Vector( Item_t ) 63 | v:push_back( Item_t( 100, 'Apple', 42 ) ) 64 | v:push_back( Item_t( 101, 'Orange', 10 ) ) 65 | v:push_back( Item_t( 102, 'Lemon', 6 ) ) 66 | assert( v:size() == 3 ) 67 | assert( v:get(0).id == 100 ) 68 | assert( v:get(2).id == 102 ) 69 | 70 | -- Now create a HashMap of Item id to Item 71 | -- Note this is stored by-value, not by-reference 72 | local map = lds.HashMap( ffi.typeof('int'), Item_t ) 73 | for i = 0, v:size()-1 do -- lds.Vector is 0-based 74 | local item = v:get(i) 75 | map:insert( item.id, item ) 76 | end 77 | assert( map:size() == 3 ) 78 | assert( map:find(100).key == 100 ) 79 | assert( map:find(100).val.inventory == 42 ) 80 | assert( map:find(102).key == 102 ) 81 | assert( map:find(102).val.inventory == 6 ) 82 | 83 | ``` 84 | 85 | ## Motivation 86 | 87 | Lua provides a powerful data structure called a *table*. A table is an heterogeneous associative array -- its keys and values can be any type (except *nil*). 88 | 89 | The [LuaJIT FFI](http://luajit.org/ext_ffi.html) enables the instantiation and manipulation of C objects directly in Lua; these objects called *cdata*. Although *cdata* may be stored in Lua tables (but [are not suitable as keys](http://luajit.org/ext_ffi_semantics.html#cdata_key)), when used this way they are subject to a [4GB memory limit and GC inefficiency](http://lua-users.org/lists/lua-l/2010-11/msg00241.html). 90 | 91 | **lds** provides the containers Array, Vector, and HashMap, which cover the common use cases of Lua *tables*. However, the memory of these containers are managed through user-specified allocators (e.g. *malloc*) and thus are not subject to the LuaJIT memory limit. It is also significantly lower load on the LuaJIT Garbage Collector. 92 | 93 | One drawback to be aware of is that **lds** containers are homogeneous. A given container has explicitly one key type and one *cdata* value type. 94 | 95 | 96 | # API 97 | 98 | Documentation for this library is pretty sparse right now -- try reading the [test scripts](https://github.com/neomantra/lds/tree/master/tests). 99 | 100 | The source is well documented and the implementation is decently readable -- much easier than an STL implementation. 101 | 102 | 103 | ## Caveats 104 | 105 | * Array and Vector indices are 0-based, **not** 1-based. 106 | 107 | * Raw memory allocators are used, not the LuaJIT __new/__gc system, so you can only store simple data. In other words, if your cdata allocates memory or other resources, you will need to manage that manually. 108 | 109 | * Right now the hash function is identity (hash(x) = x), so keys are limited to simple numeric types and pointers. 110 | 111 | * Although raw memory allocators are used, they may allocate memory in the range which LuaJIT desires. The impact of this depends on which allocator you use and your OS. 112 | 113 | 114 | ## TODO 115 | 116 | * ldoc in wiki 117 | * TODOs in HashMap 118 | 119 | ## Support 120 | 121 | Submit feature requests and bug reports at the [Issues page on GitHub](http://github.com/neomantra/lds/issues). 122 | 123 | 124 | ## Contributors 125 | 126 | * Evan Wies 127 | 128 | 129 | ## Acknowledgments 130 | 131 | Many thanks to Professor Pat Morin for a well written textbook and readable implementations, both released with open licenses. HashMap is based on his [ChainedHashTable](http://opendatastructures.org/ods-cpp/5_1_Hashing_with_Chaining.html). Read and learn at [opendatastructures.org](http://opendatastructures.org). 132 | 133 | 134 | ## LICENSE 135 | 136 | **lds** is distributed under the [MIT License](http://opensource.org/licenses/mit-license.php). 137 | 138 | > lds - LuaJIT Data Structures 139 | > 140 | > Copyright (c) 2012-2020 Evan Wies. All rights reserved 141 | > 142 | > Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 143 | > 144 | > The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 145 | > 146 | > THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 147 | 148 | -------------------------------------------------------------------------------- /lds.lua: -------------------------------------------------------------------------------- 1 | --[[ 2 | lds - LuaJIT Data Structures 3 | 4 | @copyright Copyright (c) 2012-2020 Evan Wies. All rights reserved. 5 | @license MIT License, see the COPYRIGHT file. 6 | 7 | Common module require for convenience. 8 | 9 | Individual data structures may be require'd instead, for example: 10 | local lds = require 'lds.Vector' 11 | --]] 12 | 13 | local lds = require 'lds.Array' 14 | require 'lds.Vector' 15 | require 'lds.HashMap' 16 | 17 | return lds 18 | -------------------------------------------------------------------------------- /lds/Array.lua: -------------------------------------------------------------------------------- 1 | --[[ 2 | lds - LuaJIT Data Structures 3 | 4 | @copyright Copyright (c) 2012-2020 Evan Wies. All rights reserved. 5 | @license MIT License, see the COPYRIGHT file. 6 | 7 | @type Array 8 | 9 | Fixed-sized array of FFI cdata. 10 | 11 |

Array is a fixed-sized container. 12 | Index counting starts at zero, rather than Lua's one-based indexing. 13 | 14 | Conventions: 15 | private cdata fields are prefixed with _ 16 | private class fields (via metatable) are prefixed with __ 17 | 18 | --]] 19 | 20 | local lds = require 'lds.allocator' 21 | 22 | local ffi = require 'ffi' 23 | local C = ffi.C 24 | 25 | 26 | local ArrayT__cdef = [[ 27 | struct { 28 | $ * _data; 29 | size_t _size; 30 | } 31 | ]] 32 | 33 | 34 | -- Array public methods 35 | local Array = {} 36 | 37 | 38 | ---------------------- 39 | -- Capacity functions 40 | ---------------------- 41 | 42 | --- Returns the number of elements (not bytes) in the Array. 43 | -- 44 | -- It can also be accessed witht the # operator. 45 | 46 | -- If you are seeing performance warnings with -jv (e.g. loop unroll limit), 47 | -- it can also be accessed as the field `_size`, but you should never write 48 | -- to this youself. 49 | -- 50 | -- @return Returns the number of elements (not bytes) in the Array. 51 | function Array:size() 52 | return self._size 53 | end 54 | 55 | 56 | --- Returns true if the Array is empty. 57 | -- This is only true for 0-size arrays. 58 | -- @return true if the Array size is 0, false otherwise. 59 | function Array:empty() 60 | return self._size == 0 61 | end 62 | 63 | 64 | --- Returns the size of the Array in bytes (not elements) 65 | -- @return Returns the number of bytes (not elements) in the Array. 66 | function Array:size_bytes() 67 | return self._size * self.__ct_size 68 | end 69 | 70 | 71 | ---------------------------- 72 | -- Element Access functions 73 | 74 | --- Returns the element at index `i` in the Array. 75 | -- 76 | -- @param i Index of the element to get. 77 | -- @return The element at the specified index in the Array, or `nil` if the index is out of bounds. 78 | -- See also Array:get_e, which throws an error instead. 79 | -- 80 | -- Note that the first element has an index of 0, not 1. 81 | -- 82 | function Array:get( i ) 83 | if i < 0 or i >= self._size then return nil end 84 | return self._data[i] 85 | end 86 | 87 | 88 | --- Returns the element at index `i` in the Array. 89 | -- 90 | -- @param i Index of the element to get. 91 | -- @return The element at the specified index in the Array. 92 | -- 93 | -- If this is greater than or equal to the Array size, an "ArrayT.get: index out of bounds" error is thrown. 94 | -- See also Array:get, which returns `nil` instead. 95 | -- 96 | -- Note that the first element has an index of 0, not 1. 97 | -- 98 | function Array:get_e( i ) 99 | if i < 0 or i >= self._size then lds.error("ArrayT.get: index out of bounds") end 100 | return self._data[i] 101 | end 102 | 103 | 104 | --- Returns the value of the first element of the Array. 105 | -- Returns `nil` if the Array is empty. 106 | -- @return Returns the value of the first element of the Array, or `nil` if the Array is empty. 107 | function Array:front() 108 | if self._size == 0 then return nil end 109 | return self._data[0] 110 | end 111 | 112 | 113 | --- Returns the value of the last element of the Array. 114 | -- Returns `nil` if the Array is empty. 115 | -- @return Returns the value of the last element of the Array, or `nil` if the Array is empty. 116 | function Array:back() 117 | if self._size == 0 then return nil end 118 | return self._data[self._size - 1] 119 | end 120 | 121 | 122 | --- Returns a pointer to the underlying array. 123 | --- @return Pointer to the underlying array. 124 | function Array:data() 125 | return self._data 126 | end 127 | 128 | 129 | ------------------------------ 130 | -- Modifier functions 131 | 132 | --- Set element value at index. 133 | -- Sets the element `x` at index `i` in the Array. 134 | -- 135 | -- Returns nil if the index is out of range. 136 | -- See also Array:set_e, which throws instead. 137 | -- 138 | -- @param i Index to set in the Array. 139 | -- @param x Element to set at that index 140 | -- 141 | -- Note that the first element has an index of 0, not 1. 142 | -- 143 | -- @return The previous element at the specified index in the Array, 144 | -- or nil if the index is out of range. 145 | function Array:set( i, x ) 146 | if i < 0 or i >= self._size then return nil end 147 | local prev = self._data[i] 148 | self._data[i] = x 149 | return prev 150 | end 151 | 152 | 153 | --- Set element value at index. 154 | -- Sets the element `x` at index `i` in the Array. 155 | -- 156 | -- Throws error if the index is out of range. 157 | -- See also Array:set, which returns `nil` instead. 158 | -- 159 | -- @param i Index to set in the Array. 160 | -- @param x Element to set at that index. 161 | -- If this is greater than or equal to the Array size, the error "ArrayT.set: index out of bounds" is thrown. 162 | -- 163 | -- Note that the first element has an index of 0, not 1. 164 | -- 165 | -- @return The previous element at the specified index in the Array. 166 | function Array:set_e( i, x ) 167 | if i < 0 or i >= self._size then lds.error("ArrayT.set: index out of bounds") end 168 | local prev = self._data[i] 169 | self._data[i] = x 170 | return prev 171 | end 172 | 173 | 174 | ------------------------------ 175 | -- Private methods 176 | 177 | -- Constructor method 178 | function Array:__construct( size ) 179 | local data = self.__alloc:allocate(size) 180 | if not data then lds.error('ArrayT.new allocation failed') end 181 | self._data, self._size = data, size 182 | return self -- for chaining 183 | end 184 | 185 | 186 | -- Destructor method 187 | function Array:__destruct() 188 | self.__alloc:deallocate(self._data) 189 | self._data, self._size = nil, 0 190 | return self -- for chaining 191 | end 192 | 193 | 194 | ------------------------------ 195 | -- Metatable 196 | 197 | local ArrayT__mt = { 198 | 199 | __new = function( at, size ) 200 | local self = ffi.new(at) 201 | return self:__construct(size) 202 | end, 203 | 204 | __gc = function( self ) 205 | self:__destruct() 206 | end, 207 | 208 | --- __len metamethod, returning the number of elements in the Array. 209 | -- See also Array:size() and Array._size 210 | -- @return The number of elements in the Array. 211 | __len = function( self ) 212 | return self._size 213 | end, 214 | 215 | __index = Array, 216 | } 217 | 218 | 219 | function lds.ArrayT( ct, allocator_class ) 220 | if type(ct) ~= 'cdata' then lds.error("argument 1 is not a valid 'cdata'") end 221 | allocator_class = allocator_class or lds.MallocAllocator 222 | 223 | -- clone the metatable and insert type-specific data 224 | local at_mt = lds.simple_deep_copy(ArrayT__mt) 225 | at_mt.__index.__ct = ct 226 | at_mt.__index.__ct_size = ffi.sizeof(ct) 227 | at_mt.__index.__alloc = allocator_class(ct) 228 | 229 | local at = ffi.typeof( ArrayT__cdef, ct ) 230 | return ffi.metatype( at, at_mt ) 231 | end 232 | 233 | 234 | function lds.Array( ct, size, allocator_class ) 235 | return lds.ArrayT( ct, allocator_class )( size ) 236 | end 237 | 238 | 239 | -- Return the lds API 240 | return lds 241 | -------------------------------------------------------------------------------- /lds/HashMap.lua: -------------------------------------------------------------------------------- 1 | --[[ 2 | lds - LuaJIT Data Structures 3 | 4 | @copyright Copyright (c) 2012-2020 Evan Wies. All rights reserved. 5 | @license MIT License, see the COPYRIGHT file. 6 | 7 | HashMap 8 | 9 | A map implemented as a chained hash table. 10 | 11 | Implementation is based on ChainedHashTable at opendatastructures.org: 12 | http://opendatastructures.org/ods-cpp/5_1_Hashing_with_Chaining.html 13 | 14 | TODO: fixup hash function 15 | TODO: allow 64-bitop in LuaJIT 2.1 16 | TODO: option table as: { alloc = AllocatorFactory, hash = fn, eq = fn } 17 | TODO: reserve buckets, e.g. unordered_map::reserve, max_load_factor 18 | 19 | Conventions: 20 | private cdata fields are prefixed with _ 21 | private class fields (via metatable) are prefixed with __ 22 | private methods are prefixed with HashMapT__ 23 | --]] 24 | 25 | 26 | local lds = require 'lds.hash' 27 | require 'lds.Vector' 28 | 29 | local ffi = require 'ffi' 30 | local C = ffi.C 31 | 32 | local bit = require 'bit' 33 | local blshift, brshift, bor, bxor = bit.lshift, bit.rshift, bit.bor, bit.bxor 34 | 35 | 36 | local PairT_cdef = [[ 37 | struct { 38 | $ key; 39 | $ val; 40 | } 41 | ]] 42 | 43 | 44 | local HashMapT_cdef = [[ 45 | struct { 46 | $ * _t; // lds.Vector 47 | int _tsize; 48 | int _size; 49 | int _dim; 50 | int _z; 51 | } 52 | ]] 53 | 54 | 55 | local function HashMapT__hash( self, x ) 56 | return brshift( 57 | (self._z * tonumber(x)), -- lds.hash(x)), 58 | (32 - self._dim) ) -- HashMapT__w = 32 59 | end 60 | 61 | 62 | local function HashMapT__resize( self ) 63 | self._dim = 1 64 | while blshift(1, self._dim) <= self._size do self._dim = self._dim + 1 end 65 | local shift_dim = blshift( 1, self._dim ) 66 | if shift_dim == self._tsize then return end -- don't need to resize 67 | 68 | local ptr = self.__talloc:allocate( shift_dim ) 69 | local new_t = ffi.cast( self._t, ptr ) 70 | for i = 0, shift_dim-1 do 71 | self.__vt.__construct(new_t[i]) 72 | end 73 | 74 | for i = 0, self._tsize-1 do 75 | for j = 0, self._t[i]._size-1 do 76 | local pair = self._t[i]._data[j] 77 | new_t[HashMapT__hash(self, pair.key)]:push_back(pair) 78 | end 79 | self.__vt.__destruct( self._t[i] ) 80 | end 81 | self.__talloc:deallocate( self._t ) 82 | 83 | self._t = new_t 84 | self._tsize = shift_dim 85 | end 86 | 87 | 88 | -- HashMap public methods 89 | local HashMap = {} 90 | 91 | 92 | ------------------------------ 93 | -- Capacity methods 94 | 95 | --- Returns the number of elements in the HashMap. 96 | -- 97 | -- This is the number of elements in the HashMap, not necessarily the total allocated memory. 98 | -- 99 | -- The __len metamethod returns the same value so you can use the # operator 100 | -- 101 | -- If you are seeing performance warnings with -jv (e.g. loop unroll limit), 102 | -- it can also be accessed as the field `_size`, but you should never write 103 | -- to this youself. 104 | -- 105 | -- @return Returns the number of elements in the HashMap. 106 | function HashMap:size() 107 | return self._size 108 | end 109 | 110 | 111 | --- Returns true if the HashMap is empty. 112 | -- @return true if the HashMap size is 0, false otherwise. 113 | function HashMap:empty() 114 | return self._size == 0 115 | end 116 | 117 | 118 | ------------------------------------ 119 | -- Element Access functions 120 | -- 121 | 122 | --- Get key/value pair at key `k`. 123 | -- 124 | -- Returns nil if the key is not found. 125 | -- 126 | -- @param k Key of the element to find. 127 | -- @return The key/value pair at the specified key in the HashMap. 128 | function HashMap:find( k ) 129 | local list = self._t[HashMapT__hash(self, k)] 130 | for i = 0, list._size-1 do 131 | local pair = list._data[i] 132 | if k == pair.key then return pair end 133 | end 134 | return nil 135 | end 136 | 137 | 138 | --- Returns an iterator, suitable for the Lua `for` loop, over all the pairs of the HashMap. 139 | -- Each iteration yields an single object with fields `key` and `val`. 140 | -- As this is a chained hash map. there is no logical order to the iteration. 141 | -- 142 | -- Modify the HashMap (via insert/remove/clear) during iteration 143 | -- invalidates the iterator. *BE CAREFUL!* 144 | -- 145 | -- @return iterator function 146 | function HashMap:iter() 147 | -- i: index over all the buckets 148 | -- j: index within a bucket 149 | local i, j = 0, -1 150 | return function() 151 | local t = self._t[i] 152 | j = j + 1 153 | while j >= t._size do 154 | i, j = i + 1, 0 155 | if i >= self._tsize then return nil end 156 | t = self._t[i] 157 | end 158 | return t._data[j] 159 | end 160 | end 161 | 162 | 163 | 164 | ------------------------------ 165 | -- Modifier functions 166 | 167 | 168 | --- Inserts element value `v` at key `k`. 169 | -- 170 | -- Returns the previous value, if it existed, or true if it did not. 171 | -- 172 | -- @param k Key to set in the HashMap. 173 | -- @param v Element to set at that key 174 | -- 175 | -- @return The previous element at the specified key in the HashMap, 176 | -- or true if this is a new key. 177 | function HashMap:insert( k, v ) 178 | local pair = self:find( k ) 179 | if pair then -- if it exists, update the value 180 | local old_val = pair.val 181 | pair.val = v 182 | return old_val 183 | end 184 | 185 | if (self._size + 1) > self._t._size then HashMapT__resize(self) end 186 | local j = HashMapT__hash(self, k) 187 | -- initialization of nested structs are not compiled, 188 | -- so we use a pre-allocated placeholder, `_pt_scratch`, stored in the metatable 189 | -- otherwise, this would just be: self.t[j]:push_back(self.__pt(k, v)) 190 | local pt = self.__pt_scratch 191 | pt.key, pt.val = k, v 192 | self._t[j]:push_back(pt) 193 | self._size = self._size + 1 194 | return true 195 | end 196 | 197 | 198 | --- Remove element at given key. 199 | -- @param k Key to remove 200 | -- @return The item previously at the key, or nil if absent. 201 | function HashMap:remove( k ) 202 | local list = self._t[HashMapT__hash(self, k)] 203 | for i = 0, list._size-1 do 204 | local y = list._data[i] 205 | if k == y.key then 206 | local old_val = y.val 207 | list:erase( i ) 208 | self._size = self._size - 1 209 | return old_val 210 | end 211 | end 212 | return nil 213 | end 214 | 215 | 216 | --- Clears all the elements from the HashMap. 217 | -- The allocated memory is reclaimed. 218 | function HashMap:clear() 219 | self._size = 0 220 | HashMapT__resize( self ) 221 | end 222 | 223 | 224 | ------------------------------ 225 | -- Testing functions 226 | 227 | -- Returns a Lua table with internal information about the HashMap. 228 | -- 229 | -- This table has keys: tsize, size, dim, z, and buckets 230 | -- 'buckets' is an array of tables with keys `size` and `cap`, representing 231 | -- the size and capacity of the i'th bucket. 232 | -- 233 | -- Generally for diagnostics 234 | function HashMap:get_internals() 235 | local t = { 236 | tsize = self._tsize, 237 | size = self._size, 238 | dim = self._dim, 239 | z = self._z, 240 | buckets = {}, -- size, cap 241 | } 242 | for i = 0, self._tsize-1 do 243 | t.buckets[#t.buckets+1] = { 244 | size = self._t[i]._size, 245 | cap = self._t[i]._cap, 246 | } 247 | end 248 | return t 249 | end 250 | 251 | 252 | ------------------------------ 253 | -- Metatable 254 | 255 | local HashMapT_mt = { 256 | 257 | __new = function( hmt ) 258 | local ptr = hmt.__talloc:allocate( 2 ) 259 | local hm = ffi.new( hmt, 260 | ptr, -- _t 261 | 2, -- _tsize 262 | 0, -- _size 263 | 1, -- _dim 264 | bor(math.random(0x7FFFFFFF), 1) -- _z, a random odd integer 265 | -- HashMapT__w = 32 266 | ) 267 | for i = 0, hm._tsize-1 do 268 | hm.__vt.__construct(hm._t[i]) 269 | end 270 | return hm 271 | end, 272 | 273 | __gc = function( self ) 274 | for i = 0, self._tsize-1 do 275 | self.__vt.__destruct(self._t[i]) 276 | end 277 | self.__vt.__alloc:deallocate( self._t ) 278 | end, 279 | 280 | --- __len metamethod, returning the number of elements in the HashMap. 281 | -- See also HashMap:size() and HashMap._size 282 | -- @return The number of elements in the HashMap. 283 | __len = function( self ) 284 | return self._size 285 | end, 286 | 287 | __index = HashMap, 288 | } 289 | 290 | 291 | function lds.HashMapT( ct_key, ct_val, allocator_class ) 292 | if type(ct_key) ~= 'cdata' then error("argument 1 is not a valid 'cdata'") end 293 | if type(ct_val) ~= 'cdata' then error("argument 2 is not a valid 'cdata'") end 294 | allocator_class = allocator_class or lds.MallocAllocator 295 | 296 | local pt = ffi.typeof( PairT_cdef, ct_key, ct_val ) 297 | local vt = lds.VectorT( pt, allocator_class ) 298 | local hmt = ffi.typeof( HashMapT_cdef, vt ) -- HashMap ct 299 | 300 | -- clone the metatable and insert type-specific data 301 | local hmt_mt = lds.simple_deep_copy(HashMapT_mt) 302 | hmt_mt.__index.__ct_key = ct_key -- the key ct 303 | hmt_mt.__index.__ct_val = ct_val -- the value ct 304 | hmt_mt.__index.__pt = pt -- the pair ct 305 | hmt_mt.__index.__pt_size = ffi.sizeof(pt) 306 | hmt_mt.__index.__vt = vt -- the VectorT ct 307 | hmt_mt.__index.__vt_size = ffi.sizeof(vt) 308 | hmt_mt.__index.__pt_scratch = pt() -- pre-allocated PairT 309 | hmt_mt.__index.__talloc = allocator_class(vt) 310 | 311 | return ffi.metatype( hmt, hmt_mt ) 312 | end 313 | 314 | 315 | function lds.HashMap( ct_key, ct_val, allocator ) 316 | return lds.HashMapT( ct_key, ct_val, allocator )() 317 | end 318 | 319 | 320 | -- Return the lds API 321 | return lds 322 | -------------------------------------------------------------------------------- /lds/Vector.lua: -------------------------------------------------------------------------------- 1 | --[[ 2 | lds - LuaJIT Data Structures 3 | 4 | @copyright Copyright (c) 2012-2020 Evan Wies. All rights reserved. 5 | @license MIT License, see the COPYRIGHT file. 6 | 7 | @type Vector 8 | 9 | Dynamic array of FFI cdata. 10 | 11 | Index counting starts at zero, rather than Lua's one-based indexing. 12 | 13 | Conventions: 14 | private cdata fields are prefixed with _ 15 | private class fields (via metatable) are prefixed with __ 16 | private methods are prefixed with VectorT__ 17 | 18 | --]] 19 | 20 | local lds = require 'lds.allocator' 21 | 22 | local ffi = require 'ffi' 23 | local C = ffi.C 24 | 25 | 26 | local VectorT__cdef = [[ 27 | struct { 28 | $ * _data; 29 | int _size; 30 | int _cap; 31 | } 32 | ]] 33 | 34 | 35 | -- Resizes the vector to reserve_n. 36 | -- Like libstd++, it will resize to the larger of 37 | -- reserve_n or double the capacity. 38 | local function VectorT__resize( v, reserve_n, shrink_to_fit ) 39 | local new_cap = math.max(1, reserve_n or 2*v._cap, shrink_to_fit and 1 or 2*v._cap) 40 | if v._cap >= new_cap then return end 41 | 42 | local new_data = v.__alloc:reallocate(v._data, new_cap*v.__ct_size) 43 | v._data = ffi.cast( v._data, new_data ) 44 | v._cap = new_cap 45 | end 46 | 47 | 48 | -- Vector public methods 49 | local Vector = {} 50 | 51 | 52 | ------------------------------ 53 | -- Capacity methods 54 | 55 | --- Returns the number of elements in the Vector. 56 | -- 57 | -- This is the number of elements in the Vector, not necessarily the total allocated memory. 58 | -- See Vector:capacity. 59 | -- 60 | -- The __len metamethod returns the same value so you can use the # operator 61 | -- 62 | -- If you are seeing performance warnings with -jv (e.g. loop unroll limit), 63 | -- it can also be accessed as the field `_size`, but you should never write 64 | -- to this youself. 65 | -- 66 | -- @return Returns the number of elements in the Vector. 67 | function Vector:size() 68 | return self._size 69 | end 70 | 71 | 72 | --- Returns the number of bytes used by objects in the Vector. 73 | -- 74 | -- @return Returns the number of bytes used by in the Vector. 75 | function Vector:size_bytes() 76 | return self._size * self.__ct_size 77 | end 78 | 79 | 80 | --- Returns true if the Vector is empty. 81 | -- @return true if the Vector size is 0, false otherwise. 82 | function Vector:empty() 83 | return self._size == 0 84 | end 85 | 86 | 87 | --- Returns the total number of elements the Vector can currently hold. 88 | -- This is number of elements the Vector can hold before needing to allocate more memory. 89 | -- @return The number of elements the Vector can hold before needing to allocate more memory. 90 | function Vector:capacity() 91 | return self._cap 92 | end 93 | 94 | 95 | --- Returns the number of bytes of capacity of the Vector. 96 | -- 97 | -- @return Returns the number of bytes of capacity by in the Vector. 98 | function Vector:capacity_bytes() 99 | return self._cap * self.__ct_size 100 | end 101 | 102 | 103 | --- Request a change in capacity 104 | -- 105 | -- Requests that the capacity of the allocated storage space for the elements of the vector 106 | -- container be at least enough to hold `reserve_n` elements. 107 | -- 108 | -- @param reserve_n Minimum amount desired as capacity of allocated storage, in elements of type T. 109 | function Vector:reserve( reserve_n ) 110 | VectorT__resize( self, reserve_n ) 111 | end 112 | 113 | 114 | --- Attempts to free unused memory. 115 | function Vector:shrink_to_fit() 116 | VectorT__resize(self, self._size, true ) -- true = shrink_to_fit 117 | end 118 | 119 | 120 | 121 | ------------------------------------ 122 | -- Element Access functions 123 | -- 124 | 125 | --- Get element value at index. 126 | -- Returns the element at index `i` in the vector. 127 | -- 128 | -- Returns `nil` if the index is out of range. 129 | -- See also Vector:get_e which throws an error instead. 130 | -- 131 | -- @param i Index of the element to get 132 | -- If this is greater than or equal to the vector size, "VectorT.get: index out of bounds" error is thrown. 133 | -- Notice that the first element has an index of 0, not 1. 134 | -- 135 | -- @return The element at the specified index in the Vector. 136 | function Vector:get( i ) 137 | if i < 0 or i >= self._size then return nil end 138 | return self._data[i] 139 | end 140 | 141 | 142 | --- Get element value at index. 143 | -- Returns the element at index `i` in the vector. 144 | -- 145 | -- Throws error if the index is out of range. 146 | -- See also Vector:get, which returns `nil` instead. 147 | -- 148 | -- @param i Index of the element to get. 149 | -- If this is greater than or equal to the vector size, "VectorT.get: index out of bounds" error is thrown. 150 | -- Notice that the first element has an index of 0, not 1. 151 | -- 152 | -- @return The element at the specified index in the vector. 153 | function Vector:get_e( i ) 154 | if i < 0 or i >= self._size then lds.error("VectorT.get: index out of bounds") end 155 | return self._data[i] 156 | end 157 | 158 | 159 | --- Returns the value of the first element of the Vector. 160 | -- Returns `nil` if the Vector is empty. 161 | -- @return Returns the value of the first element of the Vector. 162 | function Vector:front() 163 | if self._size == 0 then return nil end 164 | return self._data[0] 165 | end 166 | 167 | 168 | --- Returns the value of the last element of the Vector. 169 | -- Returns `nil` if the Vector is empty. 170 | -- @return Returns the value of the last element of the Vector. 171 | function Vector:back() 172 | if self._size == 0 then return nil end 173 | return self._data[self._size-1] 174 | end 175 | 176 | 177 | --- Return pointer for the underlying array. 178 | -- This can also be accessed as the field `_data`, although 179 | -- you should never write to this yourself. 180 | -- @return Pointer for the underlying array. 181 | function Vector:data() 182 | return self._data 183 | end 184 | 185 | 186 | ------------------------------ 187 | -- Modifier functions 188 | 189 | 190 | --- Set element value at index. 191 | -- Sets the element `x` at index `i` in the Vector. 192 | -- 193 | -- Returns nil if the index is out of range. 194 | -- See also Vector:set_e, which throws an error instead. 195 | -- 196 | -- @param i Index to set in the Vector. 197 | -- @param x Element to set at that index 198 | -- 199 | -- Note that the first element has an index of 0, not 1. 200 | -- 201 | -- @return The previous element at the specified index in the Vector, 202 | -- or nil if the index is out of range. 203 | function Vector:set( i, x ) 204 | if i < 0 or i >= self._size then return nil end 205 | local prev = self._data[i] 206 | self._data[i] = x 207 | return prev 208 | end 209 | 210 | 211 | --- Set element value at index. 212 | -- Sets the element `x` at index `i` in the Vector. 213 | -- 214 | -- Throws error if the index is out of range. 215 | -- See also Vector:set, which returns `nil` instead. 216 | -- 217 | -- @param i Index to set in the Vector. 218 | -- @param x Element to set at that index 219 | -- If this is greater than or equal to the Vector size, the error "VectorT.set: index out of bounds" is thrown. 220 | -- 221 | -- Note that the first element has an index of 0, not 1. 222 | -- 223 | -- @return The previous element at the specified index in the vector. 224 | function Vector:set_e( i, x ) 225 | if i < 0 or i >= self._size then lds.error("VectorT.set: index out of bounds") end 226 | local prev = self._data[i] 227 | self._data[i] = x 228 | return prev 229 | end 230 | 231 | 232 | --- Inserts given value into Vector at the specified index, move all element over. 233 | -- @param i Index to insert at 234 | -- @param x Data to be inserted. 235 | -- 236 | -- This function will insert a copy of the given value before 237 | -- the specified location. Note that this kind of operation 238 | -- could be expensive for a Vector and if it is frequently 239 | -- used the user should consider using std::list. 240 | function Vector:insert( i, x ) 241 | if type(x) == 'nil' then self:push_back(i) end -- handle default index like table.insert 242 | if i < 0 or i > self._size then lds.error("insert: index out of bounds") end 243 | if self._size + 1 > self._cap then VectorT__resize(self) end 244 | C.memmove(self._data+i+1, self._data+i, (self._size-i)*self.__ct_size) 245 | self._data[i] = x 246 | self._size = self._size + 1 247 | end 248 | 249 | 250 | --- Add data to the end of the Vector. 251 | -- @param x Data to be added. 252 | -- 253 | -- This is a typical stack operation. The function creates an 254 | -- element at the end of the Vector and assigns the given data 255 | -- to it. Due to the nature of a Vector this operation can be 256 | -- done in constant time if the Vector has preallocated space available. 257 | function Vector:push_back( x ) 258 | if self._size + 1 > self._cap then VectorT__resize(self) end 259 | self._data[self._size] = x 260 | self._size = self._size + 1 261 | end 262 | 263 | 264 | --- Remove element at given index. 265 | -- @param i Index to erase 266 | -- @return The item previously at the index. 267 | -- 268 | -- This function will erase the element at the given index and thus 269 | -- shorten the Vector by one. 270 | -- 271 | -- Note This operation could be expensive and if it is 272 | -- frequently used the user should consider using std::list. 273 | -- The user is also cautioned that this function only erases 274 | -- the element, and that if the element is itself a pointer, 275 | -- the pointed-to memory is not touched in any way. Managing 276 | -- the pointer is the user's responsibilty. 277 | function Vector:erase( i ) 278 | if type(i) == 'nil' then return self:pop_back() end -- handle default index like table.remove 279 | if i < 0 or i >= self._size then lds.error("VectorT.erase: index out of bounds") end 280 | local x = self._data[i] 281 | C.memmove(self._data+i, self._data+i+1, (self._size-i)*self.__ct_size) 282 | self._size = self._size - 1 283 | return x 284 | end 285 | 286 | 287 | --- Removes last element. 288 | -- @return The value that was previously the last element, or nil if the vector was empty. 289 | -- This is a typical stack operation. It shrinks the Vector by one. 290 | function Vector:pop_back() 291 | if self._size == 0 then return nil end 292 | local x = self._data[self._size - 1] 293 | self._size = self._size - 1 294 | return x 295 | end 296 | 297 | 298 | --- Clears all the elements from the Vector. The allocated memory is unchanged. 299 | function Vector:clear() 300 | self._size = 0 301 | end 302 | 303 | 304 | ------------------------------ 305 | -- Private methods 306 | 307 | -- Constructor method 308 | function Vector:__construct( reserve_n ) 309 | if reserve_n and reserve_n > 0 then 310 | local data = self.__alloc:allocate(n) 311 | if not data then lds.error('VectorT.new allocation failed') end 312 | self._data, self._size, self._cap = data, 0, reserve_n 313 | else 314 | self._data, self._size, self._cap = nil, 0, 0 315 | end 316 | return self -- for chaining 317 | end 318 | 319 | 320 | -- Destructor method 321 | function Vector:__destruct() 322 | self.__alloc:deallocate(self._data) 323 | self._data, self._cap, self._size = nil, 0, 0 324 | return self -- for chaining 325 | end 326 | 327 | 328 | ------------------------------ 329 | -- Metatable 330 | 331 | local VectorT__mt = { 332 | 333 | __new = function( vt, reserve_n ) 334 | local self = ffi.new(vt) 335 | return self:__construct(reserve_n) 336 | end, 337 | 338 | __gc = function( self ) 339 | self:__destruct() 340 | end, 341 | 342 | --- __len metamethod, returning the number of elements in the Vector. 343 | -- See also Vector:size() and Vector.__size 344 | -- @return The number of elements in the Vector. 345 | __len = function( self ) 346 | return self._size 347 | end, 348 | 349 | __index = Vector, 350 | } 351 | 352 | 353 | function lds.VectorT( ct, allocator_class ) 354 | if type(ct) ~= 'cdata' then lds.error("argument 1 is not a valid 'cdata'") end 355 | allocator_class = allocator_class or lds.MallocAllocator 356 | 357 | -- clone the metatable and insert type-specific data 358 | local vt_mt = lds.simple_deep_copy(VectorT__mt) 359 | vt_mt.__index.__ct = ct 360 | vt_mt.__index.__ct_size = ffi.sizeof(ct) 361 | vt_mt.__index.__alloc = allocator_class(ct) 362 | 363 | local vt = ffi.typeof( VectorT__cdef, ct ) 364 | return ffi.metatype( vt, vt_mt ) 365 | end 366 | 367 | 368 | function lds.Vector( ct, allocator_class ) 369 | return lds.VectorT( ct, allocator_class )() 370 | end 371 | 372 | 373 | -- Return the lds API 374 | return lds 375 | -------------------------------------------------------------------------------- /lds/allocator.lua: -------------------------------------------------------------------------------- 1 | --[[ 2 | lds - LuaJIT Data Structures 3 | 4 | @copyright Copyright (c) 2012-2020 Evan Wies. All rights reserved. 5 | @license MIT License, see the COPYRIGHT file. 6 | 7 | @module allocator 8 | 9 | allocator_interface = { 10 | allocate = function(self, n) -- allocates storage of n elements 11 | deallocate = function(self, p) -- deallocates storage using the allocator 12 | reallocate = function(self, p, n) -- reallocates storage at p to n elements 13 | } 14 | 15 | MallocAllocator uses the standard C malloc/free 16 | VLAAllocator uses FFI VLAs and hence the LuaJIT allocator 17 | JemallocAllocator uses jemalloc (if available) 18 | 19 | 20 | --]] 21 | 22 | local lds = require 'lds.init' 23 | 24 | local ffi = require 'ffi' 25 | local C = ffi.C 26 | 27 | 28 | ------------------------------------------------------------------------------- 29 | -- MallocAllocator 30 | -- 31 | -- An lds allocator that uses the standard C malloc/free 32 | -- 33 | 34 | ffi.cdef([[ 35 | void * malloc(size_t size); 36 | void * calloc(size_t count, size_t size); 37 | void * valloc(size_t size); 38 | void * realloc(void *ptr, size_t size); 39 | void free(void *ptr); 40 | ]]) 41 | 42 | 43 | local MallocAllocatorT__mt = { 44 | __index = { 45 | -- may be re-assigned in MallocAllocatorT 46 | allocate = function(self, n) 47 | return C.calloc( n, self._ct_size ) 48 | end, 49 | deallocate = function(self, p) 50 | if p ~= 0 then C.free(p) end 51 | end, 52 | reallocate = function(self, p, n) 53 | return C.realloc(p, n) 54 | end, 55 | } 56 | } 57 | 58 | -- which is 'malloc', 'calloc' (zeroed), or 'valloc' (page-aligned) 59 | -- calloc is the default to be consistent with LuaJIT's zero-ing behavior 60 | function lds.MallocAllocatorT( ct, which ) 61 | if type(ct) ~= 'cdata' then error('argument 1 is not a valid "cdata"') end 62 | 63 | -- clone the metatable and insert type-specific data 64 | local t_mt = lds.simple_deep_copy(MallocAllocatorT__mt) 65 | t_mt.__index._ct = ct 66 | t_mt.__index._ct_size = ffi.sizeof(ct) 67 | 68 | if which == nil or which == 'calloc' then 69 | -- keep default 70 | elseif which == 'malloc' then 71 | t_mt.__index.allocate = function(self, n) 72 | return C.malloc( n * self._ct_size ) 73 | end 74 | elseif which == 'valloc' then 75 | t_mt.__index.allocate = function(self, n) 76 | return C.malloc( n * self._ct_size ) 77 | end 78 | else 79 | error('argument 2 must be nil, "calloc", "malloc", or "valloc"') 80 | end 81 | 82 | local t_anonymous = ffi.typeof( 'struct {}' ) 83 | return ffi.metatype( t_anonymous, t_mt ) 84 | end 85 | 86 | 87 | -- which is 'malloc', 'calloc' (zeroed), or 'valloc' (page-aligned) 88 | -- calloc is the default to be consistent with LuaJIT's zero-ing behavior 89 | function lds.MallocAllocator( ct, which ) 90 | return lds.MallocAllocatorT( ct, which )() 91 | end 92 | 93 | 94 | 95 | ------------------------------------------------------------------------------- 96 | -- VLAAllocator 97 | -- 98 | -- An lds allocator that uses FFI VLAs and hence the LuaJIT allocator 99 | -- 100 | -- Note that there is some extra bookkeeping required by the allocator. 101 | -- of one 32-character string per allocated object. 102 | -- 103 | -- This allocator also does not support a true realloc, but rather does 104 | -- a new and copy. 105 | -- 106 | 107 | 108 | -- Anchor for VLA objects used by all VLAAllocators 109 | -- 110 | -- Since e are returning raw pointers, we need to keep the VLA cdata from 111 | -- being garbage collected. So we convert the address to a size_t and then 112 | -- to a string, and use that as a key for the VLA cdata. 113 | local VLAAllocator__anchors = {} 114 | 115 | 116 | VLAAllocatorT__mt = { 117 | __index = { 118 | allocate = function(self, n) 119 | local vla = ffi.new( self._vla, n ) 120 | VLAAllocator__anchors[tostring(ffi.cast(lds.size_t, vla._data))] = vla 121 | return vla._data 122 | end, 123 | deallocate = function(self, p) 124 | -- remove the stored reference and then let GC do the rest 125 | if p ~= nil then 126 | VLAAllocator__anchors[tostring(ffi.cast(lds.size_t, p))] = nil 127 | end 128 | end, 129 | reallocate = function(self, p, n) 130 | if p == nil then 131 | local vla = ffi.new( self._vla, n ) 132 | VLAAllocator__anchors[tostring(ffi.cast(lds.size_t, vla._data))] = vla 133 | return vla._data 134 | else 135 | local key_old = tostring(ffi.cast(lds.size_t, p)) 136 | local vla_old = VLAAllocator__anchors[key_old] 137 | 138 | local vla_new = ffi.new( self._vla, n ) 139 | local key_new = tostring(ffi.cast(lds.size_t, vla_new._data)) 140 | 141 | VLAAllocator__anchors[key_new] = vla_new 142 | ffi.copy(vla_new._data, p, ffi.sizeof(vla_old)) 143 | VLAAllocator__anchors[key_old] = nil 144 | return vla_new._data 145 | end 146 | end, 147 | } 148 | } 149 | 150 | 151 | function lds.VLAAllocatorT( ct ) 152 | if type(ct) ~= 'cdata' then error('argument 1 is not a valid "cdata"') end 153 | 154 | -- clone the metatable and insert type-specific data 155 | local t_mt = lds.simple_deep_copy(VLAAllocatorT__mt) 156 | t_mt.__index._ct = ct 157 | t_mt.__index._ct_size = ffi.sizeof(ct) 158 | t_mt.__index._vla = ffi.typeof( 'struct { $ _data[?]; }', ct ) 159 | 160 | local t_anonymous = ffi.typeof( 'struct {}' ) 161 | return ffi.metatype( t_anonymous, t_mt ) 162 | end 163 | 164 | 165 | function lds.VLAAllocator( ct ) 166 | return lds.VLAAllocatorT( ct )() 167 | end 168 | 169 | 170 | ------------------------------------------------------------------------------- 171 | -- JemallocAllocator 172 | -- 173 | -- An lds allocator that uses jemalloc, if available. 174 | -- 175 | 176 | -- check for jemalloc 177 | local success, J = pcall(function() return require 'lds.jemalloc' end) 178 | 179 | if success and J then 180 | 181 | lds.J = J -- make jemalloc lib immediately available to clients 182 | 183 | local JemallocAllocatorT__mt = { 184 | __index = { 185 | allocate = function(self, n) 186 | return J.mallocx( n * self._ct_size, self._flags ) 187 | end, 188 | deallocate = function(self, p) 189 | if p ~= nil then J.dallocx(p, self._flags) end 190 | end, 191 | reallocate = function(self, p, n) 192 | if p == nil then 193 | return J.mallocx( n * self._ct_size, self._flags ) 194 | else 195 | return J.rallocx(p, n, self._flags) 196 | end 197 | end, 198 | } 199 | } 200 | 201 | -- if flags is not specified, J.MALLOCX_ZERO is the default to be 202 | -- consistent with LuaJIT's allocation behavior 203 | function lds.JemallocAllocatorT( ct, flags ) 204 | if type(ct) ~= 'cdata' then error("argument 1 is not a valid 'cdata'") end 205 | 206 | -- clone the metatable and insert type-specific data 207 | local t_mt = lds.simple_deep_copy(JemallocAllocatorT__mt) 208 | t_mt.__index._ct = ct 209 | t_mt.__index._ct_size = ffi.sizeof(ct) 210 | t_mt.__index._flags = flags or J.MALLOCX_ZERO() 211 | 212 | local t_anonymous = ffi.typeof( "struct {}" ) 213 | return ffi.metatype( t_anonymous, t_mt ) 214 | end 215 | 216 | 217 | -- if flags is not specified, J.MALLOCX_ZERO is the default to be 218 | -- consistent with LuaJIT's allocation behavior 219 | function lds.JemallocAllocator( ct, flags ) 220 | return lds.JemallocAllocatorT( ct, flags )() 221 | end 222 | 223 | end -- was jemalloc required? 224 | 225 | 226 | -- Return the lds API 227 | return lds 228 | -------------------------------------------------------------------------------- /lds/hash.lua: -------------------------------------------------------------------------------- 1 | --[[ 2 | lds - LuaJIT Data Structures 3 | 4 | @copyright Copyright (c) 2012-2020 Evan Wies. All rights reserved. 5 | @license MIT License, see the COPYRIGHT file. 6 | 7 | hash functionality 8 | 9 | --]] 10 | 11 | local lds = require 'lds.init' 12 | 13 | local bit = require 'bit' 14 | local blshift, brshift, bxor = bit.lshift, bit.rshift, bit.bxor 15 | 16 | -- simple hash function 17 | function lds.hash( x ) 18 | return tonumber(x) 19 | end 20 | 21 | 22 | -- adpted from boost::hash_combine 23 | -- http://svn.boost.org/svn/boost/trunk/boost/functional/hash/hash.hpp 24 | function lds.hash_combine( seed, v ) 25 | -- magic number explanation: 26 | -- http://stackoverflow.com/questions/4948780/magic-numbers-in-boosthash-combine 27 | local magic = 0x9e3779b9 28 | local h = lds.hash(v) + magic + blshift(seed, 6) + brshift(seed, 2) 29 | return bxor(seed, h) 30 | end 31 | 32 | 33 | -- Return the lds API 34 | return lds 35 | -------------------------------------------------------------------------------- /lds/init.lua: -------------------------------------------------------------------------------- 1 | --[[ 2 | lds - LuaJIT Data Structures 3 | 4 | @copyright Copyright (c) 2012-2020 Evan Wies. All rights reserved. 5 | @license MIT License, see the COPYRIGHT file. 6 | 7 | lds module initialization 8 | 9 | --]] 10 | 11 | 12 | local ffi = require 'ffi' 13 | ffi.cdef([[ 14 | void *memmove(void *dst, const void *src, size_t len); 15 | ]]) 16 | 17 | 18 | -- API Table used by all other lds modules 19 | local lds = {} 20 | 21 | 22 | --- Error function called when the LDS library has an error 23 | -- You may replace this function. 24 | -- The default implementation calls Lua's error(). 25 | -- @param msg The error message 26 | function lds.error( msg ) 27 | error( msg ) 28 | end 29 | 30 | 31 | --- Assert function called by the LDS library 32 | -- You may replace this function. 33 | -- The default implementation is simply: if not x then lds.error(msg) end 34 | -- @param x The value to evaluate as true 35 | -- @param msg The error message 36 | function lds.assert( x, msg ) 37 | if not x then lds.error(msg) end 38 | end 39 | 40 | 41 | -- simple deep copy function 42 | -- used to copy prototypical metatables 43 | local function simple_deep_copy( x ) 44 | if type(x) ~= 'table' then return x end 45 | local t = {} 46 | for k, v in pairs(x) do 47 | t[k] = simple_deep_copy(v) 48 | end 49 | return t 50 | end 51 | lds.simple_deep_copy = simple_deep_copy 52 | 53 | 54 | -- Commonly used types 55 | lds.int32_t = ffi.typeof('int32_t') 56 | lds.uint32_t = ffi.typeof('uint32_t') 57 | lds.size_t = ffi.typeof('size_t') 58 | 59 | 60 | -- Constants 61 | lds.INT_MAX = tonumber( lds.uint32_t(-1) / 2 ) 62 | lds.INT32_MAX = tonumber( lds.uint32_t(-1) / 2 ) 63 | 64 | 65 | -- Return the lds API 66 | return lds 67 | -------------------------------------------------------------------------------- /lds/jemalloc.lua: -------------------------------------------------------------------------------- 1 | ------------------------------------------------------------------------------- 2 | -- LuaJIT bindings to the jemalloc library 3 | -- 4 | -- Copyright (c) 2014-2020 Evan Wies. 5 | -- Released under the MIT license. See the LICENSE file. 6 | -- 7 | -- Project home: https://github.com/neomantra/luajit-jemalloc 8 | -- 9 | -- Does not load jemalloc, this must be done explicitly by the client 10 | -- either with ffi.load or through a preload mechanism. 11 | -- 12 | -- Adheres to API version 3.5.0 13 | -- 14 | 15 | local ffi = require 'ffi' 16 | local C = ffi.C 17 | local pcall = pcall 18 | 19 | -- jemalloc uses an optional compile-time prefix (specified using --with-jemalloc-prefix). 20 | -- Clients must determine the prefix and assign it either to the global 21 | -- variable JEMALLOC_PREFIX (set prior to the first `require` of this library), 22 | -- or set the environment variable JEMALLOC_PREFIX. 23 | -- Defaults to no prefix, except on OSX where it is 'je_'. 24 | local JEMALLOC_PREFIX = JEMALLOC_PREFIX or 25 | os.getenv('JEMALLOC_PREFIX') or 26 | (ffi.os == 'OSX' and 'je_' or '') 27 | 28 | do 29 | local cdef_template = [[ 30 | int !_!mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp, size_t newlen); 31 | void *!_!mallocx(size_t size, int flags); 32 | void *!_!rallocx(void *ptr, size_t size, int flags); 33 | size_t !_!xallocx(void *ptr, size_t size, size_t extra, int flags); 34 | size_t !_!sallocx(void *ptr, int flags); 35 | void !_!dallocx(void *ptr, int flags); 36 | size_t !_!nallocx(size_t size, int flags); 37 | int !_!mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp, size_t newlen); 38 | int !_!mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp); 39 | int !_!mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen); 40 | void !_!malloc_stats_print(void (*write_cb) (void *, const char *), void *cbopaque, const char *opts); 41 | size_t !_!malloc_usable_size(const void *ptr); 42 | 43 | int ffs(int i); 44 | ]] 45 | local cdef_str = string.gsub(cdef_template, '!_!', JEMALLOC_PREFIX) 46 | ffi.cdef(cdef_str) 47 | end 48 | 49 | 50 | -- check if it jemalloc was pre-loaded 51 | do 52 | local mallctl_fname = JEMALLOC_PREFIX..'mallctl' 53 | if not pcall(function() return C[mallctl_fname] end) then 54 | error('jemalloc library was not pre-loaded or the prefix "'..JEMALLOC_PREFIX..'" is incorrect') 55 | end 56 | end 57 | 58 | -- our public API 59 | local J = {} 60 | 61 | 62 | do 63 | J.EINVAL, J.ENOENT, J.EPERM, J.EFAULT, J.ENOMEM = 22, 2, 1, 14, 12 64 | local abi_os = ffi.os:lower() 65 | if abi_os == 'linux' then 66 | J.EAGAIN = 11 67 | elseif abi_os == 'osx' then 68 | J.EAGAIN = 35 69 | elseif abi_os == 'bsd' then 70 | J.EAGAIN = 35 71 | else 72 | error('unsupported OS: '..abi_os) 73 | end 74 | end 75 | 76 | 77 | ------------------------------------------------------------------------------- 78 | -- malloctl support 79 | -- 80 | -- First implement mallocctl_read, since we need to 81 | -- check that we have the correct library version 82 | -- 83 | 84 | do 85 | -- all the mallctl entries can share these holders 86 | -- since they are not accessed simultaneously and only 87 | -- used duing the mallctl invocation 88 | local bool_1 = ffi.new('bool[1]') 89 | local uint64_1 = ffi.new('uint64_t[1]') 90 | local ksz_1 = ffi.new('const char*[1]') 91 | local size_1 = ffi.new('size_t[1]') 92 | local ssize_1 = ffi.new('long[1]') -- TODO: is this correct in all circumstances? 93 | 94 | local mallctl_params = { 95 | -- param = { holder, read?, write? } 96 | ['version'] = { ksz_1, true, false }, 97 | ['epoch'] = { uint64_1, true, true }, 98 | ['config.debug'] = { bool_1, true, false }, 99 | ['config.dss'] = { bool_1, true, false }, 100 | ['config.fill'] = { bool_1, true, false }, 101 | ['config.lazy_lock'] = { bool_1, true, false }, 102 | ['config.mremap'] = { bool_1, true, false }, 103 | ['config.munmap'] = { bool_1, true, false }, 104 | ['config.prof'] = { bool_1, true, false }, 105 | ['config.prof_libgcc'] = { bool_1, true, false }, 106 | ['config.prof_libunwind'] = { bool_1, true, false }, 107 | ['config.stats'] = { bool_1, true, false }, 108 | ['config.tcache'] = { bool_1, true, false }, 109 | ['config.tls'] = { bool_1, true, false }, 110 | ['config.utrace'] = { bool_1, true, false }, 111 | ['config.valgrind'] = { bool_1, true, false }, 112 | ['config.xmalloc'] = { bool_1, true, false }, 113 | ['opt.abort'] = { bool_1, true, false }, 114 | ['opt.dss'] = { ksz_1, true, false }, 115 | ['opt.lg_chunk'] = { size_1, true, false }, 116 | ['opt.narenas'] = { size_1, true, false }, 117 | ['opt.lg_dirty_mult'] = { ssize_1, true, false }, 118 | ['opt.stats_print'] = { bool_1, true, false }, 119 | ['opt.junk'] = { bool_1, true, false }, 120 | ['opt.quarantine'] = { size_1, true, false }, 121 | ['opt.redzone'] = { bool_1, true, false }, 122 | ['opt.zero'] = { bool_1, true, false }, 123 | ['opt.utrace'] = { bool_1, true, false }, 124 | ['opt.valgrind'] = { bool_1, true, false }, 125 | ['opt.xmalloc'] = { bool_1, true, false }, 126 | ['opt.tcache'] = { bool_1, true, false }, 127 | ['opt.lg_1cache_max'] = { size_1, true, false }, 128 | ['opt.prof'] = { bool_1, true, false }, 129 | ['opt.prof_prefix'] = { ksz_1, true, false }, 130 | ['opt.prof_active'] = { bool_1, true, true }, 131 | ['opt.lg_prof_sample'] = { ssize_1, true, false }, 132 | ['opt.prof_accum'] = { bool_1, true, false }, 133 | ['opt.lg_prof_interval'] = { ssize_1, true, false }, 134 | ['opt.prof_gdump'] = { bool_1, true, false }, 135 | ['opt.prof_final'] = { bool_1, true, false }, 136 | ['opt.prof_leak'] = { bool_1, true, false }, 137 | ['thread.arena'] = { uint32_1, true, true }, 138 | ['thread.allocated'] = { uint64_1, true, true }, 139 | ['thread.deallocated'] = { uint64_1, true, true }, 140 | ['thread.tcache.enabled'] = { bool_1, true, true }, 141 | ['thread.tcache.flush'] = { nil, true, false }, 142 | ['arenas.narenas'] = { uint32_1, true, false }, 143 | ['arenas.quantum'] = { size_1, true, false }, 144 | ['arenas.page'] = { size_1, true, false }, 145 | ['arenas.tcache_max'] = { size_1, true, false }, 146 | ['arenas.nbins'] = { uint32_1, true, false }, 147 | ['arenas.nhbins'] = { uint32_1, true, false }, 148 | ['arenas.nlruns'] = { size_1, true, false }, 149 | ['arenas.purge'] = { uint32_1, false, true }, 150 | ['arenas.extend'] = { uint32_1, true, false }, 151 | ['prof.active'] = { bool_1, true, true }, 152 | ['prof.dump'] = { ksz_1, false, true }, 153 | ['prof.interval'] = { uint64_1, true, false }, 154 | ['stats.allocated'] = { size_1, true, false }, 155 | ['stats.active'] = { size_1, true, false }, 156 | ['stats.mapped'] = { size_1, true, false }, 157 | ['stats.chunks.current'] = { size_1, true, false }, 158 | ['stats.chunks.total'] = { uint64_1, true, false }, 159 | ['stats.chunks.high'] = { size_1, true, false }, 160 | ['stats.huge.allocated'] = { size_1, true, false }, 161 | ['stats.huge.nmalloc'] = { uint64_1, true, false }, 162 | ['stats.huge.ndalloc'] = { uint64_1, true, false }, 163 | } 164 | 165 | --[[ 166 | -- TODO: figure this out 167 | ['thread.allocatedp'] = { (uint64_t *) r- [--enable-stats] 168 | ['thread.deallocatedp'] = { (uint64_t *) r- [--enable-stats] 169 | ['arenas.initialized'] = { (bool *) r- 170 | ['stats.cactive'] = { (size_t *) r- [--enable-stats] 171 | ['arenas.bin..size'] = { (size_t) r- 172 | ['arenas.bin..nregs'] = { (uint32_t) r- 173 | ['arenas.bin..run_size'] = { (size_t) r- 174 | ['arenas.lrun..size'] = { (size_t) r- 175 | ['arena..purge'] = { (unsigned) -- 176 | ['arena..dss'] = { (const char *) rw 177 | ['stats.arenas..dss'] = { (const char *) r- 178 | ['stats.arenas..nthreads'] = { (unsigned) r- 179 | ['stats.arenas..pactive'] = { (size_t) r- 180 | ['stats.arenas..pdirty'] = { (size_t) r- 181 | ['stats.arenas..mapped'] = { (size_t) r- [--enable-stats] 182 | ['stats.arenas..npurge'] = { (uint64_t) r- [--enable-stats] 183 | ['stats.arenas..nmadvise'] = { (uint64_t) r- [--enable-stats] 184 | ['stats.arenas..purged'] = { (uint64_t) r- [--enable-stats] 185 | ['stats.arenas..small.allocated'] = { (size_t) r- [--enable-stats] 186 | ['stats.arenas..small.nmalloc'] = { (uint64_t) r- [--enable-stats] 187 | ['stats.arenas..small.ndalloc'] = { (uint64_t) r- [--enable-stats] 188 | ['stats.arenas..small.nrequests'] = { (uint64_t) r- [--enable-stats] 189 | ['stats.arenas..large.allocated'] = { (size_t) r- [--enable-stats] 190 | ['stats.arenas..large.nmalloc'] = { (uint64_t) r- [--enable-stats] 191 | ['stats.arenas..large.ndalloc'] = { (uint64_t) r- [--enable-stats] 192 | ['stats.arenas..large.nrequests'] = { (uint64_t) r- [--enable-stats] 193 | ['stats.arenas..bins..allocated'] = { (size_t) r- [--enable-stats] 194 | ['stats.arenas..bins..nmalloc'] = { (uint64_t) r- [--enable-stats] 195 | ['stats.arenas..bins..ndalloc'] = { (uint64_t) r- [--enable-stats] 196 | ['stats.arenas..bins..nrequests'] = { (uint64_t) r- [--enable-stats] 197 | ['stats.arenas..bins..nfills'] = { (uint64_t) r- [--enable-stats 198 | ['stats.arenas..bins..nflushes'] = { (uint64_t) r- [--enable-stats 199 | ['stats.arenas..bins..nruns'] = { (uint64_t) r- [--enable-stats] 200 | ['stats.arenas..bins..nreruns'] = { (uint64_t) r- [--enable-stats] 201 | ['stats.arenas..bins..curruns'] = { (size_t) r- [--enable-stats] 202 | ['stats.arenas..lruns..nmalloc'] = { (uint64_t) r- [--enable-stats] 203 | ['stats.arenas..lruns..ndalloc'] = { (uint64_t) r- [--enable-stats] 204 | ['stats.arenas..lruns..nrequests'] = { (uint64_t) r- [--enable-stats] 205 | ['stats.arenas..lruns..curruns'] = { (size_t) r- [--enable-stats] 206 | --]] 207 | 208 | local mallctl_fname = JEMALLOC_PREFIX..'mallctl' 209 | local oldlenp = ffi.new('size_t[1]') 210 | 211 | -- TODO: load the MIBs at load-time and use them instead 212 | --- return the value on success, otherwise nil, error 213 | function J.mallctl_read( param ) 214 | local entry = mallctl_params[param] 215 | if not entry then return nil, 'invalid parameter' end 216 | if not entry[2] then return nil, 'parameter is not readable' end 217 | 218 | local oldp = entry[1] -- oldp may be nil for the non-rw entries, 219 | oldlenp[0] = oldp and ffi.sizeof(oldp) or 0 -- so set 0 size in that case 220 | 221 | local err = C[mallctl_fname]( param, oldp, oldlenp, nil, 0) 222 | if err ~= 0 then return nil, err end 223 | 224 | -- convert result to Lua string if needed 225 | if oldp == ksz_1 then 226 | return ffi.string(oldp[0]) 227 | else 228 | return oldp[0] 229 | end 230 | end 231 | 232 | -- returns true on success, otherwise false, error 233 | -- Note: relies upon LuaJIT's conversions (http://luajit.org/ext_ffi_semantics.html) 234 | -- which may throw an error if the conversion is invalid. This error is 235 | -- caught and returned as a string. 236 | function J.mallctl_write( param, value ) 237 | local entry = mallctl_params[param] 238 | if not entry then return nil, 'invalid parameter' end 239 | if not entry[3] then return nil, 'parameter is not writable' end 240 | 241 | local newp = entry[1] -- newp may be nil for the non-rw entries, 242 | local newlenp = newp and ffi.sizeof(newp) or 0 -- so set 0 size in that case 243 | 244 | -- put the value into the holder (unless it is nil) 245 | if newp then 246 | -- an invalid conversion may throw an error, pcall it 247 | local success, err = pcall(function() newp[0] = value end) 248 | if not success then 249 | return nil, err 250 | end 251 | end 252 | 253 | local err = C[mallctl_fname]( param, nil, nil, newp, newlenp ) 254 | if err ~= 0 then 255 | return nil, err 256 | else 257 | return true 258 | end 259 | end 260 | end 261 | 262 | 263 | -- version 3.5 or greater is required 264 | do 265 | local version_pattern = '^3.5' 266 | local version_str = J.mallctl_read('version') 267 | if not string.match(version_str, version_pattern) then 268 | error('jemalloc version must match "',version_pattern,'", but was "'..version_str..'"') 269 | end 270 | end 271 | 272 | 273 | function J.get_prefix() 274 | return JEMALLOC_PREFIX 275 | end 276 | 277 | 278 | ------------------------------------------------------------------------------- 279 | -- bind "standard" API to C namespace 280 | 281 | do 282 | local cdef_template = [[ 283 | void *!_!malloc(size_t size); 284 | void *!_!calloc(size_t number, size_t size); 285 | int !_!posix_memalign(void **ptr, size_t alignment, size_t size); 286 | void *!_!aligned_alloc(size_t alignment, size_t size); 287 | void *!_!realloc(void *ptr, size_t size); 288 | void !_!free(void *ptr); 289 | ]] 290 | local cdef_bound = nil 291 | 292 | -- returns true if successful 293 | -- successive invocations return the previously returned values 294 | function J.bind_standard_api() 295 | if cdef_bound == true then 296 | return true 297 | elseif cdef_bound ~= nil then 298 | return nil, cdef_bound 299 | end 300 | 301 | local cdef_str = string.gsub(cdef_template, '!_!', JEMALLOC_PREFIX) 302 | local success, err = pcall(function() ffi.cdef(cdef_str) end) 303 | if not success then 304 | cdef_bound = err 305 | return nil, err 306 | end 307 | cdef_bound = true 308 | 309 | local malloc_fname = JEMALLOC_PREFIX..'malloc' 310 | function J.malloc( size ) 311 | local ptr = C[malloc_fname]( size ) 312 | if ptr then return ptr else return nil, ffi.errno() end 313 | end 314 | 315 | local calloc_fname = JEMALLOC_PREFIX..'calloc' 316 | function J.calloc( number, size ) 317 | local ptr = C[calloc_fname]( number, size ) 318 | if ptr then return ptr else return nil, ffi.errno() end 319 | end 320 | 321 | local posix_memalign_fname = JEMALLOC_PREFIX..'posix_memalign' 322 | function J.posix_memalign( ptr, alignment, size ) 323 | local err = C[posix_memalign_fname]( ptr, alignment, size ) 324 | if err == 0 then return true else return nil, err end 325 | end 326 | 327 | local aligned_alloc_fname = JEMALLOC_PREFIX..'aligned_alloc' 328 | function J.aligned_alloc( alignment, size ) 329 | local ptr = C[aligned_alloc_fname]( alignment, size ) 330 | if ptr then return ptr else return nil, ffi.errno() end 331 | end 332 | 333 | local realloc_fname = JEMALLOC_PREFIX..'realloc' 334 | function J.realloc( ptr, size ) 335 | local ptr = C[realloc_fname]( ptr, size ) 336 | if ptr then return ptr else return nil, ffi.errno() end 337 | end 338 | 339 | local free_fname = JEMALLOC_PREFIX..'free' 340 | function J.free( ptr ) 341 | C[free_fname]( ptr ) 342 | end 343 | 344 | return true 345 | end 346 | end 347 | 348 | 349 | ------------------------------------------------------------------------------- 350 | -- bind "non-standard" API 351 | 352 | do 353 | local mallocx_fname = JEMALLOC_PREFIX..'mallocx' 354 | function J.mallocx( size, flags ) 355 | return C[mallocx_fname]( size, flags and tonumber(flags) or 0 ) 356 | end 357 | 358 | local rallocx_fname = JEMALLOC_PREFIX..'rallocx' 359 | function J.rallocx( ptr, size, flags ) 360 | return C[rallocx_fname]( ptr, size, flags and tonumber(flags) or 0 ) 361 | end 362 | 363 | local xallocx_fname = JEMALLOC_PREFIX..'xallocx' 364 | function J.xallocx( ptr, size, extra, flags ) 365 | return C[xallocx_fname]( ptr, size, flags and tonumber(flags) or 0 ) 366 | end 367 | 368 | local sallocx_fname = JEMALLOC_PREFIX..'sallocx' 369 | function J.sallocx( ptr, flags ) 370 | return C[sallocx_fname]( ptr, flags and tonumber(flags) or 0 ) 371 | end 372 | 373 | local dallocx_fname = JEMALLOC_PREFIX..'dallocx' 374 | function J.dallocx( ptr, flags) 375 | return C[dallocx_fname]( ptr, flags and tonumber(flags) or 0 ) 376 | end 377 | 378 | local nallocx_fname = JEMALLOC_PREFIX..'nallocx' 379 | function J.nallocx( size, flags) 380 | return C[nallocx_fname]( ptr, flags and tonumber(flags) or 0 ) 381 | end 382 | 383 | local malloc_usable_size_fname = JEMALLOC_PREFIX..'malloc_usable_size' 384 | function J.malloc_usable_size( ptr ) 385 | return C[malloc_usable_size_fname]( ptr ) 386 | end 387 | 388 | local malloc_stats_print_fname = JEMALLOC_PREFIX..'malloc_stats_print' 389 | function J.malloc_stats_print() -- TODO allow user-supplied callback 390 | C[malloc_stats_print_fname]( nil, nil, nil ) 391 | end 392 | 393 | -- TODO 394 | -- void (*malloc_message)(void *cbopaque, const char *s); 395 | --const char *malloc_conf; 396 | end 397 | 398 | 399 | 400 | ------------------------------------------------------------------------------- 401 | -- flag "macros" 402 | -- 403 | -- they should be combined with bit.bor 404 | -- 405 | 406 | -- /* sizeof(void *) == 2^LG_SIZEOF_PTR. */ 407 | local LG_SIZEOF_PTR = math.log(ffi.sizeof('void*')) / math.log(2) 408 | local INT_MAX = 2147483647 -- TODO: universally true? 409 | 410 | -- make bit.bor available to combine parameters 411 | J.bor = bit.bor 412 | 413 | function J.MALLOCX_LG_ALIGN( la ) 414 | return la 415 | end 416 | 417 | if LG_SIZEOF_PTR == 2 then 418 | function J.MALLOCX_ALIGN( a ) 419 | a = a or 0 420 | return (C.ffs(a)-1) 421 | end 422 | else 423 | function J.MALLOCX_ALIGN( a ) 424 | a = a or 0 425 | return (a < INT_MAX) and (C.ffs(a)-1) or (C.ffs(bit.rshift(a,32))+31) 426 | end 427 | end 428 | 429 | function J.MALLOCX_ZERO() 430 | return 0x40 431 | end 432 | 433 | -- Bias arena index bits so that 0 encodes "MALLOCX_ARENA() unspecified". 434 | function J.MALLOCX_ARENA( a ) 435 | return bit.lshift((a+1), 8) -- TODO: limit to 32 bits? 436 | end 437 | 438 | 439 | 440 | -- return public API 441 | return J 442 | -------------------------------------------------------------------------------- /rockspecs/lds-scm-1.rockspec: -------------------------------------------------------------------------------- 1 | package = 'lds' 2 | version = 'scm-1' 3 | source = { 4 | url = 'git://github.com/neomantra/lds.git', 5 | branch = 'master', 6 | } 7 | description = { 8 | summary = 'LuaJIT Data Structures', 9 | detailed = 'LuaJIT Data Structures -- hold cdata in Arrays, Vectors, and HashMaps', 10 | homepage = 'https://github.com/neomantra/lds', 11 | license = 'MIT', 12 | } 13 | dependencies = { 14 | -- TODO: specify luajit >= 2.0 15 | } 16 | build = { 17 | type = 'none', 18 | install = { 19 | lua = { 20 | ['lds'] = 'lds.lua', 21 | ['lds.allocator'] = 'lds/allocator.lua', 22 | ['lds.Array'] = 'lds/Array.lua', 23 | ['lds.hash'] = 'lds/hash.lua', 24 | ['lds.HashMap'] = 'lds/HashMap.lua', 25 | ['lds.init'] = 'lds/init.lua', 26 | ['lds.jemalloc'] = 'lds/jemalloc.lua', 27 | ['lds.Vector'] = 'lds/Vector.lua', 28 | }, 29 | }, 30 | } 31 | -------------------------------------------------------------------------------- /tests/perf_Insert.lua: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env luajit 2 | -- lds - LuaJIT Data Structures 3 | -- 4 | -- Copyright (c) 2012-2020 Evan Wies. All rights reserved. 5 | -- MIT License, see the COPYRIGHT file. 6 | -- 7 | -- Test insertion rate of various containers. 8 | -- 9 | -- To run: 10 | -- luajit tests/perf_Insert.lua NUMBER_OF_INSERTS 11 | -- 12 | 13 | -- config 14 | if not arg then 15 | arg = { 1000 } 16 | elseif #arg > 1 then 17 | print('usage: per_Insert.lua [num_inserts]') 18 | os.exit(-1) 19 | end 20 | 21 | local NUMBER_OF_INSERTS = arg[1] and tonumber(arg[1]) or 1e6 22 | 23 | 24 | -- run 25 | local ffi = require 'ffi' 26 | local C = ffi.C 27 | 28 | local lds = require 'lds.HashMap' 29 | 30 | ffi.cdef("int rand(void);"); 31 | local os_clock = os.clock 32 | local random = math.random 33 | 34 | local function benchmark( name, fn ) 35 | io.stdout:write(string.format( '%15s... ', name ) ) 36 | io.stdout:flush() 37 | collectgarbage() 38 | local start_time = os_clock() 39 | fn() 40 | local end_time = os_clock() 41 | collectgarbage() 42 | local collect_time = os_clock() 43 | io.stdout:write(string.format('run:%0.3f collect:%0.3f num:%d\n', 44 | end_time - start_time, collect_time - end_time, NUMBER_OF_INSERTS)) 45 | end 46 | 47 | 48 | ffi.cdef([[ 49 | struct Point4 { 50 | double x, y, z, w; 51 | }; 52 | ]]) 53 | local Point4_t = ffi.typeof('struct Point4') 54 | 55 | local double_t = ffi.typeof('double') 56 | local int32_t = ffi.typeof('int32_t') 57 | local uint64_t = ffi.typeof('uint64_t') 58 | 59 | local function Point4_random() 60 | return Point4_t( double_t(C.rand()), double_t(C.rand()), double_t(C.rand()), double_t(C.rand()) ) 61 | end 62 | 63 | -- [[ 64 | benchmark( 'copy random to ffi struct', function() 65 | local dest = ffi.typeof('struct{ int32_t k; struct Point4 v; }')() 66 | for _ = 0, NUMBER_OF_INSERTS do 67 | dest.k = C.rand() 68 | dest.v = Point4_random() 69 | end 70 | end) 71 | 72 | benchmark( 'copy random Lua array to a local', function() 73 | for _ = 0, NUMBER_OF_INSERTS do 74 | local x = { C.rand(), C.rand(), C.rand(), C.rand() } 75 | end 76 | end) 77 | 78 | if NUMBER_OF_INSERTS < 1e7 then -- LuaJIT runs outs of memory 79 | benchmark( 'insert random into Lua table', function() 80 | local point_table = {} 81 | for _ = 0, NUMBER_OF_INSERTS do 82 | point_table[C.rand()] = Point4_random() 83 | end 84 | end) 85 | else 86 | io.stdout:write('insert random into Lua table... skipping due to memory constraints\n') 87 | end 88 | 89 | benchmark( 'insert random into Vector', function() 90 | local point_vec = lds.Vector( Point4_t ) 91 | for _ = 0, NUMBER_OF_INSERTS do 92 | point_vec:push_back( Point4_random() ) 93 | end 94 | end) 95 | 96 | benchmark( 'insert random into HashMap', function() 97 | local point_map = lds.HashMap( int32_t, Point4_t ) 98 | for _ = 0, NUMBER_OF_INSERTS do 99 | point_map:insert( C.rand(), Point4_random() ) 100 | end 101 | end) 102 | -------------------------------------------------------------------------------- /tests/perf_unordered_map_insert.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // lds - LuaJIT Data Structures 3 | // 4 | // Copyright (c) 2012-2020 Evan Wies. All rights reserved. 5 | // MIT License, see the COPYRIGHT file. 6 | // 7 | // Inserts a number of random points 8 | // into random indexes in a boost::unordered_map. 9 | // 10 | // Build with: 11 | // g++ -o perf_unordered_map_insert -O3 tests/perf_unordered_map_insert.cpp 12 | // 13 | // I use boost in macports, so I needed to add: 14 | // -I/opt/local/include 15 | 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | 25 | int NUMBER_OF_INSERTS = 1e6; 26 | 27 | 28 | struct Point4 { 29 | Point4( double _x, double _y, double _z, double _w ) 30 | : x(_x), y(_y), z(_z), w(_w) 31 | {} 32 | 33 | static Point4 random() { 34 | return Point4( std::rand(), std::rand(), std::rand(), std::rand() ); 35 | } 36 | 37 | double x, y, z, w; 38 | }; 39 | 40 | 41 | void benchmark( const std::string& name, void(*fn)() ) 42 | { 43 | timeval start_tv, end_tv; 44 | gettimeofday( &start_tv, NULL ); 45 | 46 | fn(); 47 | 48 | gettimeofday( &end_tv, NULL ); 49 | 50 | std::printf( "%s run:%0.03f num:%d\n", 51 | name.c_str(), 52 | end_tv.tv_sec - start_tv.tv_sec + 1e-6 * (end_tv.tv_usec - start_tv.tv_usec), 53 | NUMBER_OF_INSERTS ); 54 | } 55 | 56 | 57 | void test_PointVec_insert() 58 | { 59 | typedef std::vector PointVec; 60 | PointVec point_vec; 61 | 62 | for( int i = 0; i < NUMBER_OF_INSERTS; ++i ) 63 | point_vec.push_back( Point4::random() ); 64 | } 65 | 66 | 67 | void test_DoubleSet_insert() 68 | { 69 | typedef boost::unordered_set DoubleSet; 70 | DoubleSet double_set; 71 | 72 | for( int i = 0; i < NUMBER_OF_INSERTS; ++i ) 73 | double_set.insert( std::rand() ); 74 | } 75 | 76 | 77 | void test_PointMap_insert() 78 | { 79 | typedef boost::unordered_map PointMap; 80 | PointMap point_map; 81 | 82 | for( int i = 0; i < NUMBER_OF_INSERTS; ++i ) 83 | point_map.insert( std::make_pair(std::rand(), Point4::random()) ); 84 | } 85 | 86 | 87 | int main( int argc, const char* argv[] ) 88 | { 89 | try { 90 | if( argc != 1 && argc != 2 ) 91 | throw std::invalid_argument("usage: perf_unordered_map_insert "); 92 | 93 | if( argc == 2 ) 94 | NUMBER_OF_INSERTS = boost::lexical_cast(argv[1]); 95 | } catch( const std::exception& e ) { 96 | std::cerr << e.what() << std::endl; 97 | return -1; 98 | } 99 | 100 | benchmark( "insert into vector", test_PointVec_insert ); 101 | benchmark( "insert into unordered_set", test_DoubleSet_insert ); 102 | benchmark( "insert into unordered_map",test_PointMap_insert ); 103 | 104 | return 0; 105 | } 106 | -------------------------------------------------------------------------------- /tests/run_tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # lds - LuaJIT Data Structures 3 | # 4 | # Copyright (c) 2012-2020 Evan Wies. All rights reserved. 5 | # MIT License, see the COPYRIGHT file. 6 | 7 | for t in \ 8 | test_Array \ 9 | test_Vector \ 10 | test_HashMap \ 11 | ; 12 | do 13 | echo $t 14 | luajit ./tests/$t.lua 15 | done 16 | -------------------------------------------------------------------------------- /tests/test_Array.lua: -------------------------------------------------------------------------------- 1 | --[[ 2 | lds - LuaJIT Data Structures 3 | 4 | Copyright (c) 2012-2020 Evan Wies. All rights reserved. 5 | MIT License, see the COPYRIGHT file. 6 | 7 | Exercises lds.Array 8 | --]] 9 | 10 | local ffi = require 'ffi' 11 | local lds = require 'lds.Array' 12 | require 'lds.allocator' 13 | 14 | local double_t = ffi.typeof('double') 15 | 16 | for _, ct in pairs{ lds.uint32_t, double_t, lds.size_t } do 17 | for _, alloc in pairs{ lds.MallocAllocator, 18 | lds.VLAAllocator, 19 | lds.JemallocAllocator, 20 | } do 21 | local a_t = lds.ArrayT( ct, alloc ) 22 | 23 | local a = a_t( 10 ) 24 | 25 | assert( #a == 10 ) 26 | assert( a:size() == 10 ) 27 | assert( not a:empty() ) 28 | 29 | assert( a:get(0) == 0 ) 30 | 31 | assert( a:set(3, 5) == 0 ) 32 | assert( a:get(3) == 5 ) 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /tests/test_HashMap.lua: -------------------------------------------------------------------------------- 1 | --[[ 2 | lds - LuaJIT Data Structures 3 | 4 | Copyright (c) 2012-2020 Evan Wies. All rights reserved. 5 | MIT License, see the COPYRIGHT file. 6 | 7 | Exercises lds.HashMap 8 | --]] 9 | 10 | local ffi = require 'ffi' 11 | local lds = require 'lds.HashMap' 12 | 13 | local int_t = ffi.typeof('int') 14 | local double_t = ffi.typeof('double') 15 | 16 | for _, alloc in pairs{ lds.MallocAllocator, 17 | lds.VLAAllocator, 18 | lds.JemallocAllocator, 19 | } do 20 | local hm_t = lds.HashMapT( int_t, double_t, alloc ) -- map from int to double 21 | local hm = hm_t() 22 | 23 | assert( #hm == 0 ) 24 | assert( hm:size() == 0 ) 25 | assert( hm:empty() ) 26 | 27 | assert( hm:insert(5, 24.0) == true) 28 | assert( #hm == 1 ) 29 | assert( hm:size() == 1 ) 30 | assert( not hm:empty() ) 31 | 32 | assert( hm:find(5).key == 5 ) 33 | assert( hm:find(5).val == 24.0 ) 34 | assert( hm:insert(5, 22.0) == 24.0 ) 35 | assert( hm:size() == 1 ) 36 | 37 | assert( hm:remove(5) == 22.0 ) 38 | assert( hm:remove(5) == nil ) 39 | assert( hm:size() == 0 ) 40 | 41 | hm:insert(5, 21.0) 42 | hm:insert(6, 22.0) 43 | hm:insert(7, 23.0) 44 | hm:insert(100054, 77.0) 45 | assert( hm:size() == 4 ) 46 | assert( hm:find(5).val == 21.0 ) 47 | assert( hm:find(6).val == 22.0 ) 48 | assert( hm:find(7).val == 23.0 ) 49 | assert( hm:find(100054).val == 77.0 ) 50 | 51 | -- test iterator 52 | local keys, vals = {}, {} 53 | for pair in hm:iter() do 54 | keys[#keys+1], vals[#keys+1] = pair.key, pair.val 55 | end 56 | table.sort(keys) ; table.sort(vals) 57 | assert( #keys == 4 ) 58 | assert( keys[1] == 5 ) 59 | assert( keys[2] == 6 ) 60 | assert( keys[3] == 7 ) 61 | assert( keys[4] == 100054 ) 62 | assert( #vals == 4 ) 63 | assert( vals[1] == 21 ) 64 | assert( vals[2] == 22 ) 65 | assert( vals[3] == 23 ) 66 | assert( vals[4] == 77 ) 67 | 68 | -- test get_internals 69 | local t = hm:get_internals() 70 | assert( t.size == hm:size() ) 71 | 72 | hm:clear() 73 | assert( hm:size() == 0 ) 74 | assert( hm:empty() ) 75 | end 76 | -------------------------------------------------------------------------------- /tests/test_Vector.lua: -------------------------------------------------------------------------------- 1 | --[[ 2 | lds - LuaJIT Data Structures 3 | 4 | Copyright (c) 2012-2020 Evan Wies. All rights reserved. 5 | MIT License, see the COPYRIGHT file. 6 | 7 | Exercises lds.Vector 8 | --]] 9 | 10 | local ffi = require 'ffi' 11 | local lds = require 'lds.Vector' 12 | require 'lds.allocator' 13 | 14 | local double_t = ffi.typeof('double') 15 | 16 | for _, ct in pairs{ lds.uint32_t, double_t, lds.size_t } do 17 | for _, alloc in pairs{ lds.MallocAllocator, 18 | lds.VLAAllocator, 19 | lds.JemallocAllocator, 20 | } do 21 | local dv_t = lds.VectorT( ct, alloc ) 22 | 23 | local v = dv_t() 24 | 25 | assert( #v == 0 ) 26 | assert( v:size() == 0 ) 27 | assert( v:empty() ) 28 | assert( v:get(1) == nil ) 29 | assert( v:front() == nil ) 30 | assert( v:back() == nil ) 31 | 32 | v:insert( 0, 6 ) 33 | assert( #v == 1 ) 34 | assert( v:size() == 1 ) 35 | assert( v:get(0) == 6 ) 36 | assert( v:get(0) == 6 ) 37 | assert( v:front() == 6 ) 38 | assert( v:back() == 6 ) 39 | 40 | v:insert( 0, 5 ) 41 | assert( #v == 2 ) 42 | assert( v:size() == 2 ) 43 | assert( v:get(0) == 5 ) 44 | assert( v:get(1) == 6 ) 45 | assert( v:front() == 5 ) 46 | assert( v:back() == 6 ) 47 | 48 | assert( v:set(1, 7) == 6 ) 49 | assert( v:get(1) == 7 ) 50 | assert( v:front() == 5 ) 51 | assert( v:back() == 7 ) 52 | 53 | assert( v:erase(0) == 5 ) 54 | assert( #v == 1 ) 55 | assert( v:size() == 1 ) 56 | assert( v:get(0) == 7 ) 57 | assert( v:front() == 7 ) 58 | assert( v:back() == 7 ) 59 | 60 | v:insert( 0, 5 ) 61 | v:insert( 0, 5 ) 62 | v:clear() 63 | assert( #v == 0 ) 64 | assert( v:size() == 0 ) 65 | assert( v:empty() ) 66 | assert( v:front() == nil ) 67 | assert( v:back() == nil ) 68 | 69 | v:push_back( 2 ) 70 | assert( #v == 1 ) 71 | assert( v:size() == 1 ) 72 | assert( v:get(0) == 2 ) 73 | 74 | assert( v:pop_back() == 2 ) 75 | assert( #v == 0 ) 76 | assert( v:size() == 0 ) 77 | 78 | v:shrink_to_fit() 79 | end 80 | end 81 | --------------------------------------------------------------------------------