├── luaxxhash-scm-1.rockspec ├── README.md ├── releases └── luaxxhash-1.0.0-1.rockspec ├── .gitlab-ci.yml ├── LICENSE ├── test.lua └── luaxxhash.lua /luaxxhash-scm-1.rockspec: -------------------------------------------------------------------------------- 1 | package = "luaxxhash" 2 | version = "scm-1" 3 | source = { 4 | url = "git+https://github.com/szensk/luaxxhash", 5 | } 6 | description = { 7 | summary = "A LuaJIT implementation of xxhash", 8 | detailed = [[ 9 | A LuaJIT implementation of xxhash. 10 | Doesn't require bindings. Doesn't implement digest. 11 | ]], 12 | homepage = "https://github.com/szensk/luaxxhash", 13 | license = "MIT" 14 | } 15 | build = { 16 | type = "builtin", 17 | modules = { 18 | luaxxhash = "luaxxhash.lua" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | luaxxhash 2 | ======== 3 | A LuaJIT implementation of [xxhash](https://code.google.com/p/xxhash/). Doesn't require bindings. Doesn't implement digest. 4 | 5 | Usage 6 | ===== 7 | ```lua 8 | local xxh32 = require("luaxxhash") 9 | local str = "Cats are interesting." 10 | assert(xxh32(str) == 1710037756) 11 | ``` 12 | 13 | Benchmark 14 | ========= 15 | Ran on a Intel T6400 @ 2.0GHz. 16 | ``` 17 | lua-xxhash: 54957.13/s 3959.19MB/s (C binding) 18 | luaxxhash: 12435.24/s 895.854MB/s (pure LuaJIT) 19 | pmurmur3: 5603.67/s 403.697MB/s (pure LuaJIT) 20 | ``` -------------------------------------------------------------------------------- /releases/luaxxhash-1.0.0-1.rockspec: -------------------------------------------------------------------------------- 1 | package = "luaxxhash" 2 | version = "1.0.0-1" 3 | source = { 4 | url = "git+https://github.com/szensk/luaxxhash", 5 | tag = "1.0.0", 6 | } 7 | description = { 8 | summary = "A LuaJIT implementation of xxhash", 9 | detailed = [[ 10 | A LuaJIT implementation of xxhash. 11 | Doesn't require bindings. Doesn't implement digest. 12 | ]], 13 | homepage = "https://github.com/szensk/luaxxhash", 14 | license = "MIT" 15 | } 16 | build = { 17 | type = "builtin", 18 | modules = { 19 | luaxxhash = "luaxxhash.lua" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | image: alpine:latest 2 | 3 | stages: 4 | - test 5 | 6 | test: 7 | stage: test 8 | script: 9 | - apk update && apk add luajit 10 | - luajit -v 11 | - luajit test.lua timing=false 12 | - luajit -joff test.lua timing=false 13 | 14 | luacheck: 15 | stage: test 16 | script: 17 | - apk update && apk add lua curl 18 | - curl -L https://github.com/mpeterv/luacheck/archive/0.15.1.tar.gz > luacheck.tar.gz 19 | - tar -xzf luacheck.tar.gz 20 | - rm luacheck.tar.gz 21 | - cd luacheck-0.15.1 22 | - ./install.lua /usr/local 23 | - cd .. 24 | - luacheck luaxxhash.lua 25 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | luaxxhash License 2 | -------------------------- 3 | 4 | luaxxhash is licensed under the terms of the MIT/X11 license reproduced below. 5 | 6 | =============================================================================== 7 | 8 | Copyright (C) 2014 szensk. 9 | 10 | Permission is hereby granted, free of charge, to any person obtaining a copy 11 | of this software and associated documentation files (the "Software"), to deal 12 | in the Software without restriction, including without limitation the rights 13 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 14 | copies of the Software, and to permit persons to whom the Software is 15 | furnished to do so, subject to the following conditions: 16 | 17 | The above copyright notice and this permission notice shall be included in 18 | all copies or substantial portions of the Software. 19 | 20 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 21 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 22 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 23 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 24 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 25 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 26 | THE SOFTWARE. 27 | 28 | =============================================================================== 29 | 30 | (end of COPYRIGHT) -------------------------------------------------------------------------------- /test.lua: -------------------------------------------------------------------------------- 1 | local xxh32 = require("luaxxhash") 2 | 3 | local f = io.open("sherlock.txt") 4 | local text = f:read("*all") 5 | local str = "Cats are interesting." 6 | local str1 = "a" 7 | local strt = { 8 | str1:rep(4), 9 | str1:rep(16), 10 | str1:rep(32), 11 | str1:rep(32) .. str1 .. str1:rep(4), 12 | str:rep(256), 13 | "", 14 | "dupa", 15 | str, 16 | str1, 17 | text 18 | } 19 | 20 | local answer = { 21 | 0xDA576727, 22 | 0x5DACDD8C, 23 | 0xA3234FBE, 24 | 0x3F87545B, 25 | 0xF1DB4C68, 26 | 0x02CC5D05, 27 | 0x1A47C09D, 28 | 0x65ED1AFC, 29 | 0x550D7456, 30 | 0xFAB5AEF6, 31 | } 32 | 33 | assert(xxh32('abc', 3, 0x5BD1E995) == 0xBDDEB201, "Hash produces incorrect value with seed") 34 | 35 | for i=1, #strt do 36 | local x = xxh32(strt[i]) 37 | assert(x == answer[i], "Hash produces incorrect values: " .. i .. " Hash: " .. x) 38 | end 39 | 40 | local testIndex = 10 41 | local testRepeat = 10 42 | local hashRepeat = 1000 43 | 44 | function test(f) 45 | local times = {} 46 | local throughputs = {} 47 | 48 | local len = #strt[testIndex] 49 | for i=1,testRepeat do 50 | local start = os.clock() 51 | for i=1,hashRepeat do f(strt[testIndex], len) end 52 | local time = os.clock() - start 53 | local throughput = len*hashRepeat/time/(1024*1024*1024) 54 | table.insert(times, time) 55 | table.insert(throughputs, throughput) 56 | end 57 | 58 | print( ("min: %.3fs %.2f GB/s"):format( math.min(table.unpack(times)), math.min(table.unpack(throughputs))) ) 59 | print( ("max: %.3fs %.2f GB/s"):format( math.max(table.unpack(times)), math.max(table.unpack(throughputs))) ) 60 | end 61 | 62 | if not arg[1] then 63 | test(xxh32) 64 | local ok, murmur = pcall(require, 'murmurhash3') 65 | if ok then test(murmur) end 66 | end -------------------------------------------------------------------------------- /luaxxhash.lua: -------------------------------------------------------------------------------- 1 | local ffi = require('ffi') 2 | local bit = require('bit') 3 | 4 | local rotl, xor, shr = bit.rol, bit.bxor, bit.rshift 5 | local uint32_t = ffi.typeof("uint32_t") 6 | 7 | -- Prime constants 8 | local P1 = uint32_t(0x9E3779B1) 9 | local P2 = uint32_t(0x85EBCA77) 10 | local P3 = (0xC2B2AE3D) 11 | local P4 = (0x27D4EB2F) 12 | local P5 = (0x165667B1) 13 | 14 | -- multiplication with modulo2 semantics 15 | -- see https://github.com/luapower/murmurhash3 16 | local function mmul(a, b) 17 | local type = 'uint32_t' 18 | return tonumber(ffi.cast(type, ffi.cast(type, a) * ffi.cast(type, b))) 19 | end 20 | 21 | local function xxhash32(data, len, seed) 22 | seed, len = seed or 0, len or #data 23 | local i,n = 0, 0 -- byte and word index 24 | local bytes = ffi.cast( 'const uint8_t*', data) 25 | local words = ffi.cast('const uint32_t*', data) 26 | 27 | local h32 28 | if len >= 16 then 29 | local limit = len - 16 30 | local v = ffi.new("uint32_t[4]") 31 | v[0], v[1] = seed + P1 + P2, seed + P2 32 | v[2], v[3] = seed, seed - P1 33 | while i <= limit do 34 | for j=0, 3 do 35 | v[j] = v[j] + words[n] * P2 36 | v[j] = rotl(v[j], 13); v[j] = v[j] * P1 37 | i = i + 4; n = n + 1 38 | end 39 | end 40 | h32 = rotl(v[0], 1) + rotl(v[1], 7) + rotl(v[2], 12) + rotl(v[3], 18) 41 | else 42 | h32 = seed + P5 43 | end 44 | h32 = h32 + len 45 | 46 | local limit = len - 4 47 | while i <= limit do 48 | h32 = (h32 + mmul(words[n], P3)) 49 | h32 = mmul(rotl(h32, 17), P4) 50 | i = i + 4; n = n + 1 51 | end 52 | 53 | while i < len do 54 | h32 = h32 + mmul(bytes[i], P5) 55 | h32 = mmul(rotl(h32, 11), P1) 56 | i = i + 1 57 | end 58 | 59 | h32 = xor(h32, shr(h32, 15)) 60 | h32 = mmul(h32, P2) 61 | h32 = xor(h32, shr(h32, 13)) 62 | h32 = mmul(h32, P3) 63 | return tonumber(ffi.cast("uint32_t", xor(h32, shr(h32, 16)))) 64 | end 65 | 66 | return xxhash32 67 | --------------------------------------------------------------------------------