├── .gitignore ├── .travis.yml ├── Gemfile ├── LICENSE.txt ├── README.md ├── Rakefile ├── lib ├── ruby-xxhash.rb ├── ruby-xxhash │ └── version.rb ├── ruby-xxhash32.rb └── ruby-xxhash64.rb ├── ruby-xxHash.gemspec └── spec ├── results32.yaml ├── results64.yaml ├── spec_helper.rb └── xxhash_spec.rb /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | *.rbc 3 | **/*.iml 4 | .idea/ 5 | ruby-xxhash/ 6 | .bundle 7 | .config 8 | .yardoc 9 | Gemfile.lock 10 | InstalledFiles 11 | _yardoc 12 | coverage 13 | doc/ 14 | lib/bundler/man 15 | pkg 16 | rdoc 17 | spec/reports 18 | test/tmp 19 | test/version_tmp 20 | tmp 21 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | rvm: 3 | - 1.9.3 4 | - 2.2 -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Specify your gem's dependencies in ruby-xxHash.gemspec 4 | gemspec 5 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014 Justin W Smith 2 | 3 | MIT License 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # XXhash [![Build Status](https://travis-ci.org/justinwsmith/ruby-xxhash.svg?branch=master)](https://travis-ci.org/justinwsmith/ruby-xxhash) 2 | 3 | This gem provides a pure Ruby implementation of the XXhash32 and XXhash64 hashing algorithms described here: https://code.google.com/p/xxhash/. 4 | 5 | It's intended to be (mostly) source compatible with the Gem provided by nashby: https://github.com/nashby/xxhash 6 | 7 | ## Installation 8 | 9 | Add this line to your application's Gemfile: 10 | 11 | gem 'ruby-xxHash' 12 | 13 | And then execute: 14 | 15 | $ bundle 16 | 17 | Or install it yourself as: 18 | 19 | $ gem install ruby-xxHash 20 | 21 | ## Usage 22 | 23 | ```ruby 24 | require 'ruby-xxhash' 25 | 26 | text = "test" 27 | seed = 12345 28 | 29 | XXhash.xxh32(text, seed) # => 3834992036 30 | XXhash.xxh64(text, seed) # => 7624679986283906467 31 | ``` 32 | 33 | ## Contributing 34 | 35 | 1. Fork it 36 | 2. Create your feature branch (`git checkout -b my-new-feature`) 37 | 3. Commit your changes (`git commit -am 'Add some feature'`) 38 | 4. Push to the branch (`git push origin my-new-feature`) 39 | 5. Create new Pull Request 40 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | require 'rspec/core/rake_task' 3 | 4 | RSpec::Core::RakeTask.new 5 | 6 | task :default => :spec 7 | task :test => :spec 8 | -------------------------------------------------------------------------------- /lib/ruby-xxhash.rb: -------------------------------------------------------------------------------- 1 | # Copyright 2014 Justin W. Smith 2 | 3 | require "ruby-xxhash/version" 4 | require "digest" 5 | require "ruby-xxhash32" 6 | require "ruby-xxhash64" 7 | 8 | 9 | module XXhash 10 | def self.xxh32(input, seed = 0) 11 | xxh = XXhashInternal::XXhash32.new(seed) 12 | xxh.update(input) 13 | xxh.digest 14 | end 15 | 16 | def self.xxh32_stream(io, seed = 0, chunk = 32) 17 | xxh = XXhashInternal::XXhash32.new(seed) 18 | 19 | while(data = io.read(chunk)) 20 | xxh.update(data) 21 | end 22 | 23 | xxh.digest 24 | end 25 | 26 | def self.xxh64(input, seed = 0) 27 | xxh = XXhashInternal::XXhash64.new(seed) 28 | xxh.update(input) 29 | xxh.digest 30 | end 31 | 32 | def self.xxh64_stream(io, seed = 0, chunk = 32) 33 | xxh = XXhashInternal::XXhash64.new(seed) 34 | 35 | while(data = io.read(chunk)) 36 | xxh.update(data) 37 | end 38 | 39 | xxh.digest 40 | end 41 | 42 | end 43 | 44 | 45 | module Digest 46 | class XXHash < Digest::Class 47 | attr_reader :digest_length 48 | def initialize bitlen, seed = 0 49 | case bitlen 50 | when 32 51 | @hash = XXhash::XXhashInternal::XXhash32.new(seed) 52 | when 64 53 | @hash = XXhash::XXhashInternal::XXhash64.new(seed) 54 | else 55 | raise ArgumentError, "Unsupported bit length: %s" % bitlen.inspect 56 | end 57 | @digest_length = bitlen 58 | end 59 | 60 | def update chunk 61 | @hash.update(chunk) 62 | end 63 | 64 | def digest val=nil 65 | if val 66 | @hash.update val 67 | end 68 | 69 | @hash.digest 70 | end 71 | 72 | def digest! val=nil 73 | result = digest(val) 74 | @hash.reset 75 | result 76 | end 77 | 78 | def reset 79 | @hash.reset 80 | end 81 | 82 | end 83 | 84 | class XXHash32 < Digest::XXHash 85 | def initialize seed = 0 86 | super(32, seed) 87 | end 88 | end 89 | 90 | class XXHash64 < Digest::XXHash 91 | def initialize seed = 0 92 | super(64, seed) 93 | end 94 | end 95 | 96 | end 97 | -------------------------------------------------------------------------------- /lib/ruby-xxhash/version.rb: -------------------------------------------------------------------------------- 1 | module XXhash 2 | VERSION = "0.4.0.2" 3 | end 4 | -------------------------------------------------------------------------------- /lib/ruby-xxhash32.rb: -------------------------------------------------------------------------------- 1 | 2 | 3 | module XXhash 4 | module XXhashInternal 5 | class XXhash32 6 | @@mem_total_size = 16 7 | @@prime32_1 = 2654435761 8 | @@prime32_2 = 2246822519 9 | @@prime32_3 = 3266489917 10 | @@prime32_4 = 668265263 11 | @@prime32_5 = 374761393 12 | 13 | @@thirtytwo1s = (2**32-1) 14 | 15 | def initialize seed 16 | @seed = seed 17 | reset 18 | end 19 | 20 | def reset 21 | @v1 = @seed + @@prime32_1 + @@prime32_2 22 | @v2 = @seed + @@prime32_2 23 | @v3 = @seed + 0 24 | @v4 = @seed - @@prime32_1 25 | @total_len = 0 26 | @memory = Array.new(@@mem_total_size) 27 | @memsize = 0 28 | end 29 | 30 | def update bytes 31 | if String === bytes 32 | bytes = bytes.unpack("C*") 33 | end 34 | 35 | @total_len += bytes.length 36 | 37 | p = 0 38 | 39 | while (remaining = (bytes.length - p)) > 0 40 | 41 | mem_avail = @@mem_total_size - @memsize 42 | 43 | if(remaining < mem_avail) 44 | @memory[@memsize, remaining] = bytes[p, remaining] 45 | @memsize += remaining 46 | break 47 | end 48 | 49 | @memory[@memsize, mem_avail] = bytes[p, mem_avail] 50 | 51 | i = 0 52 | [:v1, :v2, :v3, :v4].each do |m| 53 | p32 = uint32( 54 | @memory[i] | 55 | (@memory[i+1] << 8) | 56 | (@memory[i+2] << 16) | 57 | (@memory[i+3] << 24)) 58 | 59 | v = uint32(self.send(m) + p32 * @@prime32_2) 60 | v = uint32(uint32((v << 13) | (v >> (32 - 13))) * @@prime32_1) 61 | self.send((m.to_s + "=").to_sym, v) 62 | i += 4 63 | end 64 | 65 | p += mem_avail 66 | @memsize = 0 67 | end 68 | 69 | return true 70 | end 71 | 72 | def digest val=nil 73 | if val 74 | update val 75 | end 76 | 77 | if @total_len >= 16 78 | h32 = ((@v1 << 1) | (@v1 >> (32 - 1))) + 79 | ((@v2 << 7) | (@v2 >> (32 - 7))) + 80 | ((@v3 << 12) | (@v3 >> (32 - 12))) + 81 | ((@v4 << 18) | (@v4 >> (32 - 18))) 82 | else 83 | h32 = @seed + @@prime32_5 84 | end 85 | 86 | h32 = uint32(h32 + @total_len) 87 | 88 | p = 0 89 | while p <= (@memsize - 4) 90 | p32 = uint32(@memory[p] | 91 | (@memory[p+1] << 8) | 92 | (@memory[p+2] << 16) | 93 | (@memory[p+3] << 24)) 94 | h32 = uint32(h32 + p32 * @@prime32_3) 95 | h32 = uint32(uint32((h32 << 17) | (h32 >> (32 - 17))) * @@prime32_4) 96 | p += 4 97 | end 98 | 99 | while p < @memsize 100 | h32 = uint32(h32 + @memory[p] * @@prime32_5) 101 | h32 = uint32(uint32((h32 << 11) | (h32 >> (32 - 11))) * @@prime32_1) 102 | p += 1 103 | end 104 | 105 | h32 ^= h32 >> 15 106 | h32 = uint32(h32 * @@prime32_2) 107 | h32 ^= h32 >> 13 108 | h32 = uint32(h32 * @@prime32_3) 109 | h32 ^= h32 >> 16 110 | 111 | h32 112 | end 113 | 114 | private 115 | 116 | attr_accessor :v1, :v2, :v3, :v4 117 | 118 | def uint32(x) 119 | x & @@thirtytwo1s 120 | end 121 | end 122 | end 123 | end 124 | -------------------------------------------------------------------------------- /lib/ruby-xxhash64.rb: -------------------------------------------------------------------------------- 1 | require "ruby-xxhash/version" 2 | 3 | module XXhash 4 | module XXhashInternal 5 | class XXhash64 6 | @@mem_total_size = 32 7 | @@prime64_1 = 11400714785074694791 8 | @@prime64_2 = 14029467366897019727 9 | @@prime64_3 = 1609587929392839161 10 | @@prime64_4 = 9650029242287828579 11 | @@prime64_5 = 2870177450012600261 12 | 13 | @@sixtyfour1s = (2**64-1) 14 | 15 | def initialize seed 16 | @seed = seed 17 | reset 18 | end 19 | 20 | def reset 21 | @v1 = @seed + @@prime64_1 + @@prime64_2; 22 | @v2 = @seed + @@prime64_2; 23 | @v3 = @seed + 0; 24 | @v4 = @seed - @@prime64_1; 25 | @total_len = 0 26 | @memory = Array.new(@@mem_total_size) 27 | @memsize = 0 28 | end 29 | 30 | def update bytes 31 | if String === bytes 32 | bytes = bytes.unpack("C*") 33 | end 34 | 35 | @total_len += bytes.length 36 | 37 | p = 0 38 | while (remaining = (bytes.length - p)) > 0 39 | mem_avail = @@mem_total_size - @memsize 40 | 41 | if(remaining < mem_avail) 42 | @memory[@memsize, remaining] = bytes[p, remaining] 43 | @memsize += remaining 44 | break 45 | end 46 | 47 | @memory[@memsize, mem_avail] = bytes[p, mem_avail] 48 | 49 | i = 0 50 | [:v1, :v2, :v3, :v4].each do |m| 51 | p64 = uint64( 52 | @memory[i] | 53 | (@memory[i+1] << 8) | 54 | (@memory[i+2] << 16) | 55 | (@memory[i+3] << 24) | 56 | (@memory[i+4] << 32) | 57 | (@memory[i+5] << 40) | 58 | (@memory[i+6] << 48) | 59 | (@memory[i+7] << 56)) 60 | 61 | v = uint64(self.send(m) + p64 * @@prime64_2) 62 | v = uint64(uint64((v << 31) | (v >> (64 - 31))) * @@prime64_1) 63 | self.send((m.to_s + "=").to_sym, v) 64 | i += 8 65 | end 66 | 67 | p += mem_avail 68 | @memsize = 0 69 | end 70 | end 71 | 72 | def digest val=nil 73 | if val 74 | update val 75 | end 76 | 77 | if @total_len >= 32 78 | h64 = ((@v1 << 1) | (@v1 >> (64 - 1))) + 79 | ((@v2 << 7) | (@v2 >> (64 - 7))) + 80 | ((@v3 << 12) | (@v3 >> (64 - 12))) + 81 | ((@v4 << 18) | (@v4 >> (64 - 18))) 82 | 83 | [:v1, :v2, :v3, :v4].each do |m| 84 | v = uint64(self.send(m) * @@prime64_2) 85 | v = uint64((v << 31) | (v >> (64 - 31))) 86 | h64 ^= uint64(v * @@prime64_1) 87 | h64 = h64 * @@prime64_1 + @@prime64_4 88 | end 89 | else 90 | h64 = @seed + @@prime64_5 91 | end 92 | 93 | h64 = uint64(h64 + @total_len) 94 | 95 | i = 0 96 | while i <= (@memsize - 8) 97 | v = uint64( 98 | @memory[i] | 99 | (@memory[i+1] << 8) | 100 | (@memory[i+2] << 16) | 101 | (@memory[i+3] << 24) | 102 | (@memory[i+4] << 32) | 103 | (@memory[i+5] << 40) | 104 | (@memory[i+6] << 48) | 105 | (@memory[i+7] << 56)) 106 | 107 | v = uint64(v * @@prime64_2) 108 | h64 ^= uint64(uint64((v << 31) | (v >> (64 - 31))) * @@prime64_1) 109 | h64 = uint64(uint64((h64 << 27) | (h64 >> (64 - 27))) * @@prime64_1 + @@prime64_4) 110 | i += 8 111 | end 112 | 113 | if i <= (@memsize - 4) 114 | v = @memory[i] | 115 | (@memory[i+1] << 8) | 116 | (@memory[i+2] << 16) | 117 | (@memory[i+3] << 24) 118 | h64 ^= uint64(v * @@prime64_1) 119 | h64 = uint64(uint64((h64 << 23) | (h64 >> (64-23))) * @@prime64_2 + @@prime64_3) 120 | i += 4 121 | end 122 | 123 | while i < @memsize 124 | h64 ^= uint64(@memory[i] * @@prime64_5) 125 | h64 = uint64(uint64((h64 << 11) | (h64 >> (64 - 11))) * @@prime64_1) 126 | i += 1 127 | end 128 | 129 | h64 ^= h64 >> 33 130 | h64 = uint64(h64 * @@prime64_2) 131 | h64 ^= h64 >> 29 132 | h64 = uint64(h64 * @@prime64_3) 133 | h64 ^= h64 >> 32 134 | 135 | h64 136 | end 137 | 138 | private 139 | 140 | attr_accessor :v1, :v2, :v3, :v4 141 | 142 | def uint64(x) 143 | x & @@sixtyfour1s 144 | end 145 | end 146 | end 147 | end -------------------------------------------------------------------------------- /ruby-xxHash.gemspec: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | lib = File.expand_path('../lib', __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | require 'ruby-xxhash/version' 5 | 6 | Gem::Specification.new do |spec| 7 | spec.name = "ruby-xxHash" 8 | spec.version = XXhash::VERSION 9 | spec.authors = ["Justin W Smith"] 10 | spec.email = ["justin.w.smith@gmail.com"] 11 | spec.description = %q{A pure Ruby implementation of xxhash.} 12 | spec.summary = %q{A pure Ruby implementation of xxhash.} 13 | spec.homepage = "https://github.com/justinwsmith/ruby-xxhash" 14 | spec.license = "MIT" 15 | 16 | spec.files = `git ls-files`.split($/) 17 | spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) } 18 | spec.test_files = spec.files.grep(%r{^(test|spec|features)/}) 19 | spec.require_paths = ["lib"] 20 | 21 | spec.add_development_dependency "bundler" 22 | spec.add_development_dependency "rake" 23 | spec.add_development_dependency "rspec" 24 | end 25 | -------------------------------------------------------------------------------- /spec/results32.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | ? - test 3 | - 123 4 | : 2758658570 5 | ? - ! 'Lorem ipsum dolor sit amet, consectetuer adipiscing elit, ' 6 | - 0 7 | : 288417748 8 | ? - ! 'Lorem ipsum dolor sit amet, consectetuer adipiscing elit, ' 9 | - 1471 10 | : 3126436440 11 | ? - ! 'Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nibh euismod tincidunt ut laoreet dolore magna aliquam erat volutpat.' 12 | - 0 13 | : 222574394 14 | ? - ! 'Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nibh euismod tincidunt ut laoreet dolore magna aliquam erat volutpat.' 15 | - 1471 16 | : 845537824 -------------------------------------------------------------------------------- /spec/results64.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | ? - test 3 | - 123 4 | : 3134990500624303823 5 | ? - ! 'Lorem ipsum dolor sit amet, consectetuer adipiscing elit, ' 6 | - 0 7 | : 17031984226649854695 8 | ? - ! 'Lorem ipsum dolor sit amet, consectetuer adipiscing elit, ' 9 | - 1471 10 | : 13163341760735734472 11 | ? - ! 'Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nibh euismod tincidunt ut laoreet dolore magna aliquam erat volutpat.' 12 | - 0 13 | : 14967567540383795520 14 | ? - ! 'Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nibh euismod tincidunt ut laoreet dolore magna aliquam erat volutpat.' 15 | - 1471 16 | : 4189552183125498197 -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require 'bundler/setup' 2 | Bundler.setup 3 | 4 | require 'ruby-xxhash' 5 | 6 | RSpec.configure do |config| 7 | # some (optional) config here 8 | end 9 | -------------------------------------------------------------------------------- /spec/xxhash_spec.rb: -------------------------------------------------------------------------------- 1 | #Copyright (c) 2012 Vasiliy Ermolovich 2 | #Copyright (c) 2014 Justin W Smith 3 | 4 | require 'spec_helper' 5 | require 'stringio' 6 | require 'yaml' 7 | 8 | describe XXhash do 9 | hash32 = YAML.load(IO.read "spec/results32.yaml") 10 | hash64 = YAML.load(IO.read "spec/results64.yaml") 11 | 12 | hash32.each do |key, value| 13 | it 'returns correct hash' do 14 | expect(XXhash.xxh32(key[0], key[1])).to eq(value) 15 | end 16 | end 17 | 18 | hash64.each do |key, value| 19 | it 'returns correct hash' do 20 | expect(XXhash.xxh64(key[0], key[1])).to eq(value) 21 | end 22 | end 23 | 24 | describe 'StreamingHash' do 25 | 26 | hash32.each do |key, value| 27 | it 'returns correct hash' do 28 | expect(XXhash.xxh32_stream(StringIO.new(key[0]), key[1])).to eq(value) 29 | end 30 | end 31 | 32 | it 'returns same hash for streamed files' do 33 | h1 = XXhash.xxh32(File.read(__FILE__), 123) 34 | h2 = XXhash.xxh32_stream(File.open(__FILE__), 123) 35 | expect(h1).to eq(h2) 36 | end 37 | 38 | hash64.each do |key, value| 39 | it 'returns correct hash' do 40 | expect(XXhash.xxh64_stream(StringIO.new(key[0]), key[1])).to eq(value) 41 | end 42 | end 43 | 44 | it 'returns same hash for streamed files' do 45 | h1 = XXhash.xxh64(File.read(__FILE__), 123) 46 | h2 = XXhash.xxh64_stream(File.open(__FILE__), 123) 47 | expect(h1).to eq(h2) 48 | end 49 | end 50 | 51 | def use_external_hash hash, io, chunk_size=1024 52 | while chunk=io.read(chunk_size) 53 | hash.update(chunk) 54 | end 55 | hash.digest 56 | end 57 | 58 | describe 'Digest::XXHash32' do 59 | 60 | it 'returns the hash for streamed strings' do 61 | StringIO.open('test') do |io| 62 | xxhash = Digest::XXHash32.new(123) 63 | result = use_external_hash xxhash, io 64 | expect(result).to eq(2758658570) 65 | end 66 | end 67 | 68 | it 'returns the hash for streamed files' do 69 | h1 = XXhash.xxh32(File.read(__FILE__), 123) 70 | xxhash = Digest::XXHash32.new(123) 71 | result = use_external_hash xxhash, File.open(__FILE__) 72 | expect(result).to eq(h1) 73 | end 74 | 75 | it 'returns correct hash after a reset' do 76 | h1 = XXhash.xxh32(File.read(__FILE__), 123) 77 | xxhash = Digest::XXHash32.new(123) 78 | expect(xxhash.digest('test')).to eq(2758658570) 79 | xxhash.reset 80 | result = use_external_hash xxhash, File.open(__FILE__) 81 | expect(result).to eq(h1) 82 | end 83 | end 84 | 85 | describe 'Digest::XXHash64' do 86 | 87 | it 'returns the hash for streamed strings' do 88 | StringIO.open('test') do |io| 89 | xxhash = Digest::XXHash64.new(123) 90 | result = use_external_hash xxhash, io 91 | expect(result).to eq(3134990500624303823) 92 | end 93 | end 94 | 95 | it 'returns the hash for streamed files' do 96 | h1 = XXhash.xxh64(File.read(__FILE__), 123) 97 | xxhash = Digest::XXHash64.new(123) 98 | result = use_external_hash xxhash, File.open(__FILE__) 99 | expect(result).to eq(h1) 100 | end 101 | 102 | it 'returns correct hash after a reset' do 103 | h1 = XXhash.xxh64(File.read(__FILE__), 123) 104 | xxhash = Digest::XXHash64.new(123) 105 | expect(xxhash.digest('test')).to eq(3134990500624303823) 106 | xxhash.reset 107 | result = use_external_hash xxhash, File.open(__FILE__) 108 | expect(result).to eq(h1) 109 | end 110 | end 111 | 112 | end 113 | --------------------------------------------------------------------------------