├── doc_bin.png ├── example_ocr ├── Gemfile ├── Gemfile.lock ├── main.rb └── README.md ├── example_dups ├── Dockerfile ├── main.rb └── README.md ├── .gitignore ├── test_LoadError.rb ├── slim.Dockerfile ├── Gemfile ├── common.rb ├── test_rbenv.rb ├── dhash-vips.gemspec ├── .github └── workflows │ ├── dockerimage.yaml │ └── benchmark.yaml ├── LICENSE.txt ├── dev.dhash-vips.alpine.Dockerfile ├── extconf.rb ├── idhash.c ├── bin └── idhash ├── vips.ruby.alpine.Dockerfile ├── lib ├── dhash-vips-post-install-test.rb └── dhash-vips.rb ├── test.rb ├── README.md └── Rakefile /doc_bin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Nakilon/dhash-vips/HEAD/doc_bin.png -------------------------------------------------------------------------------- /example_ocr/Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | gemspec path: ".." 4 | -------------------------------------------------------------------------------- /example_dups/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM nakilonishe/dhash-vips 2 | 3 | COPY main.rb / 4 | CMD ruby main.rb 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /Gemfile.lock 2 | /pkg/ 3 | 4 | /Makefile 5 | /mkmf.log 6 | /idhash.bundle 7 | /idhash.o 8 | /idhash.so 9 | 10 | /.byebug_history 11 | .DS_Store 12 | *.jpg 13 | /test_images/*.png 14 | 15 | /temp.* 16 | -------------------------------------------------------------------------------- /test_LoadError.rb: -------------------------------------------------------------------------------- 1 | require "minitest/autorun" 2 | 3 | describe :test do 4 | it do 5 | FileUtils.move "idhash.bundle", "temp" 6 | begin 7 | require_relative "lib/dhash-vips" 8 | assert_equal :distance3_ruby, DHashVips::IDHash.method(:distance3).original_name 9 | ensure 10 | FileUtils.move "temp", "idhash.bundle" 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /slim.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ruby:2.7.2-slim 2 | COPY . /pwd 3 | RUN apt-get update && apt install --no-install-recommends -y libvips42 wget build-essential && \ 4 | mkdir /ruby && wget -O- https://cache.ruby-lang.org/pub/ruby/2.7/ruby-2.7.2.tar.gz | tar xzC /ruby --strip-components=1 && \ 5 | cd pwd && rake install && rm -rf $(pwd) && \ 6 | rm -rf /ruby && \ 7 | rm -rf /var/lib/apt/lists/* 8 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | gem "rake" 4 | 5 | gem "rmagick" 6 | gem "dhash", github: "nakilon/dhash" 7 | gem "phamilie" 8 | gem "mini_magick" 9 | gem "dhashy" 10 | gem "phash-rb", github: "nakilon/phash-rb" 11 | 12 | gem "get_process_mem" 13 | gem "mll" 14 | gem "minitest" 15 | gem "byebug", *("<11.1.0" if Gem::Version.new(RUBY_VERSION) < Gem::Version.new("2.4")) 16 | 17 | gemspec 18 | -------------------------------------------------------------------------------- /common.rb: -------------------------------------------------------------------------------- 1 | def download_if_needed path 2 | require "open-uri" 3 | require "digest" 4 | FileUtils.mkdir_p File.dirname path 5 | URI("http://gems.nakilon.pro.storage.yandexcloud.net/dhash-vips/#{File.basename path}".tap do |url| 6 | puts "downloading #{path} from #{url}" 7 | end).open do |link| 8 | File.open(path, "wb"){ |file| IO.copy_stream link, file } 9 | end unless File.exist?(path) && Digest::MD5.file(path) == File.basename(path, File.extname(path)) 10 | path 11 | end 12 | -------------------------------------------------------------------------------- /example_dups/main.rb: -------------------------------------------------------------------------------- 1 | require "dhash-vips" 2 | Dir.chdir ENV["TEST"] || "/images" 3 | 4 | pattern = %w{ *.jp*g *.png } 5 | pairs = Dir.glob(pattern).tap do |_| 6 | puts "\n#{pattern} images found: #{_.size}\n\n" 7 | end.sort.map{ |f| [DHashVips::IDHash.fingerprint(f), f] }. 8 | combination(2).map{ |(h1, f1), (h2, f2)| [DHashVips::IDHash.distance(h1, h2), f1, f2] } 9 | 10 | [ 11 | ["very similar", 0..14], 12 | ["similar", 15..19], 13 | ["probably similar", 20..24], 14 | ].each do |category, range| 15 | pairs.select{ |dist,| range.include? dist }.tap do |_| 16 | puts "#{category} image pairs: #{_.size}\n\n" 17 | end.each do |dist, f1, f2| 18 | puts "\tdistance: #{dist}\n\t#{f1}\n\t#{f2}\n\n" 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /example_ocr/Gemfile.lock: -------------------------------------------------------------------------------- 1 | PATH 2 | remote: .. 3 | specs: 4 | dhash-vips (0.1.0.0) 5 | ruby-vips (~> 2.0.16) 6 | 7 | GEM 8 | remote: https://rubygems.org/ 9 | specs: 10 | byebug (11.0.1) 11 | dhash (0.0.2) 12 | rmagick 13 | ffi (1.11.3) 14 | get_process_mem (0.2.5) 15 | ffi (~> 1.0) 16 | minitest (5.13.0) 17 | mll (2.6.4) 18 | phamilie (0.1.0) 19 | rake (13.0.1) 20 | rmagick (2.16.0) 21 | ruby-vips (2.0.16) 22 | ffi (~> 1.9) 23 | 24 | PLATFORMS 25 | ruby 26 | 27 | DEPENDENCIES 28 | byebug 29 | dhash 30 | dhash-vips! 31 | get_process_mem 32 | minitest 33 | mll 34 | phamilie 35 | rake 36 | rmagick (~> 2.16) 37 | 38 | BUNDLED WITH 39 | 2.0.2 40 | -------------------------------------------------------------------------------- /test_rbenv.rb: -------------------------------------------------------------------------------- 1 | require "open3" 2 | assert_exitstatus = lambda do |_| 3 | (string, status) = Open3.capture2e _.tap &method(:puts) 4 | unless status.exitstatus.zero? 5 | puts string 6 | abort "exitstatus = #{status.exitstatus}" 7 | end 8 | end 9 | Dir.entries("#{ENV["RBENV_ROOT"]}/sources").grep(/\A(2\.[3-9]|3\.\d)\.\d\z/).sort.reverse.uniq{ |_| _[0,3] }.reverse.each do |version| 10 | assert_exitstatus["set -e && eval \"$(rbenv init -)\" && rbenv shell #{version} && gem uninstall -a dhash-vips && ruby extconf.rb && make clean && make"] 11 | assert_exitstatus["set -e && eval \"$(rbenv init -)\" && rbenv shell #{version} && bundle install && bundle exec ruby test.rb && bundle exec ruby test_LoadError.rb"] 12 | end 13 | puts "OK" 14 | -------------------------------------------------------------------------------- /dhash-vips.gemspec: -------------------------------------------------------------------------------- 1 | Gem::Specification.new do |spec| 2 | spec.name = "dhash-vips" 3 | spec.version = "0.2.4.0" 4 | spec.summary = "dHash and IDHash perceptual image hashing/fingerprinting" 5 | 6 | spec.author = "Victor Maslov aka Nakilon" 7 | spec.email = "nakilon@gmail.com" 8 | spec.license = "MIT" 9 | spec.metadata = {"source_code_uri" => "https://github.com/nakilon/dhash-vips"} 10 | 11 | spec.add_dependency "ruby-vips", "~> 2.0", "!= 2.1.0", "!= 2.1.1" 12 | 13 | spec.require_path = "lib" 14 | spec.extensions = %w{ extconf.rb } 15 | spec.files = %w{ LICENSE.txt dhash-vips.gemspec lib/dhash-vips.rb idhash.c lib/dhash-vips-post-install-test.rb } + spec.extensions + 16 | %w{ bin/idhash } 17 | spec.executables = %w{ idhash } 18 | spec.bindir = "bin" 19 | end 20 | -------------------------------------------------------------------------------- /.github/workflows/dockerimage.yaml: -------------------------------------------------------------------------------- 1 | name: Debian Slim Docker image 2 | on: 3 | push: 4 | tags: 5 | - "v*.*.*.*" 6 | jobs: 7 | build: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v2 11 | - run: docker build -t image -f - . 2 | 3 | static VALUE idhash_distance(VALUE self, VALUE a, VALUE b) { 4 | const size_t max_words = 256 / sizeof(uint64_t); 5 | 6 | const size_t word_numbits = sizeof(uint64_t) * CHAR_BIT; 7 | size_t n; 8 | n = rb_absint_numwords(a, word_numbits, NULL); 9 | if (n > max_words || n == (size_t)-1) 10 | rb_raise(rb_eRangeError, "fingerprint #1 exceeds 256 bits"); 11 | n = rb_absint_numwords(b, word_numbits, NULL); 12 | if (n > max_words || n == (size_t)-1) 13 | rb_raise(rb_eRangeError, "fingerprint #2 exceeds 256 bits"); 14 | 15 | uint64_t as[max_words], bs[max_words]; 16 | rb_integer_pack( 17 | a, as, max_words, sizeof(uint64_t), 0, 18 | INTEGER_PACK_LSWORD_FIRST | INTEGER_PACK_NATIVE_BYTE_ORDER | INTEGER_PACK_2COMP 19 | ); 20 | rb_integer_pack( 21 | b, bs, max_words, sizeof(uint64_t), 0, 22 | INTEGER_PACK_LSWORD_FIRST | INTEGER_PACK_NATIVE_BYTE_ORDER | INTEGER_PACK_2COMP 23 | ); 24 | 25 | return INT2FIX( 26 | __builtin_popcountll((as[3] | bs[3]) & (as[1] ^ bs[1])) + 27 | __builtin_popcountll((as[2] | bs[2]) & (as[0] ^ bs[0])) 28 | ); 29 | } 30 | 31 | void Init_idhash() { 32 | VALUE m = rb_define_module("DHashVips"); 33 | VALUE mm = rb_define_module_under(m, "IDHash"); 34 | rb_define_module_function(mm, "distance3_c", idhash_distance, 2); 35 | } 36 | -------------------------------------------------------------------------------- /example_ocr/main.rb: -------------------------------------------------------------------------------- 1 | require "dhash-vips" 2 | 3 | # Courier Menlo Monaco Tahoma 4 | chars = %w{ 5 | Arial Verdana 6 | }.flat_map do |font| 7 | FileUtils.mkdir_p "chars/#{font}" 8 | (?A..?Z).map do |char| 9 | filename = "chars/#{font}/#{char.ord}.png" 10 | Vips::Image.text(char, font: font, width: 100, height: 100).invert.write_to_file filename unless File.exist? filename 11 | [DHashVips::IDHash.fingerprint(filename), char] 12 | end 13 | end 14 | unless File.exist? "monotype-arial.png" 15 | require "open-uri" 16 | File.binwrite "monotype-arial.png", open("http://gems.nakilon.pro.storage.yandexcloud.net/dhash-vips/monotype-arial.png", &:read) 17 | end 18 | 19 | split = lambda do |array| 20 | array.chunk{ |row| row.any?{ |c,| c < 255 } }.select(&:first).map(&:last) 21 | end 22 | split[Vips::Image.new_from_file("monotype-arial.png").colourspace("b-w").flatten.to_a].each do |line| 23 | split[line.transpose].map do |char| 24 | require "tempfile" 25 | temp = Tempfile.new [File.basename(File.expand_path __dir__()), ".png"] 26 | fingerprint = begin 27 | temp.write Vips::Image.new_from_array(char.transpose).write_to_buffer(".png") 28 | DHashVips::IDHash.fingerprint temp.tap(&:rewind).path 29 | ensure 30 | temp.tap(&:unlink).close 31 | end 32 | chars.min_by{ |f,| DHashVips::IDHash.distance f, fingerprint }.last 33 | end.join.tap &method(:puts) 34 | end 35 | -------------------------------------------------------------------------------- /.github/workflows/benchmark.yaml: -------------------------------------------------------------------------------- 1 | name: Benchmark 2 | on: 3 | push: 4 | branches: [master] 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v2 10 | - run: | 11 | docker run --rm -v $(pwd):/checkout -w /checkout ruby:alpine sh -c "\ 12 | wget -O /usr/local/include/CImg.h https://raw.githubusercontent.com/GreycLab/CImg/3d1fc212ffe933cb2bc841e504920e5a67e676b8/CImg.h && \ 13 | apk add --no-cache git build-base imagemagick-dev vips && \ 14 | bundle install --no-cache && \ 15 | ruby extconf.rb && make clean && make && \ 16 | bundle exec rake benchmark" 17 | timeout-minutes: 5 18 | - run: | 19 | docker run --rm -v $(pwd):/checkout -w /checkout ruby:slim sh -c "\ 20 | apt-get update && \ 21 | apt install -y --no-install-recommends wget && \ 22 | wget -O /usr/local/include/CImg.h https://raw.githubusercontent.com/GreycLab/CImg/3d1fc212ffe933cb2bc841e504920e5a67e676b8/CImg.h && \ 23 | ( \ 24 | apt install -y --no-install-recommends git build-essential libmagickcore-dev libvips libjpeg-dev ; \ 25 | apt install -y --no-install-recommends git build-essential libmagickcore-dev libvips libjpeg-dev --fix-missing \ 26 | ) && \ 27 | bundle install --no-cache && \ 28 | ruby extconf.rb && make clean && make && \ 29 | bundle exec rake benchmark" 30 | timeout-minutes: 5 31 | -------------------------------------------------------------------------------- /bin/idhash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | Signal.trap(:INT){ abort "\n(interrupted by SIGINT)" } 3 | 4 | unless 2 == ARGV.size 5 | puts "this command is to compare two images" 6 | puts "usage: #{__FILE__} " 7 | exit 8 | end 9 | 10 | require_relative "../lib/dhash-vips" 11 | ha, hb = ARGV.map{ |filename| DHashVips::IDHash.fingerprint filename } 12 | puts "distance: #{d1 = DHashVips::IDHash.distance ha, hb}" 13 | size = 2 ** 3 14 | shift = 2 * size * size 15 | ai = ha >> shift 16 | ad = ha - (ai << shift) 17 | bi = hb >> shift 18 | bd = hb - (bi << shift) 19 | 20 | _127 = shift - 1 21 | _63 = size * size - 1 22 | # width = 800 23 | # height = 800 24 | 25 | d2 = 0 26 | a, b = [[ad, ai, ARGV[0]], [bd, bi, ARGV[1]]].map do |xd, xi, path| 27 | puts File.basename path 28 | hor = Array.new(size){Array.new(size){" "}} 29 | ver = Array.new(size){Array.new(size){" "}} 30 | _127.downto(0).each_with_index do |i, ii| 31 | if i > _63 32 | y, x = (_127 - i).divmod size 33 | else 34 | x, y = (_63 - i).divmod size 35 | end 36 | if xi[i] > 0 37 | target, c = if i > _63 38 | [ver, %w{ v ^ }[xd[i]]] 39 | else 40 | [hor, %w{ > < }[xd[i]]] 41 | end 42 | target[y][x] = c 43 | end 44 | if ai[i] + bi[i] > 0 && ad[i] != bd[i] 45 | d2 += 1 46 | target = if i > _63 47 | ver 48 | else 49 | hor 50 | end 51 | target[y][x] = "\e[7m#{target[y][x]}\e[27m" 52 | end 53 | end 54 | hor.zip(ver).each{ |_| puts _.join " " } 55 | end 56 | abort "something went wrong" unless d1 * 2 == d2 57 | puts "OK" 58 | -------------------------------------------------------------------------------- /example_dups/README.md: -------------------------------------------------------------------------------- 1 | # Image duplicates search demo 2 | 3 | https://hub.docker.com/repository/docker/nakilonishe/dhash-vips-demo 4 | 5 | This is a sample Docker image that you can use to find duplicates in a folder you link. 6 | So you don't need Ruby or even git to try the gem, just Docker: 7 | 8 | ![](http://gems.nakilon.pro.storage.yandexcloud.net/dhash-vips/example_dups.png) 9 | 10 | ```none 11 | $ docker run --rm -v $(pwd)/good:/images nakilonishe/dhash-vips-demo 12 | 13 | ["*.jp*g", "*.png"] images found: 6 14 | 15 | very similar image pairs: 1 16 | 17 | distance: 11 18 | Eiffel_Tower,_view_from_the_Trocadero,_1_July_2008.jpg 19 | Eiffelturm.jpeg 20 | 21 | similar image pairs: 1 22 | 23 | distance: 18 24 | Aha_waah_taz.jpg 25 | Beauty_of_Taj_Mahal_can_only_felt_by_heart.jpg 26 | 27 | probably similar image pairs: 3 28 | 29 | distance: 20 30 | Beauty_of_Taj_Mahal_can_only_felt_by_heart.jpg 31 | Like_A_Pearl_(60650624).jpeg 32 | 33 | distance: 21 34 | Eiffel_Tower,_November_15,_2011.jpg 35 | Eiffel_Tower,_view_from_the_Trocadero,_1_July_2008.jpg 36 | 37 | distance: 22 38 | Eiffel_Tower,_November_15,_2011.jpg 39 | Eiffelturm.jpeg 40 | ``` 41 | 42 | Here thresholds are hardcoded as `[0..14, 15..19, 20..24]`. They are a bit lowered and adjusted for demo purposes -- all landscapes are a bit similar because they have a horizon line. 43 | In your programs you should find the best fitting thresholds and maybe preprocess images by smart cropping, applying filters, etc. 44 | 45 | Maybe some day I'll make it a feature (with JSON and HTML export) within a gem if anyone needs it. 46 | 47 | P.S.: don't forget that Docker `-v` needs an absolute path. And you can link it as `:ro` (read-only) if you want. The script does not write anything. 48 | -------------------------------------------------------------------------------- /example_ocr/README.md: -------------------------------------------------------------------------------- 1 | # OCR demo 2 | 3 | The IDHash is not designed for OCR in the first place but in specific cases it might work well. You might also consider using it because the algorithm is enough simple so the built solution won't be a magic black box for you. 4 | 5 | The following image was rendered at https://www.fonts.com/font/monotype/arial/light: 6 | 7 | ![](http://gems.nakilon.pro.storage.yandexcloud.net/dhash-vips/monotype-arial.png) 8 | 9 | The program renders 26 upper case chars of each font you want using your OS fonts and compares them with each char detected on the image. 10 | This isn't made to break captchas so it assumes that chars are clearly whitespace separated, not rotated, black-on-white, etc. At the end it recognizes it as: 11 | 12 | ``` 13 | $ bundle install --without development 14 | $ bundle exec ruby main.rb 15 | 16 | THEQUJCKBROWNFOXJUMFS 17 | OVERTHELAZYDOG 18 | ``` 19 | 20 | In case when you have access to the exact font that was used to render the input image it should not have errors at all but here we have some. 21 | For some reason `I` of Arial on my OS and on the website are different (and so gets confused with `T`) -- maybe the font weight matters. It's possible to improve the result by trying different fonts and combining results in smart ways. Also you can recognise space characters to split by words and then use the English dictionary to reject what does not make sense. Using the dictionary check and lots of input data you can even build a self-learning system that would build the alphabet out of fonts you provide to get as close to the unknown font as possible. 22 | 23 | So there is a lot of room for improvement as an OCR tool but this is just an example of code using the `dhash-vips` gem -- `ctrl+F` the `DHashVips::IDHash` to see lines where it's used. 24 | -------------------------------------------------------------------------------- /vips.ruby.alpine.Dockerfile: -------------------------------------------------------------------------------- 1 | ARG RUBY_ALPINE_VERSION 2 | FROM ruby:$RUBY_ALPINE_VERSION 3 | ARG RUBY_ALPINE_VERSION 4 | ENV RUBY_ALPINE_VERSION $RUBY_ALPINE_VERSION 5 | 6 | # docker build - -t vips-ruby2.3.8 --build-arg RUBY_ALPINE_VERSION=2.3.8-alpine3.8 --build-arg VIPS_VERSION=8.9.2 (a,b){ DHashVips::IDHash.distance3_ruby a, b } 5 | 6 | p as = [a.to_s(16).rjust(64,?0)].pack("H*").unpack("N*") 7 | p bs = [b.to_s(16).rjust(64,?0)].pack("H*").unpack("N*") 8 | puts as.zip(bs)[0,4].map{ |i,j| (i | j).to_s(2).rjust(32, ?0) }.zip \ 9 | as.zip(bs)[4,4].map{ |i,j| (i ^ j).to_s(2).rjust(32, ?0) } 10 | p DHashVips::IDHash.distance3_c a, b 11 | p f[a, b] 12 | fail unless 17 == f[a, b] 13 | 14 | s = [0, 1, 1<<63, (1<<63)+1, (1<<64)-1].each do |_| 15 | # p [_.to_s(16).rjust(64,?0)].pack("H*").unpack("N*").map{ |_| _.to_s(2).rjust(32, ?0) } 16 | end 17 | ss = s.repeated_permutation(4).map do |s1, s2, s3, s4| 18 | ((s1 << 192) + (s2 << 128) + (s3 << 64) + s4).tap do |_| 19 | # p [_.to_s(16).rjust(64,?0)].pack("H*").unpack("N*").map{ |_| _.to_s(2).rjust(32, ?0) } 20 | end 21 | end 22 | fail unless :distance3 == DHashVips::IDHash.method(:distance3).original_name 23 | if Gem::Version.new(RUBY_VERSION) < Gem::Version.new("2.4") 24 | check = lambda do |s1, s2| 25 | s1.is_a?(Bignum) && s2.is_a?(Bignum) 26 | end 27 | else 28 | require "rbconfig/sizeof" 29 | check = lambda do |s1, s2| 30 | # https://github.com/ruby/ruby/commit/de2f7416d2deb4166d78638a41037cb550d64484#diff-16b196bc6bfe8fba63951420f843cfb4R10 31 | _FIXNUM_MAX = (1 << (8 * RbConfig::SIZEOF["long"] - 2)) - 1 32 | s1 > _FIXNUM_MAX && s2 > _FIXNUM_MAX 33 | end 34 | end 35 | ss.product ss do |s1, s2| 36 | next unless check.call s1, s2 37 | unless f[s1, s2] == DHashVips::IDHash.distance3_c(s1, s2) 38 | p [s1, s2] 39 | p [s1.to_s(16).rjust(64,?0)].pack("H*").unpack("N*").map{ |_| _.to_s(2).rjust(32, ?0) } 40 | p [s2.to_s(16).rjust(64,?0)].pack("H*").unpack("N*").map{ |_| _.to_s(2).rjust(32, ?0) } 41 | p [f[s1, s2], DHashVips::IDHash.distance3_c(s1, s2)] 42 | fail 43 | end 44 | end 45 | 100000.times do 46 | s1, s2 = Array.new(2){ n = rand 256; ([?0] * n + [?1] * (256 - n)).shuffle.join.to_i 2 } 47 | fail unless DHashVips::IDHash.distance3(s1, s2) == DHashVips::IDHash.distance3_ruby(s1, s2) 48 | end 49 | -------------------------------------------------------------------------------- /lib/dhash-vips.rb: -------------------------------------------------------------------------------- 1 | require "vips" 2 | Vips.vector_set false 3 | 4 | module DHashVips 5 | 6 | def self.bw image 7 | (image.has_alpha? ? image.flatten(background: 255) : image).colourspace("b-w")[0] 8 | end 9 | 10 | module DHash 11 | extend self 12 | 13 | def hamming a, b 14 | (a ^ b).to_s(2).count "1" 15 | end 16 | 17 | def pixelate input, hash_size 18 | DHashVips.bw( if input.is_a? Vips::Image 19 | input.thumbnail_image(hash_size + 1, height: hash_size, size: :force) 20 | else 21 | Vips::Image.thumbnail(input, hash_size + 1, height: hash_size, size: :force) 22 | end ) 23 | end 24 | 25 | def calculate file, hash_size = 8 26 | image = pixelate file, hash_size 27 | image.cast("int").conv([[1, -1]]).crop(1, 0, hash_size, hash_size).>(0)./(255).cast("uchar").to_a.join.to_i(2) 28 | end 29 | 30 | end 31 | 32 | module IDHash 33 | 34 | def self.distance3_ruby a, b 35 | ((a ^ b) & (a | b) >> 128).to_s(2).count "1" 36 | end 37 | begin 38 | require_relative "../idhash.#{Gem::Platform.local.os == "darwin" ? "bundle" : "o"}" 39 | rescue LoadError 40 | class << self 41 | alias distance3 distance3_ruby 42 | end 43 | else 44 | # we can't just do `defined? Bignum` because it's defined but deprecated (some internal CONST_DEPRECATED flag) 45 | if Gem::Version.new(RUBY_VERSION) < Gem::Version.new("2.4") 46 | def self.distance3 a, b 47 | if a.is_a?(Bignum) && b.is_a?(Bignum) 48 | distance3_c a, b 49 | else 50 | distance3_ruby a, b 51 | end 52 | end 53 | else 54 | # https://github.com/ruby/ruby/commit/de2f7416d2deb4166d78638a41037cb550d64484#diff-16b196bc6bfe8fba63951420f843cfb4R10 55 | require "rbconfig/sizeof" 56 | FIXNUM_MAX = (1 << (8 * RbConfig::SIZEOF["long"] - 2)) - 1 57 | def self.distance3 a, b 58 | if a > FIXNUM_MAX && b > FIXNUM_MAX 59 | distance3_c a, b 60 | else 61 | distance3_ruby a, b 62 | end 63 | end 64 | end 65 | end 66 | def self.distance a, b 67 | size_a, size_b = [a, b].map do |x| 68 | # TODO write a test about possible hash sizes 69 | # they were 32 and 128, 124, 120 for MRI 2.0 70 | # but also 31, 30 happens for MRI 2.3 71 | x.size <= 32 ? 8 : 16 72 | end 73 | return distance3 a, b if [8, 8] == [size_a, size_b] 74 | fail "fingerprints were taken with different `power` param: #{size_a} and #{size_b}" if size_a != size_b 75 | ((a ^ b) & (a | b) >> 2 * size_a * size_a).to_s(2).count "1" 76 | end 77 | 78 | def self.median array 79 | h = array.size / 2 80 | return array[h] if array[h] != array[h - 1] 81 | right = array.dup 82 | left = right.shift h 83 | right.shift if right.size > left.size 84 | return right.first if left.last != right.first 85 | return right.uniq[1] if left.count(left.last) > right.count(right.first) 86 | left.last 87 | end 88 | private_class_method :median 89 | fail unless 2 == median([1, 2, 2, 2, 2, 2, 3]) 90 | fail unless 3 == median([1, 2, 2, 2, 2, 3, 3]) 91 | fail unless 3 == median([1, 1, 2, 2, 3, 3, 3]) 92 | fail unless 2 == median([1, 1, 1, 2, 3, 3, 3]) 93 | fail unless 2 == median([1, 1, 2, 2, 2, 2, 3]) 94 | fail unless 2 == median([1, 2, 2, 2, 2, 3]) 95 | fail unless 3 == median([1, 2, 2, 3, 3, 3]) 96 | fail unless 1 == median([1, 1, 1]) 97 | fail unless 1 == median([1, 1]) 98 | 99 | def self.fingerprint input, power = 3 100 | size = 2 ** power 101 | image = if input.is_a? Vips::Image 102 | input.thumbnail_image(size, height: size, size: :force) 103 | else 104 | Vips::Image.thumbnail(input, size, height: size, size: :force) 105 | end 106 | array = DHashVips.bw(image).to_enum.map &:flatten 107 | d1, i1, d2, i2 = [array, array.transpose].flat_map do |a| 108 | d = a.zip(a.rotate(1)).flat_map{ |r1, r2| r1.zip(r2).map{ |i, j| i - j } } 109 | m = median d.map(&:abs).sort 110 | [ 111 | d.map{ |c| c < 0 ? 1 : 0 }.join.to_i(2), 112 | d.map{ |c| c.abs >= m ? 1 : 0 }.join.to_i(2), 113 | ] 114 | end 115 | (((((i1 << size * size) + i2) << size * size) + d1) << size * size) + d2 116 | end 117 | 118 | end 119 | 120 | end 121 | -------------------------------------------------------------------------------- /test.rb: -------------------------------------------------------------------------------- 1 | require "minitest/autorun" 2 | 3 | require_relative "lib/dhash-vips" 4 | 5 | # TODO tests about `fingerprint(4)` 6 | 7 | [ 8 | [DHashVips::DHash, :hamming, :calculate, 2, 23, 16, 50, 7, 0x7919395919191919], 9 | 10 | # v0.2.3.0 11 | # [[0, 18, 26, 28, 28, 24, 35, 31, 42, 42, 32, 34, 33, 29, 35, 40], 12 | # [18, 0, 30, 24, 38, 34, 33, 33, 44, 42, 38, 42, 41, 37, 37, 50], 13 | # [26, 30, 0, 16, 34, 38, 29, 35, 42, 40, 36, 38, 31, 31, 31, 36], 14 | # [28, 24, 16, 0, 36, 36, 31, 35, 40, 38, 34, 34, 41, 37, 27, 34], 15 | # [28, 38, 34, 36, 0, 14, 35, 33, 40, 42, 32, 26, 27, 33, 35, 28], 16 | # [24, 34, 38, 36, 14, 0, 43, 39, 38, 40, 24, 28, 25, 31, 29, 28], 17 | # [35, 33, 29, 31, 35, 43, 0, 8, 27, 25, 35, 33, 32, 32, 28, 31], 18 | # [31, 33, 35, 35, 33, 39, 8, 0, 27, 27, 35, 33, 34, 34, 28, 33], 19 | # [42, 44, 42, 40, 40, 38, 27, 27, 0, 2, 34, 32, 31, 33, 31, 28], 20 | # [42, 42, 40, 38, 42, 40, 25, 27, 2, 0, 34, 34, 33, 35, 31, 28], 21 | # [32, 38, 36, 34, 32, 24, 35, 35, 34, 34, 0, 10, 23, 31, 25, 18], 22 | # [34, 42, 38, 34, 26, 28, 33, 33, 32, 34, 10, 0, 23, 29, 25, 16], 23 | # [33, 41, 31, 41, 27, 25, 32, 34, 31, 33, 23, 23, 0, 20, 24, 19], 24 | # [29, 37, 31, 37, 33, 31, 32, 34, 33, 35, 31, 29, 20, 0, 22, 27], 25 | # [35, 37, 31, 27, 35, 29, 28, 28, 31, 31, 25, 25, 24, 22, 0, 23], 26 | # [40, 50, 36, 34, 28, 28, 31, 33, 28, 28, 18, 16, 19, 27, 23, 0]] 27 | 28 | [DHashVips::IDHash, :distance, :fingerprint, 8, 22, 25, 70, 0, 0x1d5cdc0d1d0c1d9f5720a6fff2fe02013f00df9f005e1dc0ff670000ffff0080], 29 | 30 | # v0.2.3.0 31 | # [[0, 19, 34, 38, 57, 47, 50, 49, 45, 42, 55, 45, 60, 49, 51, 53], 32 | # [19, 0, 34, 33, 55, 47, 52, 49, 46, 49, 59, 46, 62, 54, 50, 57], 33 | # [34, 34, 0, 8, 48, 55, 46, 42, 55, 57, 45, 35, 51, 45, 46, 47], 34 | # [38, 33, 8, 0, 52, 61, 43, 38, 49, 50, 50, 37, 50, 39, 43, 51], 35 | # [57, 55, 48, 52, 0, 22, 46, 42, 70, 63, 51, 50, 35, 41, 46, 46], 36 | # [47, 47, 55, 61, 22, 0, 54, 54, 57, 53, 39, 44, 43, 41, 45, 38], 37 | # [50, 52, 46, 43, 46, 54, 0, 8, 35, 38, 51, 45, 50, 42, 43, 43], 38 | # [49, 49, 42, 38, 42, 54, 8, 0, 34, 36, 54, 49, 49, 42, 43, 44], 39 | # [45, 46, 55, 49, 70, 57, 35, 34, 0, 10, 53, 55, 56, 50, 49, 49], 40 | # [42, 49, 57, 50, 63, 53, 38, 36, 10, 0, 50, 52, 55, 48, 46, 45], 41 | # [55, 59, 45, 50, 51, 39, 51, 54, 53, 50, 0, 10, 32, 35, 40, 25], 42 | # [45, 46, 35, 37, 50, 44, 45, 49, 55, 52, 10, 0, 30, 27, 37, 26], 43 | # [60, 62, 51, 50, 35, 43, 50, 49, 56, 55, 32, 30, 0, 22, 26, 28], 44 | # [49, 54, 45, 39, 41, 41, 42, 42, 50, 48, 35, 27, 22, 0, 37, 35], 45 | # [51, 50, 46, 43, 46, 45, 43, 43, 49, 46, 40, 37, 26, 37, 0, 22], 46 | # [53, 57, 47, 51, 46, 38, 43, 44, 49, 45, 25, 26, 28, 35, 22, 0]] 47 | 48 | ].each do |lib, dm, calc, min_similar, max_similar, min_not_similar, max_not_similar, bw_exceptional, hash| 49 | 50 | describe lib do 51 | require "fileutils" 52 | require "digest" 53 | require "mll" 54 | 55 | require_relative "common" 56 | 57 | # these are false positive by idhash 58 | # 1b1d4bde376084011d027bba1c047a4b.jpg 59 | # 6d97739b4a08f965dc9239dd24382e96.jpg 60 | [ 61 | [:similar, %w{ 62 | 1d468d064d2e26b5b5de9a0241ef2d4b.jpg 92d90b8977f813af803c78107e7f698e.jpg 63 | 309666c7b45ecbf8f13e85a0bd6b0a4c.jpg 3f9f3db06db20d1d9f8188cd753f6ef4.jpg 64 | 679634ff89a31279a39f03e278bc9a01.jpg df0a3b93e9412536ee8a11255f974141.jpg 65 | 54192a3f65bd03163b04849e1577a40b.jpg 6d32f57459e5b79b5deca2a361eb8c6e.jpg 66 | 4b62e0eef58bfbc8d0d2fbf2b9d05483.jpg b8eb0ca91855b657f12fb3d627d45c53.jpg 67 | 21cd9a6986d98976b6b4655e1de7baf4.jpg 9b158c0d4953d47171a22ed84917f812.jpg 68 | 9c2c240ec02356472fb532f404d28dde.jpg fc762fa286489d8afc80adc8cdcb125e.jpg 69 | 7a833d873f8d49f12882e86af1cc6b79.jpg ac033cf01a3941dd1baa876082938bc9.jpg 70 | }, min_similar, max_similar, min_not_similar, max_not_similar], # slightly similar images 71 | [:bw_exceptional, %w{ 72 | 71662d4d4029a3b41d47d5baf681ab9a.jpg ad8a37f872956666c3077a3e9e737984.jpg 73 | }, bw_exceptional, bw_exceptional], # these are the same photo but of different size and colorspace 74 | ].each do |test_name, _images, min, max, min_not, max_not| 75 | 76 | images = _images.map{ |_| download_if_needed "test_images/#{_}" } 77 | 78 | hashes = images.map &lib.method(calc) 79 | table = MLL::table[lib.method(dm), [hashes], [hashes]] 80 | 81 | # STDERR.puts "" 82 | # require "pp" 83 | # PP.pp table, STDERR 84 | # STDERR.puts "" 85 | 86 | it test_name do 87 | similar = [] 88 | not_similar = [] 89 | hashes.size.times.to_a.repeated_combination(2) do |i, j| 90 | case 91 | when i == j 92 | assert_predicate table[i][j], :zero? 93 | when (j - i).abs == 1 && (i + j - 1) % 4 == 0 94 | # STDERR.puts [table[i][j], min, max].inspect 95 | similar.push table[i][j] 96 | else 97 | # STDERR.puts [table[i][j], min_not_similar, max_not_similar].inspect 98 | not_similar.push table[i][j] 99 | end 100 | end 101 | assert_equal [min, max], similar.minmax 102 | assert_equal [min_not, max_not], not_similar.minmax if min_not 103 | end 104 | 105 | end 106 | 107 | it "accepts Vips::Image" do 108 | # https://github.com/libvips/ruby-vips/issues/349 109 | lib.public_send calc, Vips::Image.new_from_buffer("GIF89a\x01\x00\x01\x00\x80\x01\x00\xFF\xFF\xFF\x00\x00\x00,\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;", "") 110 | end 111 | 112 | it "correct calculation" do 113 | download_if_needed "alpha_only.png" 114 | t = lib.public_send calc, "alpha_only.png" 115 | assert_equal hash, t, ->{ "0x#{hash.to_s 16} != 0x#{t.to_s 16}" } 116 | end 117 | 118 | end 119 | 120 | end 121 | 122 | describe DHashVips::IDHash do 123 | it "does not call distance3_ruby" do 124 | assert_equal :distance3, DHashVips::IDHash.method(:distance3).original_name 125 | end 126 | end 127 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![Gem version](https://badge.fury.io/rb/dhash-vips.svg) 2 | ![Benchmark](https://github.com/nakilon/dhash-vips/workflows/Benchmark/badge.svg) 3 | 4 | # dHash and IDHash gem powered by ruby-vips 5 | 6 | The **dHash** is the algorithm of image fingerprinting that can be used to measure the similarity of two images. 7 | The **IDHash** is the new algorithm that has some improvements over dHash -- I'll describe it further. 8 | 9 | All existing Ruby implementations on GitHub depended on ImageMagick. My implementation takes an advantage of speed of the libvips (the `ruby-vips` gem). For even more speed the fingerprint comparison function is also implemented as a native C extension. 10 | 11 | ## dHash 12 | 13 | The idea of dHash is that you resize the original image to 8x9 and then convert it to 8x8 array of bits -- each tells if the corresponding pixel is brighter or darker than the one on the right (or left). Then you apply the [Hamming distance](https://en.wikipedia.org/wiki/Hamming_distance) to such arrays to measure how much they are different. 14 | 15 | ## IDHash (the Important Difference Hash) 16 | 17 | The main improvement over the dHash is the "Importance" data that makes it insensitive to the resizing algorithm and possible errors due to color scheme conversion. It is an array of extra 64 bits that tells the comparing function which half of 64 bits is important (when the difference between neighbors was enough significant) and which is not. So not every bit in a fingerprint is being compared but only half of them. 18 | 19 | Other improvements are: 20 | * It subtracts not only horizontally but also vertically -- that adds 128 more bits. 21 | * Instead of resizing to 8x9 it resizes to 8x8 and puts the image on a torus. 22 | 23 | According to a benchmark the gem has the highest quality and speed compared to other gems (lower numbers are better): 24 | 25 | Fingerprint Compare 1/FMI^2 26 | this gem: 27 | IDHash default 0.087 0.111 1.111 28 | IDHash Ruby 0.087 0.416 1.111 29 | DHash 0.105 0.188 1.444 30 | 31 | other gems: 32 | Phamilie 1.328 0.161 3.000 33 | Dhash 2.337 0.196 1.222 34 | Dhashy 1.329 10.954 1.406 35 | Phash 1.566 0.220 3.000 36 | 37 | ruby 2.7.8p225 (2023-03-30 revision 1f4d455848) [arm64-darwin24] 38 | vips-8.16.1 39 | Version: ImageMagick 7.1.1-47 Q16-HDRI aarch64 22763 https://imagemagick.org 40 | Apple M4 41 | gem ruby-vips v2.2.3 42 | gem rmagick: 5.5.0 43 | gem dhash: https://github.com/nakilon/dhash.git (at master@4c49533) 44 | gem phamilie: 0.1.0 45 | gem dhashy: 1.0.7 46 | gem phash-rb: https://github.com/nakilon/phash-rb.git (at main@e4068f3) 47 | 48 | ### Example 49 | 50 | Here are two photos (by Brian Lauer): 51 | ![](http://gems.nakilon.pro.storage.yandexcloud.net/dhash-vips/idhash_example_in.png) 52 | and visualization of IDHash (`rake compare_images -- image1.jpg image2.jpg`): 53 | ![](http://gems.nakilon.pro.storage.yandexcloud.net/dhash-vips/idhash_example_out.png) 54 | 55 | Here in each of 64 cells, there are two circles that color the difference between that cell and the neighbor one. If the difference is low the Importance bit is set to zero and the circle is invisible. So there are 128 pairs of corresponding circles and when you take one, if at least one circle is visible and is of different color the line is to be drawn. Here you see 15 lines and so the distance between fingerprints will be equal to 15 (that is pretty low and can be interpreted as "images look similar"). Also, you see here that floor on this photo matters -- classic dHash won't see that it's darker than wall because it's comparing only horizontal neighbors and if one photo had no floor the distance function won't notice that. Also, it sees the Important difference between the very right and left columns because the wall has a slow but visible gradient. 56 | 57 | As of version 0.2.4.0 the gem includes a binary that you can call to get similar visualisation in terminal: 58 | 59 | ```bash 60 | idhash test_images/3f9....jpg test_images/309....jpg 61 | ``` 62 | 63 | ![screenshot](doc_bin.png) 64 | 65 | ### Remaining problems 66 | 67 | * Neither dHash nor IDHash can't automatically detect very shifted crops and rotated images but you can make a wrapper that would call the comparison function iteratively. 68 | * These algorithms are color blind because of converting an image to grayscale. If you take a photo of something in your yard the sun will create lights and shadows, but if you compare photos of something green painted on a blue wall there is a possibility the machine would see nothing painted at all. The `dhash` gem had such image in specs and that made them pretty useless (this was supposed to be a face): 69 | ![](http://gems.nakilon.pro.storage.yandexcloud.net/dhash-vips/colorblind.png) 70 | * If you have a pile of 1000000 images comparing them with each other would take a month or two. To improve the process in case of dHash that uses Hamming distance you may want to read these threads on Stackexchange network: 71 | * [How to find the closest pairs of a string of binary bins in Ruby without O^2 issues?](https://stackoverflow.com/q/8734034/322020) 72 | * [Find all pairs of values that are close under Hamming distance](https://cstheory.stackexchange.com/q/18516/27420) 73 | * [Finding the closest pair between two sets of points on the hypercube](https://cstheory.stackexchange.com/q/16322/27420) 74 | * [Would PCA work for boolean data types?](https://stats.stackexchange.com/q/159705/1125) 75 | * [Using pHash to search agaist a huge image database, what is the best approach?](https://stackoverflow.com/q/18257641/322020) 76 | * [How do I speed up this BIT_COUNT query for hamming distance?](https://stackoverflow.com/q/35065675/322020) 77 | * [Hamming distance on binary strings in SQL](https://stackoverflow.com/q/4777070/322020) 78 | 79 | ## Installation 80 | 81 | brew install vips 82 | 83 | If you have troubles, see https://jcupitt.github.io/libvips/install.html 84 | Then: 85 | 86 | gem install dhash-vips 87 | 88 | If you have troubles with the `gem ruby-vips` dependency, see https://github.com/libvips/ruby-vips 89 | 90 | ## Usage 91 | 92 | ### dHash: 93 | 94 | ```ruby 95 | require "dhash-vips" 96 | 97 | hash1 = DHashVips::DHash.calculate "photo1.jpg" 98 | hash2 = DHashVips::DHash.calculate "photo2.jpg" 99 | 100 | distance = DHashVips::DHash.hamming hash1, hash2 101 | if distance < 10 102 | puts "Images are very similar" 103 | elsif distance < 20 104 | puts "Images are slightly similar" 105 | else 106 | puts "Images are different" 107 | end 108 | ``` 109 | 110 | ### IDHash: 111 | 112 | ```ruby 113 | require "dhash-vips" 114 | 115 | hash1 = DHashVips::IDHash.fingerprint "photo1.jpg" 116 | hash2 = DHashVips::IDHash.fingerprint "photo2.jpg" 117 | 118 | distance = DHashVips::IDHash.distance hash1, hash2 119 | if distance < 20 120 | puts "Images are very similar" 121 | elsif distance < 25 122 | puts "Images are slightly similar" 123 | else 124 | puts "Images are different" 125 | end 126 | ``` 127 | 128 | ## Notes and benchmarks 129 | 130 | * The above `20` and `25` constants are found empirically and just work enough well for 8-byte hashes. To find these thresholds you can run a rake task with hardcoded test cases (pairs of photos from the same photosession are not the same but are considered to be enough 'similar' for the purpose of this benchmark): 131 | 132 | $ rake compare_quality 133 | 134 | Dhash Phamilie DHash IDHash IDHash(4) 135 | The same image: 0..0 0..0 0..0 0..0 0..0 136 | 'Jordan Voth case': 2 2 3 0 0 137 | Similar images: 1..15 14..34 1..21 7..23 52..166 138 | Different images: 10..56 22..42 10..51 22..65 117..227 139 | 1/FMI^2 = 1.222 3.0 1.444 1.111 1.266 140 | FP, FN = [2, 0] [0, 6] [4, 0] [1, 0] [1, 1] 141 | optimal threshold 16 21 22 24 128 142 | 143 | The `FMI` line (smaller number is better) here is the "quality of algorithm", i.e. the best achievable function for the ["Fowlkes–Mallows index"](https://en.wikipedia.org/wiki/Fowlkes%E2%80%93Mallows_index) value if you take the "similar" and "different" test pairs and try to draw the threshold line. For IDHash it's empirical value of 22 as you acn see above that means it's the only algorithm that allowed to separate "similar" from "different" comparisons for our test cases. 144 | The last line shows number of false positives (`FP`) and false negatives (`FN`) in case of the best achieved FMI. 145 | The [`phamilie` gem](https://github.com/toy/phamilie) is a DCT based fingerprinting tool (not a kind of dhash). 146 | 147 | * Methods were renamed from `#calculate` to `#fingerprint` and from `#hamming` to `#distance`. 148 | * The `DHash#calculate` accepts `hash_size` optional parameter that is 8 by default. The `IDHash#fingerprint`'s optional parameter is called `power` and works in a bit different way: 3 means 8 and 4 means 16 -- other sizes are not supported because they don't seem to be useful (higher fingerprint resolution makes it vulnerable to image shifts and croppings, also `#distance` becomes much slower). Because IDHash's fingerprint is more complex than DHash's one it's not that straight forward to compare them so under the hood the `#distance` method have to check the size of fingerprint. If you are sure that fingerprints were made with power=3 then to skip the check you may use the `#distance3` method directly. 149 | * The `#distance3` method will try to compile and use the Ruby C extension that is around 15 times faster than pure Ruby implementation. Native extension currently works on macOS rbenv Ruby from 2.3.8 to at least 2.7.0-preview2 installed with rbenv `-k` flag. So the full benchmark: 150 | 151 | * Ruby 2.3.8p459: 152 | 153 | load the image and calculate the fingerprint: 154 | user system total real 155 | Dhash 6.191731 0.230885 6.422616 ( 6.428763) 156 | Phamilie 5.361751 0.037524 5.399275 ( 5.402553) 157 | DHashVips::DHash 0.858045 0.144820 1.002865 ( 0.924308) 158 | DHashVips::IDHash 0.769975 0.071087 0.841062 ( 0.790470) 159 | DHashVips::IDHash 4 0.805311 0.077918 0.883229 ( 0.825897) 160 | 161 | measure the distance (32*32*2000 times): 162 | user system total real 163 | Dhash hamming 1.810000 0.000000 1.810000 ( 1.824719) 164 | Phamilie distance 1.000000 0.010000 1.010000 ( 1.006127) 165 | DHashVips::DHash hamming 1.810000 0.000000 1.810000 ( 1.817415) 166 | DHashVips::IDHash distance 1.400000 0.000000 1.400000 ( 1.401333) 167 | DHashVips::IDHash distance3_ruby 3.320000 0.010000 3.330000 ( 3.337920) 168 | DHashVips::IDHash distance3_c 0.210000 0.000000 0.210000 ( 0.212864) 169 | DHashVips::IDHash distance 4 8.300000 0.120000 8.420000 ( 8.499735) 170 | 171 | * Also note that to make `#distance` able to assume the fingerprint resolution from the size of Integer that represents it, the change in its structure was needed (left half of bits was swapped with right one), so fingerprints between versions 0.0.4.1 and 0.0.5.0 became incompatible, but you probably can convert them manually. Otherwise if we put the version or structure information inside fingerprint it would became slow to (de)serialize and store. 172 | * The version `0.2.0.0` has grayscaling bug fixed and some tweak. It made DHash a bit worse and IDHash a bit better. Fingerprints recalculation is recommended. 173 | * The version `0.2.3.0` has an important alpha layer transparency bug fix. Fingerprints recalculation is recommended. 174 | 175 | ## Possible issues 176 | 177 | * OS X El Captain and rbenv may cause environment issues that would make you do things like: 178 | 179 | $ ./ruby `rbenv which rake` compare_matrixes 180 | 181 | instead of just 182 | 183 | $ rake compare_matrixes 184 | 185 | For more information on that: https://github.com/jcupitt/ruby-vips/issues/141 186 | 187 | ## Development notes 188 | 189 | * To run unit tests in current env 190 | 191 | $ ruby extconf.rb && make clean && make # otherwise you might get silenced LoadError due to switching between rubies 192 | $ bundle exec ruby test.rb && bundle exec ruby test_LoadError.rb 193 | 194 | * To run unit tests under all available latest major rbenv ruby versions 195 | 196 | $ ruby test_rbenv.rb 197 | 198 | * Current (this is outdated) Ruby [packages](https://pkgs.alpinelinux.org/packages) for `apk add` (Alpine Linux) and existing official Ruby docker [images](https://hub.docker.com/_/ruby?tab=tags) per Alpine version: 199 | 200 | packages ruby docker hub 201 | 3.12 2.7.1 2.5.8 2.6.6 2.7.1 202 | 3.11 2.6.6 2.4.10 2.5.8 2.6.6 2.7.1 203 | 3.10 2.5.8 2.4.10 2.5.8 2.6.6 2.7.1 204 | 3.9 2.5.8 2.4.9 2.5.7 2.6.5 2.7.0p1 205 | 3.8 2.5.8 2.3.8 2.4.6 2.5.5 2.6.3 206 | 3.7 2.4.6 2.3.8 2.4.5 2.5.3 2.6.0 207 | 3.6 2.4.6 2.4.5 2.5rc 208 | 3.5 2.3.8 209 | 3.4 2.3.7 2.3.7 2.4.4 210 | 3.3 2.2.9 211 | 212 | The gem has been tested on macOS rbenv versions: 2.3.8, 2.4.9, 2.5.7, 2.6.5, 2.7.0-preview2 213 | 214 | * To quickly find out what does the dhash-vips Docker image include (TODO: write in this README about the existing Docker images): 215 | 216 | docker run --rm sh -c "cat /etc/alpine-release; ruby -v; vips -v" 217 | 218 | * You may get this: 219 | 220 | Can't install RMagick 2.16.0. Can't find MagickWand.h. 221 | 222 | because Imagemagick sucks but we need it to benchmark alternative gems, so: 223 | 224 | $ brew install imagemagick@6 225 | $ brew unlink imagemagick@7 226 | $ brew link imagemagick@6 --force 227 | 228 | * On macOS, when you do `bundle install` it may fail to install `rmagick` gem (`dhash` gem dependency) saying: 229 | 230 | ERROR: Can't install RMagick 4.0.0. Can't find magick/MagickCore.h. 231 | 232 | To resolve this do: 233 | 234 | $ brew install imagemagick@6 235 | $ LDFLAGS="-L/usr/local/opt/imagemagick@6/lib" CPPFLAGS="-I/usr/local/opt/imagemagick@6/include" bundle install 236 | 237 | * If you get `No package 'MagickCore' found` try: 238 | 239 | $ PKG_CONFIG_PATH="/usr/local/Cellar/imagemagick@6/6.9.10-74/lib/pkgconfig" bundle install 240 | 241 | * You might get: 242 | 243 | NameError: uninitialized constant Magick::Rec601LumaColorspace 244 | Did you mean? Magick::Rec601YCbCrColorspace 245 | 246 | try 247 | 248 | $ brew unlink imagemagick 249 | $ brew link imagemagick@6 250 | $ gem uninstall rmagick # select 2.x 251 | $ bundle install 252 | 253 | * Phamilie works with filenames instead of fingerprints and caches them but not distances. 254 | 255 | * Phamilie error 256 | ``` 257 | fingerprint.cpp:61:10: fatal error: 'CImg.h' file not found 258 | ``` 259 | wants you to do 260 | ```console 261 | $ brew install cimg 262 | $ gem install phamilie -- --with-cppflags="-I$(brew --prefix cimg)/include" 263 | ``` 264 | 265 | * Phashion errors: 266 | * ``` 267 | checking for sqlite3ext.h... *** extconf.rb failed *** 268 | ``` 269 | means you need my fork (`gem "phashion", github: "nakilon/phashion"`) until the https://github.com/westonplatter/phashion/pull/105 is accepted 270 | * ``` 271 | sh: convert: command not found 272 | sh: gm: command not found 273 | ``` 274 | means you need to do 275 | ```console 276 | $ brew link imagemagick@6 277 | ``` 278 | 279 | * Phashion is being excluded from benchmarks because of installation difficulties 280 | 281 | ## Credits 282 | 283 | libvips maintainers [John Cupitt](https://github.com/jcupitt) and [Kleis Auke Wolthuizen](https://github.com/kleisauke) have helped a lot. 284 | 285 | [Dmitry Davydov](https://github.com/haukot) has revived the native extension after the `BDIGIT` deprecation. 286 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | 3 | require "pp" 4 | 5 | visualize_hash = lambda do |hash| 6 | puts hash.to_s(2).rjust(64, ?0).gsub(/(?<=.)/, '\0 ').scan(/.{16}/) 7 | end 8 | 9 | desc "Compare how Vips and ImageMagick resize images to 9x8" 10 | task :compare_pixelation do |_| 11 | require_relative "lib/dhash-vips" 12 | require "dhash" 13 | 14 | ARGV.drop(1).each do |arg| 15 | FileUtils.mkdir_p "compare_pixelation/#{File.dirname arg}" 16 | 17 | puts filename = "compare_pixelation/#{arg}.dhash-vips.png" 18 | DHashVips::DHash.pixelate(arg, 8). 19 | colourspace(:srgb). # otherwise we may get `Vips::Error` `RGB color space not permitted on grayscale PNG` when the image was already bw 20 | write_to_file filename 21 | visualize_hash.call DHashVips::DHash.calculate arg 22 | 23 | puts filename = "compare_pixelation/#{arg}.dhash.png" 24 | Magick::Image.read(arg).first.quantize(256, Magick::Rec601LumaColorspace, Magick::NoDitherMethod, 8).resize!(9, 8). 25 | write filename 26 | visualize_hash.call Dhash.calculate arg 27 | end 28 | end 29 | 30 | desc "Compare how Vips resizes image to 9x8 with different kernels" 31 | task :compare_kernels do |_| 32 | require_relative "lib/dhash-vips" 33 | # require "dhash" 34 | 35 | %i{ nearest linear cubic lanczos2 lanczos3 }.each do |kernel| 36 | hashes = ARGV.drop(1).map do |arg| 37 | puts arg 38 | DHashVips::DHash.calculate(arg, 8, kernel).tap(&visualize_hash) 39 | end 40 | puts "kernel: #{kernel}, distance: #{DHashVips::DHash.hamming(*hashes)}" 41 | end 42 | end 43 | 44 | require_relative "common" 45 | 46 | desc "Compare the quality of gems" 47 | # in this test we want to know not that photos are the same but rather that they are from the same photosession 48 | task :compare_quality do 49 | require "dhash" 50 | require "phamilie" 51 | phamilie = Phamilie.new 52 | require_relative "lib/dhash-vips" 53 | require "mll" 54 | 55 | puts MLL::grid.call( [ 56 | ["", "The same image:", "'Jordan Voth case':", "Similar images:", "Different images:", "1/FMI^2 =", "FP, FN =", "optimal threshold"], 57 | *[ 58 | [Dhash, :calculate, :hamming], 59 | [phamilie, :fingerprint, :distance, nil, 0], 60 | [DHashVips::DHash, :calculate, :hamming], 61 | [DHashVips::IDHash, :fingerprint, :distance], 62 | [DHashVips::IDHash, :fingerprint, :distance, 4], 63 | ].map do |m, calc, dm, power, ii| 64 | hashes = %w{ 65 | 71662d4d4029a3b41d47d5baf681ab9a.jpg ad8a37f872956666c3077a3e9e737984.jpg 66 | 67 | 1b1d4bde376084011d027bba1c047a4b.jpg 6d97739b4a08f965dc9239dd24382e96.jpg 68 | 69 | 1d468d064d2e26b5b5de9a0241ef2d4b.jpg 92d90b8977f813af803c78107e7f698e.jpg 70 | 309666c7b45ecbf8f13e85a0bd6b0a4c.jpg 3f9f3db06db20d1d9f8188cd753f6ef4.jpg 71 | 679634ff89a31279a39f03e278bc9a01.jpg df0a3b93e9412536ee8a11255f974141.jpg 72 | 54192a3f65bd03163b04849e1577a40b.jpg 6d32f57459e5b79b5deca2a361eb8c6e.jpg 73 | 4b62e0eef58bfbc8d0d2fbf2b9d05483.jpg b8eb0ca91855b657f12fb3d627d45c53.jpg 74 | 21cd9a6986d98976b6b4655e1de7baf4.jpg 9b158c0d4953d47171a22ed84917f812.jpg 75 | 9c2c240ec02356472fb532f404d28dde.jpg fc762fa286489d8afc80adc8cdcb125e.jpg 76 | 7a833d873f8d49f12882e86af1cc6b79.jpg ac033cf01a3941dd1baa876082938bc9.jpg 77 | }.map{ |_| "compare_quality_images/#{_}" }. 78 | each(&method(:download_if_needed)). 79 | map{ |_| [_, m.public_send(calc, _, *power)] } 80 | table = MLL::table[m.method(dm), [hashes.map{|_|_[ii||1]}], [hashes.map{|_|_[ii||1]}]] 81 | report = Struct.new(:same, :bw, :sim, :not_sim).new [], [], [], [] 82 | hashes.size.times.to_a.repeated_combination(2) do |i, j| 83 | report[ 84 | case 85 | when i == j ; :same 86 | when [i, j] == [0, 1] ; :bw 87 | when i > 3 && i + 1 == j && i % 2 == 0 ; :sim 88 | else ; :not_sim 89 | end 90 | ].push table[i][j] 91 | end 92 | p report 93 | _min, max = [*report.sim, *report.not_sim].minmax 94 | fmi, fp, fn, tr = (0..max+1).map do |b| 95 | fp = report.not_sim.count{ |_| _ < b } 96 | tp = (report.sim + report.bw).count{ |_| _ < b } 97 | fn = (report.sim + report.bw).count{ |_| _ >= b } 98 | [((tp + fp) * (tp + fn)).fdiv(tp * tp), fp, fn, b] 99 | end.reject{ |_,| _.nan? }.min_by(&:first) 100 | [ 101 | "#{m.is_a?(Module) ? m.name.split("::").last : m.class}#{"(#{power})" if power}", 102 | report.same. minmax.join(".."), 103 | report.bw[0], 104 | report.sim. minmax.join(".."), 105 | report.not_sim.minmax.join(".."), 106 | fmi.round(3), 107 | [fp, fn], 108 | tr, 109 | ] 110 | end, 111 | ].transpose, spacings: [1.5, 0], alignment: :right ) 112 | end 113 | 114 | # ruby -c Rakefile && rm -f ab.png && rake compare_images -- fc762fa286489d8afc80adc8cdcb125e.jpg 9c2c240ec02356472fb532f404d28dde.jpg 2>/dev/null && ql ab.png 115 | # rm -f ab.png && ./ruby `rbenv which rake` compare_images -- 6d97739b4a08f965dc9239dd24382e96.jpg 1b1d4bde376084011d027bba1c047a4b.jpg 2>/dev/null && ql ab.png 116 | # bundle exec rake compare_images[1b1d4bde376084011d027bba1c047a4b.jpg,6d97739b4a08f965dc9239dd24382e96.jpg] 117 | desc "Visualizes the IDHash difference measurement between two images" 118 | task :compare_images do |_, args| 119 | abort "there should be two image filenames passed as arguments (and optionally the `power`)" unless (2..3) === args.extras.size 120 | abort "the optional argument should be either 3 or 4" unless [3, 4].include?(power = (args.extras[2] || 3).to_i) 121 | require_relative "lib/dhash-vips" 122 | ha, hb = args.extras.map{ |filename| DHashVips::IDHash.fingerprint(filename, power) } 123 | puts "distance: #{DHashVips::IDHash.distance ha, hb}" 124 | size = 2 ** power 125 | shift = 2 * size * size 126 | ai = ha >> shift 127 | ad = ha - (ai << shift) 128 | bi = hb >> shift 129 | bd = hb - (bi << shift) 130 | 131 | a, b = args.extras.map do |filename| 132 | image = Vips::Image.new_from_file filename 133 | image = image.resize(size.fdiv(image.width), vscale: size.fdiv(image.height)).colourspace("b-w"). 134 | resize(100, vscale: 100, kernel: :nearest).colourspace("srgb") 135 | end 136 | fail unless a.width == b.width && a.height == b.height 137 | 138 | _127 = shift - 1 139 | _63 = size * size - 1 140 | n = 0 141 | width = a.width 142 | height = a.height 143 | 144 | Vips::Operation.class_eval do 145 | old_initialize = instance_method :initialize 146 | define_method :initialize do |value| 147 | old_initialize.bind(self).(value).tap do 148 | self.instance_variable_set "@operation_name", value 149 | end 150 | end 151 | old_set = instance_method :set 152 | define_method :set do |*args| 153 | args[1].instance_variable_set "@operation_name", self.instance_variable_get("@operation_name") if args.first == "image" 154 | old_set.bind(self).(*args) 155 | end 156 | end 157 | Vips::Image.class_eval do 158 | def copy 159 | return self if caller.first.end_with?("/gems/ruby-vips-2.0.9/lib/vips/operation.rb:148:in `set'") && 160 | %w{ draw_line draw_circle }.include?(instance_variable_get "@operation_name") 161 | method_missing :copy 162 | end 163 | end 164 | 165 | require "get_process_mem" 166 | a, b = [[a, ad, ai], [b, bd, bi]].map do |image, xd, xi| 167 | _127.downto(0).each_with_index do |i, ii| 168 | mem = GetProcessMem.new(Process.pid).mb 169 | abort ">1000mb of memory consumed" if 1000 < mem 170 | if i > _63 171 | y, x = (_127 - i).divmod size 172 | else 173 | x, y = (_63 - i).divmod size 174 | end 175 | x = (width * (x + 0.5) / size).round 176 | y = (height * (y + 0.5) / size).round 177 | if i > _63 178 | (x-2..x+2).map do |x| [ 179 | [x, y , x, (y + height / size / 2 - 1) % height], 180 | [x, (y + height / size / 2 + 1) % height, x, (y + height / size ) % height], 181 | ] end 182 | else 183 | (y-2..y+2).map do |y| [ 184 | [ x , y, (x + width / size / 2 - 1) % width, y], 185 | [(x + width / size / 2 + 1) % width, y, (x + width / size ) % width, y], 186 | ] end 187 | end.each do |coords1, coords2| 188 | n += 1 189 | image = image.draw_line (1 - xd[i]) * 255, *coords1 190 | image = image.draw_line xd[i] * 255, *coords2 191 | end if ai[i] + bi[i] > 0 && ad[i] != bd[i] 192 | cx, cy = if i > _63 193 | [x, y + 30] 194 | else 195 | [x + 30, y] 196 | end 197 | image = image.draw_circle xd[i] * 255, cx, cy, 11, fill: true if xi[i] > 0 198 | image = image.draw_circle (1 - xd[i]) * 255, cx, cy, 10, fill: true if xi[i] > 0 199 | end 200 | image 201 | end 202 | puts "distance: #{n / 10}" 203 | puts "(above should be equal if raketask works correctly)" 204 | 205 | a.join(b, :horizontal, shim: 15).write_to_file "ab.png" 206 | puts "the ab.png is ready" 207 | end 208 | 209 | desc "Benchmark speed of Dhash, DHashVips::DHash, DHashVips::IDHash and Phamilie" 210 | task :compare_speed do 211 | require "dhash" 212 | require "phamilie" 213 | phamilie = Phamilie.new 214 | require_relative "lib/dhash-vips" 215 | 216 | filenames = %w{ 217 | 71662d4d4029a3b41d47d5baf681ab9a.jpg 218 | ad8a37f872956666c3077a3e9e737984.jpg 219 | 1d468d064d2e26b5b5de9a0241ef2d4b.jpg 220 | 92d90b8977f813af803c78107e7f698e.jpg 221 | 309666c7b45ecbf8f13e85a0bd6b0a4c.jpg 222 | 3f9f3db06db20d1d9f8188cd753f6ef4.jpg 223 | df0a3b93e9412536ee8a11255f974141.jpg 224 | 679634ff89a31279a39f03e278bc9a01.jpg 225 | }.flat_map do |filename| 226 | image = Vips::Image.new_from_file "images/#{filename}" 227 | [0, 1, 2, 3].map do |a| 228 | "benchmark/#{a}_#{filename}".tap do |filename| 229 | next if File.exist? filename 230 | FileUtils.mkdir_p "benchmark" 231 | image.rot(a).write_to_file filename 232 | end 233 | end 234 | end 235 | 236 | require "benchmark" 237 | puts "load the image and calculate the fingerprint:" 238 | hashes = [] 239 | Benchmark.bm 19 do |bm| 240 | [ 241 | [Dhash, :calculate], 242 | [phamilie, :fingerprint], 243 | [DHashVips::DHash, :calculate], 244 | [DHashVips::IDHash, :fingerprint], 245 | [DHashVips::IDHash, :fingerprint, 4], 246 | ].each do |m, calc, power| 247 | bm.report "#{m.is_a?(Module) ? m : m.class} #{power}" do 248 | hashes.push filenames.map{ |filename| m.send calc, filename, *power } 249 | end 250 | end 251 | end 252 | 253 | # for `distance`, `distance3_ruby` and `distance3_c` we use the same hashes 254 | # this array manipulation converts [1, 2, 3, 4, 5] into [1, 2, 3, 4, 4, 4, 5] 255 | hashes[-1, 1] = hashes[-2, 2] 256 | hashes[-1, 1] = hashes[-2, 2] 257 | 258 | puts "\nmeasure the distance (32*32*2000 times):" 259 | Benchmark.bm 32 do |bm| 260 | [ 261 | [Dhash, :hamming], 262 | [phamilie, :distance, nil, 1], 263 | [DHashVips::DHash, :hamming], 264 | [DHashVips::IDHash, :distance], 265 | [DHashVips::IDHash, :distance3_ruby], 266 | [DHashVips::IDHash, :distance3_c], 267 | [DHashVips::IDHash, :distance, 4], 268 | ].zip(hashes) do |(m, dm, power, ii), hs| 269 | bm.report "#{m.is_a?(Module) ? m : m.class} #{dm} #{power}" do 270 | _ = [hs, filenames][ii || 0] 271 | _.product _ do |h1, h2| 272 | 2000.times{ m.public_send dm, h1, h2 } 273 | end 274 | end 275 | end 276 | end 277 | 278 | end 279 | 280 | desc "Benchmarks everything about gems" 281 | task :benchmark do 282 | # TODO: better handling of the need to `ruby extconf.rb && make clean && make` 283 | system "ruby -v" 284 | puts "" 285 | 286 | system "apt-cache show libvips42 2>/dev/null | grep Version" 287 | system "vips -v 2>/dev/null" 288 | system "apt-cache show libmagickwand-dev 2>/dev/null | grep Version" 289 | system "identify -version 2>/dev/null | /usr/bin/head -1" 290 | system "identify-6 -version 2>/dev/null | /usr/bin/head -1" 291 | system "sysctl -n machdep.cpu.brand_string 2>/dev/null" 292 | system "cat /proc/cpuinfo 2>/dev/null | grep 'model name' | uniq" 293 | puts "" 294 | 295 | require_relative "lib/dhash-vips" 296 | puts "gem ruby-vips: #{Gem.loaded_specs["ruby-vips"].version}" 297 | puts "" 298 | 299 | puts "gem rmagick: #{Gem.loaded_specs["rmagick"].version}" 300 | require "dhash" ; puts "gem dhash: #{Gem.loaded_specs["dhash"].source}" 301 | require "phamilie" ; puts "gem phamilie: #{Gem.loaded_specs["phamilie"].version}" 302 | phamilie = Phamilie.new 303 | require "mini_magick" 304 | require "phash" ; puts "gem phash-rb: #{Gem.loaded_specs["phash-rb"].source}" 305 | puts "" 306 | 307 | filenames = [ 308 | %w{ benchmark_images/0/6d97739b4a08f965dc9239dd24382e96.jpg }, 309 | %w{ benchmark_images/1/7a833d873f8d49f12882e86af1cc6b79.jpg benchmark_images/1/ac033cf01a3941dd1baa876082938bc9.jpg }, 310 | %w{ benchmark_images/2/9c2c240ec02356472fb532f404d28dde.jpg benchmark_images/2/fc762fa286489d8afc80adc8cdcb125e.jpg }, 311 | %w{ benchmark_images/3/21cd9a6986d98976b6b4655e1de7baf4.jpg benchmark_images/3/9b158c0d4953d47171a22ed84917f812.jpg }, 312 | %w{ benchmark_images/4/4b62e0eef58bfbc8d0d2fbf2b9d05483.jpg benchmark_images/4/b8eb0ca91855b657f12fb3d627d45c53.jpg }, 313 | %w{ benchmark_images/5/54192a3f65bd03163b04849e1577a40b.jpg benchmark_images/5/6d32f57459e5b79b5deca2a361eb8c6e.jpg }, 314 | %w{ benchmark_images/6/679634ff89a31279a39f03e278bc9a01.jpg benchmark_images/6/df0a3b93e9412536ee8a11255f974141.jpg }, 315 | %w{ benchmark_images/7/309666c7b45ecbf8f13e85a0bd6b0a4c.jpg benchmark_images/7/3f9f3db06db20d1d9f8188cd753f6ef4.jpg }, 316 | %w{ benchmark_images/8/1d468d064d2e26b5b5de9a0241ef2d4b.jpg benchmark_images/8/92d90b8977f813af803c78107e7f698e.jpg }, 317 | %w{ benchmark_images/9/1b1d4bde376084011d027bba1c047a4b.jpg }, 318 | %w{ benchmark_images/10/71662d4d4029a3b41d47d5baf681ab9a.jpg benchmark_images/10/ad8a37f872956666c3077a3e9e737984.jpg } 319 | ].each{ |g| g.each(&method(:download_if_needed)) } 320 | puts "image groups sizes: #{filenames.map &:size}" 321 | require "benchmark" 322 | 323 | puts "step 1 / 3 (fingerprinting)" 324 | hashes = [] 325 | bm1 = [ 326 | Benchmark.realtime{ hashes.push filenames.flatten.map{ |filename| ::Dhash.calculate filename } }, 327 | Benchmark.realtime{ hashes.push filenames.flatten.map{ |filename| phamilie.fingerprint filename; filename } }, 328 | Benchmark.realtime{ hashes.push filenames.flatten.map{ |filename| ::DHashVips::IDHash.fingerprint filename } }, 329 | Benchmark.realtime{ hashes.push filenames.flatten.map{ |filename| ::DHashVips::IDHash.fingerprint filename } }, 330 | Benchmark.realtime{ hashes.push filenames.flatten.map{ |filename| ::DHashVips::DHash.calculate filename } }, 331 | Benchmark.realtime{ hashes.push filenames.flatten.map{ |filename| ::Phash::Image.new(filename).tap &:fingerprint } }, 332 | ] 333 | 334 | puts "step 2 / 3 (comparing fingerprints)" 335 | combs = filenames.flatten.size ** 2 336 | n = 10_000_000_000_000 / combs / filenames.flatten.map(&File.method(:size)).inject(:+) 337 | bm2 = [ 338 | Benchmark.realtime{ hashes[0].product(hashes[0]){ |h1, h2| n.times{ ::Dhash.hamming h1, h2 } } }, 339 | Benchmark.realtime{ hashes[1].product(hashes[1]){ |p1, p2| n.times{ phamilie.distance p1, p2 } } }, 340 | Benchmark.realtime{ hashes[2].product(hashes[2]){ |h1, h2| n.times{ ::DHashVips::IDHash.distance3 h1, h2 } } }, 341 | Benchmark.realtime{ hashes[3].product(hashes[3]){ |h1, h2| n.times{ ::DHashVips::IDHash.distance3_ruby h1, h2 } } }, 342 | Benchmark.realtime{ hashes[4].product(hashes[4]){ |h1, h2| n.times{ ::DHashVips::DHash.hamming h1, h2 } } }, 343 | Benchmark.realtime{ hashes[5].product(hashes[5]){ |h1, h2| n.times{ h1.distance_from h2 } } }, 344 | ] 345 | 346 | puts "step 3 / 3 (looking for the best threshold)" 347 | bm3 = [ 348 | ["Dhash", ->a,b{ ::Dhash.hamming a, b }], 349 | ["Phamilie", ->a,b{ phamilie.distance a, b }], 350 | ["IDHash default", ->a,b{ ::DHashVips::IDHash.distance3 a, b }], 351 | ["IDHash Ruby", ->a,b{ ::DHashVips::IDHash.distance3 a, b }], 352 | ["DHash", ->a,b{ ::DHashVips::DHash.hamming a, b }], 353 | ["Phash", ->a,b{ a.distance_from b }], 354 | ].zip(hashes).map do |(name, f), hs| 355 | report = Struct.new(:same, :sim, :not_sim).new [], [], [] 356 | hs.size.times.to_a.repeated_combination(2) do |i, j| 357 | report[ 358 | case 359 | when i == j ; :same 360 | when File.split(File.split(filenames.flatten[i]).first).last == 361 | File.split(File.split(filenames.flatten[j]).first).last && i < j ; :sim 362 | else ; :not_sim 363 | end 364 | ].push f[hs[i], hs[j]] 365 | end 366 | # p report 367 | min, max = [*report.sim, *report.not_sim].minmax 368 | p [name, min, max] 369 | fmi, fp, fn = (min..max+1).map do |b| 370 | fp = report.not_sim.count{ |_| _ < b } 371 | tp = report.sim.count{ |_| _ < b } 372 | fn = report.sim.count{ |_| _ >= b } 373 | [((tp + fp) * (tp + fn)).fdiv(tp * tp), fp, fn] 374 | end.reject{ |_,| _.nan? }.min_by(&:first) 375 | [name, fmi] 376 | end 377 | 378 | require "mll" 379 | puts MLL::grid.call %w{ \ Fingerprint Compare 1/FMI^2 }.zip(*[ 380 | bm3.map(&:first), 381 | *[bm1, bm2, bm3.map(&:last)].map{ |bm| bm.map{ |_| "%.3f" % _ } } 382 | ].transpose).transpose, spacings: [1.5, 0], alignment: :right 383 | puts "(lower numbers are better)" 384 | end 385 | --------------------------------------------------------------------------------