├── lib ├── searchkick │ ├── version.rb │ ├── middleware.rb │ ├── hash_wrapper.rb │ ├── bulk_reindex_job.rb │ ├── process_queue_job.rb │ ├── tasks.rb │ ├── indexer.rb │ ├── process_batch_job.rb │ ├── reindex_queue.rb │ ├── reindex_v2_job.rb │ ├── multi_search.rb │ ├── record_indexer.rb │ ├── record_data.rb │ ├── model.rb │ ├── bulk_indexer.rb │ ├── results.rb │ ├── logging.rb │ ├── index.rb │ └── index_options.rb └── searchkick.rb ├── .github └── ISSUE_TEMPLATE.md ├── test ├── gemfiles │ ├── cequel.gemfile │ ├── mongoid5.gemfile │ ├── activerecord42.gemfile │ ├── activerecord50.gemfile │ ├── activerecord51.gemfile │ ├── nobrainer.gemfile │ ├── apartment.gemfile │ ├── parallel_tests.gemfile │ └── mongoid6.gemfile ├── marshal_test.rb ├── errors_test.rb ├── multi_tenancy_test.rb ├── support │ └── kaminari.yml ├── routing_test.rb ├── ci │ └── before_install.sh ├── should_index_test.rb ├── reindex_v2_job_test.rb ├── model_test.rb ├── multi_search_test.rb ├── similar_test.rb ├── query_test.rb ├── order_test.rb ├── callbacks_test.rb ├── partial_reindex_test.rb ├── misspellings_test.rb ├── synonyms_test.rb ├── inheritance_test.rb ├── reindex_test.rb ├── language_test.rb ├── autocomplete_test.rb ├── suggest_test.rb ├── pagination_test.rb ├── highlight_test.rb ├── geo_shape_test.rb ├── index_test.rb ├── sql_test.rb ├── aggs_test.rb ├── boost_test.rb ├── match_test.rb ├── where_test.rb └── test_helper.rb ├── .gitignore ├── Gemfile ├── Rakefile ├── benchmark ├── Gemfile ├── search.rb └── index.rb ├── .travis.yml ├── LICENSE.txt ├── searchkick.gemspec ├── docs └── Searchkick-3-Upgrade.md ├── CONTRIBUTING.md └── CHANGELOG.md /lib/searchkick/version.rb: -------------------------------------------------------------------------------- 1 | module Searchkick 2 | VERSION = "3.0.3" 3 | end 4 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | Hi, 2 | 3 | Before creating an issue, please check out the Contributing Guide: 4 | 5 | https://github.com/ankane/searchkick/blob/master/CONTRIBUTING.md 6 | 7 | Thanks! 8 | -------------------------------------------------------------------------------- /test/gemfiles/cequel.gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Specify your gem's dependencies in searchkick.gemspec 4 | gemspec path: "../../" 5 | 6 | gem "cequel" 7 | gem "activejob" 8 | gem "redis" 9 | -------------------------------------------------------------------------------- /test/gemfiles/mongoid5.gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Specify your gem's dependencies in searchkick.gemspec 4 | gemspec path: "../../" 5 | 6 | gem "mongoid", "~> 5.0.0" 7 | gem "activejob" 8 | -------------------------------------------------------------------------------- /test/gemfiles/activerecord42.gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Specify your gem's dependencies in searchkick.gemspec 4 | gemspec path: "../../" 5 | 6 | gem "sqlite3" 7 | gem "activerecord", "~> 4.2.0" 8 | -------------------------------------------------------------------------------- /test/gemfiles/activerecord50.gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Specify your gem's dependencies in searchkick.gemspec 4 | gemspec path: "../../" 5 | 6 | gem "sqlite3" 7 | gem "activerecord", "~> 5.0.0" 8 | -------------------------------------------------------------------------------- /test/gemfiles/activerecord51.gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Specify your gem's dependencies in searchkick.gemspec 4 | gemspec path: "../../" 5 | 6 | gem "sqlite3" 7 | gem "activerecord", "~> 5.1.0" 8 | -------------------------------------------------------------------------------- /test/gemfiles/nobrainer.gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Specify your gem's dependencies in searchkick.gemspec 4 | gemspec path: "../../" 5 | 6 | gem "nobrainer", ">= 0.21.0" 7 | gem "activejob" 8 | gem "redis" 9 | -------------------------------------------------------------------------------- /test/gemfiles/apartment.gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Specify your gem's dependencies in searchkick.gemspec 4 | gemspec path: "../../" 5 | 6 | gem "sqlite3" 7 | gem "activerecord", "~> 4.2.0" 8 | gem "apartment" 9 | -------------------------------------------------------------------------------- /test/gemfiles/parallel_tests.gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Specify your gem's dependencies in searchkick.gemspec 4 | gemspec path: "../../" 5 | 6 | gem "sqlite3" 7 | gem "activerecord", "~> 5.0.0" 8 | gem "parallel_tests" 9 | -------------------------------------------------------------------------------- /test/gemfiles/mongoid6.gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Specify your gem's dependencies in searchkick.gemspec 4 | gemspec path: "../../" 5 | 6 | gem "mongoid", "~> 6.0.0" 7 | gem "activejob" 8 | gem "redis" 9 | 10 | # kaminari 11 | gem "actionpack" 12 | gem "kaminari" 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | *.rbc 3 | .bundle 4 | .config 5 | .yardoc 6 | *.lock 7 | InstalledFiles 8 | _yardoc 9 | coverage 10 | doc/ 11 | lib/bundler/man 12 | pkg 13 | rdoc 14 | spec/reports 15 | test/tmp 16 | test/version_tmp 17 | tmp 18 | *.log 19 | .DS_Store 20 | .ruby-* 21 | .idea/ 22 | *.sqlite3 23 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | # Specify your gem's dependencies in searchkick.gemspec 4 | gemspec 5 | 6 | gem "sqlite3" 7 | gem "activerecord" 8 | gem "gemoji-parser" 9 | gem "typhoeus" 10 | gem "activejob" 11 | gem "redis" 12 | gem "connection_pool" 13 | 14 | # kaminari 15 | gem "actionpack" 16 | gem "kaminari" 17 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | require "rake/testtask" 3 | 4 | begin 5 | require "parallel_tests/tasks" 6 | require "shellwords" 7 | rescue LoadError 8 | # do nothing 9 | end 10 | 11 | task default: :test 12 | Rake::TestTask.new do |t| 13 | t.libs << "test" 14 | t.pattern = "test/**/*_test.rb" 15 | t.warning = false 16 | end 17 | -------------------------------------------------------------------------------- /lib/searchkick/middleware.rb: -------------------------------------------------------------------------------- 1 | require "faraday/middleware" 2 | 3 | module Searchkick 4 | class Middleware < Faraday::Middleware 5 | def call(env) 6 | if env[:method] == :get && env[:url].path.to_s.end_with?("/_search") 7 | env[:request][:timeout] = Searchkick.search_timeout 8 | end 9 | @app.call(env) 10 | end 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /test/marshal_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | class MarshalTest < Minitest::Test 4 | def test_marshal 5 | store_names ["Product A"] 6 | assert Marshal.dump(Product.search("*").results) 7 | end 8 | 9 | def test_marshal_highlights 10 | store_names ["Product A"] 11 | assert Marshal.dump(Product.search("product", highlight: true, load: {dumpable: true}).results) 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /lib/searchkick/hash_wrapper.rb: -------------------------------------------------------------------------------- 1 | module Searchkick 2 | # Subclass of `Hashie::Mash` to wrap Hash-like structures 3 | # (responses from Elasticsearch) 4 | # 5 | # The primary goal of the subclass is to disable the 6 | # warning being printed by Hashie for re-defined 7 | # methods, such as `sort`. 8 | # 9 | class HashWrapper < ::Hashie::Mash 10 | disable_warnings if respond_to?(:disable_warnings) 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /benchmark/Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | # Specify your gem's dependencies in searchkick.gemspec 4 | gemspec path: "../" 5 | 6 | gem "sqlite3" 7 | gem "pg" 8 | gem "activerecord", "~> 5.1.0" 9 | gem "activerecord-import" 10 | gem "activejob" 11 | gem "redis" 12 | gem "sidekiq" 13 | 14 | # performance 15 | gem "typhoeus" 16 | gem "oj" 17 | 18 | # profiling 19 | gem "ruby-prof" 20 | gem "allocation_stats" 21 | gem "get_process_mem" 22 | gem "memory_profiler" 23 | gem "allocation_tracer" 24 | gem "benchmark-ips" 25 | -------------------------------------------------------------------------------- /test/errors_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | class ErrorsTest < Minitest::Test 4 | def test_bulk_import_raises_error 5 | valid_dog = Product.create(name: "2016-01-02") 6 | invalid_dog = Product.create(name: "Ol' One-Leg") 7 | index = Searchkick::Index.new "dogs", mappings: { 8 | dog: { 9 | properties: { 10 | name: {type: "date"} 11 | } 12 | } 13 | } 14 | index.store valid_dog 15 | assert_raises(Searchkick::ImportError) do 16 | index.bulk_index [valid_dog, invalid_dog] 17 | end 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /lib/searchkick/bulk_reindex_job.rb: -------------------------------------------------------------------------------- 1 | module Searchkick 2 | class BulkReindexJob < ActiveJob::Base 3 | queue_as { Searchkick.queue_name } 4 | 5 | def perform(class_name:, record_ids: nil, index_name: nil, method_name: nil, batch_id: nil, min_id: nil, max_id: nil) 6 | klass = class_name.constantize 7 | index = index_name ? Searchkick::Index.new(index_name, **klass.searchkick_options) : klass.searchkick_index 8 | record_ids ||= min_id..max_id 9 | index.import_scope( 10 | Searchkick.load_records(klass, record_ids), 11 | method_name: method_name, 12 | batch: true, 13 | batch_id: batch_id 14 | ) 15 | end 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /test/multi_tenancy_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | class MultiTenancyTest < Minitest::Test 4 | def setup 5 | skip unless defined?(Apartment) 6 | end 7 | 8 | def test_basic 9 | Apartment::Tenant.switch!("tenant1") 10 | store_names ["Product A"], Tenant 11 | Apartment::Tenant.switch!("tenant2") 12 | store_names ["Product B"], Tenant 13 | Apartment::Tenant.switch!("tenant1") 14 | assert_search "product", ["Product A"], {load: false}, Tenant 15 | Apartment::Tenant.switch!("tenant2") 16 | assert_search "product", ["Product B"], {load: false}, Tenant 17 | end 18 | 19 | def teardown 20 | Apartment::Tenant.reset if defined?(Apartment) 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /test/support/kaminari.yml: -------------------------------------------------------------------------------- 1 | en: 2 | views: 3 | pagination: 4 | first: "« First" 5 | last: "Last »" 6 | previous: "‹ Prev" 7 | next: "Next ›" 8 | truncate: "…" 9 | helpers: 10 | page_entries_info: 11 | entry: 12 | zero: "entries" 13 | one: "entry" 14 | other: "entries" 15 | one_page: 16 | display_entries: 17 | zero: "No %{entry_name} found" 18 | one: "Displaying 1 %{entry_name}" 19 | other: "Displaying all %{count} %{entry_name}" 20 | more_pages: 21 | display_entries: "Displaying %{entry_name} %{first} - %{last} of %{total} in total" 22 | -------------------------------------------------------------------------------- /lib/searchkick/process_queue_job.rb: -------------------------------------------------------------------------------- 1 | module Searchkick 2 | class ProcessQueueJob < ActiveJob::Base 3 | queue_as { Searchkick.queue_name } 4 | 5 | def perform(class_name:) 6 | model = class_name.constantize 7 | 8 | limit = model.searchkick_index.options[:batch_size] || 1000 9 | record_ids = model.searchkick_index.reindex_queue.reserve(limit: limit) 10 | if record_ids.any? 11 | Searchkick::ProcessBatchJob.perform_later( 12 | class_name: model.name, 13 | record_ids: record_ids 14 | ) 15 | # TODO when moving to reliable queuing, mark as complete 16 | 17 | if record_ids.size == limit 18 | Searchkick::ProcessQueueJob.perform_later(class_name: class_name) 19 | end 20 | end 21 | end 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /test/routing_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | class RoutingTest < Minitest::Test 4 | def test_routing_query 5 | query = Store.search("Dollar Tree", routing: "Dollar Tree", execute: false) 6 | assert_equal query.params[:routing], "Dollar Tree" 7 | end 8 | 9 | def test_routing_mappings 10 | index_options = Store.searchkick_index.index_options 11 | assert_equal index_options[:mappings]["store"][:_routing], required: true 12 | end 13 | 14 | def test_routing_correct_node 15 | store_names ["Dollar Tree"], Store 16 | assert_search "*", ["Dollar Tree"], {routing: "Dollar Tree"}, Store 17 | end 18 | 19 | def test_routing_incorrect_node 20 | store_names ["Dollar Tree"], Store 21 | assert_search "*", ["Dollar Tree"], {routing: "Boom"}, Store 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /lib/searchkick/tasks.rb: -------------------------------------------------------------------------------- 1 | namespace :searchkick do 2 | desc "reindex model" 3 | task reindex: :environment do 4 | if ENV["CLASS"] 5 | klass = ENV["CLASS"].constantize rescue nil 6 | if klass 7 | klass.reindex 8 | else 9 | abort "Could not find class: #{ENV['CLASS']}" 10 | end 11 | else 12 | abort "USAGE: rake searchkick:reindex CLASS=Product" 13 | end 14 | end 15 | 16 | if defined?(Rails) 17 | namespace :reindex do 18 | desc "reindex all models" 19 | task all: :environment do 20 | Rails.application.eager_load! 21 | Searchkick.models.each do |model| 22 | puts "Reindexing #{model.name}..." 23 | model.reindex 24 | end 25 | puts "Reindex complete" 26 | end 27 | end 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /test/ci/before_install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | gem install bundler 6 | 7 | if [[ $ELASTICSEARCH_VERSION == 1* ]]; then 8 | curl -L -O https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-$ELASTICSEARCH_VERSION.tar.gz 9 | elif [[ $ELASTICSEARCH_VERSION == 2* ]]; then 10 | curl -L -O https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/$ELASTICSEARCH_VERSION/elasticsearch-$ELASTICSEARCH_VERSION.tar.gz 11 | else 12 | curl -L -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-$ELASTICSEARCH_VERSION.tar.gz 13 | fi 14 | tar -xvf elasticsearch-$ELASTICSEARCH_VERSION.tar.gz 15 | cd elasticsearch-$ELASTICSEARCH_VERSION/bin 16 | ./elasticsearch -d 17 | wget -O- --waitretry=1 --tries=60 --retry-connrefused -v http://127.0.0.1:9200/ 18 | -------------------------------------------------------------------------------- /lib/searchkick/indexer.rb: -------------------------------------------------------------------------------- 1 | module Searchkick 2 | class Indexer 3 | attr_reader :queued_items 4 | 5 | def initialize 6 | @queued_items = [] 7 | end 8 | 9 | def queue(items) 10 | @queued_items.concat(items) 11 | perform unless Searchkick.callbacks_value == :bulk 12 | end 13 | 14 | def perform 15 | items = @queued_items 16 | @queued_items = [] 17 | if items.any? 18 | response = Searchkick.client.bulk(body: items) 19 | if response["errors"] 20 | first_with_error = response["items"].map do |item| 21 | (item["index"] || item["delete"] || item["update"]) 22 | end.find { |item| item["error"] } 23 | raise Searchkick::ImportError, "#{first_with_error["error"]} on item with id '#{first_with_error["_id"]}'" 24 | end 25 | end 26 | end 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /test/should_index_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | class ShouldIndexTest < Minitest::Test 4 | def test_basic 5 | store_names ["INDEX", "DO NOT INDEX"] 6 | assert_search "index", ["INDEX"] 7 | end 8 | 9 | def test_default_true 10 | assert Animal.new.should_index? 11 | end 12 | 13 | def test_change_to_true 14 | store_names ["DO NOT INDEX"] 15 | assert_search "index", [] 16 | product = Product.first 17 | product.name = "INDEX" 18 | product.save! 19 | Product.searchkick_index.refresh 20 | assert_search "index", ["INDEX"] 21 | end 22 | 23 | def test_change_to_false 24 | store_names ["INDEX"] 25 | assert_search "index", ["INDEX"] 26 | product = Product.first 27 | product.name = "DO NOT INDEX" 28 | product.save! 29 | Product.searchkick_index.refresh 30 | assert_search "index", [] 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /lib/searchkick/process_batch_job.rb: -------------------------------------------------------------------------------- 1 | module Searchkick 2 | class ProcessBatchJob < ActiveJob::Base 3 | queue_as { Searchkick.queue_name } 4 | 5 | def perform(class_name:, record_ids:) 6 | klass = class_name.constantize 7 | scope = Searchkick.load_records(klass, record_ids) 8 | scope = scope.search_import if scope.respond_to?(:search_import) 9 | records = scope.select(&:should_index?) 10 | 11 | # determine which records to delete 12 | delete_ids = record_ids - records.map { |r| r.id.to_s } 13 | delete_records = delete_ids.map { |id| m = klass.new; m.id = id; m } 14 | 15 | # bulk reindex 16 | index = klass.searchkick_index 17 | Searchkick.callbacks(:bulk) do 18 | index.bulk_index(records) if records.any? 19 | index.bulk_delete(delete_records) if delete_records.any? 20 | end 21 | end 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /test/reindex_v2_job_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | class ReindexV2JobTest < Minitest::Test 4 | def setup 5 | skip unless defined?(ActiveJob) 6 | super 7 | end 8 | 9 | def test_create 10 | product = Searchkick.callbacks(false) { Product.create!(name: "Boom") } 11 | Product.search_index.refresh 12 | assert_search "*", [] 13 | Searchkick::ReindexV2Job.perform_later("Product", product.id.to_s) 14 | Product.search_index.refresh 15 | assert_search "*", ["Boom"] 16 | end 17 | 18 | def test_destroy 19 | product = Searchkick.callbacks(false) { Product.create!(name: "Boom") } 20 | Product.reindex 21 | assert_search "*", ["Boom"] 22 | Searchkick.callbacks(false) { product.destroy } 23 | Searchkick::ReindexV2Job.perform_later("Product", product.id.to_s) 24 | Product.search_index.refresh 25 | assert_search "*", [] 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: ruby 3 | rvm: 2.4.2 4 | services: 5 | - mongodb 6 | - redis-server 7 | before_install: 8 | - ./test/ci/before_install.sh 9 | script: bundle exec rake test 10 | before_script: 11 | - psql -c 'create database searchkick_test;' -U postgres 12 | notifications: 13 | email: 14 | on_success: never 15 | on_failure: change 16 | gemfile: 17 | - Gemfile 18 | - test/gemfiles/activerecord51.gemfile 19 | - test/gemfiles/activerecord50.gemfile 20 | - test/gemfiles/activerecord42.gemfile 21 | - test/gemfiles/mongoid5.gemfile 22 | - test/gemfiles/mongoid6.gemfile 23 | env: 24 | - ELASTICSEARCH_VERSION=6.2.3 25 | jdk: oraclejdk8 26 | matrix: 27 | include: 28 | - gemfile: Gemfile 29 | env: ELASTICSEARCH_VERSION=5.0.1 30 | - gemfile: Gemfile 31 | env: ELASTICSEARCH_VERSION=5.6.7 32 | - gemfile: Gemfile 33 | env: ELASTICSEARCH_VERSION=6.0.0 34 | -------------------------------------------------------------------------------- /lib/searchkick/reindex_queue.rb: -------------------------------------------------------------------------------- 1 | module Searchkick 2 | class ReindexQueue 3 | attr_reader :name 4 | 5 | def initialize(name) 6 | @name = name 7 | 8 | raise Searchkick::Error, "Searchkick.redis not set" unless Searchkick.redis 9 | end 10 | 11 | def push(record_id) 12 | Searchkick.with_redis { |r| r.lpush(redis_key, record_id) } 13 | end 14 | 15 | # TODO use reliable queuing 16 | def reserve(limit: 1000) 17 | record_ids = Set.new 18 | while record_ids.size < limit && (record_id = Searchkick.with_redis { |r| r.rpop(redis_key) }) 19 | record_ids << record_id 20 | end 21 | record_ids.to_a 22 | end 23 | 24 | def clear 25 | Searchkick.with_redis { |r| r.del(redis_key) } 26 | end 27 | 28 | def length 29 | Searchkick.with_redis { |r| r.llen(redis_key) } 30 | end 31 | 32 | private 33 | 34 | def redis_key 35 | "searchkick:reindex_queue:#{name}" 36 | end 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /lib/searchkick/reindex_v2_job.rb: -------------------------------------------------------------------------------- 1 | module Searchkick 2 | class ReindexV2Job < ActiveJob::Base 3 | RECORD_NOT_FOUND_CLASSES = [ 4 | "ActiveRecord::RecordNotFound", 5 | "Mongoid::Errors::DocumentNotFound", 6 | "NoBrainer::Error::DocumentNotFound", 7 | "Cequel::Record::RecordNotFound" 8 | ] 9 | 10 | queue_as { Searchkick.queue_name } 11 | 12 | def perform(klass, id, method_name = nil) 13 | model = klass.constantize 14 | record = 15 | begin 16 | if model.respond_to?(:unscoped) 17 | model.unscoped.find(id) 18 | else 19 | model.find(id) 20 | end 21 | rescue => e 22 | # check by name rather than rescue directly so we don't need 23 | # to determine which classes are defined 24 | raise e unless RECORD_NOT_FOUND_CLASSES.include?(e.class.name) 25 | nil 26 | end 27 | 28 | unless record 29 | record = model.new 30 | record.id = id 31 | end 32 | 33 | RecordIndexer.new(record).reindex(method_name, mode: true) 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /lib/searchkick/multi_search.rb: -------------------------------------------------------------------------------- 1 | module Searchkick 2 | class MultiSearch 3 | attr_reader :queries 4 | 5 | def initialize(queries) 6 | @queries = queries 7 | end 8 | 9 | def perform 10 | if queries.any? 11 | perform_search(queries) 12 | end 13 | end 14 | 15 | private 16 | 17 | def perform_search(search_queries, perform_retry: true) 18 | responses = client.msearch(body: search_queries.flat_map { |q| [q.params.except(:body), q.body] })["responses"] 19 | 20 | retry_queries = [] 21 | search_queries.each_with_index do |query, i| 22 | if perform_retry && query.retry_misspellings?(responses[i]) 23 | query.send(:prepare) # okay, since we don't want to expose this method outside Searchkick 24 | retry_queries << query 25 | else 26 | query.handle_response(responses[i]) 27 | end 28 | end 29 | 30 | if retry_queries.any? 31 | perform_search(retry_queries, perform_retry: false) 32 | end 33 | 34 | search_queries 35 | end 36 | 37 | def client 38 | Searchkick.client 39 | end 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013-2018 Andrew Kane 2 | 3 | MIT License 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /test/model_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | class ModelTest < Minitest::Test 4 | def test_disable_callbacks_model 5 | store_names ["product a"] 6 | 7 | Searchkick.callbacks(false) do 8 | store_names ["product b"] 9 | end 10 | assert_search "product", ["product a"] 11 | 12 | Product.reindex 13 | 14 | assert_search "product", ["product a", "product b"] 15 | end 16 | 17 | def test_disable_callbacks_global 18 | # make sure callbacks default to on 19 | assert Searchkick.callbacks? 20 | 21 | store_names ["product a"] 22 | 23 | Searchkick.disable_callbacks 24 | assert !Searchkick.callbacks? 25 | 26 | store_names ["product b"] 27 | assert_search "product", ["product a"] 28 | 29 | Searchkick.enable_callbacks 30 | Product.reindex 31 | 32 | assert_search "product", ["product a", "product b"] 33 | end 34 | 35 | def test_multiple_models 36 | store_names ["Product A"] 37 | store_names ["Product B"], Speaker 38 | assert_equal Product.all.to_a + Speaker.all.to_a, Searchkick.search("product", index_name: [Product, Speaker], fields: [:name], order: "name").to_a 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /benchmark/search.rb: -------------------------------------------------------------------------------- 1 | require "bundler/setup" 2 | Bundler.require(:default) 3 | require "active_record" 4 | require "benchmark/ips" 5 | 6 | ActiveRecord::Base.default_timezone = :utc 7 | ActiveRecord::Base.time_zone_aware_attributes = true 8 | ActiveRecord::Base.establish_connection adapter: "sqlite3", database: "/tmp/searchkick" 9 | 10 | class Product < ActiveRecord::Base 11 | searchkick batch_size: 1000 12 | 13 | def search_data 14 | { 15 | name: name, 16 | color: color, 17 | store_id: store_id 18 | } 19 | end 20 | end 21 | 22 | if ENV["SETUP"] 23 | total_docs = 1000000 24 | 25 | ActiveRecord::Migration.create_table :products, force: :cascade do |t| 26 | t.string :name 27 | t.string :color 28 | t.integer :store_id 29 | end 30 | 31 | Product.import ["name", "color", "store_id"], total_docs.times.map { |i| ["Product #{i}", ["red", "blue"].sample, rand(10)] } 32 | 33 | puts "Imported" 34 | 35 | Product.reindex 36 | 37 | puts "Reindexed" 38 | end 39 | 40 | query = Product.search("product", fields: [:name], where: {color: "red", store_id: 5}, limit: 10000, load: false, execute: false) 41 | 42 | require "pp" 43 | pp query.body.as_json 44 | puts 45 | 46 | Benchmark.ips do |x| 47 | x.report { query.dup.execute } 48 | end 49 | -------------------------------------------------------------------------------- /searchkick.gemspec: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | lib = File.expand_path("../lib", __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | require "searchkick/version" 5 | 6 | Gem::Specification.new do |spec| 7 | spec.name = "searchkick" 8 | spec.version = Searchkick::VERSION 9 | spec.authors = ["Andrew Kane"] 10 | spec.email = ["andrew@chartkick.com"] 11 | spec.description = "Intelligent search made easy" 12 | spec.summary = "Searchkick learns what your users are looking for. As more people search, it gets smarter and the results get better. It’s friendly for developers - and magical for your users." 13 | spec.homepage = "https://github.com/ankane/searchkick" 14 | spec.license = "MIT" 15 | 16 | spec.files = `git ls-files`.split($/) 17 | spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } 18 | spec.test_files = spec.files.grep(%r{^(test|spec|features|benchmark)/}) 19 | spec.require_paths = ["lib"] 20 | 21 | spec.required_ruby_version = ">= 2.2.0" 22 | 23 | spec.add_dependency "activemodel", ">= 4.2" 24 | spec.add_dependency "elasticsearch", ">= 5" 25 | spec.add_dependency "hashie" 26 | 27 | spec.add_development_dependency "bundler" 28 | spec.add_development_dependency "minitest" 29 | spec.add_development_dependency "rake" 30 | end 31 | -------------------------------------------------------------------------------- /test/multi_search_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | class MultiSearchTest < Minitest::Test 4 | def test_basic 5 | store_names ["Product A"] 6 | store_names ["Store A"], Store 7 | products = Product.search("*", execute: false) 8 | stores = Store.search("*", execute: false) 9 | Searchkick.multi_search([products, stores]) 10 | assert_equal ["Product A"], products.map(&:name) 11 | assert_equal ["Store A"], stores.map(&:name) 12 | end 13 | 14 | def test_error 15 | store_names ["Product A"] 16 | products = Product.search("*", execute: false) 17 | stores = Store.search("*", order: [:bad_field], execute: false) 18 | Searchkick.multi_search([products, stores]) 19 | assert !products.error 20 | assert stores.error 21 | end 22 | 23 | def test_misspellings_below_unmet 24 | store_names ["abc", "abd", "aee"] 25 | products = Product.search("abc", misspellings: {below: 5}, execute: false) 26 | Searchkick.multi_search([products]) 27 | assert_equal ["abc", "abd"], products.map(&:name) 28 | end 29 | 30 | def test_error 31 | products = Product.search("*", order: {bad_column: :asc}, execute: false) 32 | Searchkick.multi_search([products]) 33 | assert products.error 34 | error = assert_raises(Searchkick::Error) { products.results } 35 | assert_equal error.message, "Query error - use the error method to view it" 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /test/similar_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | class SimilarTest < Minitest::Test 4 | def test_similar 5 | store_names ["Annie's Naturals Organic Shiitake & Sesame Dressing"] 6 | assert_search "Annie's Naturals Shiitake & Sesame Vinaigrette", ["Annie's Naturals Organic Shiitake & Sesame Dressing"], similar: true, fields: [:name] 7 | end 8 | 9 | def test_fields 10 | store_names ["1% Organic Milk", "2% Organic Milk", "Popcorn"] 11 | assert_equal ["2% Organic Milk"], Product.where(name: "1% Organic Milk").first.similar(fields: ["name"]).map(&:name) 12 | end 13 | 14 | def test_order 15 | store_names ["Lucerne Milk Chocolate Fat Free", "Clover Fat Free Milk"] 16 | assert_order "Lucerne Fat Free Chocolate Milk", ["Lucerne Milk Chocolate Fat Free", "Clover Fat Free Milk"], similar: true, fields: [:name] 17 | end 18 | 19 | def test_limit 20 | store_names ["1% Organic Milk", "2% Organic Milk", "Fat Free Organic Milk", "Popcorn"] 21 | assert_equal ["2% Organic Milk"], Product.where(name: "1% Organic Milk").first.similar(fields: ["name"], order: ["name"], limit: 1).map(&:name) 22 | end 23 | 24 | def test_per_page 25 | store_names ["1% Organic Milk", "2% Organic Milk", "Fat Free Organic Milk", "Popcorn"] 26 | assert_equal ["2% Organic Milk"], Product.where(name: "1% Organic Milk").first.similar(fields: ["name"], order: ["name"], per_page: 1).map(&:name) 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /test/query_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | class QueryTest < Minitest::Test 4 | def test_basic 5 | store_names ["Milk", "Apple"] 6 | query = Product.search("milk", execute: false) 7 | # query.body = {query: {match_all: {}}} 8 | # query.body = {query: {match: {name: "Apple"}}} 9 | query.body[:query] = {match_all: {}} 10 | assert_equal ["Apple", "Milk"], query.map(&:name).sort 11 | assert_equal ["Apple", "Milk"], query.execute.map(&:name).sort 12 | end 13 | 14 | def test_with_effective_min_score 15 | store_names ["Milk", "Milk2"] 16 | assert_search "milk", ["Milk"], body_options: {min_score: 1} 17 | end 18 | 19 | def test_with_uneffective_min_score 20 | store_names ["Milk", "Milk2"] 21 | assert_search "milk", ["Milk", "Milk2"], body_options: {min_score: 0.0001} 22 | end 23 | 24 | def test_default_timeout 25 | assert_equal "6s", Product.search("*", execute: false).body[:timeout] 26 | end 27 | 28 | def test_timeout_override 29 | assert_equal "1s", Product.search("*", body_options: {timeout: "1s"}, execute: false).body[:timeout] 30 | end 31 | 32 | def test_request_params 33 | assert_equal "dfs_query_then_fetch", Product.search("*", request_params: {search_type: "dfs_query_then_fetch"}, execute: false).params[:search_type] 34 | end 35 | 36 | def test_debug 37 | store_names ["Milk"] 38 | out, _ = capture_io do 39 | assert_search "milk", ["Milk"], debug: true 40 | end 41 | refute_includes out, "Error" 42 | end 43 | end 44 | -------------------------------------------------------------------------------- /test/order_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | class OrderTest < Minitest::Test 4 | def test_order_hash 5 | store_names ["Product A", "Product B", "Product C", "Product D"] 6 | assert_order "product", ["Product D", "Product C", "Product B", "Product A"], order: {name: :desc} 7 | end 8 | 9 | def test_order_string 10 | store_names ["Product A", "Product B", "Product C", "Product D"] 11 | assert_order "product", ["Product A", "Product B", "Product C", "Product D"], order: "name" 12 | end 13 | 14 | def test_order_id 15 | skip if cequel? 16 | 17 | store_names ["Product A", "Product B"] 18 | product_a = Product.where(name: "Product A").first 19 | product_b = Product.where(name: "Product B").first 20 | assert_order "product", [product_a, product_b].sort_by(&:id).map(&:name), order: {id: :asc} 21 | end 22 | 23 | def test_order_multiple 24 | store [ 25 | {name: "Product A", color: "blue", store_id: 1}, 26 | {name: "Product B", color: "red", store_id: 3}, 27 | {name: "Product C", color: "red", store_id: 2} 28 | ] 29 | assert_order "product", ["Product A", "Product B", "Product C"], order: {color: :asc, store_id: :desc} 30 | end 31 | 32 | def test_order_unmapped_type 33 | assert_order "product", [], order: {not_mapped: {unmapped_type: "long"}} 34 | end 35 | 36 | def test_order_array 37 | store [{name: "San Francisco", latitude: 37.7833, longitude: -122.4167}] 38 | assert_order "francisco", ["San Francisco"], order: [{_geo_distance: {location: "0,0"}}] 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /docs/Searchkick-3-Upgrade.md: -------------------------------------------------------------------------------- 1 | # Searchkick 3 Upgrade 2 | 3 | ## Before You Upgrade 4 | 5 | Searchkick 3 no longer uses types, since they are deprecated in Elasticsearch 6. 6 | 7 | If you use inheritance, add to your parent model: 8 | 9 | ```ruby 10 | class Animal < ApplicationRecord 11 | searchkick inheritance: true 12 | end 13 | ``` 14 | 15 | And do a full reindex before upgrading. 16 | 17 | ## Upgrading 18 | 19 | Update your Gemfile: 20 | 21 | ```ruby 22 | gem 'searchkick', '~> 3' 23 | ``` 24 | 25 | And run: 26 | 27 | ```sh 28 | bundle update searchkick 29 | ``` 30 | 31 | We recommend you don’t stem conversions anymore, so conversions for `pepper` don’t affect `peppers`, but if you want to keep the old behavior, use: 32 | 33 | ```ruby 34 | Searchkick.model_options = { 35 | stem_conversions: true 36 | } 37 | ``` 38 | 39 | Searchkick 3 disables the `_all` field by default, since Elasticsearch 6 removes the ability to reindex with it. If you’re on Elasticsearch 5 and still need it, add to your model: 40 | 41 | ```ruby 42 | class Product < ApplicationRecord 43 | searchkick _all: true 44 | end 45 | ``` 46 | 47 | If you use `record.reindex_async` or `record.reindex(async: true)`, replace it with: 48 | 49 | ```ruby 50 | record.reindex(mode: :async) 51 | ``` 52 | 53 | If you use `log: true` with `boost_by`, replace it with `modifier: "ln2p"`. 54 | 55 | If you use the `body` option and have warnings about incompatible options, remove them, as they now throw an `ArgumentError`. 56 | 57 | Check out the [changelog](https://github.com/ankane/searchkick/blob/master/CHANGELOG.md) for the full list of changes. 58 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | First, thanks for wanting to contribute. You’re awesome! :heart: 4 | 5 | ## Questions 6 | 7 | Use [Stack Overflow](https://stackoverflow.com/) with the tag `searchkick`. 8 | 9 | ## Feature Requests 10 | 11 | Create an issue. Start the title with `[Idea]`. 12 | 13 | ## Issues 14 | 15 | Think you’ve discovered an issue? 16 | 17 | 1. Search existing issues to see if it’s been reported. 18 | 2. Try the `master` branch to make sure it hasn’t been fixed. 19 | 20 | ```rb 21 | gem "searchkick", github: "ankane/searchkick" 22 | ``` 23 | 24 | 3. Try the `debug` option when searching. This can reveal useful info. 25 | 26 | ```ruby 27 | Product.search("something", debug: true) 28 | ``` 29 | 30 | If the above steps don’t help, create an issue. 31 | 32 | - Recreate the problem by forking [this gist](https://gist.github.com/ankane/f80b0923d9ae2c077f41997f7b704e5c). Include a link to your gist and the output in the issue. 33 | - For exceptions, include the complete backtrace. 34 | 35 | ## Pull Requests 36 | 37 | Fork the project and create a pull request. A few tips: 38 | 39 | - Keep changes to a minimum. If you have multiple features or fixes, submit multiple pull requests. 40 | - Follow the existing style. The code should read like it’s written by a single person. 41 | - Add one or more tests if possible. Make sure existing tests pass with: 42 | 43 | ```sh 44 | bundle exec rake test 45 | ``` 46 | 47 | Feel free to open an issue to get feedback on your idea before spending too much time on it. 48 | 49 | --- 50 | 51 | This contributing guide is released under [CCO](https://creativecommons.org/publicdomain/zero/1.0/) (public domain). Use it for your own project without attribution. 52 | -------------------------------------------------------------------------------- /lib/searchkick/record_indexer.rb: -------------------------------------------------------------------------------- 1 | module Searchkick 2 | class RecordIndexer 3 | attr_reader :record, :index 4 | 5 | def initialize(record) 6 | @record = record 7 | @index = record.class.searchkick_index 8 | end 9 | 10 | def reindex(method_name = nil, refresh: false, mode: nil) 11 | unless [true, nil, :async, :queue].include?(mode) 12 | raise ArgumentError, "Invalid value for mode" 13 | end 14 | 15 | mode ||= Searchkick.callbacks_value || index.options[:callbacks] || true 16 | 17 | case mode 18 | when :queue 19 | if method_name 20 | raise Searchkick::Error, "Partial reindex not supported with queue option" 21 | end 22 | 23 | index.reindex_queue.push(record.id.to_s) 24 | when :async 25 | unless defined?(ActiveJob) 26 | raise Searchkick::Error, "Active Job not found" 27 | end 28 | 29 | Searchkick::ReindexV2Job.perform_later( 30 | record.class.name, 31 | record.id.to_s, 32 | method_name ? method_name.to_s : nil 33 | ) 34 | else # bulk, true 35 | reindex_record(method_name) 36 | 37 | index.refresh if refresh 38 | end 39 | end 40 | 41 | private 42 | 43 | def reindex_record(method_name) 44 | if record.destroyed? || !record.persisted? || !record.should_index? 45 | begin 46 | index.remove(record) 47 | rescue Elasticsearch::Transport::Transport::Errors::NotFound 48 | # do nothing 49 | end 50 | else 51 | if method_name 52 | index.update_record(record, method_name) 53 | else 54 | index.store(record) 55 | end 56 | end 57 | end 58 | end 59 | end 60 | -------------------------------------------------------------------------------- /test/callbacks_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | class CallbacksTest < Minitest::Test 4 | def test_true_create 5 | Searchkick.callbacks(true) do 6 | store_names ["Product A", "Product B"] 7 | end 8 | Product.searchkick_index.refresh 9 | assert_search "product", ["Product A", "Product B"] 10 | end 11 | 12 | def test_false_create 13 | Searchkick.callbacks(false) do 14 | store_names ["Product A", "Product B"] 15 | end 16 | Product.searchkick_index.refresh 17 | assert_search "product", [] 18 | end 19 | 20 | def test_bulk_create 21 | Searchkick.callbacks(:bulk) do 22 | store_names ["Product A", "Product B"] 23 | end 24 | Product.searchkick_index.refresh 25 | assert_search "product", ["Product A", "Product B"] 26 | end 27 | 28 | def test_queue 29 | skip unless defined?(ActiveJob) && defined?(Redis) 30 | 31 | reindex_queue = Product.searchkick_index.reindex_queue 32 | reindex_queue.clear 33 | 34 | Searchkick.callbacks(:queue) do 35 | store_names ["Product A", "Product B"] 36 | end 37 | Product.searchkick_index.refresh 38 | assert_search "product", [], load: false, conversions: false 39 | assert_equal 2, reindex_queue.length 40 | 41 | Searchkick::ProcessQueueJob.perform_later(class_name: "Product") 42 | Product.searchkick_index.refresh 43 | assert_search "product", ["Product A", "Product B"], load: false 44 | assert_equal 0, reindex_queue.length 45 | 46 | Searchkick.callbacks(:queue) do 47 | Product.where(name: "Product B").destroy_all 48 | Product.create!(name: "Product C") 49 | end 50 | Product.searchkick_index.refresh 51 | assert_search "product", ["Product A", "Product B"], load: false 52 | assert_equal 2, reindex_queue.length 53 | 54 | Searchkick::ProcessQueueJob.perform_later(class_name: "Product") 55 | Product.searchkick_index.refresh 56 | assert_search "product", ["Product A", "Product C"], load: false 57 | assert_equal 0, reindex_queue.length 58 | end 59 | end 60 | -------------------------------------------------------------------------------- /test/partial_reindex_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | class PartialReindexTest < Minitest::Test 4 | def test_class_method 5 | store [{name: "Hi", color: "Blue"}] 6 | 7 | # normal search 8 | assert_search "hi", ["Hi"], fields: [:name], load: false 9 | assert_search "blue", ["Hi"], fields: [:color], load: false 10 | 11 | # update 12 | product = Product.first 13 | product.name = "Bye" 14 | product.color = "Red" 15 | Searchkick.callbacks(false) do 16 | product.save! 17 | end 18 | Product.searchkick_index.refresh 19 | 20 | # index not updated 21 | assert_search "hi", ["Hi"], fields: [:name], load: false 22 | assert_search "blue", ["Hi"], fields: [:color], load: false 23 | 24 | # partial reindex 25 | Product.reindex(:search_name) 26 | 27 | # name updated, but not color 28 | assert_search "bye", ["Bye"], fields: [:name], load: false 29 | assert_search "blue", ["Bye"], fields: [:color], load: false 30 | end 31 | 32 | def test_instance_method 33 | store [{name: "Hi", color: "Blue"}] 34 | 35 | # normal search 36 | assert_search "hi", ["Hi"], fields: [:name], load: false 37 | assert_search "blue", ["Hi"], fields: [:color], load: false 38 | 39 | # update 40 | product = Product.first 41 | product.name = "Bye" 42 | product.color = "Red" 43 | Searchkick.callbacks(false) do 44 | product.save! 45 | end 46 | Product.searchkick_index.refresh 47 | 48 | # index not updated 49 | assert_search "hi", ["Hi"], fields: [:name], load: false 50 | assert_search "blue", ["Hi"], fields: [:color], load: false 51 | 52 | product.reindex(:search_name, refresh: true) 53 | 54 | # name updated, but not color 55 | assert_search "bye", ["Bye"], fields: [:name], load: false 56 | assert_search "blue", ["Bye"], fields: [:color], load: false 57 | end 58 | 59 | def test_instance_method_async 60 | skip unless defined?(ActiveJob) 61 | 62 | product = Product.create!(name: "Hi") 63 | product.reindex(:search_data, mode: :async) 64 | end 65 | end 66 | -------------------------------------------------------------------------------- /test/misspellings_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | class MisspellingsTest < Minitest::Test 4 | def test_misspellings 5 | store_names ["abc", "abd", "aee"] 6 | assert_search "abc", ["abc"], misspellings: false 7 | end 8 | 9 | def test_misspellings_distance 10 | store_names ["abbb", "aabb"] 11 | assert_search "aaaa", ["aabb"], misspellings: {distance: 2} 12 | end 13 | 14 | def test_misspellings_prefix_length 15 | store_names ["ap", "api", "apt", "any", "nap", "ah", "ahi"] 16 | assert_search "ap", ["ap"], misspellings: {prefix_length: 2} 17 | assert_search "api", ["ap", "api", "apt"], misspellings: {prefix_length: 2} 18 | end 19 | 20 | def test_misspellings_prefix_length_operator 21 | store_names ["ap", "api", "apt", "any", "nap", "ah", "aha"] 22 | assert_search "ap ah", ["ap", "ah"], operator: "or", misspellings: {prefix_length: 2} 23 | assert_search "api ahi", ["ap", "api", "apt", "ah", "aha"], operator: "or", misspellings: {prefix_length: 2} 24 | end 25 | 26 | def test_misspellings_fields_operator 27 | store [ 28 | {name: "red", color: "red"}, 29 | {name: "blue", color: "blue"}, 30 | {name: "cyan", color: "blue green"}, 31 | {name: "magenta", color: "red blue"}, 32 | {name: "green", color: "green"} 33 | ] 34 | assert_search "red blue", ["red", "blue", "cyan", "magenta"], operator: "or", fields: ["color"], misspellings: false 35 | end 36 | 37 | def test_misspellings_below_unmet 38 | store_names ["abc", "abd", "aee"] 39 | assert_search "abc", ["abc", "abd"], misspellings: {below: 2} 40 | end 41 | 42 | def test_misspellings_below_unmet_result 43 | store_names ["abc", "abd", "aee"] 44 | assert Product.search("abc", misspellings: {below: 2}).misspellings? 45 | end 46 | 47 | def test_misspellings_below_met 48 | store_names ["abc", "abd", "aee"] 49 | assert_search "abc", ["abc"], misspellings: {below: 1} 50 | end 51 | 52 | def test_misspellings_below_met_result 53 | store_names ["abc", "abd", "aee"] 54 | assert !Product.search("abc", misspellings: {below: 1}).misspellings? 55 | end 56 | end 57 | -------------------------------------------------------------------------------- /test/synonyms_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | class SynonymsTest < Minitest::Test 4 | def test_bleach 5 | store_names ["Clorox Bleach", "Kroger Bleach"] 6 | assert_search "clorox", ["Clorox Bleach", "Kroger Bleach"] 7 | end 8 | 9 | def test_saran_wrap 10 | store_names ["Saran Wrap", "Kroger Plastic Wrap"] 11 | assert_search "saran wrap", ["Saran Wrap", "Kroger Plastic Wrap"] 12 | end 13 | 14 | def test_burger_buns 15 | store_names ["Hamburger Buns"] 16 | assert_search "burger buns", ["Hamburger Buns"] 17 | end 18 | 19 | def test_bandaids 20 | store_names ["Band-Aid", "Kroger 12-Pack Bandages"] 21 | assert_search "bandaids", ["Band-Aid", "Kroger 12-Pack Bandages"] 22 | end 23 | 24 | def test_qtips 25 | store_names ["Q Tips", "Kroger Cotton Swabs"] 26 | assert_search "q tips", ["Q Tips", "Kroger Cotton Swabs"] 27 | end 28 | 29 | def test_reverse 30 | store_names ["Scallions"] 31 | assert_search "green onions", ["Scallions"] 32 | end 33 | 34 | def test_exact 35 | store_names ["Green Onions", "Yellow Onions"] 36 | assert_search "scallion", ["Green Onions"] 37 | end 38 | 39 | def test_stemmed 40 | store_names ["Green Onions", "Yellow Onions"] 41 | assert_search "scallions", ["Green Onions"] 42 | end 43 | 44 | def test_word_start 45 | store_names ["Clorox Bleach", "Kroger Bleach"] 46 | assert_search "clorox", ["Clorox Bleach", "Kroger Bleach"], fields: [{name: :word_start}] 47 | end 48 | 49 | def test_wordnet 50 | # requires WordNet 51 | skip unless ENV["WORDNET"] 52 | 53 | store_names ["Creature", "Beast", "Dragon"], Animal 54 | assert_search "animal", ["Creature", "Beast"], {}, Animal 55 | end 56 | 57 | def test_directional 58 | store_names ["Lightbulb", "Green Onions", "Led"] 59 | assert_search "led", ["Lightbulb", "Led"] 60 | assert_search "Lightbulb", ["Lightbulb"] 61 | assert_search "Halogen Lamp", ["Lightbulb"] 62 | assert_search "onions", ["Green Onions"] 63 | end 64 | 65 | def test_case 66 | store_names ["Uppercase"] 67 | assert_search "lowercase", ["Uppercase"] 68 | end 69 | end 70 | -------------------------------------------------------------------------------- /test/inheritance_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | class InheritanceTest < Minitest::Test 4 | def setup 5 | skip if defined?(Cequel) 6 | super 7 | end 8 | 9 | def test_child_reindex 10 | store_names ["Max"], Cat 11 | assert Dog.reindex 12 | assert_equal 1, Animal.search("*").size 13 | end 14 | 15 | def test_child_index_name 16 | assert_equal "animals-#{Date.today.year}", Dog.searchkick_index.name 17 | end 18 | 19 | def test_child_search 20 | store_names ["Bear"], Dog 21 | store_names ["Bear"], Cat 22 | assert_equal 1, Dog.search("bear").size 23 | end 24 | 25 | def test_parent_search 26 | store_names ["Bear"], Dog 27 | store_names ["Bear"], Cat 28 | assert_equal 2, Animal.search("bear").size 29 | end 30 | 31 | def test_force_one_type 32 | store_names ["Green Bear"], Dog 33 | store_names ["Blue Bear"], Cat 34 | assert_equal ["Blue Bear"], Animal.search("bear", type: [Cat]).map(&:name) 35 | end 36 | 37 | def test_force_multiple_types 38 | store_names ["Green Bear"], Dog 39 | store_names ["Blue Bear"], Cat 40 | store_names ["Red Bear"], Animal 41 | assert_equal ["Green Bear", "Blue Bear"], Animal.search("bear", type: [Dog, Cat]).map(&:name) 42 | end 43 | 44 | def test_child_autocomplete 45 | store_names ["Max"], Cat 46 | store_names ["Mark"], Dog 47 | assert_equal ["Max"], Cat.search("ma", fields: [:name], match: :text_start).map(&:name) 48 | end 49 | 50 | def test_parent_autocomplete 51 | store_names ["Max"], Cat 52 | store_names ["Bear"], Dog 53 | assert_equal ["Bear"], Animal.search("bea", fields: [:name], match: :text_start).map(&:name).sort 54 | end 55 | 56 | # def test_child_suggest 57 | # store_names ["Shark"], Cat 58 | # store_names ["Sharp"], Dog 59 | # assert_equal ["shark"], Cat.search("shar", fields: [:name], suggest: true).suggestions 60 | # end 61 | 62 | def test_parent_suggest 63 | store_names ["Shark"], Cat 64 | store_names ["Tiger"], Dog 65 | assert_equal ["tiger"], Animal.search("tige", fields: [:name], suggest: true).suggestions.sort 66 | end 67 | 68 | def test_reindex 69 | store_names ["Bear A"], Cat 70 | store_names ["Bear B"], Dog 71 | Animal.reindex 72 | assert_equal 2, Animal.search("bear").size 73 | end 74 | 75 | # TODO move somewhere better 76 | 77 | def test_multiple_indices 78 | store_names ["Product A"] 79 | store_names ["Product B"], Animal 80 | assert_search "product", ["Product A", "Product B"], index_name: [Product.searchkick_index.name, Animal.searchkick_index.name], conversions: false 81 | end 82 | end 83 | -------------------------------------------------------------------------------- /test/reindex_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | class ReindexTest < Minitest::Test 4 | def setup 5 | super 6 | Sku.destroy_all 7 | end 8 | 9 | def test_scoped 10 | skip if nobrainer? || cequel? 11 | 12 | store_names ["Product A"] 13 | Searchkick.callbacks(false) do 14 | store_names ["Product B", "Product C"] 15 | end 16 | Product.where(name: "Product B").reindex(refresh: true) 17 | assert_search "product", ["Product A", "Product B"] 18 | end 19 | 20 | def test_associations 21 | skip if nobrainer? || cequel? 22 | 23 | store_names ["Product A"] 24 | store = Store.create!(name: "Test") 25 | Product.create!(name: "Product B", store_id: store.id) 26 | store.products.reindex(refresh: true) 27 | assert_search "product", ["Product A", "Product B"] 28 | end 29 | 30 | def test_async 31 | skip unless defined?(ActiveJob) 32 | 33 | Searchkick.callbacks(false) do 34 | store_names ["Product A"] 35 | end 36 | reindex = Product.reindex(async: true) 37 | assert_search "product", [], conversions: false 38 | 39 | index = Searchkick::Index.new(reindex[:index_name]) 40 | index.refresh 41 | assert_equal 1, index.total_docs 42 | 43 | if defined?(Redis) 44 | assert Searchkick.reindex_status(reindex[:name]) 45 | end 46 | 47 | Product.searchkick_index.promote(reindex[:index_name]) 48 | assert_search "product", ["Product A"] 49 | end 50 | 51 | def test_async_wait 52 | skip unless defined?(ActiveJob) && defined?(Redis) 53 | 54 | Searchkick.callbacks(false) do 55 | store_names ["Product A"] 56 | end 57 | 58 | capture_io do 59 | Product.reindex(async: {wait: true}) 60 | end 61 | 62 | assert_search "product", ["Product A"] 63 | end 64 | 65 | def test_async_non_integer_pk 66 | skip unless defined?(ActiveJob) 67 | 68 | Sku.create(id: SecureRandom.hex, name: "Test") 69 | reindex = Sku.reindex(async: true) 70 | assert_search "sku", [], conversions: false 71 | 72 | index = Searchkick::Index.new(reindex[:index_name]) 73 | index.refresh 74 | assert_equal 1, index.total_docs 75 | end 76 | 77 | def test_refresh_interval 78 | reindex = Product.reindex(refresh_interval: "30s", async: true, import: false) 79 | index = Searchkick::Index.new(reindex[:index_name]) 80 | assert_nil Product.search_index.refresh_interval 81 | assert_equal "30s", index.refresh_interval 82 | 83 | Product.search_index.promote(index.name, update_refresh_interval: true) 84 | assert_equal "1s", index.refresh_interval 85 | assert_equal "1s", Product.search_index.refresh_interval 86 | end 87 | end 88 | -------------------------------------------------------------------------------- /benchmark/index.rb: -------------------------------------------------------------------------------- 1 | require "bundler/setup" 2 | Bundler.require(:default) 3 | require "active_record" 4 | require "benchmark" 5 | require "active_support/notifications" 6 | 7 | ActiveSupport::Notifications.subscribe "request.searchkick" do |*args| 8 | event = ActiveSupport::Notifications::Event.new(*args) 9 | puts "Import: #{event.duration.round}ms" 10 | end 11 | 12 | ActiveJob::Base.queue_adapter = :sidekiq 13 | 14 | Searchkick.redis = Redis.new 15 | 16 | ActiveRecord::Base.default_timezone = :utc 17 | ActiveRecord::Base.time_zone_aware_attributes = true 18 | # ActiveRecord::Base.establish_connection adapter: "sqlite3", database: "/tmp/searchkick" 19 | ActiveRecord::Base.establish_connection "postgresql://localhost/searchkick_demo_development" 20 | # ActiveRecord::Base.logger = Logger.new(STDOUT) 21 | 22 | ActiveJob::Base.logger = nil 23 | 24 | class Product < ActiveRecord::Base 25 | searchkick batch_size: 1000 26 | 27 | def search_data 28 | { 29 | name: name, 30 | color: color, 31 | store_id: store_id 32 | } 33 | end 34 | end 35 | 36 | if ENV["SETUP"] 37 | total_docs = 100000 38 | 39 | ActiveRecord::Migration.create_table :products, force: :cascade do |t| 40 | t.string :name 41 | t.string :color 42 | t.integer :store_id 43 | end 44 | 45 | Product.import ["name", "color", "store_id"], total_docs.times.map { |i| ["Product #{i}", ["red", "blue"].sample, rand(10)] } 46 | 47 | puts "Imported" 48 | end 49 | 50 | result = nil 51 | report = nil 52 | stats = nil 53 | 54 | Product.searchkick_index.delete rescue nil 55 | 56 | GC.start 57 | GC.disable 58 | start_mem = GetProcessMem.new.mb 59 | 60 | time = 61 | Benchmark.realtime do 62 | # result = RubyProf.profile do 63 | # report = MemoryProfiler.report do 64 | # stats = AllocationStats.trace do 65 | reindex = Product.reindex #(async: true) 66 | # p reindex 67 | # end 68 | 69 | # 60.times do |i| 70 | # if reindex.is_a?(Hash) 71 | # docs = Searchkick::Index.new(reindex[:index_name]).total_docs 72 | # else 73 | # docs = Product.searchkick_index.total_docs 74 | # end 75 | # puts "#{i}: #{docs}" 76 | # if docs == total_docs 77 | # break 78 | # end 79 | # p Searchkick.reindex_status(reindex[:index_name]) if reindex.is_a?(Hash) 80 | # sleep(1) 81 | # # Product.searchkick_index.refresh 82 | # end 83 | end 84 | 85 | puts 86 | puts "Time: #{time.round(1)}s" 87 | 88 | if result 89 | printer = RubyProf::GraphPrinter.new(result) 90 | printer.print(STDOUT, min_percent: 5) 91 | end 92 | 93 | if report 94 | puts report.pretty_print 95 | end 96 | 97 | if stats 98 | puts result.allocations(alias_paths: true).group_by(:sourcefile, :class).to_text 99 | end 100 | -------------------------------------------------------------------------------- /test/language_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | class LanguageTest < Minitest::Test 4 | def setup 5 | skip unless ENV["LANGUAGE"] 6 | 7 | Song.destroy_all 8 | end 9 | 10 | def test_chinese 11 | # requires https://github.com/medcl/elasticsearch-analysis-ik 12 | with_options(Song, language: "chinese") do 13 | store_names ["中华人民共和国国歌"], Song 14 | assert_language_search "中华人民共和国", ["中华人民共和国国歌"] 15 | assert_language_search "国歌", ["中华人民共和国国歌"] 16 | assert_language_search "人", [] 17 | end 18 | end 19 | 20 | # experimental 21 | def test_smartcn 22 | # requires https://www.elastic.co/guide/en/elasticsearch/plugins/6.2/analysis-smartcn.html 23 | with_options(Song, language: "smartcn") do 24 | store_names ["中华人民共和国国歌"], Song 25 | assert_language_search "中华人民共和国", ["中华人民共和国国歌"] 26 | # assert_language_search "国歌", ["中华人民共和国国歌"] 27 | assert_language_search "人", [] 28 | end 29 | end 30 | 31 | def test_japanese 32 | # requires https://www.elastic.co/guide/en/elasticsearch/plugins/6.2/analysis-kuromoji.html 33 | with_options(Song, language: "japanese") do 34 | store_names ["JR新宿駅の近くにビールを飲みに行こうか"], Song 35 | assert_language_search "飲む", ["JR新宿駅の近くにビールを飲みに行こうか"] 36 | assert_language_search "jr", ["JR新宿駅の近くにビールを飲みに行こうか"] 37 | assert_language_search "新", [] 38 | end 39 | end 40 | 41 | def test_korean 42 | # requires https://github.com/open-korean-text/elasticsearch-analysis-openkoreantext 43 | with_options(Song, language: "korean") do 44 | store_names ["한국어를 처리하는 예시입니닼ㅋㅋ"], Song 45 | assert_language_search "처리", ["한국어를 처리하는 예시입니닼ㅋㅋ"] 46 | assert_language_search "한국어", ["한국어를 처리하는 예시입니닼ㅋㅋ"] 47 | assert_language_search "를", [] 48 | end 49 | end 50 | 51 | def test_polish 52 | # requires https://www.elastic.co/guide/en/elasticsearch/plugins/6.2/analysis-stempel.html 53 | with_options(Song, language: "polish") do 54 | store_names ["polski"], Song 55 | assert_language_search "polskimi", ["polski"] 56 | end 57 | end 58 | 59 | def test_ukrainian 60 | # requires https://www.elastic.co/guide/en/elasticsearch/plugins/6.2/analysis-ukrainian.html 61 | with_options(Song, language: "ukrainian") do 62 | store_names ["ресторани"], Song 63 | assert_language_search "ресторан", ["ресторани"] 64 | end 65 | end 66 | 67 | def test_vietnamese 68 | # requires https://github.com/duydo/elasticsearch-analysis-vietnamese 69 | with_options(Song, language: "vietnamese") do 70 | store_names ["công nghệ thông tin Việt Nam"], Song 71 | assert_language_search "công nghệ thông tin", ["công nghệ thông tin Việt Nam"] 72 | assert_language_search "công", [] 73 | end 74 | end 75 | 76 | def assert_language_search(term, expected) 77 | assert_search term, expected, {misspellings: false}, Song 78 | end 79 | end 80 | -------------------------------------------------------------------------------- /test/autocomplete_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | class AutocompleteTest < Minitest::Test 4 | def test_autocomplete 5 | store_names ["Hummus"] 6 | assert_search "hum", ["Hummus"], match: :text_start 7 | end 8 | 9 | def test_autocomplete_two_words 10 | store_names ["Organic Hummus"] 11 | assert_search "hum", [], match: :text_start 12 | end 13 | 14 | def test_autocomplete_fields 15 | store_names ["Hummus"] 16 | assert_search "hum", ["Hummus"], match: :text_start, fields: [:name] 17 | end 18 | 19 | def test_text_start 20 | store_names ["Where in the World is Carmen San Diego"] 21 | assert_search "where in the world is", ["Where in the World is Carmen San Diego"], fields: [{name: :text_start}] 22 | assert_search "in the world", [], fields: [{name: :text_start}] 23 | end 24 | 25 | def test_text_middle 26 | store_names ["Where in the World is Carmen San Diego"] 27 | assert_search "where in the world is", ["Where in the World is Carmen San Diego"], fields: [{name: :text_middle}] 28 | assert_search "n the wor", ["Where in the World is Carmen San Diego"], fields: [{name: :text_middle}] 29 | assert_search "men san diego", ["Where in the World is Carmen San Diego"], fields: [{name: :text_middle}] 30 | assert_search "world carmen", [], fields: [{name: :text_middle}] 31 | end 32 | 33 | def test_text_end 34 | store_names ["Where in the World is Carmen San Diego"] 35 | assert_search "men san diego", ["Where in the World is Carmen San Diego"], fields: [{name: :text_end}] 36 | assert_search "carmen san", [], fields: [{name: :text_end}] 37 | end 38 | 39 | def test_word_start 40 | store_names ["Where in the World is Carmen San Diego"] 41 | assert_search "car san wor", ["Where in the World is Carmen San Diego"], fields: [{name: :word_start}] 42 | end 43 | 44 | def test_word_middle 45 | store_names ["Where in the World is Carmen San Diego"] 46 | assert_search "orl", ["Where in the World is Carmen San Diego"], fields: [{name: :word_middle}] 47 | end 48 | 49 | def test_word_end 50 | store_names ["Where in the World is Carmen San Diego"] 51 | assert_search "rld men ego", ["Where in the World is Carmen San Diego"], fields: [{name: :word_end}] 52 | end 53 | 54 | def test_word_start_multiple_words 55 | store_names ["Dark Grey", "Dark Blue"] 56 | assert_search "dark grey", ["Dark Grey"], fields: [{name: :word_start}] 57 | end 58 | 59 | def test_word_start_exact 60 | store_names ["Back Scratcher", "Backpack"] 61 | assert_order "back", ["Back Scratcher", "Backpack"], fields: [{name: :word_start}] 62 | end 63 | 64 | def test_word_start_exact_martin 65 | store_names ["Martina", "Martin"] 66 | assert_order "martin", ["Martin", "Martina"], fields: [{name: :word_start}] 67 | end 68 | 69 | # TODO find a better place 70 | 71 | def test_exact 72 | store_names ["hi@example.org"] 73 | assert_search "hi@example.org", ["hi@example.org"], fields: [{name: :exact}] 74 | end 75 | 76 | def test_exact_case 77 | store_names ["Hello"] 78 | assert_search "hello", [], fields: [{name: :exact}] 79 | assert_search "Hello", ["Hello"], fields: [{name: :exact}] 80 | end 81 | end 82 | -------------------------------------------------------------------------------- /test/suggest_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | class SuggestTest < Minitest::Test 4 | def test_basic 5 | store_names ["Great White Shark", "Hammerhead Shark", "Tiger Shark"] 6 | assert_suggest "How Big is a Tigre Shar", "how big is a tiger shark", fields: [:name] 7 | end 8 | 9 | def test_perfect 10 | store_names ["Tiger Shark", "Great White Shark"] 11 | assert_suggest "Tiger Shark", nil, fields: [:name] # no correction 12 | end 13 | 14 | def test_phrase 15 | store_names ["Big Tiger Shark", "Tiger Sharp Teeth", "Tiger Sharp Mind"] 16 | assert_suggest "How to catch a big tiger shar", "how to catch a big tiger shark", fields: [:name] 17 | end 18 | 19 | def test_without_option 20 | store_names ["hi"] # needed to prevent ElasticsearchException - seed 668 21 | assert_raises(RuntimeError) { Product.search("hi").suggestions } 22 | end 23 | 24 | def test_multiple_fields 25 | store [ 26 | {name: "Shark", color: "Sharp"}, 27 | {name: "Shark", color: "Sharp"} 28 | ] 29 | assert_suggest_all "shar", ["shark", "sharp"] 30 | end 31 | 32 | def test_multiple_fields_highest_score_first 33 | store [ 34 | {name: "Tiger Shark", color: "Sharp"} 35 | ] 36 | assert_suggest "tiger shar", "tiger shark" 37 | end 38 | 39 | def test_multiple_fields_same_value 40 | store [ 41 | {name: "Shark", color: "Shark"} 42 | ] 43 | assert_suggest_all "shar", ["shark"] 44 | end 45 | 46 | def test_fields_option 47 | store [ 48 | {name: "Shark", color: "Sharp"} 49 | ] 50 | assert_suggest_all "shar", ["shark"], fields: [:name] 51 | end 52 | 53 | def test_fields_option_multiple 54 | store [ 55 | {name: "Shark"} 56 | ] 57 | assert_suggest "shar", "shark", fields: [:name, :unknown] 58 | end 59 | 60 | def test_fields_partial_match 61 | store_names ["Great White Shark", "Hammerhead Shark", "Tiger Shark"] 62 | assert_suggest "How Big is a Tigre Shar", "how big is a tiger shark", fields: [{name: :word_start}] 63 | end 64 | 65 | def test_fields_partial_match_boost 66 | store_names ["Great White Shark", "Hammerhead Shark", "Tiger Shark"] 67 | assert_suggest "How Big is a Tigre Shar", "how big is a tiger shark", fields: [{"name^2" => :word_start}] 68 | end 69 | 70 | def test_multiple_models 71 | skip # flaky test 72 | store_names ["Great White Shark", "Hammerhead Shark", "Tiger Shark"] 73 | assert_equal "how big is a tiger shark", Searchkick.search("How Big is a Tigre Shar", suggest: [:name], fields: [:name]).suggestions.first 74 | end 75 | 76 | def test_multiple_models_no_fields 77 | store_names ["Great White Shark", "Hammerhead Shark", "Tiger Shark"] 78 | assert_raises(ArgumentError) { Searchkick.search("How Big is a Tigre Shar", suggest: true) } 79 | end 80 | 81 | def test_star 82 | assert_equal [], Product.search("*", suggest: true).suggestions 83 | end 84 | 85 | protected 86 | 87 | def assert_suggest(term, expected, options = {}) 88 | result = Product.search(term, options.merge(suggest: true)).suggestions.first 89 | if expected.nil? 90 | assert_nil result 91 | else 92 | assert_equal expected, result 93 | end 94 | end 95 | 96 | # any order 97 | def assert_suggest_all(term, expected, options = {}) 98 | assert_equal expected.sort, Product.search(term, options.merge(suggest: true)).suggestions.sort 99 | end 100 | end 101 | -------------------------------------------------------------------------------- /lib/searchkick/record_data.rb: -------------------------------------------------------------------------------- 1 | module Searchkick 2 | class RecordData 3 | TYPE_KEYS = ["type", :type] 4 | 5 | attr_reader :index, :record 6 | 7 | def initialize(index, record) 8 | @index = index 9 | @record = record 10 | end 11 | 12 | def index_data 13 | data = record_data 14 | data[:data] = search_data 15 | {index: data} 16 | end 17 | 18 | def update_data(method_name) 19 | data = record_data 20 | data[:data] = {doc: search_data(method_name)} 21 | {update: data} 22 | end 23 | 24 | def delete_data 25 | {delete: record_data} 26 | end 27 | 28 | def search_id 29 | id = record.respond_to?(:search_document_id) ? record.search_document_id : record.id 30 | id.is_a?(Numeric) ? id : id.to_s 31 | end 32 | 33 | def document_type(ignore_type = false) 34 | index.klass_document_type(record.class, ignore_type) 35 | end 36 | 37 | private 38 | 39 | def record_data 40 | data = { 41 | _index: index.name, 42 | _id: search_id, 43 | _type: document_type 44 | } 45 | data[:_routing] = record.search_routing if record.respond_to?(:search_routing) 46 | data 47 | end 48 | 49 | def search_data(method_name = nil) 50 | partial_reindex = !method_name.nil? 51 | 52 | source = record.send(method_name || :search_data) 53 | 54 | # conversions 55 | index.conversions_fields.each do |conversions_field| 56 | if source[conversions_field] 57 | source[conversions_field] = source[conversions_field].map { |k, v| {query: k, count: v} } 58 | end 59 | end 60 | 61 | # hack to prevent generator field doesn't exist error 62 | if !partial_reindex 63 | index.suggest_fields.each do |field| 64 | if !source[field] && !source[field.to_sym] 65 | source[field] = nil 66 | end 67 | end 68 | end 69 | 70 | # locations 71 | index.locations_fields.each do |field| 72 | if source[field] 73 | if !source[field].is_a?(Hash) && (source[field].first.is_a?(Array) || source[field].first.is_a?(Hash)) 74 | # multiple locations 75 | source[field] = source[field].map { |a| location_value(a) } 76 | else 77 | source[field] = location_value(source[field]) 78 | end 79 | end 80 | end 81 | 82 | if index.options[:inheritance] 83 | if !TYPE_KEYS.any? { |tk| source.key?(tk) } 84 | source[:type] = document_type(true) 85 | end 86 | end 87 | 88 | cast_big_decimal(source) 89 | 90 | source 91 | end 92 | 93 | def location_value(value) 94 | if value.is_a?(Array) 95 | value.map(&:to_f).reverse 96 | elsif value.is_a?(Hash) 97 | {lat: value[:lat].to_f, lon: value[:lon].to_f} 98 | else 99 | value 100 | end 101 | end 102 | 103 | # change all BigDecimal values to floats due to 104 | # https://github.com/rails/rails/issues/6033 105 | # possible loss of precision :/ 106 | def cast_big_decimal(obj) 107 | case obj 108 | when BigDecimal 109 | obj.to_f 110 | when Hash 111 | obj.each do |k, v| 112 | # performance 113 | if v.is_a?(BigDecimal) 114 | obj[k] = v.to_f 115 | elsif v.is_a?(Enumerable) || 116 | obj[k] = cast_big_decimal(v) 117 | end 118 | end 119 | when Enumerable 120 | obj.map do |v| 121 | cast_big_decimal(v) 122 | end 123 | else 124 | obj 125 | end 126 | end 127 | end 128 | end 129 | -------------------------------------------------------------------------------- /test/pagination_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | class PaginationTest < Minitest::Test 4 | def test_limit 5 | store_names ["Product A", "Product B", "Product C", "Product D"] 6 | assert_order "product", ["Product A", "Product B"], order: {name: :asc}, limit: 2 7 | end 8 | 9 | def test_no_limit 10 | names = 20.times.map { |i| "Product #{i}" } 11 | store_names names 12 | assert_search "product", names 13 | end 14 | 15 | def test_offset 16 | store_names ["Product A", "Product B", "Product C", "Product D"] 17 | assert_order "product", ["Product C", "Product D"], order: {name: :asc}, offset: 2, limit: 100 18 | end 19 | 20 | def test_pagination 21 | store_names ["Product A", "Product B", "Product C", "Product D", "Product E", "Product F"] 22 | products = Product.search("product", order: {name: :asc}, page: 2, per_page: 2, padding: 1) 23 | assert_equal ["Product D", "Product E"], products.map(&:name) 24 | assert_equal "product", products.entry_name 25 | assert_equal 2, products.current_page 26 | assert_equal 1, products.padding 27 | assert_equal 2, products.per_page 28 | assert_equal 2, products.size 29 | assert_equal 2, products.length 30 | assert_equal 3, products.total_pages 31 | assert_equal 6, products.total_count 32 | assert_equal 6, products.total_entries 33 | assert_equal 2, products.limit_value 34 | assert_equal 3, products.offset_value 35 | assert_equal 3, products.offset 36 | assert_equal 3, products.next_page 37 | assert_equal 1, products.previous_page 38 | assert_equal 1, products.prev_page 39 | assert !products.first_page? 40 | assert !products.last_page? 41 | assert !products.empty? 42 | assert !products.out_of_range? 43 | assert products.any? 44 | end 45 | 46 | def test_pagination_body 47 | store_names ["Product A", "Product B", "Product C", "Product D", "Product E", "Product F"] 48 | products = Product.search("product", body: {query: {match_all: {}}, sort: [{name: "asc"}]}, page: 2, per_page: 2, padding: 1) 49 | assert_equal ["Product D", "Product E"], products.map(&:name) 50 | assert_equal "product", products.entry_name 51 | assert_equal 2, products.current_page 52 | assert_equal 1, products.padding 53 | assert_equal 2, products.per_page 54 | assert_equal 2, products.size 55 | assert_equal 2, products.length 56 | assert_equal 3, products.total_pages 57 | assert_equal 6, products.total_count 58 | assert_equal 6, products.total_entries 59 | assert_equal 2, products.limit_value 60 | assert_equal 3, products.offset_value 61 | assert_equal 3, products.offset 62 | assert_equal 3, products.next_page 63 | assert_equal 1, products.previous_page 64 | assert_equal 1, products.prev_page 65 | assert !products.first_page? 66 | assert !products.last_page? 67 | assert !products.empty? 68 | assert !products.out_of_range? 69 | assert products.any? 70 | end 71 | 72 | def test_pagination_nil_page 73 | store_names ["Product A", "Product B", "Product C", "Product D", "Product E"] 74 | products = Product.search("product", order: {name: :asc}, page: nil, per_page: 2) 75 | assert_equal ["Product A", "Product B"], products.map(&:name) 76 | assert_equal 1, products.current_page 77 | assert products.first_page? 78 | end 79 | 80 | def test_kaminari 81 | skip unless defined?(Kaminari) 82 | 83 | require "action_view" 84 | 85 | I18n.load_path = Dir["test/support/kaminari.yml"] 86 | I18n.backend.load_translations 87 | 88 | view = ActionView::Base.new 89 | 90 | store_names ["Product A"] 91 | assert_equal "Displaying 1 product", view.page_entries_info(Product.search("product")) 92 | 93 | store_names ["Product B"] 94 | assert_equal "Displaying all 2 products", view.page_entries_info(Product.search("product")) 95 | end 96 | end 97 | -------------------------------------------------------------------------------- /test/highlight_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | class HighlightTest < Minitest::Test 4 | def test_basic 5 | store_names ["Two Door Cinema Club"] 6 | assert_equal "Two Door Cinema Club", Product.search("cinema", highlight: true).highlights.first[:name] 7 | end 8 | 9 | def test_tag 10 | store_names ["Two Door Cinema Club"] 11 | assert_equal "Two Door Cinema Club", Product.search("cinema", highlight: {tag: ""}).highlights.first[:name] 12 | end 13 | 14 | def test_tag_class 15 | store_names ["Two Door Cinema Club"] 16 | assert_equal "Two Door Cinema Club", Product.search("cinema", highlight: {tag: ""}).highlights.first[:name] 17 | end 18 | 19 | def test_very_long 20 | store_names [("Two Door Cinema Club " * 100).strip] 21 | assert_equal ("Two Door Cinema Club " * 100).strip, Product.search("cinema", highlight: true).highlights.first[:name] 22 | end 23 | 24 | def test_multiple_fields 25 | store [{name: "Two Door Cinema Club", color: "Cinema Orange"}] 26 | highlights = Product.search("cinema", fields: [:name, :color], highlight: true).highlights.first 27 | assert_equal "Two Door Cinema Club", highlights[:name] 28 | assert_equal "Cinema Orange", highlights[:color] 29 | end 30 | 31 | def test_fields 32 | store [{name: "Two Door Cinema Club", color: "Cinema Orange"}] 33 | highlights = Product.search("cinema", fields: [:name, :color], highlight: {fields: [:name]}).highlights.first 34 | assert_equal "Two Door Cinema Club", highlights[:name] 35 | assert_nil highlights[:color] 36 | end 37 | 38 | def test_field_options 39 | store_names ["Two Door Cinema Club are a Northern Irish indie rock band"] 40 | fragment_size = ENV["MATCH"] == "word_start" ? 26 : 21 41 | assert_equal "Two Door Cinema Club are", Product.search("cinema", highlight: {fields: {name: {fragment_size: fragment_size}}}).highlights.first[:name] 42 | end 43 | 44 | def test_multiple_words 45 | store_names ["Hello World Hello"] 46 | assert_equal "Hello World Hello", Product.search("hello", highlight: true).highlights.first[:name] 47 | end 48 | 49 | def test_encoder 50 | store_names ["Hello"] 51 | assert_equal "<b>Hello</b>", Product.search("hello", highlight: {encoder: "html"}, misspellings: false).highlights.first[:name] 52 | end 53 | 54 | def test_word_middle 55 | store_names ["Two Door Cinema Club"] 56 | assert_equal "Two Door Cinema Club", Product.search("ine", match: :word_middle, highlight: true).highlights.first[:name] 57 | end 58 | 59 | def test_body 60 | skip if ENV["MATCH"] == "word_start" 61 | store_names ["Two Door Cinema Club"] 62 | body = { 63 | query: { 64 | match: { 65 | "name.analyzed" => "cinema" 66 | } 67 | }, 68 | highlight: { 69 | pre_tags: [""], 70 | post_tags: [""], 71 | fields: { 72 | "name.analyzed" => {} 73 | } 74 | } 75 | } 76 | assert_equal "Two Door Cinema Club", Product.search(body: body).highlights.first[:"name.analyzed"] 77 | end 78 | 79 | def test_multiple_highlights 80 | store_names ["Two Door Cinema Club Some Other Words And Much More Doors Cinema Club"] 81 | highlights = Product.search("cinema", highlight: {fragment_size: 20}).highlights(multiple: true).first[:name] 82 | assert highlights.is_a?(Array) 83 | assert_equal highlights.count, 2 84 | refute_equal highlights.first, highlights.last 85 | highlights.each do |highlight| 86 | assert highlight.include?("Cinema") 87 | end 88 | end 89 | 90 | def test_search_highlights_method 91 | store_names ["Two Door Cinema Club"] 92 | assert_equal "Two Door Cinema Club", Product.search("cinema", highlight: true).first.search_highlights[:name] 93 | end 94 | end 95 | -------------------------------------------------------------------------------- /test/geo_shape_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | class GeoShapeTest < Minitest::Test 4 | def setup 5 | Region.destroy_all 6 | store [ 7 | { 8 | name: "Region A", 9 | text: "The witch had a cat", 10 | territory: { 11 | type: "polygon", 12 | coordinates: [[[30, 40], [35, 45], [40, 40], [40, 30], [30, 30], [30, 40]]] 13 | } 14 | }, 15 | { 16 | name: "Region B", 17 | text: "and a very tall hat", 18 | territory: { 19 | type: "polygon", 20 | coordinates: [[[50, 60], [55, 65], [60, 60], [60, 50], [50, 50], [50, 60]]] 21 | } 22 | }, 23 | { 24 | name: "Region C", 25 | text: "and long ginger hair which she wore in a plait", 26 | territory: { 27 | type: "polygon", 28 | coordinates: [[[10, 20], [15, 25], [20, 20], [20, 10], [10, 10], [10, 20]]] 29 | } 30 | } 31 | ], Region 32 | end 33 | 34 | def test_circle 35 | assert_search "*", ["Region A"], { 36 | where: { 37 | territory: { 38 | geo_shape: { 39 | type: "circle", 40 | coordinates: {lat: 28.0, lon: 38.0}, 41 | radius: "444000m" 42 | } 43 | } 44 | } 45 | }, Region 46 | end 47 | 48 | def test_envelope 49 | assert_search "*", ["Region A"], { 50 | where: { 51 | territory: { 52 | geo_shape: { 53 | type: "envelope", 54 | coordinates: [[28, 42], [32, 38]] 55 | } 56 | } 57 | } 58 | }, Region 59 | end 60 | 61 | def test_polygon 62 | assert_search "*", ["Region A"], { 63 | where: { 64 | territory: { 65 | geo_shape: { 66 | type: "polygon", 67 | coordinates: [[[38, 42], [42, 42], [42, 38], [38, 38], [38, 42]]] 68 | } 69 | } 70 | } 71 | }, Region 72 | end 73 | 74 | def test_multipolygon 75 | assert_search "*", ["Region A", "Region B"], { 76 | where: { 77 | territory: { 78 | geo_shape: { 79 | type: "multipolygon", 80 | coordinates: [ 81 | [[[38, 42], [42, 42], [42, 38], [38, 38], [38, 42]]], 82 | [[[58, 62], [62, 62], [62, 58], [58, 58], [58, 62]]] 83 | ] 84 | } 85 | } 86 | } 87 | }, Region 88 | end 89 | 90 | def test_disjoint 91 | assert_search "*", ["Region B", "Region C"], { 92 | where: { 93 | territory: { 94 | geo_shape: { 95 | type: "envelope", 96 | relation: "disjoint", 97 | coordinates: [[28, 42], [32, 38]] 98 | } 99 | } 100 | } 101 | }, Region 102 | end 103 | 104 | def test_within 105 | assert_search "*", ["Region A"], { 106 | where: { 107 | territory: { 108 | geo_shape: { 109 | type: "envelope", 110 | relation: "within", 111 | coordinates: [[20, 50], [50, 20]] 112 | } 113 | } 114 | } 115 | }, Region 116 | end 117 | 118 | def test_search_math 119 | assert_search "witch", ["Region A"], { 120 | where: { 121 | territory: { 122 | geo_shape: { 123 | type: "envelope", 124 | coordinates: [[28, 42], [32, 38]] 125 | } 126 | } 127 | } 128 | }, Region 129 | end 130 | 131 | def test_search_no_match 132 | assert_search "ginger hair", [], { 133 | where: { 134 | territory: { 135 | geo_shape: { 136 | type: "envelope", 137 | coordinates: [[28, 42], [32, 38]] 138 | } 139 | } 140 | } 141 | }, Region 142 | end 143 | 144 | def test_contains 145 | assert_search "*", ["Region C"], { 146 | where: { 147 | territory: { 148 | geo_shape: { 149 | type: "envelope", 150 | relation: "contains", 151 | coordinates: [[12, 13], [13, 12]] 152 | } 153 | } 154 | } 155 | }, Region 156 | end 157 | 158 | def test_latlon 159 | assert_search "*", ["Region A"], { 160 | where: { 161 | territory: { 162 | geo_shape: { 163 | type: "envelope", 164 | coordinates: [{lat: 42, lon: 28}, {lat: 38, lon: 32}] 165 | } 166 | } 167 | } 168 | }, Region 169 | end 170 | 171 | end 172 | -------------------------------------------------------------------------------- /lib/searchkick/model.rb: -------------------------------------------------------------------------------- 1 | module Searchkick 2 | module Model 3 | def searchkick(**options) 4 | options = Searchkick.model_options.merge(options) 5 | 6 | unknown_keywords = options.keys - [:_all, :_type, :batch_size, :callbacks, :conversions, :default_fields, 7 | :filterable, :geo_shape, :highlight, :ignore_above, :index_name, :index_prefix, :inheritance, :language, 8 | :locations, :mappings, :match, :merge_mappings, :routing, :searchable, :settings, :similarity, 9 | :special_characters, :stem_conversions, :suggest, :synonyms, :text_end, 10 | :text_middle, :text_start, :word, :wordnet, :word_end, :word_middle, :word_start] 11 | raise ArgumentError, "unknown keywords: #{unknown_keywords.join(", ")}" if unknown_keywords.any? 12 | 13 | raise "Only call searchkick once per model" if respond_to?(:searchkick_index) 14 | 15 | Searchkick.models << self 16 | 17 | options[:_type] ||= -> { searchkick_index.klass_document_type(self, true) } 18 | 19 | callbacks = options.key?(:callbacks) ? options[:callbacks] : true 20 | unless [true, false, :async, :queue].include?(callbacks) 21 | raise ArgumentError, "Invalid value for callbacks" 22 | end 23 | 24 | index_name = 25 | if options[:index_name] 26 | options[:index_name] 27 | elsif options[:index_prefix].respond_to?(:call) 28 | -> { [options[:index_prefix].call, model_name.plural, Searchkick.env, Searchkick.index_suffix].compact.join("_") } 29 | else 30 | [options.key?(:index_prefix) ? options[:index_prefix] : Searchkick.index_prefix, model_name.plural, Searchkick.env, Searchkick.index_suffix].compact.join("_") 31 | end 32 | 33 | class_eval do 34 | cattr_reader :searchkick_options, :searchkick_klass 35 | 36 | class_variable_set :@@searchkick_options, options.dup 37 | class_variable_set :@@searchkick_klass, self 38 | class_variable_set :@@searchkick_index, index_name 39 | class_variable_set :@@searchkick_index_cache, {} 40 | 41 | class << self 42 | def searchkick_search(term = "*", **options, &block) 43 | Searchkick.search(term, {model: self}.merge(options), &block) 44 | end 45 | alias_method Searchkick.search_method_name, :searchkick_search if Searchkick.search_method_name 46 | 47 | def searchkick_index 48 | index = class_variable_get(:@@searchkick_index) 49 | index = index.call if index.respond_to?(:call) 50 | index_cache = class_variable_get(:@@searchkick_index_cache) 51 | index_cache[index] ||= Searchkick::Index.new(index, searchkick_options) 52 | end 53 | alias_method :search_index, :searchkick_index unless method_defined?(:search_index) 54 | 55 | def searchkick_reindex(method_name = nil, **options) 56 | scoped = (respond_to?(:current_scope) && respond_to?(:default_scoped) && current_scope && current_scope.to_sql != default_scoped.to_sql) || 57 | (respond_to?(:queryable) && queryable != unscoped.with_default_scope) 58 | 59 | searchkick_index.reindex(searchkick_klass, method_name, scoped: scoped, **options) 60 | end 61 | alias_method :reindex, :searchkick_reindex unless method_defined?(:reindex) 62 | 63 | def searchkick_index_options 64 | searchkick_index.index_options 65 | end 66 | end 67 | 68 | # always add callbacks, even when callbacks is false 69 | # so Model.callbacks block can be used 70 | if respond_to?(:after_commit) 71 | after_commit :reindex, if: -> { Searchkick.callbacks?(default: callbacks) } 72 | elsif respond_to?(:after_save) 73 | after_save :reindex, if: -> { Searchkick.callbacks?(default: callbacks) } 74 | after_destroy :reindex, if: -> { Searchkick.callbacks?(default: callbacks) } 75 | end 76 | 77 | def reindex(method_name = nil, **options) 78 | RecordIndexer.new(self).reindex(method_name, **options) 79 | end unless method_defined?(:reindex) 80 | 81 | def similar(options = {}) 82 | self.class.searchkick_index.similar_record(self, options) 83 | end unless method_defined?(:similar) 84 | 85 | def search_data 86 | data = respond_to?(:to_hash) ? to_hash : serializable_hash 87 | data.delete("id") 88 | data.delete("_id") 89 | data.delete("_type") 90 | data 91 | end unless method_defined?(:search_data) 92 | 93 | def should_index? 94 | true 95 | end unless method_defined?(:should_index?) 96 | 97 | if defined?(Cequel) && self < Cequel::Record && !method_defined?(:destroyed?) 98 | def destroyed? 99 | transient? 100 | end 101 | end 102 | end 103 | end 104 | end 105 | end 106 | -------------------------------------------------------------------------------- /lib/searchkick/bulk_indexer.rb: -------------------------------------------------------------------------------- 1 | module Searchkick 2 | class BulkIndexer 3 | attr_reader :index 4 | 5 | def initialize(index) 6 | @index = index 7 | end 8 | 9 | def import_scope(relation, resume: false, method_name: nil, async: false, batch: false, batch_id: nil, full: false, scope: nil) 10 | if scope 11 | relation = relation.send(scope) 12 | elsif relation.respond_to?(:search_import) 13 | relation = relation.search_import 14 | end 15 | 16 | if batch 17 | import_or_update relation.to_a, method_name, async 18 | Searchkick.with_redis { |r| r.srem(batches_key, batch_id) } if batch_id 19 | elsif full && async 20 | full_reindex_async(relation) 21 | elsif relation.respond_to?(:find_in_batches) 22 | if resume 23 | # use total docs instead of max id since there's not a great way 24 | # to get the max _id without scripting since it's a string 25 | 26 | # TODO use primary key and prefix with table name 27 | relation = relation.where("id > ?", total_docs) 28 | end 29 | 30 | relation = relation.select("id").except(:includes, :preload) if async 31 | 32 | relation.find_in_batches batch_size: batch_size do |items| 33 | import_or_update items, method_name, async 34 | end 35 | else 36 | each_batch(relation) do |items| 37 | import_or_update items, method_name, async 38 | end 39 | end 40 | end 41 | 42 | def bulk_index(records) 43 | Searchkick.indexer.queue(records.map { |r| RecordData.new(index, r).index_data }) 44 | end 45 | 46 | def bulk_delete(records) 47 | Searchkick.indexer.queue(records.reject { |r| r.id.blank? }.map { |r| RecordData.new(index, r).delete_data }) 48 | end 49 | 50 | def bulk_update(records, method_name) 51 | Searchkick.indexer.queue(records.map { |r| RecordData.new(index, r).update_data(method_name) }) 52 | end 53 | 54 | def batches_left 55 | Searchkick.with_redis { |r| r.scard(batches_key) } 56 | end 57 | 58 | private 59 | 60 | def import_or_update(records, method_name, async) 61 | if records.any? 62 | if async 63 | Searchkick::BulkReindexJob.perform_later( 64 | class_name: records.first.class.name, 65 | record_ids: records.map(&:id), 66 | index_name: index.name, 67 | method_name: method_name ? method_name.to_s : nil 68 | ) 69 | else 70 | records = records.select(&:should_index?) 71 | if records.any? 72 | with_retries do 73 | # call out to index for ActiveSupport notifications 74 | if method_name 75 | index.bulk_update(records, method_name) 76 | else 77 | index.bulk_index(records) 78 | end 79 | end 80 | end 81 | end 82 | end 83 | end 84 | 85 | def full_reindex_async(scope) 86 | if scope.respond_to?(:primary_key) 87 | # TODO expire Redis key 88 | primary_key = scope.primary_key 89 | 90 | starting_id = 91 | begin 92 | scope.minimum(primary_key) 93 | rescue ActiveRecord::StatementInvalid 94 | false 95 | end 96 | 97 | if starting_id.nil? 98 | # no records, do nothing 99 | elsif starting_id.is_a?(Numeric) 100 | max_id = scope.maximum(primary_key) 101 | batches_count = ((max_id - starting_id + 1) / batch_size.to_f).ceil 102 | 103 | batches_count.times do |i| 104 | batch_id = i + 1 105 | min_id = starting_id + (i * batch_size) 106 | bulk_reindex_job scope, batch_id, min_id: min_id, max_id: min_id + batch_size - 1 107 | end 108 | else 109 | scope.find_in_batches(batch_size: batch_size).each_with_index do |batch, i| 110 | batch_id = i + 1 111 | 112 | bulk_reindex_job scope, batch_id, record_ids: batch.map { |record| record.id.to_s } 113 | end 114 | end 115 | else 116 | batch_id = 1 117 | # TODO remove any eager loading 118 | scope = scope.only(:_id) if scope.respond_to?(:only) 119 | each_batch(scope) do |items| 120 | bulk_reindex_job scope, batch_id, record_ids: items.map { |i| i.id.to_s } 121 | batch_id += 1 122 | end 123 | end 124 | end 125 | 126 | def each_batch(scope) 127 | # https://github.com/karmi/tire/blob/master/lib/tire/model/import.rb 128 | # use cursor for Mongoid 129 | items = [] 130 | scope.all.each do |item| 131 | items << item 132 | if items.length == batch_size 133 | yield items 134 | items = [] 135 | end 136 | end 137 | yield items if items.any? 138 | end 139 | 140 | def bulk_reindex_job(scope, batch_id, options) 141 | Searchkick.with_redis { |r| r.sadd(batches_key, batch_id) } 142 | Searchkick::BulkReindexJob.perform_later({ 143 | class_name: scope.model_name.name, 144 | index_name: index.name, 145 | batch_id: batch_id 146 | }.merge(options)) 147 | end 148 | 149 | def with_retries 150 | retries = 0 151 | 152 | begin 153 | yield 154 | rescue Faraday::ClientError => e 155 | if retries < 1 156 | retries += 1 157 | retry 158 | end 159 | raise e 160 | end 161 | end 162 | 163 | def batches_key 164 | "searchkick:reindex:#{index.name}:batches" 165 | end 166 | 167 | def batch_size 168 | @batch_size ||= index.options[:batch_size] || 1000 169 | end 170 | end 171 | end 172 | -------------------------------------------------------------------------------- /test/index_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | class IndexTest < Minitest::Test 4 | def setup 5 | super 6 | Region.destroy_all 7 | end 8 | 9 | def test_clean_indices 10 | suffix = Searchkick.index_suffix ? "_#{Searchkick.index_suffix}" : "" 11 | old_index = Searchkick::Index.new("products_test#{suffix}_20130801000000000") 12 | different_index = Searchkick::Index.new("items_test#{suffix}_20130801000000000") 13 | 14 | old_index.delete if old_index.exists? 15 | different_index.delete if different_index.exists? 16 | 17 | # create indexes 18 | old_index.create 19 | different_index.create 20 | 21 | Product.searchkick_index.clean_indices 22 | 23 | assert Product.searchkick_index.exists? 24 | assert different_index.exists? 25 | assert !old_index.exists? 26 | end 27 | 28 | def test_clean_indices_old_format 29 | suffix = Searchkick.index_suffix ? "_#{Searchkick.index_suffix}" : "" 30 | old_index = Searchkick::Index.new("products_test#{suffix}_20130801000000") 31 | old_index.create 32 | 33 | Product.searchkick_index.clean_indices 34 | 35 | assert !old_index.exists? 36 | end 37 | 38 | def test_retain 39 | Product.reindex 40 | assert_equal 1, Product.searchkick_index.all_indices.size 41 | Product.reindex(retain: true) 42 | assert_equal 2, Product.searchkick_index.all_indices.size 43 | end 44 | 45 | def test_total_docs 46 | store_names ["Product A"] 47 | assert_equal 1, Product.searchkick_index.total_docs 48 | end 49 | 50 | def test_mapping 51 | store_names ["Dollar Tree"], Store 52 | assert_equal [], Store.search(body: {query: {match: {name: "dollar"}}}).map(&:name) 53 | assert_equal ["Dollar Tree"], Store.search(body: {query: {match: {name: "Dollar Tree"}}}).map(&:name) 54 | end 55 | 56 | def test_body 57 | store_names ["Dollar Tree"], Store 58 | assert_equal [], Store.search(body: {query: {match: {name: "dollar"}}}).map(&:name) 59 | assert_equal ["Dollar Tree"], Store.search(body: {query: {match: {name: "Dollar Tree"}}}, load: false).map(&:name) 60 | end 61 | 62 | def test_body_incompatible_options 63 | assert_raises(ArgumentError) do 64 | Store.search(body: {query: {match: {name: "dollar"}}}, where: {id: 1}) 65 | end 66 | end 67 | 68 | def test_block 69 | store_names ["Dollar Tree"] 70 | products = 71 | Product.search "boom" do |body| 72 | body[:query] = {match_all: {}} 73 | end 74 | assert_equal ["Dollar Tree"], products.map(&:name) 75 | end 76 | 77 | def test_tokens 78 | assert_equal ["dollar", "dollartre", "tree"], Product.searchkick_index.tokens("Dollar Tree", analyzer: "searchkick_index") 79 | end 80 | 81 | def test_tokens_analyzer 82 | assert_equal ["dollar", "tree"], Product.searchkick_index.tokens("Dollar Tree", analyzer: "searchkick_search2") 83 | end 84 | 85 | def test_record_not_found 86 | store_names ["Product A", "Product B"] 87 | Product.where(name: "Product A").delete_all 88 | assert_output nil, "[searchkick] WARNING: Records in search index do not exist in database\n" do 89 | assert_search "product", ["Product B"] 90 | end 91 | ensure 92 | Product.reindex 93 | end 94 | 95 | def test_bad_mapping 96 | Product.searchkick_index.delete 97 | store_names ["Product A"] 98 | error = assert_raises(Searchkick::InvalidQueryError) { Product.search "test" } 99 | assert_equal "Bad mapping - run Product.reindex", error.message 100 | ensure 101 | Product.reindex 102 | end 103 | 104 | def test_remove_blank_id 105 | store_names ["Product A"] 106 | Product.searchkick_index.remove(Product.new) 107 | assert_search "product", ["Product A"] 108 | ensure 109 | Product.reindex 110 | end 111 | 112 | def test_missing_index 113 | assert_raises(Searchkick::MissingIndexError) { Product.search("test", index_name: "not_found") } 114 | end 115 | 116 | def test_unsupported_version 117 | raises_exception = ->(_) { raise Elasticsearch::Transport::Transport::Error, "[500] No query registered for [multi_match]" } 118 | Searchkick.client.stub :search, raises_exception do 119 | assert_raises(Searchkick::UnsupportedVersionError) { Product.search("test") } 120 | end 121 | end 122 | 123 | def test_invalid_body 124 | assert_raises(Searchkick::InvalidQueryError) { Product.search(body: {boom: true}) } 125 | end 126 | 127 | def test_transaction 128 | skip unless defined?(ActiveRecord) 129 | Product.transaction do 130 | store_names ["Product A"] 131 | raise ActiveRecord::Rollback 132 | end 133 | assert_search "*", [] 134 | end 135 | 136 | def test_filterable 137 | # skip for 5.0 since it throws 138 | # Cannot search on field [alt_description] since it is not indexed. 139 | store [{name: "Product A", alt_description: "Hello"}] 140 | assert_raises(Searchkick::InvalidQueryError) do 141 | assert_search "*", [], where: {alt_description: "Hello"} 142 | end 143 | end 144 | 145 | def test_filterable_non_string 146 | store [{name: "Product A", store_id: 1}] 147 | assert_search "*", ["Product A"], where: {store_id: 1} 148 | end 149 | 150 | def test_large_value 151 | skip if nobrainer? 152 | large_value = 1000.times.map { "hello" }.join(" ") 153 | store [{name: "Product A", text: large_value}], Region 154 | assert_search "product", ["Product A"], {}, Region 155 | assert_search "hello", ["Product A"], {fields: [:name, :text]}, Region 156 | assert_search "hello", ["Product A"], {}, Region 157 | end 158 | 159 | def test_very_large_value 160 | skip if nobrainer? 161 | large_value = 10000.times.map { "hello" }.join(" ") 162 | store [{name: "Product A", text: large_value}], Region 163 | assert_search "product", ["Product A"], {}, Region 164 | assert_search "hello", ["Product A"], {fields: [:name, :text]}, Region 165 | # values that exceed ignore_above are not included in _all field :( 166 | # assert_search "hello", ["Product A"], {}, Region 167 | end 168 | end 169 | -------------------------------------------------------------------------------- /test/sql_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | class SqlTest < Minitest::Test 4 | def test_operator 5 | store_names ["Honey"] 6 | assert_search "fresh honey", [] 7 | assert_search "fresh honey", ["Honey"], operator: "or" 8 | end 9 | 10 | def test_operator_scoring 11 | store_names ["Big Red Circle", "Big Green Circle", "Small Orange Circle"] 12 | assert_order "big red circle", ["Big Red Circle", "Big Green Circle", "Small Orange Circle"], operator: "or" 13 | end 14 | 15 | def test_fields_operator 16 | store [ 17 | {name: "red", color: "red"}, 18 | {name: "blue", color: "blue"}, 19 | {name: "cyan", color: "blue green"}, 20 | {name: "magenta", color: "red blue"}, 21 | {name: "green", color: "green"} 22 | ] 23 | assert_search "red blue", ["red", "blue", "cyan", "magenta"], operator: "or", fields: ["color"] 24 | end 25 | 26 | def test_fields 27 | store [ 28 | {name: "red", color: "light blue"}, 29 | {name: "blue", color: "red fish"} 30 | ] 31 | assert_search "blue", ["red"], fields: ["color"] 32 | end 33 | 34 | def test_non_existent_field 35 | store_names ["Milk"] 36 | assert_search "milk", [], fields: ["not_here"] 37 | end 38 | 39 | def test_fields_both_match 40 | store [ 41 | {name: "Blue A", color: "red"}, 42 | {name: "Blue B", color: "light blue"} 43 | ] 44 | assert_first "blue", "Blue B", fields: [:name, :color] 45 | end 46 | 47 | def test_big_decimal 48 | store [ 49 | {name: "Product", latitude: 80.0} 50 | ] 51 | assert_search "product", ["Product"], where: {latitude: {gt: 79}} 52 | end 53 | 54 | # body_options 55 | 56 | def test_body_options_should_merge_into_body 57 | query = Product.search("*", body_options: {min_score: 1.0}, execute: false) 58 | assert_equal 1.0, query.body[:min_score] 59 | end 60 | 61 | # load 62 | 63 | def test_load_default 64 | store_names ["Product A"] 65 | assert_kind_of Product, Product.search("product").first 66 | end 67 | 68 | def test_load_false 69 | store_names ["Product A"] 70 | assert_kind_of Hash, Product.search("product", load: false).first 71 | end 72 | 73 | def test_load_false_methods 74 | store_names ["Product A"] 75 | assert_equal "Product A", Product.search("product", load: false).first.name 76 | end 77 | 78 | def test_load_false_with_includes 79 | store_names ["Product A"] 80 | assert_kind_of Hash, Product.search("product", load: false, includes: [:store]).first 81 | end 82 | 83 | def test_load_false_nested_object 84 | aisle = {"id" => 1, "name" => "Frozen"} 85 | store [{name: "Product A", aisle: aisle}] 86 | assert_equal aisle, Product.search("product", load: false).first.aisle.to_hash 87 | end 88 | 89 | # select 90 | 91 | def test_select 92 | store [{name: "Product A", store_id: 1}] 93 | result = Product.search("product", load: false, select: [:name, :store_id]).first 94 | assert_equal %w(id name store_id), result.keys.reject { |k| k.start_with?("_") }.sort 95 | assert_equal "Product A", result.name 96 | assert_equal 1, result.store_id 97 | end 98 | 99 | def test_select_array 100 | store [{name: "Product A", user_ids: [1, 2]}] 101 | result = Product.search("product", load: false, select: [:user_ids]).first 102 | assert_equal [1, 2], result.user_ids 103 | end 104 | 105 | def test_select_single_field 106 | store [{name: "Product A", store_id: 1}] 107 | result = Product.search("product", load: false, select: :name).first 108 | assert_equal %w(id name), result.keys.reject { |k| k.start_with?("_") }.sort 109 | assert_equal "Product A", result.name 110 | assert_nil result.store_id 111 | end 112 | 113 | def test_select_all 114 | store [{name: "Product A", user_ids: [1, 2]}] 115 | hit = Product.search("product", select: true).hits.first 116 | assert_equal hit["_source"]["name"], "Product A" 117 | assert_equal hit["_source"]["user_ids"], [1, 2] 118 | end 119 | 120 | def test_select_none 121 | store [{name: "Product A", user_ids: [1, 2]}] 122 | hit = Product.search("product", select: []).hits.first 123 | assert_nil hit["_source"] 124 | hit = Product.search("product", select: false).hits.first 125 | assert_nil hit["_source"] 126 | end 127 | 128 | def test_select_includes 129 | store [{name: "Product A", user_ids: [1, 2]}] 130 | result = Product.search("product", load: false, select: {includes: [:name]}).first 131 | assert_equal %w(id name), result.keys.reject { |k| k.start_with?("_") }.sort 132 | assert_equal "Product A", result.name 133 | assert_nil result.store_id 134 | end 135 | 136 | def test_select_excludes 137 | store [{name: "Product A", user_ids: [1, 2], store_id: 1}] 138 | result = Product.search("product", load: false, select: {excludes: [:name]}).first 139 | assert_nil result.name 140 | assert_equal [1, 2], result.user_ids 141 | assert_equal 1, result.store_id 142 | end 143 | 144 | def test_select_include_and_excludes 145 | # let's take this to the next level 146 | store [{name: "Product A", user_ids: [1, 2], store_id: 1}] 147 | result = Product.search("product", load: false, select: {includes: [:store_id], excludes: [:name]}).first 148 | assert_equal 1, result.store_id 149 | assert_nil result.name 150 | assert_nil result.user_ids 151 | end 152 | 153 | # nested 154 | 155 | def test_nested_search 156 | store [{name: "Product A", aisle: {"id" => 1, "name" => "Frozen"}}], Speaker 157 | assert_search "frozen", ["Product A"], {fields: ["aisle.name"]}, Speaker 158 | end 159 | 160 | # other tests 161 | 162 | def test_includes 163 | skip unless defined?(ActiveRecord) 164 | store_names ["Product A"] 165 | assert Product.search("product", includes: [:store]).first.association(:store).loaded? 166 | end 167 | 168 | def test_model_includes 169 | skip unless defined?(ActiveRecord) 170 | 171 | store_names ["Product A"] 172 | store_names ["Store A"], Store 173 | 174 | associations = {Product => [:store], Store => [:products]} 175 | result = Searchkick.search("*", index_name: [Product, Store], model_includes: associations) 176 | 177 | assert_equal 2, result.length 178 | 179 | result.group_by(&:class).each_pair do |klass, records| 180 | assert records.first.association(associations[klass].first).loaded? 181 | end 182 | end 183 | 184 | def test_scope_results 185 | skip unless defined?(ActiveRecord) 186 | 187 | store_names ["Product A", "Product B"] 188 | assert_search "product", ["Product A"], scope_results: ->(r) { r.where(name: "Product A") } 189 | end 190 | end 191 | -------------------------------------------------------------------------------- /test/aggs_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | class AggsTest < Minitest::Test 4 | def setup 5 | super 6 | store [ 7 | {name: "Product Show", latitude: 37.7833, longitude: 12.4167, store_id: 1, in_stock: true, color: "blue", price: 21, created_at: 2.days.ago}, 8 | {name: "Product Hide", latitude: 29.4167, longitude: -98.5000, store_id: 2, in_stock: false, color: "green", price: 25, created_at: 2.days.from_now}, 9 | {name: "Product B", latitude: 43.9333, longitude: -122.4667, store_id: 2, in_stock: false, color: "red", price: 5, created_at: Time.now}, 10 | {name: "Foo", latitude: 43.9333, longitude: 12.4667, store_id: 3, in_stock: false, color: "yellow", price: 15, created_at: Time.now} 11 | ] 12 | end 13 | 14 | def test_basic 15 | assert_equal ({1 => 1, 2 => 2}), store_agg(aggs: [:store_id]) 16 | end 17 | 18 | def test_where 19 | assert_equal ({1 => 1}), store_agg(aggs: {store_id: {where: {in_stock: true}}}) 20 | end 21 | 22 | def test_order 23 | agg = Product.search("Product", aggs: {color: {order: {"_term" => "desc"}}}).aggs["color"] 24 | assert_equal %w(red green blue), agg["buckets"].map { |b| b["key"] } 25 | end 26 | 27 | def test_field 28 | assert_equal ({1 => 1, 2 => 2}), store_agg(aggs: {store_id: {}}) 29 | assert_equal ({1 => 1, 2 => 2}), store_agg(aggs: {store_id: {field: "store_id"}}) 30 | assert_equal ({1 => 1, 2 => 2}), store_agg({aggs: {store_id_new: {field: "store_id"}}}, "store_id_new") 31 | end 32 | 33 | def test_min_doc_count 34 | assert_equal ({2 => 2}), store_agg(aggs: {store_id: {min_doc_count: 2}}) 35 | end 36 | 37 | def test_no_aggs 38 | assert_nil Product.search("*").aggs 39 | end 40 | 41 | def test_limit 42 | agg = Product.search("Product", aggs: {store_id: {limit: 1}}).aggs["store_id"] 43 | assert_equal 1, agg["buckets"].size 44 | # assert_equal 3, agg["doc_count"] 45 | assert_equal(1, agg["sum_other_doc_count"]) 46 | end 47 | 48 | def test_ranges 49 | price_ranges = [{to: 10}, {from: 10, to: 20}, {from: 20}] 50 | agg = Product.search("Product", aggs: {price: {ranges: price_ranges}}).aggs["price"] 51 | 52 | assert_equal 3, agg["buckets"].size 53 | assert_equal 10.0, agg["buckets"][0]["to"] 54 | assert_equal 20.0, agg["buckets"][2]["from"] 55 | assert_equal 1, agg["buckets"][0]["doc_count"] 56 | assert_equal 0, agg["buckets"][1]["doc_count"] 57 | assert_equal 2, agg["buckets"][2]["doc_count"] 58 | end 59 | 60 | def test_date_ranges 61 | ranges = [{to: 1.day.ago}, {from: 1.day.ago, to: 1.day.from_now}, {from: 1.day.from_now}] 62 | agg = Product.search("Product", aggs: {created_at: {date_ranges: ranges}}).aggs["created_at"] 63 | 64 | assert_equal 1, agg["buckets"][0]["doc_count"] 65 | assert_equal 1, agg["buckets"][1]["doc_count"] 66 | assert_equal 1, agg["buckets"][2]["doc_count"] 67 | end 68 | 69 | def test_query_where 70 | assert_equal ({1 => 1}), store_agg(where: {in_stock: true}, aggs: [:store_id]) 71 | end 72 | 73 | def test_two_wheres 74 | assert_equal ({2 => 1}), store_agg(where: {color: "red"}, aggs: {store_id: {where: {in_stock: false}}}) 75 | end 76 | 77 | def test_where_override 78 | assert_equal ({}), store_agg(where: {color: "red"}, aggs: {store_id: {where: {in_stock: false, color: "blue"}}}) 79 | assert_equal ({2 => 1}), store_agg(where: {color: "blue"}, aggs: {store_id: {where: {in_stock: false, color: "red"}}}) 80 | end 81 | 82 | def test_skip 83 | assert_equal ({1 => 1, 2 => 2}), store_agg(where: {store_id: 2}, aggs: [:store_id]) 84 | end 85 | 86 | def test_skip_complex 87 | assert_equal ({1 => 1, 2 => 1}), store_agg(where: {store_id: 2, price: {gt: 5}}, aggs: [:store_id]) 88 | end 89 | 90 | def test_multiple 91 | assert_equal ({"store_id" => {1 => 1, 2 => 2}, "color" => {"blue" => 1, "green" => 1, "red" => 1}}), store_multiple_aggs(aggs: [:store_id, :color]) 92 | end 93 | 94 | def test_smart_aggs_false 95 | assert_equal ({2 => 2}), store_agg(where: {color: "red"}, aggs: {store_id: {where: {in_stock: false}}}, smart_aggs: false) 96 | assert_equal ({2 => 2}), store_agg(where: {color: "blue"}, aggs: {store_id: {where: {in_stock: false}}}, smart_aggs: false) 97 | end 98 | 99 | def test_aggs_group_by_date 100 | store [{name: "Old Product", created_at: 3.years.ago}] 101 | products = 102 | Product.search("Product", { 103 | where: { 104 | created_at: {lt: Time.now} 105 | }, 106 | aggs: { 107 | products_per_year: { 108 | date_histogram: { 109 | field: :created_at, 110 | interval: :year 111 | } 112 | } 113 | } 114 | }) 115 | 116 | assert_equal 4, products.aggs["products_per_year"]["buckets"].size 117 | end 118 | 119 | def test_aggs_avg 120 | products = 121 | Product.search("*", { 122 | aggs: { 123 | avg_price: { 124 | avg: { 125 | field: :price 126 | } 127 | } 128 | } 129 | }) 130 | assert_equal 16.5, products.aggs["avg_price"]["value"] 131 | end 132 | 133 | def test_aggs_cardinality 134 | products = 135 | Product.search("*", { 136 | aggs: { 137 | total_stores: { 138 | cardinality: { 139 | field: :store_id 140 | } 141 | } 142 | } 143 | }) 144 | assert_equal 3, products.aggs["total_stores"]["value"] 145 | end 146 | 147 | def test_aggs_min_max 148 | products = 149 | Product.search("*", { 150 | aggs: { 151 | min_price: { 152 | min: { 153 | field: :price 154 | } 155 | }, 156 | max_price: { 157 | max: { 158 | field: :price 159 | } 160 | } 161 | } 162 | }) 163 | assert_equal 5, products.aggs["min_price"]["value"] 164 | assert_equal 25, products.aggs["max_price"]["value"] 165 | end 166 | 167 | def test_aggs_sum 168 | products = 169 | Product.search("*", { 170 | aggs: { 171 | sum_price: { 172 | sum: { 173 | field: :price 174 | } 175 | } 176 | } 177 | }) 178 | assert_equal 66, products.aggs["sum_price"]["value"] 179 | end 180 | 181 | def test_body_options 182 | products = 183 | Product.search("*", 184 | body_options: { 185 | aggs: { 186 | price: { 187 | histogram: {field: :price, interval: 10} 188 | } 189 | } 190 | } 191 | ) 192 | 193 | expected = [ 194 | {"key" => 0.0, "doc_count" => 1}, 195 | {"key" => 10.0, "doc_count" => 1}, 196 | {"key" => 20.0, "doc_count" => 2} 197 | ] 198 | assert_equal products.aggs["price"]["buckets"], expected 199 | end 200 | 201 | protected 202 | 203 | def buckets_as_hash(agg) 204 | Hash[agg["buckets"].map { |v| [v["key"], v["doc_count"]] }] 205 | end 206 | 207 | def store_agg(options, agg_key = "store_id") 208 | buckets = Product.search("Product", options).aggs[agg_key] 209 | buckets_as_hash(buckets) 210 | end 211 | 212 | def store_multiple_aggs(options) 213 | Hash[Product.search("Product", options).aggs.map do |field, filtered_agg| 214 | [field, buckets_as_hash(filtered_agg)] 215 | end] 216 | end 217 | end 218 | -------------------------------------------------------------------------------- /lib/searchkick/results.rb: -------------------------------------------------------------------------------- 1 | require "forwardable" 2 | 3 | module Searchkick 4 | class Results 5 | include Enumerable 6 | extend Forwardable 7 | 8 | attr_reader :klass, :response, :options 9 | 10 | def_delegators :results, :each, :any?, :empty?, :size, :length, :slice, :[], :to_ary 11 | 12 | def initialize(klass, response, options = {}) 13 | @klass = klass 14 | @response = response 15 | @options = options 16 | end 17 | 18 | def results 19 | @results ||= begin 20 | if options[:load] 21 | # results can have different types 22 | results = {} 23 | 24 | hits.group_by { |hit, _| hit["_type"] }.each do |type, grouped_hits| 25 | klass = (!options[:index_name] && @klass) || type.camelize.constantize 26 | results[type] = results_query(klass, grouped_hits).to_a.index_by { |r| r.id.to_s } 27 | end 28 | 29 | # sort 30 | results = 31 | hits.map do |hit| 32 | result = results[hit["_type"]][hit["_id"].to_s] 33 | if result && !(options[:load].is_a?(Hash) && options[:load][:dumpable]) 34 | if hit["highlight"] && !result.respond_to?(:search_highlights) 35 | highlights = Hash[hit["highlight"].map { |k, v| [(options[:json] ? k : k.sub(/\.#{@options[:match_suffix]}\z/, "")).to_sym, v.first] }] 36 | result.define_singleton_method(:search_highlights) do 37 | highlights 38 | end 39 | end 40 | end 41 | result 42 | end.compact 43 | 44 | if results.size != hits.size 45 | warn "[searchkick] WARNING: Records in search index do not exist in database" 46 | end 47 | 48 | results 49 | else 50 | hits.map do |hit| 51 | result = 52 | if hit["_source"] 53 | hit.except("_source").merge(hit["_source"]) 54 | elsif hit["fields"] 55 | hit.except("fields").merge(hit["fields"]) 56 | else 57 | hit 58 | end 59 | 60 | if hit["highlight"] 61 | highlight = Hash[hit["highlight"].map { |k, v| [base_field(k), v.first] }] 62 | options[:highlighted_fields].map { |k| base_field(k) }.each do |k| 63 | result["highlighted_#{k}"] ||= (highlight[k] || result[k]) 64 | end 65 | end 66 | 67 | result["id"] ||= result["_id"] # needed for legacy reasons 68 | HashWrapper.new(result) 69 | end 70 | end 71 | end 72 | end 73 | 74 | def suggestions 75 | if response["suggest"] 76 | response["suggest"].values.flat_map { |v| v.first["options"] }.sort_by { |o| -o["score"] }.map { |o| o["text"] }.uniq 77 | elsif options[:term] == "*" 78 | [] 79 | else 80 | raise "Pass `suggest: true` to the search method for suggestions" 81 | end 82 | end 83 | 84 | def aggregations 85 | response["aggregations"] 86 | end 87 | 88 | def aggs 89 | @aggs ||= begin 90 | if aggregations 91 | aggregations.dup.each do |field, filtered_agg| 92 | buckets = filtered_agg[field] 93 | # move the buckets one level above into the field hash 94 | if buckets 95 | filtered_agg.delete(field) 96 | filtered_agg.merge!(buckets) 97 | end 98 | end 99 | end 100 | end 101 | end 102 | 103 | def took 104 | response["took"] 105 | end 106 | 107 | def error 108 | response["error"] 109 | end 110 | 111 | def model_name 112 | klass.model_name 113 | end 114 | 115 | def entry_name(options = {}) 116 | if options.empty? 117 | # backward compatibility 118 | model_name.human.downcase 119 | else 120 | default = options[:count] == 1 ? model_name.human : model_name.human.pluralize 121 | model_name.human(options.reverse_merge(default: default)) 122 | end 123 | end 124 | 125 | def total_count 126 | response["hits"]["total"] 127 | end 128 | alias_method :total_entries, :total_count 129 | 130 | def current_page 131 | options[:page] 132 | end 133 | 134 | def per_page 135 | options[:per_page] 136 | end 137 | alias_method :limit_value, :per_page 138 | 139 | def padding 140 | options[:padding] 141 | end 142 | 143 | def total_pages 144 | (total_count / per_page.to_f).ceil 145 | end 146 | alias_method :num_pages, :total_pages 147 | 148 | def offset_value 149 | (current_page - 1) * per_page + padding 150 | end 151 | alias_method :offset, :offset_value 152 | 153 | def previous_page 154 | current_page > 1 ? (current_page - 1) : nil 155 | end 156 | alias_method :prev_page, :previous_page 157 | 158 | def next_page 159 | current_page < total_pages ? (current_page + 1) : nil 160 | end 161 | 162 | def first_page? 163 | previous_page.nil? 164 | end 165 | 166 | def last_page? 167 | next_page.nil? 168 | end 169 | 170 | def out_of_range? 171 | current_page > total_pages 172 | end 173 | 174 | def hits 175 | if error 176 | raise Searchkick::Error, "Query error - use the error method to view it" 177 | else 178 | @response["hits"]["hits"] 179 | end 180 | end 181 | 182 | def with_hit 183 | results.zip(hits) 184 | end 185 | 186 | def highlights(multiple: false) 187 | hits.map do |hit| 188 | Hash[hit["highlight"].map { |k, v| [(options[:json] ? k : k.sub(/\.#{@options[:match_suffix]}\z/, "")).to_sym, multiple ? v : v.first] }] 189 | end 190 | end 191 | 192 | def with_highlights(multiple: false) 193 | results.zip(highlights(multiple: multiple)) 194 | end 195 | 196 | def misspellings? 197 | @options[:misspellings] 198 | end 199 | 200 | private 201 | 202 | def results_query(records, hits) 203 | ids = hits.map { |hit| hit["_id"] } 204 | if options[:includes] || options[:model_includes] 205 | included_relations = [] 206 | combine_includes(included_relations, options[:includes]) 207 | combine_includes(included_relations, options[:model_includes][records]) if options[:model_includes] 208 | 209 | records = 210 | if defined?(NoBrainer::Document) && records < NoBrainer::Document 211 | if Gem.loaded_specs["nobrainer"].version >= Gem::Version.new("0.21") 212 | records.eager_load(included_relations) 213 | else 214 | records.preload(included_relations) 215 | end 216 | else 217 | records.includes(included_relations) 218 | end 219 | end 220 | 221 | if options[:scope_results] 222 | records = options[:scope_results].call(records) 223 | end 224 | 225 | Searchkick.load_records(records, ids) 226 | end 227 | 228 | def combine_includes(result, inc) 229 | if inc 230 | if inc.is_a?(Array) 231 | result.concat(inc) 232 | else 233 | result << inc 234 | end 235 | end 236 | end 237 | 238 | def base_field(k) 239 | k.sub(/\.(analyzed|word_start|word_middle|word_end|text_start|text_middle|text_end|exact)\z/, "") 240 | end 241 | end 242 | end 243 | -------------------------------------------------------------------------------- /lib/searchkick.rb: -------------------------------------------------------------------------------- 1 | require "active_model" 2 | require "active_support/core_ext/hash/deep_merge" 3 | require "elasticsearch" 4 | require "hashie" 5 | 6 | require "searchkick/bulk_indexer" 7 | require "searchkick/index" 8 | require "searchkick/indexer" 9 | require "searchkick/hash_wrapper" 10 | require "searchkick/middleware" 11 | require "searchkick/model" 12 | require "searchkick/multi_search" 13 | require "searchkick/query" 14 | require "searchkick/reindex_queue" 15 | require "searchkick/record_data" 16 | require "searchkick/record_indexer" 17 | require "searchkick/results" 18 | require "searchkick/version" 19 | 20 | require "searchkick/logging" if defined?(ActiveSupport::Notifications) 21 | 22 | begin 23 | require "rake" 24 | rescue LoadError 25 | # do nothing 26 | end 27 | require "searchkick/tasks" if defined?(Rake) 28 | 29 | begin 30 | require "rake" 31 | rescue LoadError 32 | # do nothing 33 | end 34 | require "searchkick/tasks" if defined?(Rake) 35 | 36 | # background jobs 37 | begin 38 | require "active_job" 39 | rescue LoadError 40 | # do nothing 41 | end 42 | if defined?(ActiveJob) 43 | require "searchkick/bulk_reindex_job" 44 | require "searchkick/process_batch_job" 45 | require "searchkick/process_queue_job" 46 | require "searchkick/reindex_v2_job" 47 | end 48 | 49 | module Searchkick 50 | class Error < StandardError; end 51 | class MissingIndexError < Error; end 52 | class UnsupportedVersionError < Error; end 53 | class InvalidQueryError < Elasticsearch::Transport::Transport::Errors::BadRequest; end 54 | class DangerousOperation < Error; end 55 | class ImportError < Error; end 56 | 57 | class << self 58 | attr_accessor :search_method_name, :wordnet_path, :timeout, :models, :client_options, :redis, :index_prefix, :index_suffix, :queue_name, :model_options 59 | attr_writer :client, :env, :search_timeout 60 | attr_reader :aws_credentials 61 | end 62 | self.search_method_name = :search 63 | self.wordnet_path = "/var/lib/wn_s.pl" 64 | self.timeout = 10 65 | self.models = [] 66 | self.client_options = {} 67 | self.queue_name = :searchkick 68 | self.model_options = {} 69 | 70 | def self.client 71 | @client ||= begin 72 | require "typhoeus/adapters/faraday" if defined?(Typhoeus) 73 | 74 | Elasticsearch::Client.new({ 75 | url: ENV["ELASTICSEARCH_URL"], 76 | transport_options: {request: {timeout: timeout}, headers: {content_type: "application/json"}}, 77 | retry_on_failure: 2 78 | }.deep_merge(client_options)) do |f| 79 | f.use Searchkick::Middleware 80 | f.request signer_middleware_key, signer_middleware_aws_params if aws_credentials 81 | end 82 | end 83 | end 84 | 85 | def self.env 86 | @env ||= ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development" 87 | end 88 | 89 | def self.search_timeout 90 | @search_timeout || timeout 91 | end 92 | 93 | def self.server_version 94 | @server_version ||= client.info["version"]["number"] 95 | end 96 | 97 | def self.server_below?(version) 98 | Gem::Version.new(server_version.split("-")[0]) < Gem::Version.new(version.split("-")[0]) 99 | end 100 | 101 | def self.search(term = "*", model: nil, **options, &block) 102 | options = options.dup 103 | klass = model 104 | 105 | # make Searchkick.search(index_name: [Product]) and Product.search equivalent 106 | unless klass 107 | index_name = Array(options[:index_name]) 108 | if index_name.size == 1 && index_name.first.respond_to?(:searchkick_index) 109 | klass = index_name.first 110 | options.delete(:index_name) 111 | end 112 | end 113 | 114 | query = Searchkick::Query.new(klass, term, options) 115 | block.call(query.body) if block 116 | if options[:execute] == false 117 | query 118 | else 119 | query.execute 120 | end 121 | end 122 | 123 | def self.multi_search(queries) 124 | Searchkick::MultiSearch.new(queries).perform 125 | end 126 | 127 | # callbacks 128 | 129 | def self.enable_callbacks 130 | self.callbacks_value = nil 131 | end 132 | 133 | def self.disable_callbacks 134 | self.callbacks_value = false 135 | end 136 | 137 | def self.callbacks?(default: true) 138 | if callbacks_value.nil? 139 | default 140 | else 141 | callbacks_value != false 142 | end 143 | end 144 | 145 | def self.callbacks(value) 146 | if block_given? 147 | previous_value = callbacks_value 148 | begin 149 | self.callbacks_value = value 150 | result = yield 151 | indexer.perform if callbacks_value == :bulk 152 | result 153 | ensure 154 | self.callbacks_value = previous_value 155 | end 156 | else 157 | self.callbacks_value = value 158 | end 159 | end 160 | 161 | def self.aws_credentials=(creds) 162 | begin 163 | require "faraday_middleware/aws_signers_v4" 164 | rescue LoadError 165 | require "faraday_middleware/aws_sigv4" 166 | end 167 | @aws_credentials = creds 168 | @client = nil # reset client 169 | end 170 | 171 | def self.reindex_status(index_name) 172 | if redis 173 | batches_left = Searchkick::Index.new(index_name).batches_left 174 | { 175 | completed: batches_left == 0, 176 | batches_left: batches_left 177 | } 178 | else 179 | raise Searchkick::Error, "Redis not configured" 180 | end 181 | end 182 | 183 | def self.with_redis 184 | if redis 185 | if redis.respond_to?(:with) 186 | redis.with do |r| 187 | yield r 188 | end 189 | else 190 | yield redis 191 | end 192 | end 193 | end 194 | 195 | # private 196 | def self.load_records(records, ids) 197 | records = 198 | if records.respond_to?(:primary_key) 199 | # ActiveRecord 200 | records.where(records.primary_key => ids) if records.primary_key 201 | elsif records.respond_to?(:queryable) 202 | # Mongoid 3+ 203 | records.queryable.for_ids(ids) 204 | elsif records.respond_to?(:unscoped) && :id.respond_to?(:in) 205 | # Nobrainer 206 | records.unscoped.where(:id.in => ids) 207 | elsif records.respond_to?(:key_column_names) 208 | records.where(records.key_column_names.first => ids) 209 | end 210 | 211 | raise Searchkick::Error, "Not sure how to load records" if !records 212 | 213 | records 214 | end 215 | 216 | # private 217 | def self.indexer 218 | Thread.current[:searchkick_indexer] ||= Searchkick::Indexer.new 219 | end 220 | 221 | # private 222 | def self.callbacks_value 223 | Thread.current[:searchkick_callbacks_enabled] 224 | end 225 | 226 | # private 227 | def self.callbacks_value=(value) 228 | Thread.current[:searchkick_callbacks_enabled] = value 229 | end 230 | 231 | # private 232 | def self.signer_middleware_key 233 | defined?(FaradayMiddleware::AwsSignersV4) ? :aws_signers_v4 : :aws_sigv4 234 | end 235 | 236 | # private 237 | def self.signer_middleware_aws_params 238 | if signer_middleware_key == :aws_sigv4 239 | {service: "es", region: "us-east-1"}.merge(aws_credentials) 240 | else 241 | { 242 | credentials: aws_credentials[:credentials] || Aws::Credentials.new(aws_credentials[:access_key_id], aws_credentials[:secret_access_key]), 243 | service_name: "es", 244 | region: aws_credentials[:region] || "us-east-1" 245 | } 246 | end 247 | end 248 | end 249 | 250 | # TODO find better ActiveModel hook 251 | ActiveModel::Callbacks.include(Searchkick::Model) 252 | 253 | ActiveSupport.on_load(:active_record) do 254 | extend Searchkick::Model 255 | end 256 | -------------------------------------------------------------------------------- /lib/searchkick/logging.rb: -------------------------------------------------------------------------------- 1 | # based on https://gist.github.com/mnutt/566725 2 | require "active_support/core_ext/module/attr_internal" 3 | 4 | module Searchkick 5 | module QueryWithInstrumentation 6 | def execute_search 7 | name = searchkick_klass ? "#{searchkick_klass.name} Search" : "Search" 8 | event = { 9 | name: name, 10 | query: params 11 | } 12 | ActiveSupport::Notifications.instrument("search.searchkick", event) do 13 | super 14 | end 15 | end 16 | end 17 | 18 | module IndexWithInstrumentation 19 | def store(record) 20 | event = { 21 | name: "#{record.searchkick_klass.name} Store", 22 | id: search_id(record) 23 | } 24 | if Searchkick.callbacks_value == :bulk 25 | super 26 | else 27 | ActiveSupport::Notifications.instrument("request.searchkick", event) do 28 | super 29 | end 30 | end 31 | end 32 | 33 | def remove(record) 34 | name = record && record.searchkick_klass ? "#{record.searchkick_klass.name} Remove" : "Remove" 35 | event = { 36 | name: name, 37 | id: search_id(record) 38 | } 39 | if Searchkick.callbacks_value == :bulk 40 | super 41 | else 42 | ActiveSupport::Notifications.instrument("request.searchkick", event) do 43 | super 44 | end 45 | end 46 | end 47 | 48 | def update_record(record, method_name) 49 | event = { 50 | name: "#{record.searchkick_klass.name} Update", 51 | id: search_id(record) 52 | } 53 | if Searchkick.callbacks_value == :bulk 54 | super 55 | else 56 | ActiveSupport::Notifications.instrument("request.searchkick", event) do 57 | super 58 | end 59 | end 60 | end 61 | 62 | def bulk_index(records) 63 | if records.any? 64 | event = { 65 | name: "#{records.first.searchkick_klass.name} Import", 66 | count: records.size 67 | } 68 | event[:id] = search_id(records.first) if records.size == 1 69 | if Searchkick.callbacks_value == :bulk 70 | super 71 | else 72 | ActiveSupport::Notifications.instrument("request.searchkick", event) do 73 | super 74 | end 75 | end 76 | end 77 | end 78 | alias_method :import, :bulk_index 79 | 80 | def bulk_update(records, *args) 81 | if records.any? 82 | event = { 83 | name: "#{records.first.searchkick_klass.name} Update", 84 | count: records.size 85 | } 86 | event[:id] = search_id(records.first) if records.size == 1 87 | if Searchkick.callbacks_value == :bulk 88 | super 89 | else 90 | ActiveSupport::Notifications.instrument("request.searchkick", event) do 91 | super 92 | end 93 | end 94 | end 95 | end 96 | 97 | def bulk_delete(records) 98 | if records.any? 99 | event = { 100 | name: "#{records.first.searchkick_klass.name} Delete", 101 | count: records.size 102 | } 103 | event[:id] = search_id(records.first) if records.size == 1 104 | if Searchkick.callbacks_value == :bulk 105 | super 106 | else 107 | ActiveSupport::Notifications.instrument("request.searchkick", event) do 108 | super 109 | end 110 | end 111 | end 112 | end 113 | end 114 | 115 | module IndexerWithInstrumentation 116 | def perform 117 | if Searchkick.callbacks_value == :bulk 118 | event = { 119 | name: "Bulk", 120 | count: queued_items.size 121 | } 122 | ActiveSupport::Notifications.instrument("request.searchkick", event) do 123 | super 124 | end 125 | else 126 | super 127 | end 128 | end 129 | end 130 | 131 | module SearchkickWithInstrumentation 132 | def multi_search(searches) 133 | event = { 134 | name: "Multi Search", 135 | body: searches.flat_map { |q| [q.params.except(:body).to_json, q.body.to_json] }.map { |v| "#{v}\n" }.join 136 | } 137 | ActiveSupport::Notifications.instrument("multi_search.searchkick", event) do 138 | super 139 | end 140 | end 141 | end 142 | 143 | # https://github.com/rails/rails/blob/master/activerecord/lib/active_record/log_subscriber.rb 144 | class LogSubscriber < ActiveSupport::LogSubscriber 145 | def self.runtime=(value) 146 | Thread.current[:searchkick_runtime] = value 147 | end 148 | 149 | def self.runtime 150 | Thread.current[:searchkick_runtime] ||= 0 151 | end 152 | 153 | def self.reset_runtime 154 | rt = runtime 155 | self.runtime = 0 156 | rt 157 | end 158 | 159 | def search(event) 160 | self.class.runtime += event.duration 161 | return unless logger.debug? 162 | 163 | payload = event.payload 164 | name = "#{payload[:name]} (#{event.duration.round(1)}ms)" 165 | type = payload[:query][:type] 166 | index = payload[:query][:index].is_a?(Array) ? payload[:query][:index].join(",") : payload[:query][:index] 167 | 168 | # no easy way to tell which host the client will use 169 | host = Searchkick.client.transport.hosts.first 170 | debug " #{color(name, YELLOW, true)} curl #{host[:protocol]}://#{host[:host]}:#{host[:port]}/#{CGI.escape(index)}#{type ? "/#{type.map { |t| CGI.escape(t) }.join(',')}" : ''}/_search?pretty -H 'Content-Type: application/json' -d '#{payload[:query][:body].to_json}'" 171 | end 172 | 173 | def request(event) 174 | self.class.runtime += event.duration 175 | return unless logger.debug? 176 | 177 | payload = event.payload 178 | name = "#{payload[:name]} (#{event.duration.round(1)}ms)" 179 | 180 | debug " #{color(name, YELLOW, true)} #{payload.except(:name).to_json}" 181 | end 182 | 183 | def multi_search(event) 184 | self.class.runtime += event.duration 185 | return unless logger.debug? 186 | 187 | payload = event.payload 188 | name = "#{payload[:name]} (#{event.duration.round(1)}ms)" 189 | 190 | # no easy way to tell which host the client will use 191 | host = Searchkick.client.transport.hosts.first 192 | debug " #{color(name, YELLOW, true)} curl #{host[:protocol]}://#{host[:host]}:#{host[:port]}/_msearch?pretty -H 'Content-Type: application/json' -d '#{payload[:body]}'" 193 | end 194 | end 195 | 196 | # https://github.com/rails/rails/blob/master/activerecord/lib/active_record/railties/controller_runtime.rb 197 | module ControllerRuntime 198 | extend ActiveSupport::Concern 199 | 200 | protected 201 | 202 | attr_internal :searchkick_runtime 203 | 204 | def process_action(action, *args) 205 | # We also need to reset the runtime before each action 206 | # because of queries in middleware or in cases we are streaming 207 | # and it won't be cleaned up by the method below. 208 | Searchkick::LogSubscriber.reset_runtime 209 | super 210 | end 211 | 212 | def cleanup_view_runtime 213 | searchkick_rt_before_render = Searchkick::LogSubscriber.reset_runtime 214 | runtime = super 215 | searchkick_rt_after_render = Searchkick::LogSubscriber.reset_runtime 216 | self.searchkick_runtime = searchkick_rt_before_render + searchkick_rt_after_render 217 | runtime - searchkick_rt_after_render 218 | end 219 | 220 | def append_info_to_payload(payload) 221 | super 222 | payload[:searchkick_runtime] = (searchkick_runtime || 0) + Searchkick::LogSubscriber.reset_runtime 223 | end 224 | 225 | module ClassMethods 226 | def log_process_action(payload) 227 | messages = super 228 | runtime = payload[:searchkick_runtime] 229 | messages << ("Searchkick: %.1fms" % runtime.to_f) if runtime.to_f > 0 230 | messages 231 | end 232 | end 233 | end 234 | end 235 | 236 | Searchkick::Query.prepend(Searchkick::QueryWithInstrumentation) 237 | Searchkick::Index.prepend(Searchkick::IndexWithInstrumentation) 238 | Searchkick::Indexer.prepend(Searchkick::IndexerWithInstrumentation) 239 | Searchkick.singleton_class.prepend(Searchkick::SearchkickWithInstrumentation) 240 | Searchkick::LogSubscriber.attach_to :searchkick 241 | ActiveSupport.on_load(:action_controller) do 242 | include Searchkick::ControllerRuntime 243 | end 244 | -------------------------------------------------------------------------------- /test/boost_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | class BoostTest < Minitest::Test 4 | # conversions 5 | 6 | def test_conversions 7 | store [ 8 | {name: "Tomato A", conversions: {"tomato" => 1}}, 9 | {name: "Tomato B", conversions: {"tomato" => 2}}, 10 | {name: "Tomato C", conversions: {"tomato" => 3}} 11 | ] 12 | assert_order "tomato", ["Tomato C", "Tomato B", "Tomato A"] 13 | assert_equal_scores "tomato", conversions: false 14 | end 15 | 16 | def test_multiple_conversions 17 | store [ 18 | {name: "Speaker A", conversions_a: {"speaker" => 1}, conversions_b: {"speaker" => 6}}, 19 | {name: "Speaker B", conversions_a: {"speaker" => 2}, conversions_b: {"speaker" => 5}}, 20 | {name: "Speaker C", conversions_a: {"speaker" => 3}, conversions_b: {"speaker" => 4}} 21 | ], Speaker 22 | 23 | assert_equal_scores "speaker", {conversions: false}, Speaker 24 | assert_equal_scores "speaker", {}, Speaker 25 | assert_equal_scores "speaker", {conversions: ["conversions_a", "conversions_b"]}, Speaker 26 | assert_equal_scores "speaker", {conversions: ["conversions_b", "conversions_a"]}, Speaker 27 | assert_order "speaker", ["Speaker C", "Speaker B", "Speaker A"], {conversions: "conversions_a"}, Speaker 28 | assert_order "speaker", ["Speaker A", "Speaker B", "Speaker C"], {conversions: "conversions_b"}, Speaker 29 | end 30 | 31 | def test_multiple_conversions_with_boost_term 32 | store [ 33 | {name: "Speaker A", conversions_a: {"speaker" => 4, "speaker_1" => 1}}, 34 | {name: "Speaker B", conversions_a: {"speaker" => 3, "speaker_1" => 2}}, 35 | {name: "Speaker C", conversions_a: {"speaker" => 2, "speaker_1" => 3}}, 36 | {name: "Speaker D", conversions_a: {"speaker" => 1, "speaker_1" => 4}} 37 | ], Speaker 38 | 39 | assert_order "speaker", ["Speaker A", "Speaker B", "Speaker C", "Speaker D"], {conversions: "conversions_a"}, Speaker 40 | assert_order "speaker", ["Speaker D", "Speaker C", "Speaker B", "Speaker A"], {conversions: "conversions_a", conversions_term: "speaker_1"}, Speaker 41 | end 42 | 43 | def test_conversions_case 44 | store [ 45 | {name: "Tomato A", conversions: {"tomato" => 1, "TOMATO" => 1, "tOmAtO" => 1}}, 46 | {name: "Tomato B", conversions: {"tomato" => 2}} 47 | ] 48 | assert_order "tomato", ["Tomato A", "Tomato B"] 49 | end 50 | 51 | # global boost 52 | 53 | def test_boost 54 | store [ 55 | {name: "Tomato A"}, 56 | {name: "Tomato B", orders_count: 10}, 57 | {name: "Tomato C", orders_count: 100} 58 | ] 59 | assert_order "tomato", ["Tomato C", "Tomato B", "Tomato A"], boost: "orders_count" 60 | end 61 | 62 | def test_boost_zero 63 | store [ 64 | {name: "Zero Boost", orders_count: 0} 65 | ] 66 | assert_order "zero", ["Zero Boost"], boost: "orders_count" 67 | end 68 | 69 | def test_conversions_weight 70 | store [ 71 | {name: "Product Boost", orders_count: 20}, 72 | {name: "Product Conversions", conversions: {"product" => 10}} 73 | ] 74 | assert_order "product", ["Product Conversions", "Product Boost"], boost: "orders_count" 75 | end 76 | 77 | def test_boost_fields 78 | store [ 79 | {name: "Red", color: "White"}, 80 | {name: "White", color: "Red Red Red"} 81 | ] 82 | assert_order "red", ["Red", "White"], fields: ["name^10", "color"] 83 | end 84 | 85 | def test_boost_fields_decimal 86 | store [ 87 | {name: "Red", color: "White"}, 88 | {name: "White", color: "Red Red Red"} 89 | ] 90 | assert_order "red", ["Red", "White"], fields: ["name^10.5", "color"] 91 | end 92 | 93 | def test_boost_fields_word_start 94 | store [ 95 | {name: "Red", color: "White"}, 96 | {name: "White", color: "Red Red Red"} 97 | ] 98 | assert_order "red", ["Red", "White"], fields: [{"name^10" => :word_start}, "color"] 99 | end 100 | 101 | # for issue #855 102 | def test_apostrophes 103 | store_names ["Valentine's Day Special"] 104 | assert_search "Valentines", ["Valentine's Day Special"], fields: ["name^5"] 105 | assert_search "Valentine's", ["Valentine's Day Special"], fields: ["name^5"] 106 | assert_search "Valentine", ["Valentine's Day Special"], fields: ["name^5"] 107 | end 108 | 109 | def test_boost_by 110 | store [ 111 | {name: "Tomato A"}, 112 | {name: "Tomato B", orders_count: 10}, 113 | {name: "Tomato C", orders_count: 100} 114 | ] 115 | assert_order "tomato", ["Tomato C", "Tomato B", "Tomato A"], boost_by: [:orders_count] 116 | assert_order "tomato", ["Tomato C", "Tomato B", "Tomato A"], boost_by: {orders_count: {factor: 10}} 117 | end 118 | 119 | def test_boost_by_missing 120 | store [ 121 | {name: "Tomato A"}, 122 | {name: "Tomato B", orders_count: 10}, 123 | ] 124 | 125 | assert_order "tomato", ["Tomato A", "Tomato B"], boost_by: {orders_count: {missing: 100}} 126 | end 127 | 128 | def test_boost_by_boost_mode_multiply 129 | store [ 130 | {name: "Tomato A", found_rate: 0.9}, 131 | {name: "Tomato B"}, 132 | {name: "Tomato C", found_rate: 0.5} 133 | ] 134 | 135 | assert_order "tomato", ["Tomato B", "Tomato A", "Tomato C"], boost_by: {found_rate: {boost_mode: "multiply"}} 136 | end 137 | 138 | def test_boost_where 139 | store [ 140 | {name: "Tomato A"}, 141 | {name: "Tomato B", user_ids: [1, 2]}, 142 | {name: "Tomato C", user_ids: [3]} 143 | ] 144 | assert_first "tomato", "Tomato B", boost_where: {user_ids: 2} 145 | assert_first "tomato", "Tomato B", boost_where: {user_ids: 1..2} 146 | assert_first "tomato", "Tomato B", boost_where: {user_ids: [1, 4]} 147 | assert_first "tomato", "Tomato B", boost_where: {user_ids: {value: 2, factor: 10}} 148 | assert_first "tomato", "Tomato B", boost_where: {user_ids: {value: [1, 4], factor: 10}} 149 | assert_order "tomato", ["Tomato C", "Tomato B", "Tomato A"], boost_where: {user_ids: [{value: 1, factor: 10}, {value: 3, factor: 20}]} 150 | end 151 | 152 | def test_boost_by_recency 153 | store [ 154 | {name: "Article 1", created_at: 2.days.ago}, 155 | {name: "Article 2", created_at: 1.day.ago}, 156 | {name: "Article 3", created_at: Time.now} 157 | ] 158 | assert_order "article", ["Article 3", "Article 2", "Article 1"], boost_by_recency: {created_at: {scale: "7d", decay: 0.5}} 159 | end 160 | 161 | def test_boost_by_recency_origin 162 | store [ 163 | {name: "Article 1", created_at: 2.days.ago}, 164 | {name: "Article 2", created_at: 1.day.ago}, 165 | {name: "Article 3", created_at: Time.now} 166 | ] 167 | assert_order "article", ["Article 1", "Article 2", "Article 3"], boost_by_recency: {created_at: {origin: 2.days.ago, scale: "7d", decay: 0.5}} 168 | end 169 | 170 | def test_boost_by_distance 171 | store [ 172 | {name: "San Francisco", latitude: 37.7833, longitude: -122.4167}, 173 | {name: "San Antonio", latitude: 29.4167, longitude: -98.5000}, 174 | {name: "San Marino", latitude: 43.9333, longitude: 12.4667} 175 | ] 176 | assert_order "san", ["San Francisco", "San Antonio", "San Marino"], boost_by_distance: {field: :location, origin: [37, -122], scale: "1000mi"} 177 | end 178 | 179 | def test_boost_by_distance_hash 180 | store [ 181 | {name: "San Francisco", latitude: 37.7833, longitude: -122.4167}, 182 | {name: "San Antonio", latitude: 29.4167, longitude: -98.5000}, 183 | {name: "San Marino", latitude: 43.9333, longitude: 12.4667} 184 | ] 185 | assert_order "san", ["San Francisco", "San Antonio", "San Marino"], boost_by_distance: {field: :location, origin: {lat: 37, lon: -122}, scale: "1000mi"} 186 | end 187 | 188 | def test_boost_by_distance_v2 189 | store [ 190 | {name: "San Francisco", latitude: 37.7833, longitude: -122.4167}, 191 | {name: "San Antonio", latitude: 29.4167, longitude: -98.5000}, 192 | {name: "San Marino", latitude: 43.9333, longitude: 12.4667} 193 | ] 194 | assert_order "san", ["San Francisco", "San Antonio", "San Marino"], boost_by_distance: {location: {origin: [37, -122], scale: "1000mi"}} 195 | end 196 | 197 | def test_boost_by_distance_v2_hash 198 | store [ 199 | {name: "San Francisco", latitude: 37.7833, longitude: -122.4167}, 200 | {name: "San Antonio", latitude: 29.4167, longitude: -98.5000}, 201 | {name: "San Marino", latitude: 43.9333, longitude: 12.4667} 202 | ] 203 | assert_order "san", ["San Francisco", "San Antonio", "San Marino"], boost_by_distance: {location: {origin: {lat: 37, lon: -122}, scale: "1000mi"}} 204 | end 205 | 206 | def test_boost_by_distance_v2_factor 207 | store [ 208 | {name: "San Francisco", latitude: 37.7833, longitude: -122.4167, found_rate: 0.1}, 209 | {name: "San Antonio", latitude: 29.4167, longitude: -98.5000, found_rate: 0.99}, 210 | {name: "San Marino", latitude: 43.9333, longitude: 12.4667, found_rate: 0.2} 211 | ] 212 | 213 | assert_order "san", ["San Antonio","San Francisco", "San Marino"], boost_by: {found_rate: {factor: 100}}, boost_by_distance: {location: {origin: [37, -122], scale: "1000mi"}} 214 | assert_order "san", ["San Francisco", "San Antonio", "San Marino"], boost_by: {found_rate: {factor: 100}}, boost_by_distance: {location: {origin: [37, -122], scale: "1000mi", factor: 100}} 215 | end 216 | 217 | def test_boost_by_indices 218 | skip if cequel? 219 | 220 | store_names ["Rex"], Animal 221 | store_names ["Rexx"], Product 222 | 223 | assert_order "Rex", ["Rexx", "Rex"], {index_name: [Animal, Product], indices_boost: {Animal => 1, Product => 200}, fields: [:name]}, Store 224 | end 225 | end 226 | -------------------------------------------------------------------------------- /test/match_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | class MatchTest < Minitest::Test 4 | # exact 5 | 6 | def test_match 7 | store_names ["Whole Milk", "Fat Free Milk", "Milk"] 8 | assert_search "milk", ["Milk", "Whole Milk", "Fat Free Milk"] 9 | end 10 | 11 | def test_case 12 | store_names ["Whole Milk", "Fat Free Milk", "Milk"] 13 | assert_search "MILK", ["Milk", "Whole Milk", "Fat Free Milk"] 14 | end 15 | 16 | def test_cheese_space_in_index 17 | store_names ["Pepper Jack Cheese Skewers"] 18 | assert_search "pepperjack cheese skewers", ["Pepper Jack Cheese Skewers"] 19 | end 20 | 21 | def test_operator 22 | store_names ["fresh", "honey"] 23 | assert_search "fresh honey", ["fresh", "honey"], {operator: "or"} 24 | assert_search "fresh honey", [], {operator: "and"} 25 | assert_search "fresh honey", ["fresh", "honey"], {operator: :or} 26 | end 27 | 28 | # def test_cheese_space_in_query 29 | # store_names ["Pepperjack Cheese Skewers"] 30 | # assert_search "pepper jack cheese skewers", ["Pepperjack Cheese Skewers"] 31 | # end 32 | 33 | def test_middle_token 34 | store_names ["Dish Washer Amazing Organic Soap"] 35 | assert_search "dish soap", ["Dish Washer Amazing Organic Soap"] 36 | end 37 | 38 | def test_middle_token_wine 39 | store_names ["Beringer Wine Founders Estate Chardonnay"] 40 | assert_search "beringer chardonnay", ["Beringer Wine Founders Estate Chardonnay"] 41 | end 42 | 43 | def test_percent 44 | # Note: "2% Milk" doesn't get matched in ES below 5.1.1 45 | # This could be a bug since it has an edit distance of 1 46 | store_names ["1% Milk", "Whole Milk"] 47 | assert_search "1%", ["1% Milk"] 48 | end 49 | 50 | # ascii 51 | 52 | def test_jalapenos 53 | store_names ["Jalapeño"] 54 | assert_search "jalapeno", ["Jalapeño"] 55 | end 56 | 57 | def test_swedish 58 | store_names ["ÅÄÖ"] 59 | assert_search "aao", ["ÅÄÖ"] 60 | end 61 | 62 | # stemming 63 | 64 | def test_stemming 65 | store_names ["Whole Milk", "Fat Free Milk", "Milk"] 66 | assert_search "milks", ["Milk", "Whole Milk", "Fat Free Milk"] 67 | end 68 | 69 | # fuzzy 70 | 71 | def test_misspelling_sriracha 72 | store_names ["Sriracha"] 73 | assert_search "siracha", ["Sriracha"] 74 | end 75 | 76 | def test_misspelling_multiple 77 | store_names ["Greek Yogurt", "Green Onions"] 78 | assert_search "greed", ["Greek Yogurt", "Green Onions"] 79 | end 80 | 81 | def test_short_word 82 | store_names ["Finn"] 83 | assert_search "fin", ["Finn"] 84 | end 85 | 86 | def test_edit_distance_two 87 | store_names ["Bingo"] 88 | assert_search "bin", [] 89 | assert_search "bingooo", [] 90 | assert_search "mango", [] 91 | end 92 | 93 | def test_edit_distance_one 94 | store_names ["Bingo"] 95 | assert_search "bing", ["Bingo"] 96 | assert_search "bingoo", ["Bingo"] 97 | assert_search "ringo", ["Bingo"] 98 | end 99 | 100 | def test_edit_distance_long_word 101 | store_names ["thisisareallylongword"] 102 | assert_search "thisisareallylongwor", ["thisisareallylongword"] # missing letter 103 | assert_search "thisisareelylongword", [] # edit distance = 2 104 | end 105 | 106 | def test_misspelling_tabasco 107 | store_names ["Tabasco"] 108 | assert_search "tobasco", ["Tabasco"] 109 | end 110 | 111 | def test_misspelling_zucchini 112 | store_names ["Zucchini"] 113 | assert_search "zuchini", ["Zucchini"] 114 | end 115 | 116 | def test_misspelling_ziploc 117 | store_names ["Ziploc"] 118 | assert_search "zip lock", ["Ziploc"] 119 | end 120 | 121 | def test_misspelling_zucchini_transposition 122 | store_names ["zucchini"] 123 | assert_search "zuccihni", ["zucchini"] 124 | 125 | # need to specify field 126 | # as transposition option isn't supported for multi_match queries 127 | # until Elasticsearch 6.1 128 | assert_search "zuccihni", [], misspellings: {transpositions: false}, fields: [:name] 129 | end 130 | 131 | def test_misspelling_lasagna 132 | store_names ["lasagna"] 133 | assert_search "lasanga", ["lasagna"], misspellings: {transpositions: true} 134 | assert_search "lasgana", ["lasagna"], misspellings: {transpositions: true} 135 | assert_search "lasaang", [], misspellings: {transpositions: true} # triple transposition, shouldn't work 136 | assert_search "lsagana", [], misspellings: {transpositions: true} # triple transposition, shouldn't work 137 | end 138 | 139 | def test_misspelling_lasagna_pasta 140 | store_names ["lasagna pasta"] 141 | assert_search "lasanga", ["lasagna pasta"], misspellings: {transpositions: true} 142 | assert_search "lasanga pasta", ["lasagna pasta"], misspellings: {transpositions: true} 143 | assert_search "lasanga pasat", ["lasagna pasta"], misspellings: {transpositions: true} # both words misspelled with a transposition should still work 144 | end 145 | 146 | def test_misspellings_word_start 147 | store_names ["Sriracha"] 148 | assert_search "siracha", ["Sriracha"], fields: [{name: :word_start}] 149 | end 150 | 151 | # spaces 152 | 153 | def test_spaces_in_field 154 | store_names ["Red Bull"] 155 | assert_search "redbull", ["Red Bull"] 156 | end 157 | 158 | def test_spaces_in_query 159 | store_names ["Dishwasher"] 160 | assert_search "dish washer", ["Dishwasher"] 161 | end 162 | 163 | def test_spaces_three_words 164 | store_names ["Dish Washer Soap", "Dish Washer"] 165 | assert_search "dish washer soap", ["Dish Washer Soap"] 166 | end 167 | 168 | def test_spaces_stemming 169 | store_names ["Almond Milk"] 170 | assert_search "almondmilks", ["Almond Milk"] 171 | end 172 | 173 | # butter 174 | 175 | def test_exclude_butter 176 | store_names ["Butter Tub", "Peanut Butter Tub"] 177 | assert_search "butter", ["Butter Tub"], exclude: ["peanut butter"] 178 | end 179 | 180 | def test_exclude_butter_word_start 181 | store_names ["Butter Tub", "Peanut Butter Tub"] 182 | assert_search "butter", ["Butter Tub"], exclude: ["peanut butter"], match: :word_start 183 | end 184 | 185 | def test_exclude_butter_exact 186 | store_names ["Butter Tub", "Peanut Butter Tub"] 187 | assert_search "butter", [], exclude: ["peanut butter"], fields: [{name: :exact}] 188 | end 189 | 190 | def test_exclude_same_exact 191 | store_names ["Butter Tub", "Peanut Butter Tub"] 192 | assert_search "Butter Tub", ["Butter Tub"], exclude: ["Peanut Butter Tub"], fields: [{name: :exact}] 193 | end 194 | 195 | def test_exclude_egg_word_start 196 | store_names ["eggs", "eggplant"] 197 | assert_search "egg", ["eggs"], exclude: ["eggplant"], match: :word_start 198 | end 199 | 200 | def test_exclude_string 201 | store_names ["Butter Tub", "Peanut Butter Tub"] 202 | assert_search "butter", ["Butter Tub"], exclude: "peanut butter" 203 | end 204 | 205 | # other 206 | 207 | def test_all 208 | store_names ["Product A", "Product B"] 209 | assert_search "*", ["Product A", "Product B"] 210 | end 211 | 212 | def test_no_arguments 213 | store_names [] 214 | assert_equal [], Product.search.to_a 215 | end 216 | 217 | def test_no_term 218 | store_names ["Product A"] 219 | assert_equal ["Product A"], Product.search(where: {name: "Product A"}).map(&:name) 220 | end 221 | 222 | def test_to_be_or_not_to_be 223 | store_names ["to be or not to be"] 224 | assert_search "to be", ["to be or not to be"] 225 | end 226 | 227 | def test_apostrophe 228 | store_names ["Ben and Jerry's"] 229 | assert_search "ben and jerrys", ["Ben and Jerry's"] 230 | end 231 | 232 | def test_apostrophe_search 233 | store_names ["Ben and Jerrys"] 234 | assert_search "ben and jerry's", ["Ben and Jerrys"] 235 | end 236 | 237 | def test_ampersand_index 238 | store_names ["Ben & Jerry's"] 239 | assert_search "ben and jerrys", ["Ben & Jerry's"] 240 | end 241 | 242 | def test_ampersand_search 243 | store_names ["Ben and Jerry's"] 244 | assert_search "ben & jerrys", ["Ben and Jerry's"] 245 | end 246 | 247 | def test_phrase 248 | store_names ["Fresh Honey", "Honey Fresh"] 249 | assert_search "fresh honey", ["Fresh Honey"], match: :phrase 250 | end 251 | 252 | def test_phrase_again 253 | store_names ["Social entrepreneurs don't have it easy raising capital"] 254 | assert_search "social entrepreneurs don't have it easy raising capital", ["Social entrepreneurs don't have it easy raising capital"], match: :phrase 255 | end 256 | 257 | def test_phrase_order 258 | store_names ["Wheat Bread", "Whole Wheat Bread"] 259 | assert_order "wheat bread", ["Wheat Bread", "Whole Wheat Bread"], match: :phrase, fields: [:name] 260 | end 261 | 262 | def test_dynamic_fields 263 | store_names ["Red Bull"], Speaker 264 | assert_search "redbull", ["Red Bull"], {fields: [:name]}, Speaker 265 | end 266 | 267 | def test_unsearchable 268 | skip 269 | store [ 270 | {name: "Unsearchable", description: "Almond"} 271 | ] 272 | assert_search "almond", [] 273 | end 274 | 275 | def test_unsearchable_where 276 | store [ 277 | {name: "Unsearchable", description: "Almond"} 278 | ] 279 | assert_search "*", ["Unsearchable"], where: {description: "Almond"} 280 | end 281 | 282 | def test_emoji 283 | skip unless defined?(EmojiParser) 284 | store_names ["Banana"] 285 | assert_search "🍌", ["Banana"], emoji: true 286 | end 287 | 288 | def test_emoji_multiple 289 | skip unless defined?(EmojiParser) 290 | store_names ["Ice Cream Cake"] 291 | assert_search "🍨🍰", ["Ice Cream Cake"], emoji: true 292 | end 293 | end 294 | -------------------------------------------------------------------------------- /lib/searchkick/index.rb: -------------------------------------------------------------------------------- 1 | require "searchkick/index_options" 2 | 3 | module Searchkick 4 | class Index 5 | include IndexOptions 6 | 7 | attr_reader :name, :options 8 | 9 | def initialize(name, options = {}) 10 | @name = name 11 | @options = options 12 | @klass_document_type = {} # cache 13 | end 14 | 15 | def create(body = {}) 16 | client.indices.create index: name, body: body 17 | end 18 | 19 | def delete 20 | if !Searchkick.server_below?("6.0.0") && alias_exists? 21 | # can't call delete directly on aliases in ES 6 22 | indices = client.indices.get_alias(name: name).keys 23 | client.indices.delete index: indices 24 | else 25 | client.indices.delete index: name 26 | end 27 | end 28 | 29 | def exists? 30 | client.indices.exists index: name 31 | end 32 | 33 | def refresh 34 | client.indices.refresh index: name 35 | end 36 | 37 | def alias_exists? 38 | client.indices.exists_alias name: name 39 | end 40 | 41 | def mapping 42 | client.indices.get_mapping index: name 43 | end 44 | 45 | def settings 46 | client.indices.get_settings index: name 47 | end 48 | 49 | def refresh_interval 50 | settings.values.first["settings"]["index"]["refresh_interval"] 51 | end 52 | 53 | def update_settings(settings) 54 | client.indices.put_settings index: name, body: settings 55 | end 56 | 57 | def tokens(text, options = {}) 58 | client.indices.analyze(body: {text: text}.merge(options), index: name)["tokens"].map { |t| t["token"] } 59 | end 60 | 61 | def total_docs 62 | response = 63 | client.search( 64 | index: name, 65 | body: { 66 | query: {match_all: {}}, 67 | size: 0 68 | } 69 | ) 70 | 71 | response["hits"]["total"] 72 | end 73 | 74 | def promote(new_name, update_refresh_interval: false) 75 | if update_refresh_interval 76 | new_index = Searchkick::Index.new(new_name, @options) 77 | settings = options[:settings] || {} 78 | refresh_interval = (settings[:index] && settings[:index][:refresh_interval]) || "1s" 79 | new_index.update_settings(index: {refresh_interval: refresh_interval}) 80 | end 81 | 82 | old_indices = 83 | begin 84 | client.indices.get_alias(name: name).keys 85 | rescue Elasticsearch::Transport::Transport::Errors::NotFound 86 | {} 87 | end 88 | actions = old_indices.map { |old_name| {remove: {index: old_name, alias: name}} } + [{add: {index: new_name, alias: name}}] 89 | client.indices.update_aliases body: {actions: actions} 90 | end 91 | alias_method :swap, :promote 92 | 93 | def retrieve(record) 94 | client.get( 95 | index: name, 96 | type: document_type(record), 97 | id: search_id(record) 98 | )["_source"] 99 | end 100 | 101 | def all_indices(unaliased: false) 102 | indices = 103 | begin 104 | client.indices.get_aliases 105 | rescue Elasticsearch::Transport::Transport::Errors::NotFound 106 | {} 107 | end 108 | indices = indices.select { |_k, v| v.empty? || v["aliases"].empty? } if unaliased 109 | indices.select { |k, _v| k =~ /\A#{Regexp.escape(name)}_\d{14,17}\z/ }.keys 110 | end 111 | 112 | # remove old indices that start w/ index_name 113 | def clean_indices 114 | indices = all_indices(unaliased: true) 115 | indices.each do |index| 116 | Searchkick::Index.new(index).delete 117 | end 118 | indices 119 | end 120 | 121 | # record based 122 | # use helpers for notifications 123 | 124 | def store(record) 125 | bulk_indexer.bulk_index([record]) 126 | end 127 | 128 | def remove(record) 129 | bulk_indexer.bulk_delete([record]) 130 | end 131 | 132 | def update_record(record, method_name) 133 | bulk_indexer.bulk_update([record], method_name) 134 | end 135 | 136 | def bulk_delete(records) 137 | bulk_indexer.bulk_delete(records) 138 | end 139 | 140 | def bulk_index(records) 141 | bulk_indexer.bulk_index(records) 142 | end 143 | alias_method :import, :bulk_index 144 | 145 | def bulk_update(records, method_name) 146 | bulk_indexer.bulk_update(records, method_name) 147 | end 148 | 149 | def search_id(record) 150 | RecordData.new(self, record).search_id 151 | end 152 | 153 | def document_type(record) 154 | RecordData.new(self, record).document_type 155 | end 156 | 157 | def similar_record(record, **options) 158 | like_text = retrieve(record).to_hash 159 | .keep_if { |k, _| !options[:fields] || options[:fields].map(&:to_s).include?(k) } 160 | .values.compact.join(" ") 161 | 162 | # TODO deep merge method 163 | options[:where] ||= {} 164 | options[:where][:_id] ||= {} 165 | options[:where][:_id][:not] = record.id.to_s 166 | options[:per_page] ||= 10 167 | options[:similar] = true 168 | 169 | # TODO use index class instead of record class 170 | Searchkick.search(like_text, model: record.class, **options) 171 | end 172 | 173 | # queue 174 | 175 | def reindex_queue 176 | Searchkick::ReindexQueue.new(name) 177 | end 178 | 179 | # reindex 180 | 181 | def reindex(relation, method_name, scoped:, full: false, scope: nil, **options) 182 | refresh = options.fetch(:refresh, !scoped) 183 | 184 | if method_name 185 | # update 186 | import_scope(relation, method_name: method_name, scope: scope) 187 | self.refresh if refresh 188 | true 189 | elsif scoped && !full 190 | # reindex association 191 | import_scope(relation, scope: scope) 192 | self.refresh if refresh 193 | true 194 | else 195 | # full reindex 196 | reindex_scope(relation, scope: scope, **options) 197 | end 198 | end 199 | 200 | def create_index(index_options: nil) 201 | index_options ||= self.index_options 202 | index = Searchkick::Index.new("#{name}_#{Time.now.strftime('%Y%m%d%H%M%S%L')}", @options) 203 | index.create(index_options) 204 | index 205 | end 206 | 207 | def import_scope(relation, **options) 208 | bulk_indexer.import_scope(relation, **options) 209 | end 210 | 211 | def batches_left 212 | bulk_indexer.batches_left 213 | end 214 | 215 | # other 216 | 217 | def klass_document_type(klass, ignore_type = false) 218 | @klass_document_type[[klass, ignore_type]] ||= begin 219 | if !ignore_type && klass.searchkick_klass.searchkick_options[:_type] 220 | type = klass.searchkick_klass.searchkick_options[:_type] 221 | type = type.call if type.respond_to?(:call) 222 | type 223 | else 224 | klass.model_name.to_s.underscore 225 | end 226 | end 227 | end 228 | 229 | # should not be public 230 | def conversions_fields 231 | @conversions_fields ||= begin 232 | conversions = Array(options[:conversions]) 233 | conversions.map(&:to_s) + conversions.map(&:to_sym) 234 | end 235 | end 236 | 237 | def suggest_fields 238 | @suggest_fields ||= Array(options[:suggest]).map(&:to_s) 239 | end 240 | 241 | def locations_fields 242 | @locations_fields ||= begin 243 | locations = Array(options[:locations]) 244 | locations.map(&:to_s) + locations.map(&:to_sym) 245 | end 246 | end 247 | 248 | protected 249 | 250 | def client 251 | Searchkick.client 252 | end 253 | 254 | def bulk_indexer 255 | @bulk_indexer ||= BulkIndexer.new(self) 256 | end 257 | 258 | # https://gist.github.com/jarosan/3124884 259 | # http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/ 260 | def reindex_scope(relation, import: true, resume: false, retain: false, async: false, refresh_interval: nil, scope: nil) 261 | if resume 262 | index_name = all_indices.sort.last 263 | raise Searchkick::Error, "No index to resume" unless index_name 264 | index = Searchkick::Index.new(index_name, @options) 265 | else 266 | clean_indices unless retain 267 | 268 | index_options = relation.searchkick_index_options 269 | index_options.deep_merge!(settings: {index: {refresh_interval: refresh_interval}}) if refresh_interval 270 | index = create_index(index_options: index_options) 271 | end 272 | 273 | import_options = { 274 | resume: resume, 275 | async: async, 276 | full: true, 277 | scope: scope 278 | } 279 | 280 | # check if alias exists 281 | alias_exists = alias_exists? 282 | if alias_exists 283 | # import before promotion 284 | index.import_scope(relation, **import_options) if import 285 | 286 | # get existing indices to remove 287 | unless async 288 | promote(index.name, update_refresh_interval: !refresh_interval.nil?) 289 | clean_indices unless retain 290 | end 291 | else 292 | delete if exists? 293 | promote(index.name, update_refresh_interval: !refresh_interval.nil?) 294 | 295 | # import after promotion 296 | index.import_scope(relation, **import_options) if import 297 | end 298 | 299 | if async 300 | if async.is_a?(Hash) && async[:wait] 301 | puts "Created index: #{index.name}" 302 | puts "Jobs queued. Waiting..." 303 | loop do 304 | sleep 3 305 | status = Searchkick.reindex_status(index.name) 306 | break if status[:completed] 307 | puts "Batches left: #{status[:batches_left]}" 308 | end 309 | # already promoted if alias didn't exist 310 | if alias_exists 311 | puts "Jobs complete. Promoting..." 312 | promote(index.name, update_refresh_interval: !refresh_interval.nil?) 313 | end 314 | clean_indices unless retain 315 | puts "SUCCESS!" 316 | end 317 | 318 | {index_name: index.name} 319 | else 320 | index.refresh 321 | true 322 | end 323 | rescue Elasticsearch::Transport::Transport::Errors::BadRequest => e 324 | if e.message.include?("No handler for type [text]") 325 | raise UnsupportedVersionError, "This version of Searchkick requires Elasticsearch 5 or greater" 326 | end 327 | 328 | raise e 329 | end 330 | end 331 | end 332 | -------------------------------------------------------------------------------- /test/where_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | class WhereTest < Minitest::Test 4 | def test_where 5 | now = Time.now 6 | store [ 7 | {name: "Product A", store_id: 1, in_stock: true, backordered: true, created_at: now, orders_count: 4, user_ids: [1, 2, 3]}, 8 | {name: "Product B", store_id: 2, in_stock: true, backordered: false, created_at: now - 1, orders_count: 3, user_ids: [1]}, 9 | {name: "Product C", store_id: 3, in_stock: false, backordered: true, created_at: now - 2, orders_count: 2, user_ids: [1, 3]}, 10 | {name: "Product D", store_id: 4, in_stock: false, backordered: false, created_at: now - 3, orders_count: 1} 11 | ] 12 | assert_search "product", ["Product A", "Product B"], where: {in_stock: true} 13 | 14 | # due to precision 15 | unless cequel? 16 | # date 17 | assert_search "product", ["Product A"], where: {created_at: {gt: now - 1}} 18 | assert_search "product", ["Product A", "Product B"], where: {created_at: {gte: now - 1}} 19 | assert_search "product", ["Product D"], where: {created_at: {lt: now - 2}} 20 | assert_search "product", ["Product C", "Product D"], where: {created_at: {lte: now - 2}} 21 | end 22 | 23 | # integer 24 | assert_search "product", ["Product A"], where: {store_id: {lt: 2}} 25 | assert_search "product", ["Product A", "Product B"], where: {store_id: {lte: 2}} 26 | assert_search "product", ["Product D"], where: {store_id: {gt: 3}} 27 | assert_search "product", ["Product C", "Product D"], where: {store_id: {gte: 3}} 28 | 29 | # range 30 | assert_search "product", ["Product A", "Product B"], where: {store_id: 1..2} 31 | assert_search "product", ["Product A"], where: {store_id: 1...2} 32 | assert_search "product", ["Product A", "Product B"], where: {store_id: [1, 2]} 33 | assert_search "product", ["Product B", "Product C", "Product D"], where: {store_id: {not: 1}} 34 | assert_search "product", ["Product B", "Product C", "Product D"], where: {store_id: {_not: 1}} 35 | assert_search "product", ["Product C", "Product D"], where: {store_id: {not: [1, 2]}} 36 | assert_search "product", ["Product C", "Product D"], where: {store_id: {_not: [1, 2]}} 37 | assert_search "product", ["Product A"], where: {user_ids: {lte: 2, gte: 2}} 38 | 39 | # or 40 | assert_search "product", ["Product A", "Product B", "Product C"], where: {or: [[{in_stock: true}, {store_id: 3}]]} 41 | assert_search "product", ["Product A", "Product B", "Product C"], where: {or: [[{orders_count: [2, 4]}, {store_id: [1, 2]}]]} 42 | assert_search "product", ["Product A", "Product D"], where: {or: [[{orders_count: 1}, {created_at: {gte: now - 1}, backordered: true}]]} 43 | 44 | # _or 45 | assert_search "product", ["Product A", "Product B", "Product C"], where: {_or: [{in_stock: true}, {store_id: 3}]} 46 | assert_search "product", ["Product A", "Product B", "Product C"], where: {_or: [{orders_count: [2, 4]}, {store_id: [1, 2]}]} 47 | assert_search "product", ["Product A", "Product D"], where: {_or: [{orders_count: 1}, {created_at: {gte: now - 1}, backordered: true}]} 48 | 49 | # _and 50 | assert_search "product", ["Product A"], where: {_and: [{in_stock: true}, {backordered: true}]} 51 | 52 | # _not 53 | assert_search "product", ["Product B", "Product C"], where: {_not: {_or: [{orders_count: 1}, {created_at: {gte: now - 1}, backordered: true}]}} 54 | 55 | # all 56 | assert_search "product", ["Product A", "Product C"], where: {user_ids: {all: [1, 3]}} 57 | assert_search "product", [], where: {user_ids: {all: [1, 2, 3, 4]}} 58 | 59 | # any / nested terms 60 | assert_search "product", ["Product B", "Product C"], where: {user_ids: {not: [2], in: [1, 3]}} 61 | assert_search "product", ["Product B", "Product C"], where: {user_ids: {_not: [2], in: [1, 3]}} 62 | 63 | # not / exists 64 | assert_search "product", ["Product D"], where: {user_ids: nil} 65 | assert_search "product", ["Product A", "Product B", "Product C"], where: {user_ids: {not: nil}} 66 | assert_search "product", ["Product A", "Product B", "Product C"], where: {user_ids: {_not: nil}} 67 | assert_search "product", ["Product A", "Product C", "Product D"], where: {user_ids: [3, nil]} 68 | assert_search "product", ["Product B"], where: {user_ids: {not: [3, nil]}} 69 | assert_search "product", ["Product B"], where: {user_ids: {_not: [3, nil]}} 70 | end 71 | 72 | def test_regexp 73 | store_names ["Product A"] 74 | assert_search "*", ["Product A"], where: {name: /Pro.+/} 75 | end 76 | 77 | def test_alternate_regexp 78 | store_names ["Product A", "Item B"] 79 | assert_search "*", ["Product A"], where: {name: {regexp: "Pro.+"}} 80 | end 81 | 82 | def test_special_regexp 83 | store_names ["Product ", "Item "] 84 | assert_search "*", ["Product "], where: {name: /Pro.+<.+/} 85 | end 86 | 87 | def test_where_string 88 | store [ 89 | {name: "Product A", color: "RED"} 90 | ] 91 | assert_search "product", ["Product A"], where: {color: "RED"} 92 | end 93 | 94 | def test_where_nil 95 | store [ 96 | {name: "Product A"}, 97 | {name: "Product B", color: "red"} 98 | ] 99 | assert_search "product", ["Product A"], where: {color: nil} 100 | end 101 | 102 | def test_where_id 103 | store_names ["Product A"] 104 | product = Product.first 105 | assert_search "product", ["Product A"], where: {id: product.id.to_s} 106 | end 107 | 108 | def test_where_empty 109 | store_names ["Product A"] 110 | assert_search "product", ["Product A"], where: {} 111 | end 112 | 113 | def test_where_empty_array 114 | store_names ["Product A"] 115 | assert_search "product", [], where: {store_id: []} 116 | end 117 | 118 | # http://elasticsearch-users.115913.n3.nabble.com/Numeric-range-quey-or-filter-in-an-array-field-possible-or-not-td4042967.html 119 | # https://gist.github.com/jprante/7099463 120 | def test_where_range_array 121 | store [ 122 | {name: "Product A", user_ids: [11, 23, 13, 16, 17, 23]}, 123 | {name: "Product B", user_ids: [1, 2, 3, 4, 5, 6, 7, 8, 9]}, 124 | {name: "Product C", user_ids: [101, 230, 150, 200]} 125 | ] 126 | assert_search "product", ["Product A"], where: {user_ids: {gt: 10, lt: 24}} 127 | end 128 | 129 | def test_where_range_array_again 130 | store [ 131 | {name: "Product A", user_ids: [19, 32, 42]}, 132 | {name: "Product B", user_ids: [13, 40, 52]} 133 | ] 134 | assert_search "product", ["Product A"], where: {user_ids: {gt: 26, lt: 36}} 135 | end 136 | 137 | def test_near 138 | store [ 139 | {name: "San Francisco", latitude: 37.7833, longitude: -122.4167}, 140 | {name: "San Antonio", latitude: 29.4167, longitude: -98.5000} 141 | ] 142 | assert_search "san", ["San Francisco"], where: {location: {near: [37.5, -122.5]}} 143 | end 144 | 145 | def test_near_hash 146 | store [ 147 | {name: "San Francisco", latitude: 37.7833, longitude: -122.4167}, 148 | {name: "San Antonio", latitude: 29.4167, longitude: -98.5000} 149 | ] 150 | assert_search "san", ["San Francisco"], where: {location: {near: {lat: 37.5, lon: -122.5}}} 151 | end 152 | 153 | def test_near_within 154 | store [ 155 | {name: "San Francisco", latitude: 37.7833, longitude: -122.4167}, 156 | {name: "San Antonio", latitude: 29.4167, longitude: -98.5000}, 157 | {name: "San Marino", latitude: 43.9333, longitude: 12.4667} 158 | ] 159 | assert_search "san", ["San Francisco", "San Antonio"], where: {location: {near: [37, -122], within: "2000mi"}} 160 | end 161 | 162 | def test_near_within_hash 163 | store [ 164 | {name: "San Francisco", latitude: 37.7833, longitude: -122.4167}, 165 | {name: "San Antonio", latitude: 29.4167, longitude: -98.5000}, 166 | {name: "San Marino", latitude: 43.9333, longitude: 12.4667} 167 | ] 168 | assert_search "san", ["San Francisco", "San Antonio"], where: {location: {near: {lat: 37, lon: -122}, within: "2000mi"}} 169 | end 170 | 171 | def test_geo_polygon 172 | store [ 173 | {name: "San Francisco", latitude: 37.7833, longitude: -122.4167}, 174 | {name: "San Antonio", latitude: 29.4167, longitude: -98.5000}, 175 | {name: "San Marino", latitude: 43.9333, longitude: 12.4667} 176 | ] 177 | polygon = [ 178 | {lat: 42.185695, lon: -125.496146}, 179 | {lat: 42.185695, lon: -94.125535}, 180 | {lat: 27.122789, lon: -94.125535}, 181 | {lat: 27.12278, lon: -125.496146} 182 | ] 183 | assert_search "san", ["San Francisco", "San Antonio"], where: {location: {geo_polygon: {points: polygon}}} 184 | end 185 | 186 | def test_top_left_bottom_right 187 | store [ 188 | {name: "San Francisco", latitude: 37.7833, longitude: -122.4167}, 189 | {name: "San Antonio", latitude: 29.4167, longitude: -98.5000} 190 | ] 191 | assert_search "san", ["San Francisco"], where: {location: {top_left: [38, -123], bottom_right: [37, -122]}} 192 | end 193 | 194 | def test_top_left_bottom_right_hash 195 | store [ 196 | {name: "San Francisco", latitude: 37.7833, longitude: -122.4167}, 197 | {name: "San Antonio", latitude: 29.4167, longitude: -98.5000} 198 | ] 199 | assert_search "san", ["San Francisco"], where: {location: {top_left: {lat: 38, lon: -123}, bottom_right: {lat: 37, lon: -122}}} 200 | end 201 | 202 | def test_top_right_bottom_left 203 | store [ 204 | {name: "San Francisco", latitude: 37.7833, longitude: -122.4167}, 205 | {name: "San Antonio", latitude: 29.4167, longitude: -98.5000} 206 | ] 207 | assert_search "san", ["San Francisco"], where: {location: {top_right: [38, -122], bottom_left: [37, -123]}} 208 | end 209 | 210 | def test_top_right_bottom_left_hash 211 | store [ 212 | {name: "San Francisco", latitude: 37.7833, longitude: -122.4167}, 213 | {name: "San Antonio", latitude: 29.4167, longitude: -98.5000} 214 | ] 215 | assert_search "san", ["San Francisco"], where: {location: {top_right: {lat: 38, lon: -122}, bottom_left: {lat: 37, lon: -123}}} 216 | end 217 | 218 | def test_multiple_locations 219 | store [ 220 | {name: "San Francisco", latitude: 37.7833, longitude: -122.4167}, 221 | {name: "San Antonio", latitude: 29.4167, longitude: -98.5000} 222 | ] 223 | assert_search "san", ["San Francisco"], where: {multiple_locations: {near: [37.5, -122.5]}} 224 | end 225 | 226 | def test_multiple_locations_with_term_filter 227 | store [ 228 | {name: "San Francisco", latitude: 37.7833, longitude: -122.4167}, 229 | {name: "San Antonio", latitude: 29.4167, longitude: -98.5000} 230 | ] 231 | assert_search "san", [], where: {multiple_locations: {near: [37.5, -122.5]}, name: "San Antonio"} 232 | assert_search "san", ["San Francisco"], where: {multiple_locations: {near: [37.5, -122.5]}, name: "San Francisco"} 233 | end 234 | 235 | def test_multiple_locations_hash 236 | store [ 237 | {name: "San Francisco", latitude: 37.7833, longitude: -122.4167}, 238 | {name: "San Antonio", latitude: 29.4167, longitude: -98.5000} 239 | ] 240 | assert_search "san", ["San Francisco"], where: {multiple_locations: {near: {lat: 37.5, lon: -122.5}}} 241 | end 242 | 243 | def test_nested 244 | store [ 245 | {name: "Product A", details: {year: 2016}} 246 | ] 247 | assert_search "product", ["Product A"], where: {"details.year" => 2016} 248 | end 249 | end 250 | -------------------------------------------------------------------------------- /test/test_helper.rb: -------------------------------------------------------------------------------- 1 | require "bundler/setup" 2 | Bundler.require(:default) 3 | require "minitest/autorun" 4 | require "minitest/pride" 5 | require "logger" 6 | require "active_support/core_ext" if defined?(NoBrainer) 7 | require "active_support/notifications" 8 | 9 | Searchkick.index_suffix = ENV["TEST_ENV_NUMBER"] 10 | 11 | ENV["RACK_ENV"] = "test" 12 | 13 | Minitest::Test = Minitest::Unit::TestCase unless defined?(Minitest::Test) 14 | 15 | if !defined?(ParallelTests) || ParallelTests.first_process? 16 | File.delete("elasticsearch.log") if File.exist?("elasticsearch.log") 17 | end 18 | 19 | Searchkick.client.transport.logger = Logger.new("elasticsearch.log") 20 | Searchkick.search_timeout = 5 21 | 22 | if defined?(Redis) 23 | if defined?(ConnectionPool) 24 | Searchkick.redis = ConnectionPool.new { Redis.new } 25 | else 26 | Searchkick.redis = Redis.new 27 | end 28 | end 29 | 30 | puts "Running against Elasticsearch #{Searchkick.server_version}" 31 | 32 | I18n.config.enforce_available_locales = true 33 | 34 | if defined?(ActiveJob) 35 | ActiveJob::Base.logger = nil 36 | ActiveJob::Base.queue_adapter = :inline 37 | end 38 | 39 | ActiveSupport::LogSubscriber.logger = ActiveSupport::Logger.new(STDOUT) if ENV["NOTIFICATIONS"] 40 | 41 | def nobrainer? 42 | defined?(NoBrainer) 43 | end 44 | 45 | def cequel? 46 | defined?(Cequel) 47 | end 48 | 49 | if defined?(Mongoid) 50 | Mongoid.logger.level = Logger::INFO 51 | Mongo::Logger.logger.level = Logger::INFO if defined?(Mongo::Logger) 52 | 53 | Mongoid.configure do |config| 54 | config.connect_to "searchkick_test" 55 | end 56 | 57 | class Product 58 | include Mongoid::Document 59 | include Mongoid::Timestamps 60 | 61 | field :name 62 | field :store_id, type: Integer 63 | field :in_stock, type: Boolean 64 | field :backordered, type: Boolean 65 | field :orders_count, type: Integer 66 | field :found_rate, type: BigDecimal 67 | field :price, type: Integer 68 | field :color 69 | field :latitude, type: BigDecimal 70 | field :longitude, type: BigDecimal 71 | field :description 72 | field :alt_description 73 | end 74 | 75 | class Store 76 | include Mongoid::Document 77 | has_many :products 78 | 79 | field :name 80 | end 81 | 82 | class Region 83 | include Mongoid::Document 84 | 85 | field :name 86 | field :text 87 | end 88 | 89 | class Speaker 90 | include Mongoid::Document 91 | 92 | field :name 93 | end 94 | 95 | class Animal 96 | include Mongoid::Document 97 | 98 | field :name 99 | end 100 | 101 | class Dog < Animal 102 | end 103 | 104 | class Cat < Animal 105 | end 106 | 107 | class Sku 108 | include Mongoid::Document 109 | 110 | field :name 111 | end 112 | 113 | class Song 114 | include Mongoid::Document 115 | 116 | field :name 117 | end 118 | elsif defined?(NoBrainer) 119 | NoBrainer.configure do |config| 120 | config.app_name = :searchkick 121 | config.environment = :test 122 | end 123 | 124 | class Product 125 | include NoBrainer::Document 126 | include NoBrainer::Document::Timestamps 127 | 128 | field :id, type: Object 129 | field :name, type: Text 130 | field :in_stock, type: Boolean 131 | field :backordered, type: Boolean 132 | field :orders_count, type: Integer 133 | field :found_rate 134 | field :price, type: Integer 135 | field :color, type: String 136 | field :latitude 137 | field :longitude 138 | field :description, type: String 139 | field :alt_description, type: String 140 | 141 | belongs_to :store, validates: false 142 | end 143 | 144 | class Store 145 | include NoBrainer::Document 146 | 147 | field :id, type: Object 148 | field :name, type: String 149 | end 150 | 151 | class Region 152 | include NoBrainer::Document 153 | 154 | field :id, type: Object 155 | field :name, type: String 156 | field :text, type: Text 157 | end 158 | 159 | class Speaker 160 | include NoBrainer::Document 161 | 162 | field :id, type: Object 163 | field :name, type: String 164 | end 165 | 166 | class Animal 167 | include NoBrainer::Document 168 | 169 | field :id, type: Object 170 | field :name, type: String 171 | end 172 | 173 | class Dog < Animal 174 | end 175 | 176 | class Cat < Animal 177 | end 178 | 179 | class Sku 180 | include NoBrainer::Document 181 | 182 | field :id, type: String 183 | field :name, type: String 184 | end 185 | 186 | class Song 187 | include NoBrainer::Document 188 | 189 | field :id, type: Object 190 | field :name, type: String 191 | end 192 | elsif defined?(Cequel) 193 | cequel = 194 | Cequel.connect( 195 | host: "127.0.0.1", 196 | port: 9042, 197 | keyspace: "searchkick_test", 198 | default_consistency: :all 199 | ) 200 | # cequel.logger = ActiveSupport::Logger.new(STDOUT) 201 | cequel.schema.drop! if cequel.schema.exists? 202 | cequel.schema.create! 203 | Cequel::Record.connection = cequel 204 | 205 | class Product 206 | include Cequel::Record 207 | 208 | key :id, :uuid, auto: true 209 | column :name, :text, index: true 210 | column :store_id, :int 211 | column :in_stock, :boolean 212 | column :backordered, :boolean 213 | column :orders_count, :int 214 | column :found_rate, :decimal 215 | column :price, :int 216 | column :color, :text 217 | column :latitude, :decimal 218 | column :longitude, :decimal 219 | column :description, :text 220 | column :alt_description, :text 221 | column :created_at, :timestamp 222 | end 223 | 224 | class Store 225 | include Cequel::Record 226 | 227 | key :id, :timeuuid, auto: true 228 | column :name, :text 229 | 230 | # has issue with id serialization 231 | def search_data 232 | { 233 | name: name 234 | } 235 | end 236 | end 237 | 238 | class Region 239 | include Cequel::Record 240 | 241 | key :id, :timeuuid, auto: true 242 | column :name, :text 243 | column :text, :text 244 | end 245 | 246 | class Speaker 247 | include Cequel::Record 248 | 249 | key :id, :timeuuid, auto: true 250 | column :name, :text 251 | end 252 | 253 | class Animal 254 | include Cequel::Record 255 | 256 | key :id, :timeuuid, auto: true 257 | column :name, :text 258 | 259 | # has issue with id serialization 260 | def search_data 261 | { 262 | name: name 263 | } 264 | end 265 | end 266 | 267 | class Dog < Animal 268 | end 269 | 270 | class Cat < Animal 271 | end 272 | 273 | class Sku 274 | include Cequel::Record 275 | 276 | key :id, :uuid 277 | column :name, :text 278 | end 279 | 280 | class Song 281 | include Cequel::Record 282 | 283 | key :id, :timeuuid, auto: true 284 | column :name, :text 285 | end 286 | 287 | [Product, Store, Region, Speaker, Animal, Sku, Song].each(&:synchronize_schema) 288 | else 289 | require "active_record" 290 | 291 | # for debugging 292 | # ActiveRecord::Base.logger = Logger.new(STDOUT) 293 | 294 | # rails does this in activerecord/lib/active_record/railtie.rb 295 | ActiveRecord::Base.default_timezone = :utc 296 | ActiveRecord::Base.time_zone_aware_attributes = true 297 | 298 | # migrations 299 | ActiveRecord::Base.establish_connection adapter: "sqlite3", database: ":memory:" 300 | 301 | ActiveRecord::Base.raise_in_transactional_callbacks = true if ActiveRecord::VERSION::STRING.start_with?("4.2.") 302 | 303 | if defined?(Apartment) 304 | class Rails 305 | def self.env 306 | ENV["RACK_ENV"] 307 | end 308 | end 309 | 310 | tenants = ["tenant1", "tenant2"] 311 | Apartment.configure do |config| 312 | config.tenant_names = tenants 313 | config.database_schema_file = false 314 | config.excluded_models = ["Product", "Store", "Animal", "Dog", "Cat"] 315 | end 316 | 317 | class Tenant < ActiveRecord::Base 318 | searchkick index_prefix: -> { Apartment::Tenant.current } 319 | end 320 | 321 | tenants.each do |tenant| 322 | begin 323 | Apartment::Tenant.create(tenant) 324 | rescue Apartment::TenantExists 325 | # do nothing 326 | end 327 | Apartment::Tenant.switch!(tenant) 328 | 329 | ActiveRecord::Migration.create_table :tenants, force: true do |t| 330 | t.string :name 331 | t.timestamps null: true 332 | end 333 | 334 | Tenant.reindex 335 | end 336 | 337 | Apartment::Tenant.reset 338 | end 339 | 340 | ActiveRecord::Migration.create_table :products do |t| 341 | t.string :name 342 | t.integer :store_id 343 | t.boolean :in_stock 344 | t.boolean :backordered 345 | t.integer :orders_count 346 | t.decimal :found_rate 347 | t.integer :price 348 | t.string :color 349 | t.decimal :latitude, precision: 10, scale: 7 350 | t.decimal :longitude, precision: 10, scale: 7 351 | t.text :description 352 | t.text :alt_description 353 | t.timestamps null: true 354 | end 355 | 356 | ActiveRecord::Migration.create_table :stores do |t| 357 | t.string :name 358 | end 359 | 360 | ActiveRecord::Migration.create_table :regions do |t| 361 | t.string :name 362 | t.text :text 363 | end 364 | 365 | ActiveRecord::Migration.create_table :speakers do |t| 366 | t.string :name 367 | end 368 | 369 | ActiveRecord::Migration.create_table :animals do |t| 370 | t.string :name 371 | t.string :type 372 | end 373 | 374 | ActiveRecord::Migration.create_table :skus, id: :uuid do |t| 375 | t.string :name 376 | end 377 | 378 | ActiveRecord::Migration.create_table :songs do |t| 379 | t.string :name 380 | end 381 | 382 | class Product < ActiveRecord::Base 383 | belongs_to :store 384 | end 385 | 386 | class Store < ActiveRecord::Base 387 | has_many :products 388 | end 389 | 390 | class Region < ActiveRecord::Base 391 | end 392 | 393 | class Speaker < ActiveRecord::Base 394 | end 395 | 396 | class Animal < ActiveRecord::Base 397 | end 398 | 399 | class Dog < Animal 400 | end 401 | 402 | class Cat < Animal 403 | end 404 | 405 | class Sku < ActiveRecord::Base 406 | end 407 | 408 | class Song < ActiveRecord::Base 409 | end 410 | end 411 | 412 | class Product 413 | searchkick \ 414 | synonyms: [ 415 | ["clorox", "bleach"], 416 | ["scallion", "greenonion"], 417 | ["saran wrap", "plastic wrap"], 418 | ["qtip", "cottonswab"], 419 | ["burger", "hamburger"], 420 | ["bandaid", "bandag"], 421 | ["UPPERCASE", "lowercase"], 422 | "lightbulb => led,lightbulb", 423 | "lightbulb => halogenlamp" 424 | ], 425 | suggest: [:name, :color], 426 | conversions: [:conversions], 427 | locations: [:location, :multiple_locations], 428 | text_start: [:name], 429 | text_middle: [:name], 430 | text_end: [:name], 431 | word_start: [:name], 432 | word_middle: [:name], 433 | word_end: [:name], 434 | highlight: [:name], 435 | filterable: [:name, :color, :description], 436 | similarity: "BM25", 437 | match: ENV["MATCH"] ? ENV["MATCH"].to_sym : nil 438 | 439 | attr_accessor :conversions, :user_ids, :aisle, :details 440 | 441 | def search_data 442 | serializable_hash.except("id", "_id").merge( 443 | conversions: conversions, 444 | user_ids: user_ids, 445 | location: {lat: latitude, lon: longitude}, 446 | multiple_locations: [{lat: latitude, lon: longitude}, {lat: 0, lon: 0}], 447 | aisle: aisle, 448 | details: details 449 | ) 450 | end 451 | 452 | def should_index? 453 | name != "DO NOT INDEX" 454 | end 455 | 456 | def search_name 457 | { 458 | name: name 459 | } 460 | end 461 | end 462 | 463 | class Store 464 | searchkick \ 465 | routing: true, 466 | merge_mappings: true, 467 | mappings: { 468 | store: { 469 | properties: { 470 | name: {type: "keyword"} 471 | } 472 | } 473 | } 474 | 475 | def search_document_id 476 | id 477 | end 478 | 479 | def search_routing 480 | name 481 | end 482 | end 483 | 484 | class Region 485 | searchkick \ 486 | geo_shape: { 487 | territory: {tree: "quadtree", precision: "10km"} 488 | } 489 | 490 | attr_accessor :territory 491 | 492 | def search_data 493 | { 494 | name: name, 495 | text: text, 496 | territory: territory 497 | } 498 | end 499 | end 500 | 501 | class Speaker 502 | searchkick \ 503 | conversions: ["conversions_a", "conversions_b"] 504 | 505 | attr_accessor :conversions_a, :conversions_b, :aisle 506 | 507 | def search_data 508 | serializable_hash.except("id", "_id").merge( 509 | conversions_a: conversions_a, 510 | conversions_b: conversions_b, 511 | aisle: aisle 512 | ) 513 | end 514 | end 515 | 516 | class Animal 517 | searchkick \ 518 | inheritance: true, 519 | text_start: [:name], 520 | suggest: [:name], 521 | index_name: -> { "#{name.tableize}-#{Date.today.year}#{Searchkick.index_suffix}" }, 522 | callbacks: defined?(ActiveJob) ? :async : true 523 | # wordnet: true 524 | end 525 | 526 | class Sku 527 | searchkick callbacks: defined?(ActiveJob) ? :async : true 528 | end 529 | 530 | class Song 531 | searchkick 532 | end 533 | 534 | Product.searchkick_index.delete if Product.searchkick_index.exists? 535 | Product.reindex 536 | Product.reindex # run twice for both index paths 537 | Product.create!(name: "Set mapping") 538 | 539 | Store.reindex 540 | Animal.reindex 541 | Speaker.reindex 542 | Region.reindex 543 | 544 | class Minitest::Test 545 | def setup 546 | Product.destroy_all 547 | Store.destroy_all 548 | Animal.destroy_all 549 | Speaker.destroy_all 550 | end 551 | 552 | protected 553 | 554 | def store(documents, klass = Product) 555 | documents.shuffle.each do |document| 556 | klass.create!(document) 557 | end 558 | klass.searchkick_index.refresh 559 | end 560 | 561 | def store_names(names, klass = Product) 562 | store names.map { |name| {name: name} }, klass 563 | end 564 | 565 | # no order 566 | def assert_search(term, expected, options = {}, klass = Product) 567 | assert_equal expected.sort, klass.search(term, options).map(&:name).sort 568 | end 569 | 570 | def assert_order(term, expected, options = {}, klass = Product) 571 | assert_equal expected, klass.search(term, options).map(&:name) 572 | end 573 | 574 | def assert_equal_scores(term, options = {}, klass = Product) 575 | assert_equal 1, klass.search(term, options).hits.map { |a| a["_score"] }.uniq.size 576 | end 577 | 578 | def assert_first(term, expected, options = {}, klass = Product) 579 | assert_equal expected, klass.search(term, options).map(&:name).first 580 | end 581 | 582 | def with_options(klass, options) 583 | previous_options = klass.searchkick_options.dup 584 | begin 585 | klass.searchkick_options.merge!(options) 586 | klass.reindex 587 | yield 588 | ensure 589 | klass.searchkick_options.clear 590 | klass.searchkick_options.merge!(previous_options) 591 | end 592 | end 593 | end 594 | -------------------------------------------------------------------------------- /lib/searchkick/index_options.rb: -------------------------------------------------------------------------------- 1 | module Searchkick 2 | module IndexOptions 3 | def index_options 4 | options = @options 5 | language = options[:language] 6 | language = language.call if language.respond_to?(:call) 7 | index_type = options[:_type] 8 | index_type = index_type.call if index_type.respond_to?(:call) 9 | 10 | if options[:mappings] && !options[:merge_mappings] 11 | settings = options[:settings] || {} 12 | mappings = options[:mappings] 13 | else 14 | below60 = Searchkick.server_below?("6.0.0") 15 | below62 = Searchkick.server_below?("6.2.0") 16 | 17 | default_type = "text" 18 | default_analyzer = :searchkick_index 19 | keyword_mapping = {type: "keyword"} 20 | 21 | all = options.key?(:_all) ? options[:_all] : false 22 | index_true_value = true 23 | index_false_value = false 24 | 25 | keyword_mapping[:ignore_above] = options[:ignore_above] || 30000 26 | 27 | settings = { 28 | analysis: { 29 | analyzer: { 30 | searchkick_keyword: { 31 | type: "custom", 32 | tokenizer: "keyword", 33 | filter: ["lowercase"] + (options[:stem_conversions] ? ["searchkick_stemmer"] : []) 34 | }, 35 | default_analyzer => { 36 | type: "custom", 37 | # character filters -> tokenizer -> token filters 38 | # https://www.elastic.co/guide/en/elasticsearch/guide/current/analysis-intro.html 39 | char_filter: ["ampersand"], 40 | tokenizer: "standard", 41 | # synonym should come last, after stemming and shingle 42 | # shingle must come before searchkick_stemmer 43 | filter: ["standard", "lowercase", "asciifolding", "searchkick_index_shingle", "searchkick_stemmer"] 44 | }, 45 | searchkick_search: { 46 | type: "custom", 47 | char_filter: ["ampersand"], 48 | tokenizer: "standard", 49 | filter: ["standard", "lowercase", "asciifolding", "searchkick_search_shingle", "searchkick_stemmer"] 50 | }, 51 | searchkick_search2: { 52 | type: "custom", 53 | char_filter: ["ampersand"], 54 | tokenizer: "standard", 55 | filter: ["standard", "lowercase", "asciifolding", "searchkick_stemmer"] 56 | }, 57 | # https://github.com/leschenko/elasticsearch_autocomplete/blob/master/lib/elasticsearch_autocomplete/analyzers.rb 58 | searchkick_autocomplete_search: { 59 | type: "custom", 60 | tokenizer: "keyword", 61 | filter: ["lowercase", "asciifolding"] 62 | }, 63 | searchkick_word_search: { 64 | type: "custom", 65 | tokenizer: "standard", 66 | filter: ["lowercase", "asciifolding"] 67 | }, 68 | searchkick_suggest_index: { 69 | type: "custom", 70 | tokenizer: "standard", 71 | filter: ["lowercase", "asciifolding", "searchkick_suggest_shingle"] 72 | }, 73 | searchkick_text_start_index: { 74 | type: "custom", 75 | tokenizer: "keyword", 76 | filter: ["lowercase", "asciifolding", "searchkick_edge_ngram"] 77 | }, 78 | searchkick_text_middle_index: { 79 | type: "custom", 80 | tokenizer: "keyword", 81 | filter: ["lowercase", "asciifolding", "searchkick_ngram"] 82 | }, 83 | searchkick_text_end_index: { 84 | type: "custom", 85 | tokenizer: "keyword", 86 | filter: ["lowercase", "asciifolding", "reverse", "searchkick_edge_ngram", "reverse"] 87 | }, 88 | searchkick_word_start_index: { 89 | type: "custom", 90 | tokenizer: "standard", 91 | filter: ["lowercase", "asciifolding", "searchkick_edge_ngram"] 92 | }, 93 | searchkick_word_middle_index: { 94 | type: "custom", 95 | tokenizer: "standard", 96 | filter: ["lowercase", "asciifolding", "searchkick_ngram"] 97 | }, 98 | searchkick_word_end_index: { 99 | type: "custom", 100 | tokenizer: "standard", 101 | filter: ["lowercase", "asciifolding", "reverse", "searchkick_edge_ngram", "reverse"] 102 | } 103 | }, 104 | filter: { 105 | searchkick_index_shingle: { 106 | type: "shingle", 107 | token_separator: "" 108 | }, 109 | # lucky find http://web.archiveorange.com/archive/v/AAfXfQ17f57FcRINsof7 110 | searchkick_search_shingle: { 111 | type: "shingle", 112 | token_separator: "", 113 | output_unigrams: false, 114 | output_unigrams_if_no_shingles: true 115 | }, 116 | searchkick_suggest_shingle: { 117 | type: "shingle", 118 | max_shingle_size: 5 119 | }, 120 | searchkick_edge_ngram: { 121 | type: "edgeNGram", 122 | min_gram: 1, 123 | max_gram: 50 124 | }, 125 | searchkick_ngram: { 126 | type: "nGram", 127 | min_gram: 1, 128 | max_gram: 50 129 | }, 130 | searchkick_stemmer: { 131 | # use stemmer if language is lowercase, snowball otherwise 132 | type: language == language.to_s.downcase ? "stemmer" : "snowball", 133 | language: language || "English" 134 | } 135 | }, 136 | char_filter: { 137 | # https://www.elastic.co/guide/en/elasticsearch/guide/current/custom-analyzers.html 138 | # &_to_and 139 | ampersand: { 140 | type: "mapping", 141 | mappings: ["&=> and "] 142 | } 143 | } 144 | } 145 | } 146 | 147 | case language 148 | when "chinese" 149 | settings[:analysis][:analyzer].merge!( 150 | default_analyzer => { 151 | type: "ik_smart" 152 | }, 153 | searchkick_search: { 154 | type: "ik_smart" 155 | }, 156 | searchkick_search2: { 157 | type: "ik_max_word" 158 | } 159 | ) 160 | 161 | settings[:analysis][:filter].delete(:searchkick_stemmer) 162 | when "japanese" 163 | settings[:analysis][:analyzer].merge!( 164 | default_analyzer => { 165 | type: "kuromoji" 166 | }, 167 | searchkick_search: { 168 | type: "kuromoji" 169 | }, 170 | searchkick_search2: { 171 | type: "kuromoji" 172 | } 173 | ) 174 | when "korean" 175 | settings[:analysis][:analyzer].merge!( 176 | default_analyzer => { 177 | type: "openkoreantext-analyzer" 178 | }, 179 | searchkick_search: { 180 | type: "openkoreantext-analyzer" 181 | }, 182 | searchkick_search2: { 183 | type: "openkoreantext-analyzer" 184 | } 185 | ) 186 | when "vietnamese" 187 | settings[:analysis][:analyzer].merge!( 188 | default_analyzer => { 189 | type: "vi_analyzer" 190 | }, 191 | searchkick_search: { 192 | type: "vi_analyzer" 193 | }, 194 | searchkick_search2: { 195 | type: "vi_analyzer" 196 | } 197 | ) 198 | when "polish", "ukrainian", "smartcn" 199 | settings[:analysis][:analyzer].merge!( 200 | default_analyzer => { 201 | type: language 202 | }, 203 | searchkick_search: { 204 | type: language 205 | }, 206 | searchkick_search2: { 207 | type: language 208 | } 209 | ) 210 | end 211 | 212 | if Searchkick.env == "test" 213 | settings[:number_of_shards] = 1 214 | settings[:number_of_replicas] = 0 215 | end 216 | 217 | if options[:similarity] 218 | settings[:similarity] = {default: {type: options[:similarity]}} 219 | end 220 | 221 | unless below62 222 | settings[:index] = { 223 | max_ngram_diff: 49, 224 | max_shingle_diff: 4 225 | } 226 | end 227 | 228 | settings.deep_merge!(options[:settings] || {}) 229 | 230 | # synonyms 231 | synonyms = options[:synonyms] || [] 232 | 233 | synonyms = synonyms.call if synonyms.respond_to?(:call) 234 | 235 | if synonyms.any? 236 | settings[:analysis][:filter][:searchkick_synonym] = { 237 | type: "synonym", 238 | # only remove a single space from synonyms so three-word synonyms will fail noisily instead of silently 239 | synonyms: synonyms.select { |s| s.size > 1 }.map { |s| s.is_a?(Array) ? s.map { |s2| s2.sub(/\s+/, "") }.join(",") : s }.map(&:downcase) 240 | } 241 | # choosing a place for the synonym filter when stemming is not easy 242 | # https://groups.google.com/forum/#!topic/elasticsearch/p7qcQlgHdB8 243 | # TODO use a snowball stemmer on synonyms when creating the token filter 244 | 245 | # http://elasticsearch-users.115913.n3.nabble.com/synonym-multi-words-search-td4030811.html 246 | # I find the following approach effective if you are doing multi-word synonyms (synonym phrases): 247 | # - Only apply the synonym expansion at index time 248 | # - Don't have the synonym filter applied search 249 | # - Use directional synonyms where appropriate. You want to make sure that you're not injecting terms that are too general. 250 | settings[:analysis][:analyzer][default_analyzer][:filter].insert(4, "searchkick_synonym") if below60 251 | settings[:analysis][:analyzer][default_analyzer][:filter] << "searchkick_synonym" 252 | 253 | %w(word_start word_middle word_end).each do |type| 254 | settings[:analysis][:analyzer]["searchkick_#{type}_index".to_sym][:filter].insert(2, "searchkick_synonym") 255 | end 256 | end 257 | 258 | if options[:wordnet] 259 | settings[:analysis][:filter][:searchkick_wordnet] = { 260 | type: "synonym", 261 | format: "wordnet", 262 | synonyms_path: Searchkick.wordnet_path 263 | } 264 | 265 | settings[:analysis][:analyzer][default_analyzer][:filter].insert(4, "searchkick_wordnet") 266 | settings[:analysis][:analyzer][default_analyzer][:filter] << "searchkick_wordnet" 267 | 268 | %w(word_start word_middle word_end).each do |type| 269 | settings[:analysis][:analyzer]["searchkick_#{type}_index".to_sym][:filter].insert(2, "searchkick_wordnet") 270 | end 271 | end 272 | 273 | if options[:special_characters] == false 274 | settings[:analysis][:analyzer].each_value do |analyzer_settings| 275 | analyzer_settings[:filter].reject! { |f| f == "asciifolding" } 276 | end 277 | end 278 | 279 | mapping = {} 280 | 281 | # conversions 282 | Array(options[:conversions]).each do |conversions_field| 283 | mapping[conversions_field] = { 284 | type: "nested", 285 | properties: { 286 | query: {type: default_type, analyzer: "searchkick_keyword"}, 287 | count: {type: "integer"} 288 | } 289 | } 290 | end 291 | 292 | mapping_options = Hash[ 293 | [:suggest, :word, :text_start, :text_middle, :text_end, :word_start, :word_middle, :word_end, :highlight, :searchable, :filterable] 294 | .map { |type| [type, (options[type] || []).map(&:to_s)] } 295 | ] 296 | 297 | word = options[:word] != false && (!options[:match] || options[:match] == :word) 298 | 299 | mapping_options[:searchable].delete("_all") 300 | 301 | analyzed_field_options = {type: default_type, index: index_true_value, analyzer: default_analyzer} 302 | 303 | mapping_options.values.flatten.uniq.each do |field| 304 | fields = {} 305 | 306 | if options.key?(:filterable) && !mapping_options[:filterable].include?(field) 307 | fields[field] = {type: default_type, index: index_false_value} 308 | else 309 | fields[field] = keyword_mapping 310 | end 311 | 312 | if !options[:searchable] || mapping_options[:searchable].include?(field) 313 | if word 314 | fields["analyzed"] = analyzed_field_options 315 | 316 | if mapping_options[:highlight].include?(field) 317 | fields["analyzed"][:term_vector] = "with_positions_offsets" 318 | end 319 | end 320 | 321 | mapping_options.except(:highlight, :searchable, :filterable, :word).each do |type, f| 322 | if options[:match] == type || f.include?(field) 323 | fields[type] = {type: default_type, index: index_true_value, analyzer: "searchkick_#{type}_index"} 324 | end 325 | end 326 | end 327 | 328 | mapping[field] = fields[field].merge(fields: fields.except(field)) 329 | end 330 | 331 | (options[:locations] || []).map(&:to_s).each do |field| 332 | mapping[field] = { 333 | type: "geo_point" 334 | } 335 | end 336 | 337 | options[:geo_shape] = options[:geo_shape].product([{}]).to_h if options[:geo_shape].is_a?(Array) 338 | (options[:geo_shape] || {}).each do |field, shape_options| 339 | mapping[field] = shape_options.merge(type: "geo_shape") 340 | end 341 | 342 | if options[:inheritance] 343 | mapping[:type] = keyword_mapping 344 | end 345 | 346 | routing = {} 347 | if options[:routing] 348 | routing = {required: true} 349 | unless options[:routing] == true 350 | routing[:path] = options[:routing].to_s 351 | end 352 | end 353 | 354 | dynamic_fields = { 355 | # analyzed field must be the default field for include_in_all 356 | # http://www.elasticsearch.org/guide/reference/mapping/multi-field-type/ 357 | # however, we can include the not_analyzed field in _all 358 | # and the _all index analyzer will take care of it 359 | "{name}" => keyword_mapping 360 | } 361 | 362 | if below60 && all 363 | dynamic_fields["{name}"][:include_in_all] = !options[:searchable] 364 | end 365 | 366 | if options.key?(:filterable) 367 | dynamic_fields["{name}"] = {type: default_type, index: index_false_value} 368 | end 369 | 370 | unless options[:searchable] 371 | if options[:match] && options[:match] != :word 372 | dynamic_fields[options[:match]] = {type: default_type, index: index_true_value, analyzer: "searchkick_#{options[:match]}_index"} 373 | end 374 | 375 | if word 376 | dynamic_fields["analyzed"] = analyzed_field_options 377 | end 378 | end 379 | 380 | # http://www.elasticsearch.org/guide/reference/mapping/multi-field-type/ 381 | multi_field = dynamic_fields["{name}"].merge(fields: dynamic_fields.except("{name}")) 382 | 383 | mappings = { 384 | index_type => { 385 | properties: mapping, 386 | _routing: routing, 387 | # https://gist.github.com/kimchy/2898285 388 | dynamic_templates: [ 389 | { 390 | string_template: { 391 | match: "*", 392 | match_mapping_type: "string", 393 | mapping: multi_field 394 | } 395 | } 396 | ] 397 | } 398 | } 399 | 400 | if below60 401 | all_enabled = all && (!options[:searchable] || options[:searchable].to_a.map(&:to_s).include?("_all")) 402 | mappings[index_type][:_all] = all_enabled ? analyzed_field_options : {enabled: false} 403 | end 404 | 405 | mappings = mappings.deep_merge(options[:mappings] || {}) 406 | end 407 | 408 | { 409 | settings: settings, 410 | mappings: mappings 411 | } 412 | end 413 | end 414 | end 415 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## 3.0.4 [unreleased] 2 | 3 | - Friendlier error message for bad mapping with partial matches 4 | - Warn when records in search index do not exist in database 5 | 6 | ## 3.0.3 7 | 8 | - Added support for pagination with `body` option 9 | - Added `boost_by_recency` option 10 | - Fixed "Model Search Data" output for `debug` option 11 | - Fixed `reindex_status` error 12 | - Fixed error with optional operators in Ruby regexp 13 | - Fixed deprecation warnings for Elasticsearch 6.2+ 14 | 15 | ## 3.0.2 16 | 17 | - Added support for Korean and Vietnamese 18 | - Fixed `Unsupported argument type: Symbol` for async partial reindex 19 | - Fixed infinite recursion with multi search and misspellings below 20 | - Do not raise an error when `id` is indexed 21 | 22 | ## 3.0.1 23 | 24 | - Added `scope` option for partial reindex 25 | - Added support for Japanese, Polish, and Ukrainian 26 | 27 | ## 3.0.0 28 | 29 | - Added support for Chinese 30 | - No longer requires fields to query for Elasticsearch 6 31 | - Results can be marshaled by default (unless using `highlight` option) 32 | 33 | Breaking changes 34 | 35 | - Removed support for Elasticsearch 2 36 | - Removed support for ActiveRecord < 4.2 and Mongoid < 5 37 | - Types are no longer used 38 | - The `_all` field is disabled by default in Elasticsearch 5 39 | - Conversions are not stemmed by default 40 | - An `ArgumentError` is raised instead of a warning when options are incompatible with the `body` option 41 | - Removed `log` option from `boost_by` 42 | - Removed `Model.enable_search_callbacks`, `Model.disable_search_callbacks`, and `Model.search_callbacks?` 43 | - Removed `reindex_async` method, as `reindex` now defaults to callbacks mode specified on the model 44 | - Removed `async` option from `record.reindex` 45 | - Removed `search_hit` method - use `with_hit` instead 46 | - Removed `each_with_hit` - use `with_hit.each` instead 47 | - Removed `with_details` - use `with_highlights` instead 48 | - Bumped default `limit` to 10,000 49 | 50 | ## 2.5.0 51 | 52 | - Try requests 3 times before raising error 53 | - Better exception when trying to access results for failed multi-search query 54 | - More efficient aggregations with `where` clauses 55 | - Added support for `faraday_middleware-aws-sigv4` 56 | - Added `credentials` option to `aws_credentials` 57 | - Added `modifier` option to `boost_by` 58 | - Added `scope_results` option 59 | - Added `factor` option to `boost_by_distance` 60 | 61 | ## 2.4.0 62 | 63 | - Fixed `similar` for Elasticsearch 6 64 | - Added `inheritance` option 65 | - Added `_type` option 66 | - Fixed `Must specify fields to search` error when searching `*` 67 | 68 | ## 2.3.2 69 | 70 | - Added `_all` and `default_fields` options 71 | - Added global `index_prefix` option 72 | - Added `wait` option to async reindex 73 | - Added `model_includes` option 74 | - Added `missing` option for `boost_by` 75 | - Raise error for `reindex_status` when Redis not configured 76 | - Warn when incompatible options used with `body` option 77 | - Fixed bug where `routing` and `type` options were silently ignored with `body` option 78 | - Fixed `reindex(async: true)` for non-numeric primary keys in Postgres 79 | 80 | ## 2.3.1 81 | 82 | - Added support for `reindex(async: true)` for non-numeric primary keys 83 | - Added `conversions_term` option 84 | - Added support for passing fields to `suggest` option 85 | - Fixed `page_view_entries` for Kaminari 86 | 87 | ## 2.3.0 88 | 89 | - Fixed analyzer on dynamically mapped fields 90 | - Fixed error with `similar` method and `_all` field 91 | - Throw error when fields are needed 92 | - Added `queue_name` option 93 | - No longer require synonyms to be lowercase 94 | 95 | ## 2.2.1 96 | 97 | - Added `avg`, `cardinality`, `max`, `min`, and `sum` aggregations 98 | - Added `load: {dumpable: true}` option 99 | - Added `index_suffix` option 100 | - Accept string for `exclude` option 101 | 102 | ## 2.2.0 103 | 104 | - Fixed bug with text values longer than 256 characters and `_all` field - see [#850](https://github.com/ankane/searchkick/issues/850) 105 | - Fixed issue with `_all` field in `searchable` 106 | - Fixed `exclude` option with `word_start` 107 | 108 | ## 2.1.1 109 | 110 | - Fixed duplicate notifications 111 | - Added support for `connection_pool` 112 | - Added `exclude` option 113 | 114 | ## 2.1.0 115 | 116 | - Background reindexing and queues are officially supported 117 | - Log updates and deletes 118 | 119 | ## 2.0.4 120 | 121 | - Added support for queuing updates [experimental] 122 | - Added `refresh_interval` option to `reindex` 123 | - Prefer `search_index` over `searchkick_index` 124 | 125 | ## 2.0.3 126 | 127 | - Added `async` option to `reindex` [experimental] 128 | - Added `misspellings?` method to results 129 | 130 | ## 2.0.2 131 | 132 | - Added `retain` option to `reindex` 133 | - Added support for attributes in highlight tags 134 | - Fixed potentially silent errors in reindex job 135 | - Improved syntax for `boost_by_distance` 136 | 137 | ## 2.0.1 138 | 139 | - Added `search_hit` and `search_highlights` methods to models 140 | - Improved reindex performance 141 | 142 | ## 2.0.0 143 | 144 | - Added support for `reindex` on associations 145 | 146 | Breaking changes 147 | 148 | - Removed support for Elasticsearch 1 as it reaches [end of life](https://www.elastic.co/support/eol) 149 | - Removed facets, legacy options, and legacy methods 150 | - Invalid options now throw an `ArgumentError` 151 | - The `query` and `json` options have been removed in favor of `body` 152 | - The `include` option has been removed in favor of `includes` 153 | - The `personalize` option has been removed in favor of `boost_where` 154 | - The `partial` option has been removed in favor of `operator` 155 | - Renamed `select_v2` to `select` (legacy `select` no longer available) 156 | - The `_all` field is disabled if `searchable` option is used (for performance) 157 | - The `partial_reindex(:method_name)` method has been replaced with `reindex(:method_name)` 158 | - The `unsearchable` and `only_analyzed` options have been removed in favor of `searchable` and `filterable` 159 | - `load: false` no longer returns an array in Elasticsearch 2 160 | 161 | ## 1.5.1 162 | 163 | - Added `client_options` 164 | - Added `refresh` option to `reindex` method 165 | - Improved syntax for partial reindex 166 | 167 | ## 1.5.0 168 | 169 | - Added support for geo shape indexing and queries 170 | - Added `_and`, `_or`, `_not` to `where` option 171 | 172 | ## 1.4.2 173 | 174 | - Added support for directional synonyms 175 | - Easier AWS setup 176 | - Fixed `total_docs` method for ES 5+ 177 | - Fixed exception on update errors 178 | 179 | ## 1.4.1 180 | 181 | - Added `partial_reindex` method 182 | - Added `debug` option to `search` method 183 | - Added `profile` option 184 | 185 | ## 1.4.0 186 | 187 | - Official support for Elasticsearch 5 188 | - Boost exact matches for partial matching 189 | - Added `searchkick_debug` method 190 | - Added `geo_polygon` filter 191 | 192 | ## 1.3.6 193 | 194 | - Fixed `Job adapter not found` error 195 | 196 | ## 1.3.5 197 | 198 | - Added support for Elasticsearch 5.0 beta 199 | - Added `request_params` option 200 | - Added `filterable` option 201 | 202 | ## 1.3.4 203 | 204 | - Added `resume` option to reindex 205 | - Added search timeout to payload 206 | 207 | ## 1.3.3 208 | 209 | - Fix for namespaced models (broken in 1.3.2) 210 | 211 | ## 1.3.2 212 | 213 | - Added `body_options` option 214 | - Added `date_histogram` aggregation 215 | - Added `indices_boost` option 216 | - Added support for multiple conversions 217 | 218 | ## 1.3.1 219 | 220 | - Fixed error with Ruby 2.0 221 | - Fixed error with indexing large fields 222 | 223 | ## 1.3.0 224 | 225 | - Added support for Elasticsearch 5.0 alpha 226 | - Added support for phrase matches 227 | - Added support for procs for `index_prefix` option 228 | 229 | ## 1.2.1 230 | 231 | - Added `multi_search` method 232 | - Added support for routing for Elasticsearch 2 233 | - Added support for `search_document_id` and `search_document_type` in models 234 | - Fixed error with instrumentation for searching multiple models 235 | - Fixed instrumentation for bulk updates 236 | 237 | ## 1.2.0 238 | 239 | - Fixed deprecation warnings with `alias_method_chain` 240 | - Added `analyzed_only` option for large text fields 241 | - Added `encoder` option to highlight 242 | - Fixed issue in `similar` method with `per_page` option 243 | - Added basic support for multiple models 244 | 245 | ## 1.1.2 246 | 247 | - Added bulk updates with `callbacks` method 248 | - Added `bulk_delete` method 249 | - Added `search_timeout` option 250 | - Fixed bug with new location format for `boost_by_distance` 251 | 252 | ## 1.1.1 253 | 254 | - Added support for `{lat: lat, lon: lon}` as preferred format for locations 255 | 256 | ## 1.1.0 257 | 258 | - Added `below` option to misspellings to improve performance 259 | - Fixed synonyms for `word_*` partial matches 260 | - Added `searchable` option 261 | - Added `similarity` option 262 | - Added `match` option 263 | - Added `word` option 264 | - Added highlighted fields to `load: false` 265 | 266 | ## 1.0.3 267 | 268 | - Added support for Elasticsearch 2.1 269 | 270 | ## 1.0.2 271 | 272 | - Throw `Searchkick::ImportError` for errors when importing records 273 | - Errors now inherit from `Searchkick::Error` 274 | - Added `order` option to aggregations 275 | - Added `mapping` method 276 | 277 | ## 1.0.1 278 | 279 | - Added aggregations method to get raw response 280 | - Use `execute: false` for lazy loading 281 | - Return nil when no aggs 282 | - Added emoji search 283 | 284 | ## 1.0.0 285 | 286 | - Added support for Elasticsearch 2.0 287 | - Added support for aggregations 288 | - Added ability to use misspellings for partial matches 289 | - Added `fragment_size` option for highlight 290 | - Added `took` method to results 291 | 292 | Breaking changes 293 | 294 | - Raise `Searchkick::DangerousOperation` error when calling reindex with scope 295 | - Enabled misspellings by default for partial matches 296 | - Enabled transpositions by default for misspellings 297 | 298 | ## 0.9.1 299 | 300 | - `and` now matches `&` 301 | - Added `transpositions` option to misspellings 302 | - Added `boost_mode` and `log` options to `boost_by` 303 | - Added `prefix_length` option to `misspellings` 304 | - Added ability to set env 305 | 306 | ## 0.9.0 307 | 308 | - Much better performance for where queries if no facets 309 | - Added basic support for regex 310 | - Added support for routing 311 | - Made `Searchkick.disable_callbacks` thread-safe 312 | 313 | ## 0.8.7 314 | 315 | - Fixed Mongoid import 316 | 317 | ## 0.8.6 318 | 319 | - Added support for NoBrainer 320 | - Added `stem_conversions: false` option 321 | - Added support for multiple `boost_where` values on the same field 322 | - Added support for array of values for `boost_where` 323 | - Fixed suggestions with partial match boost 324 | - Fixed redefining existing instance methods in models 325 | 326 | ## 0.8.5 327 | 328 | - Added support for Elasticsearch 1.4 329 | - Added `unsearchable` option 330 | - Added `select: true` option 331 | - Added `body` option 332 | 333 | ## 0.8.4 334 | 335 | - Added `boost_by_distance` 336 | - More flexible highlight options 337 | - Better `env` logic 338 | 339 | ## 0.8.3 340 | 341 | - Added support for ActiveJob 342 | - Added `timeout` setting 343 | - Fixed import with no records 344 | 345 | ## 0.8.2 346 | 347 | - Added `async` to `callbacks` option 348 | - Added `wordnet` option 349 | - Added `edit_distance` option to eventually replace `distance` option 350 | - Catch misspelling of `misspellings` option 351 | - Improved logging 352 | 353 | ## 0.8.1 354 | 355 | - Added `search_method_name` option 356 | - Fixed `order` for array of hashes 357 | - Added support for Mongoid 2 358 | 359 | ## 0.8.0 360 | 361 | - Added support for Elasticsearch 1.2 362 | 363 | ## 0.7.9 364 | 365 | - Added `tokens` method 366 | - Added `json` option 367 | - Added exact matches 368 | - Added `prev_page` for Kaminari pagination 369 | - Added `import` option to reindex 370 | 371 | ## 0.7.8 372 | 373 | - Added `boost_by` and `boost_where` options 374 | - Added ability to boost fields - `name^10` 375 | - Added `select` option for `load: false` 376 | 377 | ## 0.7.7 378 | 379 | - Added support for automatic failover 380 | - Fixed `operator` option (and default) for partial matches 381 | 382 | ## 0.7.6 383 | 384 | - Added `stats` option to facets 385 | - Added `padding` option 386 | 387 | ## 0.7.5 388 | 389 | - Do not throw errors when index becomes out of sync with database 390 | - Added custom exception types 391 | - Fixed `offset` and `offset_value` 392 | 393 | ## 0.7.4 394 | 395 | - Fixed reindex with inheritance 396 | 397 | ## 0.7.3 398 | 399 | - Fixed multi-index searches 400 | - Fixed suggestions for partial matches 401 | - Added `offset` and `length` for improved pagination 402 | 403 | ## 0.7.2 404 | 405 | - Added smart facets 406 | - Added more fields to `load: false` result 407 | - Fixed logging for multi-index searches 408 | - Added `first_page?` and `last_page?` for improved Kaminari support 409 | 410 | ## 0.7.1 411 | 412 | - Fixed huge issue w/ zero-downtime reindexing on 0.90 413 | 414 | ## 0.7.0 415 | 416 | - Added support for Elasticsearch 1.1 417 | - Dropped support for Elasticsearch below 0.90.4 (unfortunate side effect of above) 418 | 419 | ## 0.6.3 420 | 421 | - Removed patron since no support for Windows 422 | - Added error if `searchkick` is called multiple times 423 | 424 | ## 0.6.2 425 | 426 | - Added logging 427 | - Fixed index_name option 428 | - Added ability to use proc as the index name 429 | 430 | ## 0.6.1 431 | 432 | - Fixed huge issue w/ zero-downtime reindexing on 0.90 and elasticsearch-ruby 1.0 433 | - Restore load: false behavior 434 | - Restore total_entries method 435 | 436 | ## 0.6.0 437 | 438 | - Moved to elasticsearch-ruby 439 | - Added support for modifying the query and viewing the response 440 | - Added support for page_entries_info method 441 | 442 | ## 0.5.3 443 | 444 | - Fixed bug w/ word_* queries 445 | 446 | ## 0.5.2 447 | 448 | - Use after_commit hook for ActiveRecord to prevent data inconsistencies 449 | 450 | ## 0.5.1 451 | 452 | - Replaced stop words with common terms query 453 | - Added language option 454 | - Fixed bug with empty array in where clause 455 | - Fixed bug with MongoDB integer _id 456 | - Fixed reindex bug when callbacks disabled 457 | 458 | ## 0.5.0 459 | 460 | - Better control over partial matches 461 | - Added merge_mappings option 462 | - Added batch_size option 463 | - Fixed bug with nil where clauses 464 | 465 | ## 0.4.2 466 | 467 | - Added `should_index?` method to control which records are indexed 468 | - Added ability to temporarily disable callbacks 469 | - Added custom mappings 470 | 471 | ## 0.4.1 472 | 473 | - Fixed issue w/ inheritance mapping 474 | 475 | ## 0.4.0 476 | 477 | - Added support for Mongoid 4 478 | - Added support for multiple locations 479 | 480 | ## 0.3.5 481 | 482 | - Added facet ranges 483 | - Added all operator 484 | 485 | ## 0.3.4 486 | 487 | - Added highlighting 488 | - Added :distance option to misspellings 489 | - Fixed issue w/ BigDecimal serialization 490 | 491 | ## 0.3.3 492 | 493 | - Better error messages 494 | - Added where: {field: nil} queries 495 | 496 | ## 0.3.2 497 | 498 | - Added support for single table inheritance 499 | - Removed Tire::Model::Search 500 | 501 | ## 0.3.1 502 | 503 | - Added index_prefix option 504 | - Fixed ES issue with incorrect facet counts 505 | - Added option to turn off special characters 506 | 507 | ## 0.3.0 508 | 509 | - Fixed reversed coordinates 510 | - Added bounded by a box queries 511 | - Expanded `or` queries 512 | 513 | ## 0.2.8 514 | 515 | - Added option to disable callbacks 516 | - Fixed bug with facets with Elasticsearch 0.90.5 517 | 518 | ## 0.2.7 519 | 520 | - Added limit to facet 521 | - Improved similar items 522 | 523 | ## 0.2.6 524 | 525 | - Added option to disable misspellings 526 | 527 | ## 0.2.5 528 | 529 | - Added geospartial searches 530 | - Create alias before importing document if no alias exists 531 | - Fixed exception when :per_page option is a string 532 | - Check `RAILS_ENV` if `RACK_ENV` is not set 533 | 534 | ## 0.2.4 535 | 536 | - Use `to_hash` instead of `as_json` for default `search_data` method 537 | - Works for Mongoid 1.3 538 | - Use one shard in test environment for consistent scores 539 | 540 | ## 0.2.3 541 | 542 | - Setup Travis 543 | - Clean old indices before reindex 544 | - Search for `*` returns all results 545 | - Fixed pagination 546 | - Added `similar` method 547 | 548 | ## 0.2.2 549 | 550 | - Clean old indices after reindex 551 | - More expansions for fuzzy queries 552 | 553 | ## 0.2.1 554 | 555 | - Added Rails logger 556 | - Only fetch ids when `load: true` 557 | 558 | ## 0.2.0 559 | 560 | - Added autocomplete 561 | - Added “Did you mean” suggestions 562 | - Added personalized searches 563 | 564 | ## 0.1.4 565 | 566 | - Bug fix 567 | 568 | ## 0.1.3 569 | 570 | - Changed edit distance to one for misspellings 571 | - Raise errors when indexing fails 572 | - Fixed pagination 573 | - Fixed :include option 574 | 575 | ## 0.1.2 576 | 577 | - Launch 578 | --------------------------------------------------------------------------------