├── .rubocop ├── .rspec ├── rubocop.gemfile ├── bin ├── console └── setup ├── lib ├── db_text_search │ ├── version.rb │ ├── full_text │ │ ├── mysql_adapter.rb │ │ ├── sqlite_adapter.rb │ │ ├── postgres_adapter.rb │ │ └── abstract_adapter.rb │ ├── case_insensitive │ │ ├── insensitive_column_adapter.rb │ │ ├── abstract_adapter.rb │ │ ├── lower_adapter.rb │ │ └── collate_nocase_adapter.rb │ ├── query_building.rb │ ├── full_text.rb │ └── case_insensitive.rb └── db_text_search.rb ├── shared.gemfile ├── spec ├── gemfiles │ ├── rubocop.gemfile │ ├── rails_5_2.gemfile │ ├── rails_6_1.gemfile │ ├── rails_7_0.gemfile │ ├── rails_main.gemfile │ └── rails_6_0.gemfile ├── db_text_search_spec.rb ├── db_text_search │ ├── full_text_spec.rb │ └── case_insensitive_spec.rb └── spec_helper.rb ├── .gitignore ├── Gemfile ├── .codeclimate.yml ├── .simplecov ├── RELEASE-CHECKLIST.md ├── LICENSE.txt ├── .rubocop.yml ├── db_text_search.gemspec ├── script └── create-db-users ├── CHANGES.md ├── Rakefile ├── CODE_OF_CONDUCT.md ├── .travis.yml └── README.md /.rubocop: -------------------------------------------------------------------------------- 1 | -D 2 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --format documentation 2 | --color 3 | -------------------------------------------------------------------------------- /rubocop.gemfile: -------------------------------------------------------------------------------- 1 | gem 'rubocop', '= 0.49.1' 2 | -------------------------------------------------------------------------------- /bin/console: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | require 'bundler/setup' 3 | require 'db_text_search' 4 | require 'irb' 5 | IRB.start 6 | -------------------------------------------------------------------------------- /bin/setup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | IFS=$'\n\t' 4 | set -vx 5 | 6 | bundle install 7 | script/create-db-users 8 | -------------------------------------------------------------------------------- /lib/db_text_search/version.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module DbTextSearch 4 | # Gem version 5 | VERSION = '1.0.0' 6 | end 7 | -------------------------------------------------------------------------------- /shared.gemfile: -------------------------------------------------------------------------------- 1 | unless ENV['TRAVIS'] 2 | group :test, :development do 3 | gem 'byebug', platform: :mri, require: false 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /spec/gemfiles/rubocop.gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | source 'https://rubygems.org' 4 | 5 | eval_gemfile '../../rubocop.gemfile' 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle/ 2 | /.yardoc 3 | /Gemfile.lock 4 | *.gemfile.lock 5 | /_yardoc/ 6 | /coverage/ 7 | /doc/ 8 | /pkg/ 9 | /spec/reports/ 10 | /tmp/ 11 | /log/ 12 | -------------------------------------------------------------------------------- /spec/db_text_search_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe DbTextSearch do 6 | it 'has a version number' do 7 | expect(DbTextSearch::VERSION).not_to be nil 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | source 'https://rubygems.org' 4 | 5 | gemspec 6 | 7 | gem 'rails', '~> 7.0.0' 8 | 9 | eval_gemfile File.expand_path('shared.gemfile', __dir__) 10 | eval_gemfile File.expand_path('rubocop.gemfile', __dir__) 11 | -------------------------------------------------------------------------------- /.codeclimate.yml: -------------------------------------------------------------------------------- 1 | --- 2 | engines: 3 | duplication: 4 | enabled: true 5 | config: 6 | languages: 7 | - ruby 8 | fixme: 9 | enabled: true 10 | rubocop: 11 | enabled: true 12 | ratings: 13 | paths: 14 | - "**.rb" 15 | exclude_paths: 16 | - script/ 17 | - spec/ 18 | -------------------------------------------------------------------------------- /spec/gemfiles/rails_5_2.gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | gemspec path: '../../' 3 | eval_gemfile '../../shared.gemfile' 4 | gem 'rails', '~> 5.2.0' 5 | 6 | # https://github.com/rails/rails/blob/v5.2.2/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb#L12 7 | gem 'sqlite3', '~> 1.3.6' 8 | -------------------------------------------------------------------------------- /.simplecov: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | SimpleCov.start do 3 | add_filter '/spec/' 4 | add_filter '/lib/db_text_search/case_insensitive/abstract_adapter.rb' 5 | add_filter '/lib/db_text_search/full_text/abstract_adapter.rb' 6 | add_group 'Lib', 'lib/' 7 | formatter SimpleCov::Formatter::HTMLFormatter unless ENV['TRAVIS'] 8 | end 9 | -------------------------------------------------------------------------------- /RELEASE-CHECKLIST.md: -------------------------------------------------------------------------------- 1 | The checklist for releasing a new version of db_text_search. 2 | 3 | Pre-requisites for the releaser: 4 | 5 | * Push access to RubyGems. 6 | 7 | Release checklist: 8 | 9 | - [ ] Update gem version in `version.rb` and `README.md`. 10 | - [ ] Update `CHANGELOG.md`. 11 | - [ ] Wait for the Travis build to come back green. 12 | - [ ] Tag the release and push it to rubygems: 13 | 14 | ```bash 15 | rake release 16 | ``` 17 | - [ ] Copy the release notes from the changelog to [GitHub Releases](https://github.com/thredded/db_text_search/releases). 18 | -------------------------------------------------------------------------------- /spec/gemfiles/rails_6_1.gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | gemspec path: '../../' 3 | eval_gemfile '../../shared.gemfile' 4 | gem 'rails', '~> 6.1.0' 5 | 6 | # https://github.com/rails/rails/blob/v6.1.4/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb#L13 7 | gem 'sqlite3', '~> 1.4' 8 | 9 | # https://github.com/rails/rails/blob/v6.1.4/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb#L4 10 | gem 'pg' 11 | 12 | # https://github.com/rails/rails/blob/v6.1.4/activerecord/lib/active_record/connection_adapters/mysql2_adapter.rb#L6 13 | gem "mysql2", "~> 0.5" 14 | -------------------------------------------------------------------------------- /spec/gemfiles/rails_7_0.gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | gemspec path: '../../' 3 | eval_gemfile '../../shared.gemfile' 4 | gem 'rails', '~> 7.0.0' 5 | 6 | # https://github.com/rails/rails/blob/v7.0.2/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb#L13 7 | gem 'sqlite3', '~> 1.4' 8 | 9 | # https://github.com/rails/rails/blob/v7.0.2/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb#L4 10 | gem 'pg' 11 | 12 | # https://github.com/rails/rails/blob/v7.0.2/activerecord/lib/active_record/connection_adapters/mysql2_adapter.rb#L6 13 | gem "mysql2", "~> 0.5" 14 | -------------------------------------------------------------------------------- /spec/gemfiles/rails_main.gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | gemspec path: '../../' 3 | eval_gemfile '../../shared.gemfile' 4 | gem 'rails', github: "rails/rails", branch: "main" 5 | 6 | # https://github.com/rails/rails/blob/main/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb#L13 7 | gem 'sqlite3', '~> 1.4' 8 | 9 | # https://github.com/rails/rails/blob/main/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb#L4 10 | gem 'pg' 11 | 12 | # https://github.com/rails/rails/blob/main/activerecord/lib/active_record/connection_adapters/mysql2_adapter.rb#L6 13 | gem "mysql2", "~> 0.5" 14 | -------------------------------------------------------------------------------- /spec/gemfiles/rails_6_0.gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | gemspec path: '../../' 3 | eval_gemfile '../../shared.gemfile' 4 | gem 'rails', '~> 6.0.0' 5 | 6 | # https://github.com/rails/rails/blob/v6.0.4/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb#L13 7 | gem 'sqlite3', '~> 1.4' 8 | 9 | # https://github.com/rails/rails/blob/v6.0.4/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb#L4 10 | gem 'pg', '>= 0.18', '< 2.0' 11 | 12 | # https://github.com/rails/rails/blob/v6.0.4/activerecord/lib/active_record/connection_adapters/mysql2_adapter.rb#L6 13 | gem 'mysql2', '>= 0.4.4' 14 | 15 | -------------------------------------------------------------------------------- /lib/db_text_search/full_text/mysql_adapter.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'db_text_search/full_text/abstract_adapter' 4 | module DbTextSearch 5 | class FullText 6 | # Provides basic FTS support for MySQL. 7 | # 8 | # Runs a `MATCH AGAINST` query against a `FULLTEXT` index. 9 | # 10 | # @note MySQL v5.6.4+ is required. 11 | # @api private 12 | class MysqlAdapter < AbstractAdapter 13 | # (see AbstractAdapter#search) 14 | def search(terms, pg_ts_config:) 15 | @scope.where("MATCH (#{quoted_scope_column}) AGAINST (?)", terms.uniq.join(' ')) 16 | end 17 | 18 | # (see AbstractAdapter.add_index) 19 | def self.add_index(connection, table_name, column_name, name:, pg_ts_config:) 20 | connection.add_index table_name, column_name, name: name, type: :fulltext 21 | end 22 | end 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /lib/db_text_search/full_text/sqlite_adapter.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'db_text_search/full_text/abstract_adapter' 4 | module DbTextSearch 5 | class FullText 6 | # Provides very basic FTS support for SQLite. 7 | # 8 | # Runs a `LIKE %term%` query for each term, joined with `AND`. 9 | # Cannot use an index. 10 | # 11 | # @note .add_index is a no-op. 12 | # @api private 13 | class SqliteAdapter < AbstractAdapter 14 | # (see AbstractAdapter#search) 15 | def search(terms, pg_ts_config:) 16 | quoted_col = quoted_scope_column 17 | terms.map(&:downcase).uniq.inject(@scope) do |scope, term| 18 | scope.where("#{quoted_col} COLLATE NOCASE LIKE ?", "%#{sanitize_sql_like term}%") 19 | end 20 | end 21 | 22 | # A no-op, as we just use LIKE for sqlite. 23 | def self.add_index(_connection, _table_name, _column_name, name:, pg_ts_config:); end 24 | end 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /lib/db_text_search/case_insensitive/insensitive_column_adapter.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'db_text_search/case_insensitive/abstract_adapter' 4 | module DbTextSearch 5 | class CaseInsensitive 6 | # Provides case-insensitive string-in-set querying for case-insensitive columns. 7 | # @api private 8 | class InsensitiveColumnAdapter < AbstractAdapter 9 | # (see AbstractAdapter#in) 10 | def in(values) 11 | @scope.where(@column => values) 12 | end 13 | 14 | # (see AbstractAdapter#prefix) 15 | def prefix(query) 16 | @scope.where "#{quoted_scope_column} LIKE ?", "#{sanitize_sql_like(query)}%" 17 | end 18 | 19 | # (see AbstractAdapter#column_for_order) 20 | def column_for_order(asc_or_desc) 21 | Arel.sql("#{quoted_scope_column} #{asc_or_desc}") 22 | end 23 | 24 | # (see AbstractAdapter.add_index) 25 | def self.add_index(connection, table_name, column_name, options = {}) 26 | connection.add_index table_name, column_name, **options 27 | end 28 | end 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /lib/db_text_search/full_text/postgres_adapter.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'db_text_search/full_text/abstract_adapter' 4 | module DbTextSearch 5 | class FullText 6 | # Provides basic FTS support for PostgreSQL. 7 | # 8 | # Runs a `@@ plainto_tsquery` query against a `gist(to_tsvector(...))` index. 9 | # 10 | # @see DbTextSearch::FullText::DEFAULT_PG_TS_CONFIG 11 | # @api private 12 | class PostgresAdapter < AbstractAdapter 13 | # (see AbstractAdapter#search) 14 | def search(terms, pg_ts_config:) 15 | @scope.where("to_tsvector(#{pg_ts_config}, #{quoted_scope_column}) @@ plainto_tsquery(#{pg_ts_config}, ?)", 16 | terms.uniq.join(' ')) 17 | end 18 | 19 | # (see AbstractAdapter.add_index) 20 | def self.add_index(connection, table_name, column_name, name:, pg_ts_config:) 21 | expression = "USING gist(to_tsvector(#{pg_ts_config}, #{connection.quote_column_name column_name}))" 22 | connection.exec_query quoted_create_index(connection, table_name, name: name, expression: expression) 23 | end 24 | end 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Gleb Mazovetskiy 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /lib/db_text_search/full_text/abstract_adapter.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module DbTextSearch 4 | class FullText 5 | # A base class for FullText adapters. 6 | # @abstract 7 | # @api private 8 | class AbstractAdapter 9 | include ::DbTextSearch::QueryBuilding 10 | 11 | # @param scope [ActiveRecord::Relation, Class] 12 | # @param column [Symbol] name 13 | def initialize(scope, column) 14 | @scope = scope 15 | @column = column 16 | end 17 | 18 | # @param terms [Array] 19 | # @param pg_ts_config [String] a pg text search config 20 | # @return [ActiveRecord::Relation] 21 | # @abstract 22 | def search(terms, pg_ts_config:) 23 | fail 'abstract' 24 | end 25 | 26 | # Add an index for full text search. 27 | # 28 | # @param connection [ActiveRecord::ConnectionAdapters::AbstractAdapter] 29 | # @param table_name [String, Symbol] 30 | # @param column_name [String, Symbol] 31 | # @param name [String, Symbol] index name 32 | # @param pg_ts_config [String] for Postgres, the TS config to use; ignored for non-postgres. 33 | # @abstract 34 | def self.add_index(connection, table_name, column_name, name:, pg_ts_config:) 35 | fail 'abstract' 36 | end 37 | end 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | AllCops: 2 | TargetRubyVersion: 2.1 3 | Lint/HandleExceptions: 4 | Exclude: 5 | - 'spec/spec_helper.rb' 6 | Lint/UnusedMethodArgument: 7 | AllowUnusedKeywordArguments: true 8 | Exclude: 9 | - 'lib/db_text_search/case_insensitive/abstract_adapter.rb' 10 | - 'lib/db_text_search/full_text/abstract_adapter.rb' 11 | Metrics/CyclomaticComplexity: 12 | Max: 7 13 | Metrics/MethodLength: 14 | Max: 16 15 | Metrics/ModuleLength: 16 | Max: 130 17 | Exclude: 18 | - 'spec/**/*.rb' 19 | Metrics/BlockLength: 20 | Exclude: 21 | - 'spec/**/*.rb' 22 | Layout/AlignParameters: 23 | Enabled: false 24 | Layout/CaseIndentation: 25 | IndentOneStep: true 26 | Layout/IndentHash: 27 | IndentationWidth: 4 28 | Layout/SpaceInsideHashLiteralBraces: 29 | Enabled: false 30 | Layout/FirstParameterIndentation: 31 | IndentationWidth: 4 32 | Layout/MultilineOperationIndentation: 33 | EnforcedStyle: indented 34 | IndentationWidth: 4 35 | Layout/MultilineMethodCallIndentation: 36 | EnforcedStyle: indented 37 | IndentationWidth: 4 38 | Layout/SpaceAroundOperators: 39 | Enabled: false 40 | Style/BlockDelimiters: 41 | Enabled: false 42 | Style/Lambda: 43 | Enabled: false 44 | Style/MutableConstant: 45 | Enabled: false 46 | Style/SignalException: 47 | EnforcedStyle: semantic 48 | Style/UnneededPercentQ: 49 | Enabled: false 50 | Metrics/LineLength: 51 | Max: 120 52 | -------------------------------------------------------------------------------- /lib/db_text_search/query_building.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module DbTextSearch 4 | # Common methods for building SQL that use @scope and @column instance variables. 5 | # @api private 6 | module QueryBuilding 7 | def self.included(base) # :nodoc: 8 | base.extend ClassMethods 9 | end 10 | 11 | # @return [String] SQL-quoted string suitable for use in a LIKE statement, with % and _ escaped. 12 | def sanitize_sql_like(string, escape_character = '\\') 13 | pattern = Regexp.union(escape_character, '%', '_') 14 | string.gsub(pattern) { |x| [escape_character, x].join } 15 | end 16 | 17 | protected 18 | 19 | # @return [String] SQL-quoted scope table name. 20 | def quoted_scope_table 21 | @scope.connection.quote_table_name(@scope.table_name) 22 | end 23 | 24 | # @return [String] SQL-quoted column (without the table name). 25 | def quoted_column 26 | @scope.connection.quote_column_name(@column) 27 | end 28 | 29 | # @return [String] SQL-quoted column fully-qualified with the scope table name. 30 | def quoted_scope_column 31 | "#{quoted_scope_table}.#{quoted_column}" 32 | end 33 | 34 | # Common methods for building SQL 35 | # @api private 36 | module ClassMethods 37 | protected 38 | 39 | # @return [String] a CREATE INDEX statement 40 | def quoted_create_index(connection, table_name, name:, expression:, unique: false) 41 | "CREATE #{'UNIQUE ' if unique}INDEX #{name} ON #{connection.quote_table_name(table_name)} #{expression}" 42 | end 43 | end 44 | end 45 | end 46 | -------------------------------------------------------------------------------- /lib/db_text_search.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'active_record' 4 | require 'active_support/core_ext/hash/keys' 5 | 6 | require 'db_text_search/version' 7 | require 'db_text_search/case_insensitive' 8 | require 'db_text_search/full_text' 9 | 10 | # DbTextSearch provides a unified interface on top of ActiveRecord for SQLite, MySQL, and PostgreSQL to do: 11 | # * Case-insensitive string-in-set querying, and CI index creation. 12 | # * Basic full-text search for a list of terms, and FTS index creation. 13 | # @see DbTextSearch::CaseInsensitive 14 | # @see DbTextSearch::FullText 15 | module DbTextSearch 16 | # Call the appropriate proc based on the adapter name. 17 | # @param connection [ActiveRecord::ConnectionAdapters::AbstractAdapter] 18 | # @param mysql [Proc] 19 | # @param postgres [Proc] 20 | # @param sqlite [Proc] 21 | # @return the called proc return value. 22 | # @api private 23 | def self.match_adapter(connection, mysql:, postgres:, sqlite:) 24 | case connection.adapter_name 25 | when /mysql/i 26 | mysql.call 27 | when /postg/i # match all postgres and postgis adapters 28 | postgres.call 29 | when /sqlite/i 30 | sqlite.call 31 | else 32 | unsupported_adapter! connection 33 | end 34 | end 35 | 36 | # Raises an ArgumentError with "Unsupported adapter #{connection.adapter_name}" 37 | # @param connection [ActiveRecord::ConnectionAdapters::AbstractAdapter] 38 | # @api private 39 | def self.unsupported_adapter!(connection) 40 | fail ArgumentError, "Unsupported adapter #{connection.adapter_name}" 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /db_text_search.gemspec: -------------------------------------------------------------------------------- 1 | lib = File.expand_path('../lib', __FILE__) 2 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 3 | require 'db_text_search/version' 4 | 5 | Gem::Specification.new do |s| 6 | s.name = 'db_text_search' 7 | s.version = DbTextSearch::VERSION 8 | s.authors = ['Gleb Mazovetskiy'] 9 | s.email = ['glex.spb@gmail.com'] 10 | 11 | s.summary = 'A unified interface on top of ActiveRecord for SQLite, MySQL, and PostgreSQL'\ 12 | 'for case-insensitive string search and basic full-text search.' 13 | s.description = 'Different relational databases treat text search very differently. DbTextSearch provides '\ 14 | 'a unified interface on top of ActiveRecord for SQLite, MySQL, and PostgreSQL to do '\ 15 | 'case-insensitive string-in-set querying and CI index creation, and '\ 16 | 'basic full-text search for a list of terms, and FTS index creation.' 17 | s.homepage = 'https://github.com/thredded/db_text_search' 18 | s.license = 'MIT' 19 | 20 | s.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec|bin|script)/|^\.|Rakefile|Gemfile}) } 21 | 22 | s.require_paths = ['lib'] 23 | s.required_ruby_version = '>= 2.1', '< 4.0' 24 | 25 | s.add_dependency 'activerecord', '>= 5.2.0' 26 | 27 | s.add_development_dependency 'mysql2', '>= 0.3.20' 28 | s.add_development_dependency 'pg', '>= 0.18.4' 29 | s.add_development_dependency 'sqlite3', '>= 1.3.11' 30 | 31 | s.add_development_dependency 'rake', '~> 13.0' 32 | s.add_development_dependency 'rspec', '~> 3.4' 33 | s.add_development_dependency 'simplecov' 34 | end 35 | -------------------------------------------------------------------------------- /script/create-db-users: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | GREEN='\033[0;32m' 4 | RESET_COLOR='\033[0m' 5 | 6 | if [ -n "$1" ]; then cat <<'HELP'; exit; fi 7 | Usage: script/create-db-users 8 | Create the db_text_search database users for all the supported databases. 9 | If the `DB` environment variable is set, do the above only for that database. 10 | HELP 11 | 12 | USER='db_text_search' 13 | PASS='db_text_search' 14 | 15 | set -e 16 | log() { if [ -t 1 ]; then echo -e >&2 "${GREEN}create-db-users: $@${RESET_COLOR}"; else echo >&2 "$@"; fi } 17 | 18 | create_mysql_user() { 19 | if mysql -s -u"$USER" -p"$PASS" -e '' 2>/dev/null; then return; fi 20 | log "Creating MySQL '$USER' user. MySQL root password required." 21 | if [ -n "$TRAVIS" ]; then 22 | mysql_flags='' 23 | fi 24 | mysql --verbose -uroot $mysql_flags </dev/null; then 36 | log "sudo required:" 37 | cmd="sudo -u ${PG_DAEMON_USER:-postgres} psql postgres" 38 | fi 39 | $cmd --echo-all <] 13 | # @param column [Symbol] name 14 | def initialize(scope, column) 15 | @scope = scope 16 | @column = column 17 | end 18 | 19 | # @param values [Array] 20 | # @return [ActiveRecord::Relation] 21 | # @abstract 22 | def in(values) 23 | fail 'abstract' 24 | end 25 | 26 | # @param query [String] 27 | # @return [ActiveRecord::Relation] 28 | # @abstract 29 | def prefix(query) 30 | fail 'abstract' 31 | end 32 | 33 | # @param asc_or_desc [Symbol] 34 | # @return [Arel::Collectors::SQLString] 35 | # @abstract 36 | def column_for_order(asc_or_desc) 37 | fail 'abstract' 38 | end 39 | 40 | # Add an index for case-insensitive string search. 41 | # 42 | # @param connection [ActiveRecord::ConnectionAdapters::AbstractAdapter] 43 | # @param table_name [String, Symbol] 44 | # @param column_name [String, Symbol] 45 | # @param options [Hash] passed down to ActiveRecord::ConnectionAdapters::SchemaStatements#add_index. 46 | # @option options name [String] index name 47 | # @option options unique [Boolean] default: false 48 | # @abstract 49 | def self.add_index(connection, table_name, column_name, options = {}) 50 | fail 'abstract' 51 | end 52 | end 53 | end 54 | end 55 | -------------------------------------------------------------------------------- /lib/db_text_search/case_insensitive/lower_adapter.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'db_text_search/case_insensitive/abstract_adapter' 4 | module DbTextSearch 5 | class CaseInsensitive 6 | # Provides case-insensitive string-in-set querying by applying the database LOWER function. 7 | # @api private 8 | class LowerAdapter < AbstractAdapter 9 | # (see AbstractAdapter#in) 10 | def in(values) 11 | conn = @scope.connection 12 | @scope.where "LOWER(#{quoted_scope_column}) IN (#{values.map { |v| "LOWER(#{conn.quote(v)})" }.join(', ')})" 13 | end 14 | 15 | # (see AbstractAdapter#prefix) 16 | def prefix(query) 17 | @scope.where "LOWER(#{quoted_scope_column}) LIKE LOWER(?)", "#{sanitize_sql_like(query)}%" 18 | end 19 | 20 | # (see AbstractAdapter#column_for_order) 21 | def column_for_order(asc_or_desc) 22 | Arel.sql("LOWER(#{quoted_scope_column}) #{asc_or_desc}") 23 | end 24 | 25 | # (see AbstractAdapter.add_index) 26 | def self.add_index(connection, table_name, column_name, options = {}) 27 | unsupported = -> { DbTextSearch.unsupported_adapter! connection } 28 | DbTextSearch.match_adapter( 29 | connection, 30 | # TODO: Switch to native Rails support once it lands. 31 | # https://github.com/rails/rails/pull/18499 32 | postgres: -> { 33 | options = options.dup 34 | options[:name] ||= "#{table_name}_#{column_name}_lower" 35 | options[:expression] = "(LOWER(#{connection.quote_column_name(column_name)}) text_pattern_ops)" 36 | connection.exec_query(quoted_create_index(connection, table_name, **options)) 37 | }, 38 | mysql: unsupported, 39 | sqlite: unsupported 40 | ) 41 | end 42 | end 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /lib/db_text_search/case_insensitive/collate_nocase_adapter.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'db_text_search/case_insensitive/abstract_adapter' 4 | module DbTextSearch 5 | class CaseInsensitive 6 | # Provides case-insensitive string-in-set querying via COLLATE NOCASE. 7 | # @api private 8 | class CollateNocaseAdapter < AbstractAdapter 9 | # (see AbstractAdapter#in) 10 | def in(values) 11 | conn = @scope.connection 12 | @scope.where "#{quoted_scope_column} COLLATE NOCASE IN (#{values.map { |v| conn.quote(v.to_s) }.join(', ')})" 13 | end 14 | 15 | # (see AbstractAdapter#prefix) 16 | def prefix(query) 17 | escape = '\\' 18 | escaped_query = "#{sanitize_sql_like(query, escape)}%" 19 | # assuming case_sensitive_prefix mode to be disabled, prefix it is by default. 20 | # this is to avoid adding COLLATE NOCASE here, which prevents index use in SQLite LIKE. 21 | @scope.where "#{quoted_scope_column} LIKE ?#{" ESCAPE '#{escape}'" if escaped_query.include?(escape)}", 22 | escaped_query 23 | end 24 | 25 | # (see AbstractAdapter#column_for_order) 26 | def column_for_order(asc_or_desc) 27 | Arel.sql("#{quoted_scope_column} COLLATE NOCASE #{asc_or_desc}") 28 | end 29 | 30 | # (see AbstractAdapter.add_index) 31 | def self.add_index(connection, table_name, column_name, options = {}) 32 | # TODO: Switch to the native Rails solution once it's landed, as the current one requires SQL dump format. 33 | # https://github.com/rails/rails/pull/18499 34 | options = options.dup 35 | options[:name] ||= "#{column_name}_nocase" 36 | options[:expression] = "(#{connection.quote_column_name(column_name)} COLLATE NOCASE)" 37 | connection.exec_query quoted_create_index(connection, table_name, **options) 38 | end 39 | end 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /spec/db_text_search/full_text_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | module DbTextSearch 6 | RSpec.describe FullText do 7 | let!(:post_midsummer) { Post.create!(content: 'Love looks not with the eyes, but with the mind;') } 8 | let!(:post_richard) { Post.create!(content: 'An honest tale speeds best, being plainly told.') } 9 | let!(:post_well) { Post.create!(content: 'Love all, trust a few, do wrong to none.') } 10 | 11 | around { |ex| force_index { ex.call } } 12 | it '#find(terms) with index' do 13 | index_name = :index_posts_content_fts 14 | FullText.add_index(Post.connection, :posts, :content, name: index_name) 15 | finder = FullText.new(Post, :content) 16 | expect(finder.search('love').to_a).to eq [post_midsummer, post_well] 17 | expect(finder.search(%w[honest plainly]).to_a).to eq [post_richard] 18 | expect(finder.search(%w[trust]).to_a).to eq [post_well] 19 | expect(finder.search('Shakespeare').to_a).to be_empty 20 | unless Post.connection.adapter_name =~ /sqlite/i 21 | expect(finder.search('love')).to use_index(index_name) 22 | end 23 | end 24 | 25 | describe '.add_index' do 26 | it 'fails with ArgumentError on an unknown adapter' do 27 | mock_connection = Struct.new(:adapter_name).new('AnInvalidAdapter') 28 | expect { 29 | FullText.add_index mock_connection, :posts, :content 30 | }.to raise_error(ArgumentError, 'Unsupported adapter AnInvalidAdapter') 31 | end 32 | end 33 | 34 | class Post < ActiveRecord::Base 35 | end 36 | 37 | before :all do 38 | ActiveRecord::Schema.define do 39 | self.verbose = false 40 | create_table :posts do |t| 41 | t.text :content, null: false 42 | end 43 | end 44 | ActiveRecord::Base.connection.schema_cache.clear! 45 | end 46 | 47 | after :all do 48 | ActiveRecord::Migration.drop_table :posts 49 | ActiveRecord::Base.connection.schema_cache.clear! 50 | end 51 | end 52 | end 53 | -------------------------------------------------------------------------------- /lib/db_text_search/full_text.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'db_text_search/full_text/postgres_adapter' 4 | require 'db_text_search/full_text/mysql_adapter' 5 | require 'db_text_search/full_text/sqlite_adapter' 6 | 7 | module DbTextSearch 8 | # Provides basic full-text search for a list of terms, and FTS index creation. 9 | class FullText 10 | # The default Postgres text search config. 11 | DEFAULT_PG_TS_CONFIG = %q('english') 12 | 13 | # @param scope [ActiveRecord::Relation, Class] 14 | # @param column [Symbol] name 15 | def initialize(scope, column) 16 | @adapter = self.class.adapter_class(scope.connection, scope.table_name, column).new(scope, column) 17 | @scope = scope 18 | end 19 | 20 | # @param term_or_terms [String, Array] 21 | # @param pg_ts_config [String] for Postgres, the TS config to use; ignored for non-postgres. 22 | # @return [ActiveRecord::Relation] 23 | def search(term_or_terms, pg_ts_config: DEFAULT_PG_TS_CONFIG) 24 | values = Array(term_or_terms) 25 | return @scope.none if values.empty? 26 | @adapter.search(values, pg_ts_config: pg_ts_config) 27 | end 28 | 29 | # Add an index for full text search. 30 | # 31 | # @param connection [ActiveRecord::ConnectionAdapters::AbstractAdapter] 32 | # @param table_name [String, Symbol] 33 | # @param column_name [String, Symbol] 34 | # @param name [String, Symbol] index name 35 | # @param pg_ts_config [String] for Postgres, the TS config to use; ignored for non-postgres. 36 | def self.add_index(connection, table_name, column_name, name: "#{table_name}_#{column_name}_fts", 37 | pg_ts_config: DEFAULT_PG_TS_CONFIG) 38 | adapter_class(connection, table_name, column_name) 39 | .add_index(connection, table_name, column_name, name: name, pg_ts_config: pg_ts_config) 40 | end 41 | 42 | # @param connection [ActiveRecord::ConnectionAdapters::AbstractAdapter] 43 | # @param _table_name [String, Symbol] 44 | # @param _column_name [String, Symbol] 45 | # @return [Class] 46 | # @api private 47 | def self.adapter_class(connection, _table_name, _column_name) 48 | DbTextSearch.match_adapter( 49 | connection, 50 | mysql: -> { MysqlAdapter }, 51 | postgres: -> { PostgresAdapter }, 52 | sqlite: -> { SqliteAdapter } 53 | ) 54 | end 55 | end 56 | end 57 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'bundler/gem_tasks' 2 | 3 | require 'rspec/core/rake_task' 4 | 5 | RSpec::Core::RakeTask.new(:spec) 6 | task default: :spec 7 | 8 | # Common methods for the test_all_dbs, test_all_gemfiles, and test_all Rake tasks. 9 | module TestTasks 10 | module_function 11 | 12 | def run_all(envs, cmd = 'bundle install --quiet && bundle exec rspec', success_message:) 13 | statuses = envs.map { |env| run(env, cmd) } 14 | failed = statuses.reject(&:first).map(&:last) 15 | if failed.empty? 16 | $stderr.puts success_message 17 | else 18 | $stderr.puts "❌ FAILING (#{failed.size}):\n#{failed.map { |env| to_bash_cmd_with_env(cmd, env) } * "\n"}" 19 | exit 1 20 | end 21 | end 22 | 23 | def run(env, cmd) 24 | require 'pty' 25 | require 'English' 26 | Bundler.with_clean_env do 27 | $stderr.puts to_bash_cmd_with_env(cmd, env) 28 | PTY.spawn(env, cmd) do |r, _w, pid| 29 | begin 30 | r.each_line { |l| puts l } 31 | rescue Errno::EIO 32 | # Errno:EIO error means that the process has finished giving output. 33 | next 34 | ensure 35 | ::Process.wait pid 36 | end 37 | end 38 | [$CHILD_STATUS && $CHILD_STATUS.exitstatus.zero?, env] 39 | end 40 | end 41 | 42 | def gemfiles 43 | Dir.glob('./spec/gemfiles/*.gemfile').sort.reject { |path| path.include?('rubocop.gemfile') } 44 | end 45 | 46 | def dbs 47 | %w[sqlite3 mysql2 postgresql] 48 | end 49 | 50 | def to_bash_cmd_with_env(cmd, env) 51 | "(export #{env.map { |k, v| "#{k}=#{v}" }.join(' ')}; #{cmd})" 52 | end 53 | end 54 | 55 | desc 'Test all Gemfiles from spec/*.gemfile' 56 | task :test_all_gemfiles do 57 | envs = TestTasks.gemfiles.map { |gemfile| {'BUNDLE_GEMFILE' => gemfile} } 58 | TestTasks.run_all envs, success_message: "✓ Tests pass with all #{envs.size} gemfiles" 59 | end 60 | 61 | desc 'Test all supported databases' 62 | task :test_all_dbs do 63 | envs = TestTasks.dbs.map { |db| {'DB' => db} } 64 | TestTasks.run_all envs, 'bundle exec rspec', success_message: "✓ Tests pass with all #{envs.size} databases" 65 | end 66 | 67 | desc 'Test all databases x gemfiles' 68 | task :test_all do 69 | dbs = TestTasks.dbs 70 | gemfiles = TestTasks.gemfiles 71 | TestTasks.run_all dbs.flat_map { |db| gemfiles.map { |gemfile| {'DB' => db, 'BUNDLE_GEMFILE' => gemfile} } }, 72 | success_message: "✓ Tests pass with all #{dbs.size} databases x #{gemfiles.size} gemfiles" 73 | end 74 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Code of Conduct 2 | 3 | As contributors and maintainers of this project, and in the interest of 4 | fostering an open and welcoming community, we pledge to respect all people who 5 | contribute through reporting issues, posting feature requests, updating 6 | documentation, submitting pull requests or patches, and other activities. 7 | 8 | We are committed to making participation in this project a harassment-free 9 | experience for everyone, regardless of level of experience, gender, gender 10 | identity and expression, sexual orientation, disability, personal appearance, 11 | body size, race, ethnicity, age, religion, or nationality. 12 | 13 | Examples of unacceptable behavior by participants include: 14 | 15 | * The use of sexualized language or imagery 16 | * Personal attacks 17 | * Trolling or insulting/derogatory comments 18 | * Public or private harassment 19 | * Publishing other's private information, such as physical or electronic 20 | addresses, without explicit permission 21 | * Other unethical or unprofessional conduct 22 | 23 | Project maintainers have the right and responsibility to remove, edit, or 24 | reject comments, commits, code, wiki edits, issues, and other contributions 25 | that are not aligned to this Code of Conduct, or to ban temporarily or 26 | permanently any contributor for other behaviors that they deem inappropriate, 27 | threatening, offensive, or harmful. 28 | 29 | By adopting this Code of Conduct, project maintainers commit themselves to 30 | fairly and consistently applying these principles to every aspect of managing 31 | this project. Project maintainers who do not follow or enforce the Code of 32 | Conduct may be permanently removed from the project team. 33 | 34 | This code of conduct applies both within project spaces and in public spaces 35 | when an individual is representing the project or its community. 36 | 37 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 38 | reported by contacting a project maintainer at glex.spb@gmail.com. All 39 | complaints will be reviewed and investigated and will result in a response that 40 | is deemed necessary and appropriate to the circumstances. Maintainers are 41 | obligated to maintain confidentiality with regard to the reporter of an 42 | incident. 43 | 44 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 45 | version 1.3.0, available at 46 | [http://contributor-covenant.org/version/1/3/0/][version] 47 | 48 | [homepage]: http://contributor-covenant.org 49 | [version]: http://contributor-covenant.org/version/1/3/0/ -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | os: linux 3 | dist: bionic 4 | sudo: false 5 | 6 | before_install: 7 | - bundle config set --local path ../../vendor/bundle without debug 8 | 9 | before_script: 10 | - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter 11 | - chmod +x ./cc-test-reporter 12 | - ./cc-test-reporter before-build 13 | 14 | script: 15 | - bundle exec rspec --format d 16 | 17 | after_script: 18 | - ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT 19 | 20 | matrix: 21 | include: 22 | - name: Rubocop 23 | gemfile: spec/gemfiles/rubocop.gemfile 24 | rvm: 2.7 25 | script: bundle exec rubocop 26 | 27 | - gemfile: spec/gemfiles/rails_5_2.gemfile 28 | rvm: 2.6 29 | env: DB=sqlite3 30 | - gemfile: spec/gemfiles/rails_5_2.gemfile 31 | rvm: 2.6 32 | env: DB=mysql2 DB_USERNAME=root DB_PASSWORD="" 33 | services: mysql 34 | - gemfile: spec/gemfiles/rails_5_2.gemfile 35 | rvm: 2.6 36 | env: DB=postgresql DB_USERNAME=postgres DB_PASSWORD="" 37 | services: postgresql 38 | 39 | - gemfile: spec/gemfiles/rails_6_0.gemfile 40 | rvm: 2.7 41 | env: DB=sqlite3 42 | - gemfile: spec/gemfiles/rails_6_0.gemfile 43 | rvm: 2.7 44 | env: DB=mysql2 DB_USERNAME=root DB_PASSWORD="" 45 | services: mysql 46 | - gemfile: spec/gemfiles/rails_6_0.gemfile 47 | rvm: 2.7 48 | env: DB=postgresql DB_USERNAME=postgres DB_PASSWORD="" 49 | services: postgresql 50 | 51 | - gemfile: spec/gemfiles/rails_6_1.gemfile 52 | rvm: 3.0 53 | env: DB=sqlite3 54 | - gemfile: spec/gemfiles/rails_6_1.gemfile 55 | rvm: 3.0 56 | env: DB=mysql2 DB_USERNAME=root DB_PASSWORD="" 57 | services: mysql 58 | - gemfile: spec/gemfiles/rails_6_1.gemfile 59 | rvm: 3.0 60 | env: DB=postgresql DB_USERNAME=postgres DB_PASSWORD="" 61 | services: postgresql 62 | 63 | - gemfile: spec/gemfiles/rails_7_0.gemfile 64 | rvm: 3.0 65 | env: DB=sqlite3 66 | - gemfile: spec/gemfiles/rails_7_0.gemfile 67 | rvm: 3.0 68 | env: DB=mysql2 DB_USERNAME=root DB_PASSWORD="" 69 | services: mysql 70 | - gemfile: spec/gemfiles/rails_7_0.gemfile 71 | rvm: 3.0 72 | env: DB=postgresql DB_USERNAME=postgres DB_PASSWORD="" 73 | services: postgresql 74 | 75 | - gemfile: spec/gemfiles/rails_main.gemfile 76 | rvm: 3.0 77 | env: DB=sqlite3 78 | - gemfile: spec/gemfiles/rails_main.gemfile 79 | rvm: 3.0 80 | env: DB=mysql2 DB_USERNAME=root DB_PASSWORD="" 81 | services: mysql 82 | - gemfile: spec/gemfiles/rails_main.gemfile 83 | rvm: 3.0 84 | env: DB=postgresql DB_USERNAME=postgres DB_PASSWORD="" 85 | services: postgresql 86 | 87 | cache: bundler 88 | bundler_args: --path ../../vendor/bundle --without debug 89 | 90 | env: 91 | global: 92 | - COVERAGE=1 TRAVIS=1 93 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__) 4 | ENV['RAILS_ENV'] = ENV['RACK_ENV'] = 'test' 5 | 6 | if ENV['COVERAGE'] && !%w[rbx jruby].include?(RUBY_ENGINE) 7 | require 'simplecov' 8 | SimpleCov.command_name 'RSpec' 9 | end 10 | 11 | require 'db_text_search' 12 | require 'fileutils' 13 | 14 | FileUtils.mkpath 'log' unless File.directory? 'log' 15 | ActiveRecord::Base.logger = Logger.new('log/test-queries.log') 16 | 17 | ENV['DB'] ||= 'sqlite3' 18 | case ENV['DB'] 19 | when 'mysql2', 'postgresql' 20 | system({'DB' => ENV['DB']}, 'script/create-db-users') unless ENV['TRAVIS'] 21 | config = { 22 | # Host 127.0.0.1 required for default postgres installation on Ubuntu. 23 | host: '127.0.0.1', 24 | database: 'db_text_search_gem_test', 25 | encoding: 'utf8', 26 | min_messages: 'WARNING', 27 | adapter: ENV['DB'], 28 | username: ENV['DB_USERNAME'] || 'db_text_search', 29 | password: ENV['DB_PASSWORD'] || 'db_text_search' 30 | } 31 | if ENV['DB'] == 'postgresql' 32 | begin 33 | # Must be required before establish_connection. 34 | require 'schema_plus_pg_indexes' 35 | rescue LoadError 36 | # Nothing to do here, optional dependency 37 | end 38 | end 39 | ActiveRecord::Tasks::DatabaseTasks.create(config.stringify_keys) 40 | ActiveRecord::Base.establish_connection(config) 41 | when 'sqlite3' 42 | ActiveRecord::Base.establish_connection(adapter: 'sqlite3', database: ':memory:') 43 | else 44 | fail "Unknown DB adapter #{ENV['DB']}" 45 | end 46 | 47 | def force_index 48 | enable_force_index, disable_force_index = DbTextSearch.match_adapter( 49 | ActiveRecord::Base.connection, 50 | postgres: -> { ['SET enable_seqscan=off', 'SET enable_seqscan=on'] }, 51 | mysql: -> { ['SET max_seeks_for_key=1', 'SET max_seeks_for_key=18446744073709551615'] }, 52 | sqlite: -> {} 53 | ) 54 | begin 55 | ActiveRecord::Base.connection.execute(enable_force_index).tap { |r| r && r.clear } if enable_force_index 56 | yield 57 | ensure 58 | ActiveRecord::Base.connection.execute(disable_force_index).tap { |r| r && r.clear } if disable_force_index 59 | end 60 | end 61 | 62 | def explain_index_expr(index_name) 63 | DbTextSearch.match_adapter( 64 | ActiveRecord::Base.connection, 65 | mysql: -> { /\b(ref|index|range|fulltext)\b.*\b#{Regexp.escape index_name}\b/ }, 66 | postgres: -> { "Index Scan using #{index_name}" }, 67 | sqlite: -> { /USING (?:COVERING )?INDEX #{Regexp.escape index_name}\b/ } 68 | ) 69 | end 70 | 71 | def psql_su_cmd 72 | system(%q(psql postgres -c '' 2>/dev/null)) ? 'psql' : 'sudo -u postgres psql -U postgres' 73 | end 74 | 75 | RSpec::Matchers.define :use_index do |index_name| 76 | match do |scope| 77 | expect(scope.explain).to match(explain_index_expr(index_name)) 78 | end 79 | 80 | failure_message do |scope| 81 | "expected EXPLAIN result to include #{explain_index_expr(index_name).inspect}:\n#{scope.explain}" 82 | end 83 | end 84 | -------------------------------------------------------------------------------- /lib/db_text_search/case_insensitive.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'db_text_search/case_insensitive/insensitive_column_adapter' 4 | require 'db_text_search/case_insensitive/lower_adapter' 5 | require 'db_text_search/case_insensitive/collate_nocase_adapter' 6 | 7 | module DbTextSearch 8 | # Provides case-insensitive string-in-set querying, LIKE querying, and CI index creation. 9 | class CaseInsensitive 10 | # @param scope [ActiveRecord::Relation, Class] 11 | # @param column [Symbol] name 12 | def initialize(scope, column) 13 | @adapter = self.class.adapter_class(scope.connection, scope.table_name, column).new(scope, column) 14 | @scope = scope 15 | end 16 | 17 | # @param value_or_values [String, Array] 18 | # @return [ActiveRecord::Relation] 19 | def in(value_or_values) 20 | values = Array(value_or_values) 21 | return @scope.none if values.empty? 22 | @adapter.in(values) 23 | end 24 | 25 | # @param query [String] 26 | # @return [ActiveRecord::Relation] the scope of records with matching prefix. 27 | def prefix(query) 28 | return @scope.none if query.empty? 29 | @adapter.prefix(query) 30 | end 31 | 32 | # @param asc_or_desc [Symbol] 33 | # @return [Arel::Collectors::SQLString] a string to be used within an `.order()` 34 | def column_for_order(asc_or_desc) 35 | fail 'Pass either :asc or :desc' unless %i[asc desc].include?(asc_or_desc) 36 | @adapter.column_for_order(asc_or_desc) 37 | end 38 | 39 | # Adds a case-insensitive column to the given table. 40 | # @param connection [ActiveRecord::ConnectionAdapters::AbstractAdapter] 41 | # @param table_name [String, Symbol] 42 | # @param column_name [String, Symbol] 43 | # @param options [Hash] passed to ActiveRecord::ConnectionAdapters::SchemaStatements#add_index 44 | def self.add_ci_text_column(connection, table_name, column_name, options = {}) 45 | type, options = DbTextSearch.match_adapter( 46 | connection, 47 | mysql: -> { [:text, options] }, 48 | postgres: -> { 49 | connection.enable_extension 'citext' 50 | [(ActiveRecord::VERSION::STRING >= '4.2.0' ? :citext : 'CITEXT'), options] 51 | }, 52 | sqlite: -> { 53 | if ActiveRecord::VERSION::MAJOR >= 5 54 | [:text, options.merge(collation: 'NOCASE')] 55 | else 56 | ['TEXT COLLATE NOCASE', options] 57 | end 58 | } 59 | ) 60 | connection.add_column table_name, column_name, type, **options 61 | end 62 | 63 | # Add an index for case-insensitive string search. 64 | # 65 | # @param connection [ActiveRecord::ConnectionAdapters::AbstractAdapter] 66 | # @param table_name [String, Symbol] 67 | # @param column_name [String, Symbol] 68 | # @param options [Hash] 69 | # @option options name [String] index name 70 | # @option options unique [Boolean] default: false 71 | def self.add_index(connection, table_name, column_name, options = {}) 72 | adapter_class(connection, table_name, column_name).add_index(connection, table_name, column_name, options) 73 | end 74 | 75 | # @param connection [ActiveRecord::ConnectionAdapters::AbstractAdapter] 76 | # @param table_name [String, Symbol] 77 | # @param column_name [String, Symbol] 78 | # @return [Class] 79 | # @api private 80 | def self.adapter_class(connection, table_name, column_name) 81 | lower_or_insensitive = -> { 82 | column_case_sensitive?(connection, table_name, column_name) ? LowerAdapter : InsensitiveColumnAdapter 83 | } 84 | DbTextSearch.match_adapter( 85 | connection, 86 | mysql: lower_or_insensitive, 87 | postgres: lower_or_insensitive, 88 | # Always use COLLATE NOCASE for SQLite, as we can't check if the column is case-sensitive. 89 | # It has no performance impact apart from slightly longer query strings for case-insensitive columns. 90 | sqlite: -> { CollateNocaseAdapter } 91 | ) 92 | end 93 | 94 | # @param connection [ActiveRecord::ConnectionAdapters::AbstractAdapter] 95 | # @param table_name [String, Symbol] 96 | # @param column_name [String, Symbol] 97 | # @return [Boolean] 98 | # @note sqlite not supported. 99 | # @api private 100 | def self.column_case_sensitive?(connection, table_name, column_name) # rubocop:disable Metrics/AbcSize 101 | column = connection.schema_cache.columns(table_name).detect { |c| c.name == column_name.to_s } 102 | fail "Column #{column_name.to_s.inspect} not found on table #{table_name.inspect}" if column.nil? 103 | DbTextSearch.match_adapter( 104 | connection, 105 | mysql: -> { column.case_sensitive? }, 106 | postgres: -> { column.sql_type !~ /citext/i }, 107 | sqlite: -> { DbTextSearch.unsupported_adapter! connection } 108 | ) 109 | end 110 | end 111 | end 112 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DbTextSearch [![Build Status](https://travis-ci.org/thredded/db_text_search.svg?branch=main)](https://travis-ci.org/thredded/db_text_search) [![Code Climate](https://codeclimate.com/github/thredded/db_text_search/badges/gpa.svg)](https://codeclimate.com/github/thredded/db_text_search) [![Test Coverage](https://codeclimate.com/github/thredded/db_text_search/badges/coverage.svg)](https://codeclimate.com/github/thredded/db_text_search/coverage) 2 | 3 | Different relational databases treat text search very differently. 4 | DbTextSearch provides a unified interface on top of ActiveRecord for SQLite, MySQL, and PostgreSQL to do: 5 | 6 | * Case-insensitive string-in-set querying, prefix querying, and case-insensitive index creation. 7 | * Basic full-text search for a list of terms, and FTS index creation. 8 | 9 | ## Installation 10 | 11 | Add this line to your application's Gemfile: 12 | 13 | ```ruby 14 | gem 'db_text_search', '~> 1.0' 15 | ``` 16 | 17 | ## Usage 18 | 19 | ### Case-insensitive string matching 20 | 21 | Add an index in a migration to an existing CI (case-insensitive) or CS (case-sensitive) column: 22 | 23 | ```ruby 24 | DbTextSearch::CaseInsensitive.add_index connection, :users, :username 25 | # Options: name, unique 26 | ``` 27 | 28 | Or, create a new CI column: 29 | 30 | ```ruby 31 | DbTextSearch::CaseInsensitive.add_ci_text_column connection, :users, :username 32 | ``` 33 | 34 | Perform a search for records with column that case-insensitively equals to one of the strings in a given set: 35 | 36 | ```ruby 37 | # Find all confirmed users that have either the username Alice or Bob (case-insensitively): 38 | DbTextSearch::CaseInsensitive.new(User.confirmed, :username).in(%w(Alice Bob)) 39 | #=> ActiveRecord::Relation 40 | ``` 41 | 42 | Perform a case-insensitive prefix search: 43 | 44 | ```ruby 45 | DbTextSearch::CaseInsensitive.new(User.confirmed, :username).prefix('Jo') 46 | ``` 47 | 48 | See also: [API documentation][api-docs]. 49 | 50 | ### Full text search 51 | 52 | Add an index: 53 | 54 | ```ruby 55 | DbTextSearch::FullText.add_index connection, :posts, :content 56 | # Options: name 57 | ``` 58 | 59 | Perform a full-text search: 60 | 61 | ```ruby 62 | DbTextSearch::FullText.new(Post.published, :content).search('peace') 63 | DbTextSearch::FullText.new(Post.published, :content).search(%w(love kaori)) 64 | ``` 65 | 66 | ## Under the hood 67 | 68 | ### Case-insensitive string matching 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 |
Case-insensitive equality methods
Column typeSQLiteMySQLPostgreSQL
Detected typesSearch / indexDetected typesSearch / indexDetected typesSearch / index
CIalways treated as CS COLLATE NOCASEdefault defaultCITEXT default
CSnon-ci collations LOWER
no index		default LOWER
88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 |
Case-insensitive prefix matching (using LIKE)
Column typeSQLiteMySQLPostgreSQL
CI 97 | default, cannot always use an index,
98 | even for prefix queries 99 | 		defaultcannot use an index
CScannot use an indexLOWER(column text_pattern_ops)
109 | 110 | 111 | ### Full-text search 112 | 113 | #### MySQL 114 | 115 | A `FULLTEXT` index, and a `MATCH AGAINST` query. MySQL v5.6.4+ is required. 116 | 117 | #### PostgreSQL 118 | 119 | A `gist(to_tsvector(...))` index, and a `@@ plainto_tsquery` query. 120 | Methods also accept an optional `pg_ts_config` argument (default: `"'english'"`) that is ignored for other databases. 121 | 122 | #### SQLite 123 | 124 | **No index**, a `LIKE %term%` query for each term joined with `AND`. 125 | 126 | ## Development 127 | 128 | Make sure you have a working installation of SQLite, MySQL, and PostgreSQL. 129 | After checking out the repo, run `bin/setup` to install dependencies. 130 | Then, run `rake test_all` to run the tests with all databases and gemfiles. 131 | 132 | See the Rakefile for other available test tasks. 133 | 134 | You can also run `bin/console` for an interactive prompt that will allow you to experiment. 135 | 136 | ## Contributing 137 | 138 | Bug reports and pull requests are welcome on GitHub at https://github.com/thredded/db_text_search. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct. 139 | 140 | ## License 141 | 142 | The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT). 143 | 144 | [api-docs]: http://www.rubydoc.info/gems/db_text_search 145 | -------------------------------------------------------------------------------- /spec/db_text_search/case_insensitive_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | module DbTextSearch 6 | RSpec.describe CaseInsensitive do 7 | column_cases = [['case-insensitive', :ci_name], ['case-sensitive', :cs_name]] 8 | describe '#in(value)' do 9 | let!(:records) { %w[ABC abC abc].map { |name| Name.create!(ci_name: name, cs_name: name) } } 10 | column_cases.each do |(column_desc, column)| 11 | it "works with a #{column_desc} column" do 12 | finder = CaseInsensitive.new(Name, column) 13 | expect(finder.in('aBc').to_a).to eq records 14 | end 15 | end 16 | end 17 | 18 | describe '#prefix(query)' do 19 | let!(:records) { %w[Joe john jack jill x%zz].map { |name| Name.create!(ci_name: name, cs_name: name) } } 20 | column_cases.each do |(column_desc, column)| 21 | it "works with a #{column_desc} column" do 22 | finder = CaseInsensitive.new(Name, column) 23 | expect(finder.prefix('Jo').to_a).to eq records.first(2) 24 | expect(finder.prefix('x%Z').to_a).to eq [records.last] 25 | end 26 | end 27 | end 28 | 29 | describe '#column_for_order(asc_or_desc)' do 30 | let!(:records) { %w[ABCz abCa abce].map { |name| Name.create!(ci_name: name, cs_name: name) } } 31 | column_cases.each do |(column_desc, column)| 32 | it "orders asc correctly with a #{column_desc} column" do 33 | case_insensitive = CaseInsensitive.new(Name, column) 34 | expect(Name.all.order(case_insensitive.column_for_order(:asc)).map(&:cs_name)).to eq(%w[abCa abce ABCz]) 35 | end 36 | end 37 | end 38 | 39 | describe '.add_index' do 40 | column_cases.each do |(column_desc, column)| 41 | index_name = :an_index 42 | 43 | it 'does not use an index when there is none (sanity check)' do 44 | force_index { expect(CaseInsensitive.new(Name, column).in('aBc')).to_not use_index(index_name) } 45 | end 46 | 47 | describe "adds a usable index on a #{column_desc} column" do 48 | before :all do 49 | if Name.connection.adapter_name =~ /mysql/i && column == :cs_name 50 | skip 'MySQL case-insensitive index creation for case-sensitive columns is not yet implemented' 51 | end 52 | CaseInsensitive.add_index Name.connection, :names, column, name: index_name 53 | end 54 | 55 | after :all do 56 | if Name.connection.adapter_name =~ /mysql/i && column == :cs_name 57 | next 58 | end 59 | if Name.connection.adapter_name =~ /postg/i 60 | # Work around https://github.com/rails/rails/issues/24359 61 | Name.connection.exec_query "DROP INDEX #{index_name}" 62 | else 63 | ActiveRecord::Migration.remove_index :names, name: index_name 64 | end 65 | end 66 | 67 | it 'uses an index for #find' do 68 | force_index { expect(CaseInsensitive.new(Name, column).in('aBc')).to use_index(index_name) } 69 | end 70 | it 'uses an index for #prefix' do 71 | if Name.connection.adapter_name =~ /postg/i && column == :ci_name 72 | skip 'PostgreSQL does not use a LIKE index on citext columns' 73 | end 74 | force_index { expect(CaseInsensitive.new(Name, column).prefix('A')).to use_index(index_name) } 75 | end 76 | end 77 | end 78 | 79 | describe CaseInsensitive::LowerAdapter do 80 | it 'throws an error for MySQL case-insensitive index for case-sensitive column' do 81 | mock_connection = Struct.new(:adapter_name).new('MySQL') 82 | expect { 83 | CaseInsensitive::LowerAdapter.add_index(mock_connection, :names, :name) 84 | }.to raise_error(ArgumentError, 'Unsupported adapter MySQL') 85 | end 86 | 87 | it 'throws an error for an invalid adapter' do 88 | mock_connection = Struct.new(:adapter_name).new('AnInvalidAdapter') 89 | expect { 90 | CaseInsensitive::LowerAdapter.add_index(mock_connection, :names, :name) 91 | }.to raise_error(ArgumentError, 'Unsupported adapter AnInvalidAdapter') 92 | end 93 | end 94 | end 95 | 96 | describe '.add_ci_text_column' do 97 | column = :ci_text 98 | before :all do 99 | CaseInsensitive.add_ci_text_column Name.connection, :names, column 100 | Name.reset_column_information 101 | ActiveRecord::Base.connection.schema_cache.clear! 102 | end 103 | after :all do 104 | ActiveRecord::Migration.remove_column :names, column 105 | Name.reset_column_information 106 | ActiveRecord::Base.connection.schema_cache.clear! 107 | end 108 | it 'adds a case-insensitive column' do 109 | if Name.connection.adapter_name =~ /sqlite/i 110 | # check not implemented, so just check that a search uses index 111 | ActiveRecord::Migration.add_index :names, column, name: :"#{column}_index" 112 | finder = CaseInsensitive.new(Name, column) 113 | record = Name.create!(ci_name: 'Abc', cs_name: 'Abc', column => 'Abc') 114 | expect(finder.in('aBc')).to use_index(:"#{column}_index") 115 | expect(finder.in('aBc').first).to eq record 116 | else 117 | expect(CaseInsensitive.column_case_sensitive?(Name.connection, :names, column)).to be_falsey 118 | end 119 | end 120 | it 'fails with ArgumentError on an unknown adapter' do 121 | mock_connection = Struct.new(:adapter_name).new('AnInvalidAdapter') 122 | expect { 123 | CaseInsensitive.add_ci_text_column mock_connection, :names, column 124 | }.to raise_error(ArgumentError, 'Unsupported adapter AnInvalidAdapter') 125 | end 126 | end 127 | 128 | describe '.column_case_sensitive?' do 129 | it 'is truthy for a case-sensitive column' do 130 | skip 'not implemented for SQLite' if Name.connection.adapter_name =~ /sqlite/i 131 | expect(CaseInsensitive.column_case_sensitive?(Name.connection, :names, :cs_name)).to be_truthy 132 | end 133 | 134 | it 'is falsey for a case-insensitive column' do 135 | skip 'not implemented for SQLite' if Name.connection.adapter_name =~ /sqlite/i 136 | expect(CaseInsensitive.column_case_sensitive?(Name.connection, :names, :ci_name)).to be_falsey 137 | end 138 | 139 | it 'fails with ArgumentError on an unknown adapter and sqlite' do 140 | %w[SQLite UnknownAdapter].each do |adapter_name| 141 | mock_connection = double('Connection', 142 | adapter_name: adapter_name, 143 | schema_cache: double(columns: [double('Column', name: 'a_column')])) 144 | expect { 145 | CaseInsensitive.column_case_sensitive? mock_connection, :names, :a_column 146 | }.to raise_error(ArgumentError, "Unsupported adapter #{adapter_name}") 147 | end 148 | end 149 | end 150 | 151 | class Name < ActiveRecord::Base 152 | end 153 | 154 | before do 155 | Name.delete_all 156 | end 157 | 158 | before :all do 159 | ActiveRecord::Schema.define do 160 | self.verbose = false 161 | create_table :names do |t| 162 | DbTextSearch.match_adapter( 163 | connection, 164 | mysql: -> { 165 | t.string :ci_name 166 | t.column :cs_name, 'VARCHAR(191) COLLATE utf8_bin' 167 | }, 168 | postgres: -> { 169 | begin 170 | connection.enable_extension 'citext' 171 | rescue ActiveRecord::StatementInvalid 172 | raise "Please run the command below to enable the 'citext' Postgres extension:\n" \ 173 | "#{psql_su_cmd} -d #{ActiveRecord::Base.connection_config[:database]} " \ 174 | "-c 'CREATE EXTENSION citext;'" 175 | end 176 | t.column :ci_name, 'CITEXT' 177 | t.string :cs_name 178 | }, 179 | sqlite: -> { 180 | t.column :ci_name, 'VARCHAR(191) COLLATE NOCASE' 181 | t.string :cs_name 182 | } 183 | ) 184 | end 185 | end 186 | ActiveRecord::Base.connection.schema_cache.clear! 187 | end 188 | 189 | after :all do 190 | ActiveRecord::Migration.drop_table :names 191 | ActiveRecord::Base.connection.schema_cache.clear! 192 | end 193 | end 194 | end 195 | --------------------------------------------------------------------------------