├── .circleci └── config.yml ├── .gitignore ├── Gemfile ├── HISTORY ├── LICENCE ├── Procfile.support ├── README.markdown ├── Rakefile ├── bin └── loadsphinx ├── lib ├── riddle.rb └── riddle │ ├── 0.9.8.rb │ ├── 0.9.9.rb │ ├── 0.9.9 │ ├── client.rb │ ├── client │ │ └── filter.rb │ └── configuration │ │ └── searchd.rb │ ├── 1.10.rb │ ├── 1.10 │ └── client.rb │ ├── 2.0.1.rb │ ├── 2.0.1 │ └── client.rb │ ├── 2.1.0.rb │ ├── auto_version.rb │ ├── client.rb │ ├── client │ ├── filter.rb │ ├── message.rb │ └── response.rb │ ├── command_failed_error.rb │ ├── command_result.rb │ ├── configuration.rb │ ├── configuration │ ├── common.rb │ ├── distributed_index.rb │ ├── index.rb │ ├── index_settings.rb │ ├── indexer.rb │ ├── parser.rb │ ├── realtime_index.rb │ ├── remote_index.rb │ ├── searchd.rb │ ├── section.rb │ ├── source.rb │ ├── sql_source.rb │ ├── template_index.rb │ ├── tsv_source.rb │ └── xml_source.rb │ ├── controller.rb │ ├── execute_command.rb │ ├── query.rb │ └── query │ ├── delete.rb │ ├── insert.rb │ └── select.rb ├── riddle.gemspec └── spec ├── fixtures ├── .gitignore ├── data │ ├── 0.9.9 │ │ ├── anchor.bin │ │ ├── any.bin │ │ ├── boolean.bin │ │ ├── comment.bin │ │ ├── distinct.bin │ │ ├── field_weights.bin │ │ ├── filter.bin │ │ ├── filter_array.bin │ │ ├── filter_array_exclude.bin │ │ ├── filter_boolean.bin │ │ ├── filter_floats.bin │ │ ├── filter_floats_exclude.bin │ │ ├── filter_range.bin │ │ ├── filter_range_exclude.bin │ │ ├── group.bin │ │ ├── index.bin │ │ ├── index_weights.bin │ │ ├── keywords_with_hits.bin │ │ ├── keywords_without_hits.bin │ │ ├── overrides.bin │ │ ├── phrase.bin │ │ ├── rank_mode.bin │ │ ├── select.bin │ │ ├── simple.bin │ │ ├── sort.bin │ │ ├── update_simple.bin │ │ └── weights.bin │ ├── 1.10 │ │ ├── anchor.bin │ │ ├── any.bin │ │ ├── boolean.bin │ │ ├── comment.bin │ │ ├── distinct.bin │ │ ├── field_weights.bin │ │ ├── filter.bin │ │ ├── filter_array.bin │ │ ├── filter_array_exclude.bin │ │ ├── filter_boolean.bin │ │ ├── filter_floats.bin │ │ ├── filter_floats_exclude.bin │ │ ├── filter_range.bin │ │ ├── filter_range_exclude.bin │ │ ├── group.bin │ │ ├── index.bin │ │ ├── index_weights.bin │ │ ├── keywords_with_hits.bin │ │ ├── keywords_without_hits.bin │ │ ├── overrides.bin │ │ ├── phrase.bin │ │ ├── rank_mode.bin │ │ ├── select.bin │ │ ├── simple.bin │ │ ├── sort.bin │ │ ├── update_simple.bin │ │ └── weights.bin │ ├── 2.0.1 │ │ ├── anchor.bin │ │ ├── any.bin │ │ ├── boolean.bin │ │ ├── comment.bin │ │ ├── distinct.bin │ │ ├── field_weights.bin │ │ ├── filter.bin │ │ ├── filter_array.bin │ │ ├── filter_array_exclude.bin │ │ ├── filter_boolean.bin │ │ ├── filter_floats.bin │ │ ├── filter_floats_exclude.bin │ │ ├── filter_range.bin │ │ ├── filter_range_exclude.bin │ │ ├── group.bin │ │ ├── index.bin │ │ ├── index_weights.bin │ │ ├── keywords_with_hits.bin │ │ ├── keywords_without_hits.bin │ │ ├── overrides.bin │ │ ├── phrase.bin │ │ ├── rank_mode.bin │ │ ├── select.bin │ │ ├── simple.bin │ │ ├── sort.bin │ │ ├── update_simple.bin │ │ └── weights.bin │ └── 2.1.0 │ │ ├── anchor.bin │ │ ├── any.bin │ │ ├── boolean.bin │ │ ├── comment.bin │ │ ├── distinct.bin │ │ ├── field_weights.bin │ │ ├── filter.bin │ │ ├── filter_array.bin │ │ ├── filter_array_exclude.bin │ │ ├── filter_boolean.bin │ │ ├── filter_floats.bin │ │ ├── filter_floats_exclude.bin │ │ ├── filter_range.bin │ │ ├── filter_range_exclude.bin │ │ ├── group.bin │ │ ├── index.bin │ │ ├── index_weights.bin │ │ ├── keywords_with_hits.bin │ │ ├── keywords_without_hits.bin │ │ ├── overrides.bin │ │ ├── phrase.bin │ │ ├── rank_mode.bin │ │ ├── select.bin │ │ ├── simple.bin │ │ ├── sort.bin │ │ ├── update_simple.bin │ │ └── weights.bin ├── sphinx │ └── configuration.erb └── sql │ ├── conf.example.yml │ ├── data.sql │ ├── data.tsv │ └── structure.sql ├── functional ├── connection_spec.rb ├── escaping_spec.rb ├── excerpt_spec.rb ├── keywords_spec.rb ├── merging_spec.rb ├── parsing_spec.rb ├── persistance_spec.rb ├── search_spec.rb ├── status_spec.rb └── update_spec.rb ├── riddle ├── auto_version_spec.rb ├── client_spec.rb ├── configuration_spec.rb ├── controller_spec.rb ├── query │ ├── delete_spec.rb │ ├── insert_spec.rb │ └── select_spec.rb └── query_spec.rb ├── riddle_spec.rb ├── spec_helper.rb ├── support ├── binary_fixtures.rb ├── jruby_client.rb ├── mri_client.rb └── sphinx.rb └── unit ├── client_spec.rb ├── configuration ├── common_spec.rb ├── distributed_index_spec.rb ├── index_spec.rb ├── indexer_spec.rb ├── realtime_index_spec.rb ├── searchd_spec.rb ├── source_spec.rb ├── sql_source_spec.rb ├── template_index_spec.rb ├── tsv_source_spec.rb └── xml_source_spec.rb ├── configuration_spec.rb ├── filter_spec.rb ├── message_spec.rb ├── response_spec.rb └── riddle_spec.rb /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | 3 | orbs: 4 | ruby: circleci/ruby@1.0 5 | 6 | workflows: 7 | test: 8 | jobs: 9 | - test: 10 | name: "Sphinx 2.2" 11 | sphinx_version: 2.2.11 12 | sphinx_engine: sphinx 13 | debian: jessie 14 | ruby: '2.4.6' 15 | - test: 16 | name: "Sphinx 3.4" 17 | sphinx_version: 3.4.1 18 | sphinx_engine: sphinx 19 | debian: buster 20 | matrix: 21 | parameters: 22 | ruby: [ '2.4.10', '2.5.9', '2.6.9', '2.7.5', '3.0.3' ] 23 | - test: 24 | name: "Manticore 3.5" 25 | sphinx_version: 3.5.4 26 | sphinx_engine: manticore 27 | debian: buster 28 | matrix: 29 | parameters: 30 | ruby: [ '2.4.10', '2.5.9', '2.6.9', '2.7.5', '3.0.3' ] 31 | - test: 32 | name: "Manticore 4.0" 33 | sphinx_version: 4.0.2 34 | sphinx_engine: manticore 35 | debian: buster 36 | matrix: 37 | parameters: 38 | ruby: [ '2.4.10', '2.5.9', '2.6.9', '2.7.5', '3.0.3' ] 39 | 40 | jobs: 41 | test: 42 | parameters: 43 | ruby: 44 | type: string 45 | sphinx_version: 46 | type: string 47 | sphinx_engine: 48 | type: string 49 | debian: 50 | type: string 51 | 52 | docker: 53 | - image: circleci/ruby:<< parameters.ruby >>-<< parameters.debian >> 54 | 55 | - image: circleci/mysql:5.7 56 | environment: 57 | MYSQL_ROOT_PASSWORD: riddle 58 | MYSQL_DATABASE: riddle 59 | 60 | working_directory: ~/app 61 | 62 | steps: 63 | - checkout 64 | 65 | - restore_cache: 66 | keys: 67 | - v1-dependencies-<< parameters.ruby >> 68 | 69 | - run: 70 | name: install bundler 71 | command: | 72 | if [ "<< parameters.ruby >>" == "2.7.5" ]; then 73 | export BUNDLER_VERSION=2.1.4 74 | elif [ "<< parameters.ruby >>" == "3.0.3" ]; then 75 | export BUNDLER_VERSION=2.1.4 76 | else 77 | export BUNDLER_VERSION=1.17.3 78 | fi 79 | export BUNDLE_PATH=vendor/bundle 80 | gem install bundler:$BUNDLER_VERSION 81 | 82 | - run: 83 | name: install dependencies 84 | command: | 85 | bundle install --jobs=4 --retry=3 --path vendor/bundle 86 | bundle update 87 | 88 | - save_cache: 89 | paths: 90 | - ./vendor/bundle 91 | key: v1-dependencies-<< parameters.ruby >> 92 | 93 | - run: 94 | name: set up sphinx 95 | command: "./bin/loadsphinx << parameters.sphinx_version >> << parameters.sphinx_engine >>" 96 | 97 | - run: 98 | name: wait for MySQL to be ready 99 | command: | 100 | for i in `seq 1 10`; 101 | do 102 | nc -z 127.0.0.1 3306 && echo Success && exit 0 103 | echo -n . 104 | sleep 1 105 | done 106 | echo Failed waiting for MySQL && exit 1 107 | 108 | - run: 109 | name: tests 110 | environment: 111 | CI: "true" 112 | MYSQL_HOST: 127.0.0.1 113 | MYSQL_USER: root 114 | MYSQL_PASSWORD: riddle 115 | SPHINX_VERSION: << parameters.sphinx_version >> 116 | SPHINX_ENGINE: << parameters.sphinx_engine >> 117 | command: bundle exec rspec 118 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | *.rbc 3 | *.swp 4 | *.tmproj 5 | coverage 6 | Gemfile.lock 7 | .overmind.env 8 | data 9 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | source 'http://rubygems.org' 4 | 5 | gemspec 6 | 7 | gem 'mysql2', '~> 0.5.2', :platform => :ruby 8 | gem 'jdbc-mysql', '~> 5.1.47', :platform => :jruby 9 | -------------------------------------------------------------------------------- /HISTORY: -------------------------------------------------------------------------------- 1 | 2.4.3 - December 20th 2021 2 | - Use File.exist? instead of the deprecated File.exists?. 3 | - Switched CI over to CircleCI, and testing against newer Sphinx/Manticore releases. 4 | 5 | 2.4.2 - April 4th 2020 6 | - Ensure parsing of files with a common section are rendered the same way. 7 | 8 | 2.4.1 - January 4th 2020 9 | - Fix typo of stopword_step setting. 10 | 11 | 2.4.0 - July 28th 2019 12 | - Escape slashes and single quotes for attribute and field values in INSERT/REPLACE statements, rather than removing them (#103). 13 | - Remove support for Sphinx 2.0. 14 | - Add Manticore 2.7 and 2.8. 15 | - Test suite improvements. 16 | - Gemspec improvements (Olle Jonsson #105) 17 | 18 | 2.3.2 - September 23rd 2018 19 | - Fix attempt to modify a frozen string when generating configuration. 20 | - Test against Manticore 2.6.3. 21 | 22 | 2.3.1 - March 25th 2018 23 | - Handle parsing of invalid configuration files without endlessly looping. 24 | - Test against mysql2 0.5. 25 | - Test against Sphinx 3.0. 26 | 27 | 2.3.0 - January 14th 2018 28 | - Add controller method for merging indices. 29 | - Add support for sockets in searchd configuration. 30 | - Fix handling of command errors when executed via backticks (verbose). 31 | 32 | 2.2.2 - December 2nd 2017 33 | - Fix frozen string concatenation for searchd/indexer commands. 34 | 35 | 2.2.1 - December 2nd 2017 36 | - Check if Mysql2::Client is defined before using it (not just Mysql2). 37 | 38 | 2.2.0 - June 20th 2017 39 | - Add compatibility for MRI's frozen string literals setting. 40 | - Stop packaging test files with gem releases (@dimko). 41 | 42 | 2.1.0 - January 5th 2017 43 | - Wrap string attribute filter values in single quotes. 44 | - Remove direct references to FixNum to avoid MRI 2.4 warnings. 45 | - Remove escaped line-endings while parsing. 46 | - Escape word-operators like MAYBE, NEAR etc (Jonathan del Strother) 47 | 48 | 2.0.0 - September 25th 2016 49 | - Start and stop commands now accept a verbose option. 50 | - Failed commands (as determined by a non-zero status code) raise a Riddle::CommandFailedError exception. 51 | - A missing configuration file when running the start and stop commands now raises a Riddle::NoConfigurationFileError instance instead of a generic RuntimeError instance. 52 | - Riddle::Controller now returns Riddle::CommandResult instances for indexing, start and stop commands, which includes status code and (non-verbose) output. 53 | - Handle group_concat attribute types (@crazyshot, @bibendi). 54 | - Additional searchd settings: query_log_min_msec, agent_conect_timeout, agent_query_timeout, agent_retry_count, agent_retry_delay. 55 | - Default to 2.1.0 or newer support. 56 | - Move plugin_dir option from searchd to common. 57 | 58 | 1.5.12 - June 1st 2015 59 | - Adding ? as an escaped character (Alexey Nikitin). 60 | - Adding contributor code of conduct. 61 | - Spec fixes, and updating escape_column to not escape JSON expressions that make use of dot or bracket notation (Daniel Vandersluis). 62 | - Fix stop action to allow exception propagation (Dejan Simic). 63 | 64 | 1.5.11 - April 19th 2014 65 | - Riddle::Query.escape covers = and & characters. 66 | - Hold onto address and port settings when crafting the equivalent listen setting, but don't render them. 67 | - Allow for Sphinx's common settings section (Trevor Smith). Optional and initially disabled. 68 | - Allow for multiple attributes in GROUP BY clauses (J. Garcia). 69 | - Riddle::Query.escape covers < and > characters. 70 | - The parser should not presume indexer and searchd sections exist. 71 | 72 | 1.5.10 - January 11th 2014 73 | - SELECT values can be prepended as well as the existing append support. 74 | - New settings for Sphinx 2.2.1. 75 | - Template index type for Sphinx 2.2.1. 76 | - TSV source types for Sphinx 2.2.1. 77 | - Support for HAVING, GROUP-n-BEST in SELECT statements. 78 | - Dates in filters are converted to (UTC) timestamp integers. 79 | - Default to * in SELECT queries only if nothing else is supplied. 80 | - Fix licence, URL in gemspec (Ken Dreyer). 81 | - Handle empty arrays for filter elegantly (Bryan Ricker). 82 | - Add a contributing section to the README (Ken Dreyer). 83 | - Don't automatically escape function references in SphinxQL ORDER clauses. 84 | 85 | 1.5.9 - October 20th 2013 86 | - Adding all known Sphinx settings to configuration classes as of Sphinx 2.1.2, including JSON settings. 87 | - Convert date objects in INSERT/REPLACE queries to timestamps, just like time objects. 88 | - Don't escape references to id in SphinxQL INSERT/REPLACE queries. 89 | 90 | 1.5.8 - August 26th 2013 91 | - Reworked escaping to be consistent and always query-safe (Demian Ferreiro). 92 | - Escape column names in SphinxQL WHERE, INSERT, ORDER BY and GROUP BY clauses and statements (Jason Rust). 93 | 94 | 1.5.7 - July 9th 2013 95 | - Respect Riddle::OutOfBoundsError instances, instead of wrapping them in ResponseError. 96 | - Handle boolean values for snippets options. 97 | - Don't modify snippets parameters (Demian Ferreiro). 98 | - rt_attr_multi and rt_attr_multi_64 settings for real-time indices. 99 | - Arrays in INSERT/REPLACE statements are wrapped in parentheses with values separated by commas. Required for MVA values in real-time indices. 100 | - Clear out the query queue before running a single query. 101 | 102 | 1.5.6 - May 7th 2013 103 | - Wrap underlying parse errors within Riddle::ResponseError instances when parsing responses. 104 | - Add lemmatization options (Kirill Lazarev). 105 | - Ignore configuration lines that are only comments when parsing configurations. 106 | - Construct GROUP ORDER and ORDER in SphinxQL in the correct order (Grzegorz Derebecki). 107 | 108 | 1.5.5 - February 23rd 2013 109 | - Added Riddle::Query.escape for SphinxQL queries. 110 | - Fixed failover handling (Ngan Pham). 111 | - Improved encoding default check (Darcy Brown). 112 | - Removing REE support (as it is no longer supported either). 113 | - Client key is used for binary protocol persistent connections (if set). 114 | - Escaping single quotes in SphinxQL snippets calls. 115 | - Regex fix for matching {'s (Rob Golkosky). 116 | 117 | 1.5.4 - January 2nd 2013 118 | - RT indices get most of the same settings as SQL indices. 119 | - Escape single quotes in SphinxQL match queries, given we're wrapping them in single quotes. 120 | - Remove unnecessary characters from string values for SphinxQL inserts. 121 | - Convert time objects to integers for SphinxQL inserts. 122 | - Include 'orphan' sources (which aren't used directly by indices, but could be parents of sources that are used) when generating configuration files. 123 | - Use parent source type if child source has no type supplied. 124 | - Ignore comments when parsing Sphinx configuration files. 125 | 126 | 1.5.3 - August 10th 2012 127 | - Sphinx 2.0.5 support. 128 | - :with_all and :without_all support for SphinxQL. 129 | - Allow setting of prefix and infix fields directly. 130 | - Configuration parser 131 | - Adding rotate command to the controller. 132 | 133 | 1.5.2 - May 14th 2012 134 | - Fixing 64-bit MVA support. 135 | - Grouped searches now sort by weight instead of by group as a default. You can change this setting via Riddle::Client#group_clause. 136 | - Use a local array of servers - don't clear the main set (S. Christoffer Eliesen). 137 | - Fixing VersionError reference for Sphinx 0.9.9 or better (S. Christoffer Eliesen). 138 | - Consistent documentation for default port: 9312 (Aleksey Morozov). 139 | - Sphinx 2.0.4 support (Ilia Lobsanov). 140 | - Handle single-value float filters (by translating them to a range) (Steven Bristol). 141 | - Sphinx 2.0.2-dev handled as Sphinx 2.0.1. 142 | - Sphinx 2.0.3 support. 143 | - String options handled appropriately for SphinxQL excerpts calls. 144 | 145 | 1.5.1 - January 2nd 2012 146 | - If no known servers work, raise an appropriate error. 147 | - Sphinx 2.1.0-dev support. 148 | 149 | 1.5.0 - November 4th 2011 150 | - Handle exclusive filters in SphinxQL SELECT commands. 151 | - Allow for native Ruby objects in SphinxQL UPDATE commands. 152 | - Handle options of hashes in SphinxQL SELECT commands. 153 | - Allow for SphinxQL select clauses. 154 | - Improving SphinxQL filter handling of native Ruby objects. 155 | - Switch plural index references from indexes to indices, to distinguish beside indexes (the action). 156 | - Rescue against timeouts and connection resets. 157 | - Fixing reference to TCPSocket. 158 | - Handle port numbers as integers for listen setting (Ngan Pham). 159 | - Provide the option to start searchd with the nodetach flag (Aaron Gilbralter). 160 | - Don't shuffle servers (if there's more than one) - let developers (or Thinking Sphinx) manage that (Ngan Pham). 161 | 162 | 1.4.0 - August 2nd 2011 163 | - Checking against both Windows platforms for Ruby (Paul Gibler) 164 | - Encoding improvements (Alexey Artamonov) 165 | - More Rubyish syntax (James Cook) 166 | - Handling Ruby encodings (James Cook) 167 | - Coreseek support (saberma) 168 | - Section restructure for better inheritance (Alexey Artamonov) 169 | - MySQL41 connection support 170 | - requiring 'thread' for Mutex use 171 | 172 | 1.3.3 - May 25th 2011 173 | - Using MySQL2 library for SphinxQL interface 174 | - Adding Sphinx 2.0.x settings 175 | - SphinxQL support 176 | - Speed improvements for hash lookups (Enrico Thierbach) 177 | - Handle race conditions of segfaults while returning responses (Jason Lambert) 178 | - 2.0.x support 179 | 180 | 1.3.2 - May 12th 2011 181 | - client_key support 182 | 183 | 1.3.1 - May 9th 2011 184 | - Don't output warnings or exit when version isn't detected - presume Thinking Sphinx will handle that. 185 | - Confirm configuration file exists before attempting to start/stop Sphinx. 186 | - Use a Mutex instead of the current Thread. 187 | 188 | 1.3.0 - May 7th 2011 189 | - Attempts at untested 2.0.x and client_key support 190 | - Using Bundler, MySQL2 and Ruby 1.9.2 in development 191 | - Allow for Sphinx versions compiled from source and SVN (Greg Weber) 192 | 193 | 1.2.2 - December 22nd 2011 194 | -------------------------------------------------------------------------------- /LICENCE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2008-2010 Pat Allan 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /Procfile.support: -------------------------------------------------------------------------------- 1 | mysql: $(brew --prefix mysql@5.7)/bin/mysqld --datadir=$(PWD)/data/mysql --port ${MYSQL_PORT:-3306} --socket=mysql.sock 2 | -------------------------------------------------------------------------------- /README.markdown: -------------------------------------------------------------------------------- 1 | # Riddle 2 | 3 | [![Build Status](https://travis-ci.org/pat/riddle.svg?branch=develop)](https://travis-ci.org/pat/riddle) 4 | 5 | Riddle is a Ruby library interfacing with the [Sphinx](http://sphinxsearch.com/) full-text search tool. It is written by [Pat Allan](http://freelancing-gods.com), and has been influenced by both Dmytro Shteflyuk's Ruby client and the original PHP client. It can be used for interactions with Sphinx's command-line tools `searchd` and `indexer`, sending search queries via the binary protocol, and programmatically generating Sphinx configuration files. 6 | 7 | The syntax here, while closer to a usual Ruby approach than the PHP client, is quite old (Riddle was first published in 2007). While it would be nice to re-work things, it's really not a priority, given the bulk of Riddle's code is for Sphinx's deprecated binary protocol. 8 | 9 | ## Installation 10 | 11 | Riddle is available as a gem, so you can install it directly: 12 | 13 | gem install riddle 14 | 15 | Or include it in a Gemfile: 16 | 17 | gem 'riddle', '~> 2.4' 18 | 19 | ## Usage 20 | 21 | As of version 1.0.0, Riddle supports multiple versions of Sphinx in the one gem - you'll need to require your specific version after a normal require, though. The latest distinct version is `2.1.0`: 22 | 23 | require 'riddle' 24 | require 'riddle/2.1.0' 25 | 26 | The full list of versions available are `0.9.8` (the initial base), `0.9.9`, `1.10`, `2.0.1`, and `2.1.0`. If you're using something more modern than 2.1.0, then just require that, and the rest should be fine (changes to the binary protocol since then are minimal). 27 | 28 | ### Configuration 29 | 30 | Riddle's structure for generating Sphinx configuration is very direct mapping to Sphinx's configuration options. First, create an instance of `Riddle::Configuration`: 31 | 32 | config = Riddle::Configuration.new 33 | 34 | This configuration instance has methods `indexer`, `searchd` and `common`, which return separate inner-configuration objects with methods mapping to the equivalent [Sphinx settings](http://sphinxsearch.com/docs/current.html#conf-reference). So, you may want to do the following: 35 | 36 | config.indexer.mem_limit = '128M' 37 | config.searchd.log = '/my/log/file.log' 38 | 39 | Similarly, there are two further methods `indices` and `sources`, which are arrays meant to hold instances of index and source inner-configuration objects respectively (all of which have methods matching their Sphinx settings). The available index classes are: 40 | 41 | * `Riddle::Configuration::DistributedIndex` 42 | * `Riddle::Configuration::Index` 43 | * `Riddle::Configuration::RealtimeIndex` 44 | * `Riddle::Configuration::RemoteIndex` 45 | * `Riddle::Configuration::TemplateIndex` 46 | 47 | All of these index classes should be initialised with their name, and in the case of plain indices, their source objects. Remote indices take an address, port and name as their initialiser parameters. 48 | 49 | index = Riddle::Configuration::Index.new 'articles', article_source_a, article_source_b 50 | index.path = '/path/to/index/files" 51 | index.docinfo = 'external' 52 | 53 | The available source classes are: 54 | 55 | * `Riddle::Configuration::SQLSource` 56 | * `Riddle::Configuration::TSVSource` 57 | * `Riddle::Configuration::XMLSource` 58 | 59 | The initialising parameters are the name of the source, and the type of source: 60 | 61 | source = Riddle::Configuration::SQLSource.new 'article_source', 'mysql' 62 | source.sql_query = "SELECT id, title, body FROM articles" 63 | source.sql_host = "127.0.0.1" 64 | 65 | Once you have created your configuration object tree, you can then generate the string representation and perhaps save it to a file: 66 | 67 | File.write "sphinx.conf", configuration.render 68 | 69 | It's also possible to parse an existing Sphinx configuration file into a configuration option tree: 70 | 71 | configuration = Riddle::Configuration.parse! File.read('sphinx.conf') 72 | 73 | ### Indexing and Starting/Stopping the Daemon 74 | 75 | using Sphinx's command-line tools `indexer` and `searchd` via Riddle is all done via an instance of `Riddle::Controller`: 76 | 77 | configuration_file = "/path/to/sphinx.conf" 78 | configuration = Riddle::Configuration.parse! File.read(configuration_file) 79 | controller = Riddle::Controller.new configuration, configuration_file 80 | 81 | # set the path where the indexer and searchd binaries are located: 82 | controller.bin_path = '/usr/local/bin' 83 | 84 | # set different binary names if you're running a custom Sphinx installation: 85 | controller.searchd_binary_name = 'sphinxsearchd' 86 | controller.indexer_binary_name = 'sphinxindexer' 87 | 88 | # process all indices: 89 | controller.index 90 | # process specific indices: 91 | controller.index 'articles', 'books' 92 | # rotate old index files out for the new ones: 93 | controller.rotate 94 | 95 | # start the daemon: 96 | controller.start 97 | # start the daemon and do not detach the process: 98 | controller.start :nodetach => true 99 | # stop the daemon: 100 | controller.stop 101 | 102 | The index, start and stop methods all accept a hash of options, and the :verbose option is respected in each case. 103 | 104 | Each of these methods will return an instance of `Riddle::CommandResult` - or, if the command fails (as judged by the process status code), a `Riddle::CommandFailedError` exception is raised. These exceptions respond to the `command_result` method with the corresponding details. 105 | 106 | ### SphinxQL Queries 107 | 108 | Riddle does not have any code to send SphinxQL queries and commands to Sphinx. Because Sphinx uses the mysql41 protocol (thus, mimicing a MySQL database server), I recommend using the [mysql2](https://github.com/brianmario/mysql2) gem instead. The [connection code](https://github.com/pat/thinking-sphinx/blob/develop/lib/thinking_sphinx/connection.rb) in Thinking Sphinx may provide some inspiration on this. 109 | 110 | ### Binary Protocol Searching 111 | 112 | Sphinx's legacy binary protocol does not have many of the more recent Sphinx features - such as real-time indices - as these are only available in the SphinxQL/mysql41 protocol. However, Riddle can still be used for the binary protocol if you wish. 113 | 114 | To get started, just instantiate a Client object: 115 | 116 | client = Riddle::Client.new # defaults to localhost and port 9312 117 | client = Riddle::Client.new "sphinxserver.domain.tld", 3333 # custom settings 118 | 119 | And then set the parameters to what you want, before running a query: 120 | 121 | client.match_mode = :extended 122 | client.query "Pat Allan @state Victoria" 123 | 124 | The results from a query are similar to the other clients - but here's the details. It's a hash with 125 | the following keys: 126 | 127 | * `:matches` 128 | * `:fields` 129 | * `:attributes` 130 | * `:attribute_names` 131 | * `:words` 132 | * `:total` 133 | * `:total_found` 134 | * `:time` 135 | * `:status` 136 | * `:warning` (if appropriate) 137 | * `:error` (if appropriate) 138 | 139 | The key `:matches` returns an array of hashes - the actual search results. Each hash has the document id (`:doc`), the result weighting (`:weight`), and a hash of the attributes for the document (`:attributes`). 140 | 141 | The `:fields` and `:attribute_names` keys return list of fields and attributes for the documents. The key `:attributes` will return a hash of attribute name and type pairs, and `:words` returns a hash of hashes representing the words from the search, with the number of documents and hits for each, along the lines of: 142 | 143 | results[:words]["Pat"] #=> {:docs => 12, :hits => 15} 144 | 145 | `:total`, `:total_found` and `:time` return the number of matches available, the total number of matches (which may be greater than the maximum available), and the time in milliseconds that the query took to run. 146 | 147 | `:status` is the error code for the query - and if there was a related warning, it will be under the `:warning` key. Fatal errors will be described under `:error`. 148 | 149 | ## Contributing 150 | 151 | Please note that this project has a [Contributor Code of Conduct](http://contributor-covenant.org/version/1/0/0/). By participating in this project you agree to abide by its terms. 152 | 153 | Riddle uses the [git-flow](http://jeffkreeftmeijer.com/2010/why-arent-you-using-git-flow/) process for development. The `main` branch is the latest released code (in a gem). The `develop` branch is what's coming in the next release. (There may be occasional feature and hotfix branches, although these are generally not pushed to GitHub.) 154 | 155 | When submitting a patch to Riddle, please submit your pull request against the `develop` branch. 156 | 157 | ## Contributors 158 | 159 | Thanks to the following people who have contributed to Riddle in some shape or form: 160 | 161 | * Andrew Aksyonoff 162 | * Brad Greenlee 163 | * Lachie Cox 164 | * Jeremy Seitz 165 | * Mark Lane 166 | * Xavier Noria 167 | * Henrik Nye 168 | * Kristopher Chambers 169 | * Rob Anderton 170 | * Dylan Egan 171 | * Jerry Vos 172 | * Piotr Sarnacki 173 | * Tim Preston 174 | * Amir Yalon 175 | * Sam Goldstein 176 | * Matt Todd 177 | * Paco Guzmán 178 | * Greg Weber 179 | * Enrico Thierbach 180 | * Jason Lambert 181 | * Saberma 182 | * James Cook 183 | * Alexey Artamonov 184 | * Paul Gibler 185 | * Ngan Pham 186 | * Aaron Gilbralter 187 | * Steven Bristol 188 | * Ilia Lobsanov 189 | * Aleksey Morozov 190 | * S\. Christoffer Eliesen 191 | * Rob Golkosky 192 | * Darcy Brown 193 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'rubygems' 4 | require 'bundler' 5 | 6 | Bundler::GemHelper.install_tasks 7 | Bundler.require :default, :development 8 | 9 | require 'rspec/core/rake_task' 10 | 11 | RSpec::Core::RakeTask.new 12 | 13 | RSpec::Core::RakeTask.new(:rcov) do |spec| 14 | spec.rcov_opts = ['--exclude', 'spec', '--exclude', 'gems'] 15 | spec.rcov = true 16 | end 17 | 18 | YARD::Rake::YardocTask.new 19 | 20 | task :default => :spec 21 | -------------------------------------------------------------------------------- /bin/loadsphinx: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | version=$1 4 | engine=$2 5 | 6 | set -e 7 | 8 | load_sphinx () { 9 | distro="xenial" 10 | 11 | case $version in 12 | 2.1.9) 13 | url="http://sphinxsearch.com/files/sphinxsearch_2.1.9-release-0ubuntu11~trusty_amd64.deb" 14 | format="deb" 15 | distro="trusty";; 16 | 2.2.11) 17 | url="http://sphinxsearch.com/files/sphinxsearch_2.2.11-release-1~jessie_amd64.deb" 18 | format="deb";; 19 | 3.0.3) 20 | url="http://sphinxsearch.com/files/sphinx-3.0.3-facc3fb-linux-amd64.tar.gz" 21 | format="gz";; 22 | 3.1.1) 23 | url="http://sphinxsearch.com/files/sphinx-3.1.1-612d99f-linux-amd64.tar.gz" 24 | format="gz";; 25 | 3.2.1) 26 | url="http://sphinxsearch.com/files/sphinx-3.2.1-f152e0b-linux-amd64.tar.gz" 27 | format="gz";; 28 | 3.3.1) 29 | url="http://sphinxsearch.com/files/sphinx-3.3.1-b72d67b-linux-amd64.tar.gz" 30 | format="gz";; 31 | 3.4.1) 32 | url="http://sphinxsearch.com/files/sphinx-3.4.1-efbcc65-linux-amd64.tar.gz" 33 | format="gz";; 34 | *) 35 | echo "No Sphinx version $version available" 36 | exit 1;; 37 | esac 38 | 39 | if [ "$distro" == "trusty" ]; then 40 | curl --location http://launchpadlibrarian.net/247512886/libmysqlclient18_5.6.28-1ubuntu3_amd64.deb -o libmysql.deb 41 | sudo apt-get install ./libmysql.deb 42 | fi 43 | 44 | if [ "$format" == "deb" ]; then 45 | curl --location $url -o sphinx.deb 46 | sudo apt-get install libodbc1 47 | sudo dpkg -i ./sphinx.deb 48 | sudo apt-get install -f 49 | else 50 | curl $url -o sphinx.tar.gz 51 | tar -zxvf sphinx.tar.gz 52 | sudo mv sphinx-$version/bin/* /usr/local/bin/. 53 | fi 54 | } 55 | 56 | load_manticore () { 57 | url="https://github.com/manticoresoftware/manticore/releases/download/$version/manticore_$version.deb" 58 | 59 | case $version in 60 | 2.6.4) 61 | url="https://github.com/manticoresoftware/manticoresearch/releases/download/2.6.4/manticore_2.6.4-180503-37308c3-release-stemmer.xenial_amd64-bin.deb";; 62 | 2.7.5) 63 | url="https://github.com/manticoresoftware/manticoresearch/releases/download/2.7.5/manticore_2.7.5-181204-4a31c54-release-stemmer.xenial_amd64-bin.deb";; 64 | 2.8.2) 65 | url="https://github.com/manticoresoftware/manticoresearch/releases/download/2.8.2/manticore_2.8.2-190402-4e81114d-release-stemmer.stretch_amd64-bin.deb";; 66 | 3.4.2) 67 | url="https://github.com/manticoresoftware/manticoresearch/releases/download/3.4.2/manticore_3.4.2-200410-6903305-release.xenial_amd64-bin.deb";; 68 | 3.5.4) 69 | url="https://repo.manticoresearch.com/repository/manticoresearch_buster/dists/buster/main/binary-amd64/manticore_3.5.4-210107-f70faec5_amd64.deb";; 70 | 4.0.2) 71 | url="https://repo.manticoresearch.com/repository/manticoresearch_buster/dists/buster/main/binary-amd64/manticore_4.0.2-210921-af497f245_amd64.deb";; 72 | *) 73 | echo "No Manticore version $version available" 74 | exit 1;; 75 | esac 76 | 77 | curl --location $url -o manticore.deb 78 | sudo dpkg -i ./manticore.deb 79 | sudo apt-get install -f 80 | } 81 | 82 | if [ "$engine" == "sphinx" ]; then 83 | load_sphinx 84 | else 85 | load_manticore 86 | fi 87 | -------------------------------------------------------------------------------- /lib/riddle.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'date' 4 | require 'thread' 5 | require 'socket' 6 | require 'stringio' 7 | require 'timeout' 8 | 9 | module Riddle #:nodoc: 10 | @@mutex = Mutex.new 11 | @@escape_pattern = /[\(\)\|\-!@~"&\/\?]/ 12 | @@use_encoding = defined?(::Encoding) && 13 | ::Encoding.respond_to?(:default_external) 14 | 15 | class ConnectionError < StandardError #:nodoc: 16 | # 17 | end 18 | 19 | def self.encode(data, encoding = @@use_encoding && ::Encoding.default_external) 20 | if @@use_encoding 21 | data.force_encoding(encoding) 22 | else 23 | data 24 | end 25 | end 26 | 27 | def self.mutex 28 | @@mutex 29 | end 30 | 31 | def self.escape_pattern 32 | @@escape_pattern 33 | end 34 | 35 | def self.escape_pattern=(pattern) 36 | mutex.synchronize do 37 | @@escape_pattern = pattern 38 | end 39 | end 40 | 41 | def self.escape(string) 42 | string.gsub(escape_pattern) { |char| "\\#{char}" } 43 | end 44 | 45 | def self.loaded_version 46 | @@sphinx_version 47 | end 48 | 49 | def self.loaded_version=(version) 50 | @@sphinx_version = version 51 | end 52 | 53 | def self.version_warning 54 | return if loaded_version 55 | 56 | STDERR.puts %Q{ 57 | Riddle cannot detect Sphinx on your machine, and so can't determine which 58 | version of Sphinx you are planning on using. Please use one of the following 59 | lines after "require 'riddle'" to avoid this warning. 60 | 61 | require 'riddle/0.9.8' 62 | # or 63 | require 'riddle/0.9.9' 64 | # or 65 | require 'riddle/1.10' 66 | 67 | } 68 | end 69 | 70 | end 71 | 72 | require 'riddle/auto_version' 73 | require 'riddle/client' 74 | require 'riddle/command_failed_error' 75 | require 'riddle/command_result' 76 | require 'riddle/configuration' 77 | require 'riddle/controller' 78 | require 'riddle/execute_command' 79 | require 'riddle/query' 80 | 81 | Riddle.loaded_version = nil 82 | Riddle::AutoVersion.configure 83 | -------------------------------------------------------------------------------- /lib/riddle/0.9.8.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | Riddle.loaded_version = '0.9.8' 4 | -------------------------------------------------------------------------------- /lib/riddle/0.9.9.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | Riddle.loaded_version = '0.9.9' 4 | 5 | require 'riddle/0.9.9/client' 6 | require 'riddle/0.9.9/client/filter' 7 | require 'riddle/0.9.9/configuration/searchd' 8 | 9 | Riddle.escape_pattern = /[\(\)\|\-!@~"&\/\\\^\$=]/ 10 | -------------------------------------------------------------------------------- /lib/riddle/0.9.9/client.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | Riddle::Client::Versions[:search] = 0x116 4 | Riddle::Client::Versions[:update] = 0x102 5 | 6 | class Riddle::Client 7 | private 8 | 9 | def initialise_connection(available_server) 10 | socket = initialise_socket(available_server) 11 | 12 | # Send version 13 | socket.send [1].pack('N'), 0 14 | 15 | # Checking version 16 | version = socket.recv(4).unpack('N*').first 17 | if version < 1 18 | socket.close 19 | raise Riddle::VersionError, "Can only connect to searchd version 1.0 or better, not version #{version}" 20 | end 21 | 22 | socket 23 | end 24 | 25 | def update_message(index, attributes, values_by_doc) 26 | message = Message.new 27 | 28 | message.append_string index 29 | message.append_int attributes.length 30 | attributes.each_with_index do |attribute, index| 31 | message.append_string attribute 32 | message.append_boolean values_by_doc.values.first[index].is_a?(Array) 33 | end 34 | 35 | message.append_int values_by_doc.length 36 | values_by_doc.each do |key,values| 37 | message.append_64bit_int key # document ID 38 | values.each do |value| 39 | case value 40 | when Array 41 | message.append_int value.length 42 | message.append_ints *value 43 | else 44 | message.append_int value 45 | end 46 | end 47 | end 48 | 49 | message.to_s 50 | end 51 | end 52 | -------------------------------------------------------------------------------- /lib/riddle/0.9.9/client/filter.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Riddle::Client::Filter 4 | # 5 | 6 | private 7 | 8 | def append_integer_range(message, range) 9 | message.append_64bit_ints self.values.first, self.values.last 10 | end 11 | 12 | def append_array(message, array) 13 | message.append_64bit_ints *array.collect { |val| 14 | case val 15 | when TrueClass 16 | 1 17 | when FalseClass 18 | 0 19 | else 20 | val 21 | end 22 | } 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /lib/riddle/0.9.9/configuration/searchd.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Riddle 4 | class Configuration 5 | class Searchd 6 | NUMBER = 1.class 7 | 8 | def valid? 9 | set_listen 10 | 11 | !( @listen.nil? || @listen.empty? || @pid_file.nil? ) 12 | end 13 | 14 | private 15 | 16 | def set_listen 17 | @listen = @listen.to_s if @listen.is_a?(NUMBER) 18 | 19 | return unless @listen.nil? || @listen.empty? 20 | 21 | @listen = [] 22 | @listen << @port.to_s if @port 23 | @listen << "9306:mysql41" if @mysql41.is_a?(TrueClass) 24 | @listen << "#{@mysql41}:mysql41" if @mysql41.is_a?(NUMBER) 25 | 26 | if @listen.empty? && @address 27 | @listen << @address 28 | else 29 | @listen = @listen.collect { |line| "#{@address}:#{line}" } if @address 30 | end 31 | 32 | @listen += Array(@socket) if @socket 33 | end 34 | 35 | def settings 36 | @listen.nil? ? super : super - [:address, :port] 37 | end 38 | end 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /lib/riddle/1.10.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'riddle/0.9.9' 4 | 5 | Riddle.loaded_version = '1.10' 6 | 7 | require 'riddle/1.10/client' -------------------------------------------------------------------------------- /lib/riddle/1.10/client.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | Riddle::Client::Versions[:search] = 0x117 4 | Riddle::Client::Versions[:excerpt] = 0x102 5 | 6 | class Riddle::Client 7 | private 8 | 9 | # Generation of the message to send to Sphinx for an excerpts request. 10 | def excerpts_message(options) 11 | message = Message.new 12 | 13 | message.append [0, excerpt_flags(options)].pack('N2') # 0 = mode 14 | message.append_string options[:index] 15 | message.append_string options[:words] 16 | 17 | # options 18 | message.append_string options[:before_match] 19 | message.append_string options[:after_match] 20 | message.append_string options[:chunk_separator] 21 | message.append_ints options[:limit], options[:around] 22 | message.append_ints options[:limit_passages], options[:limit_words] 23 | message.append_ints options[:start_passage_id] 24 | message.append_string options[:html_strip_mode] 25 | 26 | if Versions[:excerpt] >= 0x103 27 | message.append_string options[:passage_boundary] 28 | end 29 | 30 | message.append_array options[:docs] 31 | 32 | message.to_s 33 | end 34 | end -------------------------------------------------------------------------------- /lib/riddle/2.0.1.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'riddle/0.9.9' 4 | require 'riddle/1.10' 5 | 6 | Riddle.loaded_version = '2.0.1' 7 | 8 | require 'riddle/2.0.1/client' 9 | -------------------------------------------------------------------------------- /lib/riddle/2.0.1/client.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | Riddle::Client::Versions[:search] = 0x118 4 | Riddle::Client::Versions[:excerpt] = 0x103 -------------------------------------------------------------------------------- /lib/riddle/2.1.0.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'riddle/0.9.9' 4 | require 'riddle/1.10' 5 | require 'riddle/2.0.1' 6 | 7 | Riddle.loaded_version = '2.1.0' 8 | 9 | Riddle::Client::Versions[:search] = 0x119 10 | Riddle::Client::Versions[:excerpt] = 0x104 11 | 12 | Riddle::Client::RankModes[:expr] = 8 13 | Riddle::Client::RankModes[:total] = 9 14 | 15 | Riddle::Client::AttributeTypes[:multi] = 0x40000001 16 | Riddle::Client::AttributeTypes[:multi_64] = 0x40000002 17 | 18 | Riddle::Client::AttributeHandlers[Riddle::Client::AttributeTypes[:multi_64]] = :next_64bit_int_array 19 | -------------------------------------------------------------------------------- /lib/riddle/auto_version.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Riddle::AutoVersion 4 | def self.configure 5 | controller = Riddle::Controller.new nil, '' 6 | version = ENV['SPHINX_VERSION'] || controller.sphinx_version 7 | 8 | case version 9 | when '0.9.8', '0.9.9' 10 | require "riddle/#{version}" 11 | when /1.10/ 12 | require 'riddle/1.10' 13 | when /2.0.[12]/ 14 | require 'riddle/2.0.1' 15 | else 16 | require 'riddle/2.1.0' 17 | end 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /lib/riddle/client/filter.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Riddle 4 | class Client 5 | class Filter 6 | attr_accessor :attribute, :values, :exclude 7 | 8 | # Attribute name, values (which can be an array or a range), and whether 9 | # the filter should be exclusive. 10 | def initialize(attribute, values, exclude=false) 11 | @attribute, @values, @exclude = attribute, values, exclude 12 | end 13 | 14 | def exclude? 15 | self.exclude 16 | end 17 | 18 | # Returns the message for this filter to send to the Sphinx service 19 | def query_message 20 | message = Message.new 21 | 22 | message.append_string self.attribute.to_s 23 | case self.values 24 | when Range 25 | if self.values.first.is_a?(Float) && self.values.last.is_a?(Float) 26 | message.append_int FilterTypes[:float_range] 27 | message.append_floats self.values.first, self.values.last 28 | else 29 | message.append_int FilterTypes[:range] 30 | append_integer_range message, self.values 31 | end 32 | when Array 33 | if self.values.first.is_a?(Float) && self.values.length == 1 34 | message.append_int FilterTypes[:float_range] 35 | message.append_floats self.values.first, self.values.first 36 | else 37 | message.append_int FilterTypes[:values] 38 | message.append_int self.values.length 39 | append_array message, self.values 40 | end 41 | end 42 | message.append_int self.exclude? ? 1 : 0 43 | 44 | message.to_s 45 | end 46 | 47 | private 48 | 49 | def append_integer_range(message, range) 50 | message.append_ints self.values.first, self.values.last 51 | end 52 | 53 | # Using to_f is a hack from the PHP client - to workaround 32bit signed 54 | # ints on x32 platforms 55 | def append_array(message, array) 56 | message.append_ints *array.collect { |val| 57 | case val 58 | when TrueClass 59 | 1.0 60 | when FalseClass 61 | 0.0 62 | else 63 | val.to_f 64 | end 65 | } 66 | end 67 | end 68 | end 69 | end 70 | -------------------------------------------------------------------------------- /lib/riddle/client/message.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Riddle 4 | class Client 5 | # This class takes care of the translation of ints, strings and arrays to 6 | # the format required by the Sphinx service. 7 | class Message 8 | def initialize 9 | @message = StringIO.new String.new(""), "w" 10 | @message.set_encoding 'ASCII-8BIT' 11 | @size_method = @message.respond_to?(:bytesize) ? :bytesize : :length 12 | end 13 | 14 | # Append raw data (only use if you know what you're doing) 15 | def append(*args) 16 | args.each { |arg| @message << arg } 17 | end 18 | 19 | # Append a string's length, then the string itself 20 | def append_string(str) 21 | string = Riddle.encode(str.dup, 'ASCII-8BIT') 22 | @message << [string.send(@size_method)].pack('N') + string 23 | end 24 | 25 | # Append an integer 26 | def append_int(int) 27 | @message << [int.to_i].pack('N') 28 | end 29 | 30 | def append_64bit_int(int) 31 | @message << [int.to_i >> 32, int.to_i & 0xFFFFFFFF].pack('NN') 32 | end 33 | 34 | # Append a float 35 | def append_float(float) 36 | @message << [float].pack('f').unpack('L*').pack("N") 37 | end 38 | 39 | def append_boolean(bool) 40 | append_int(bool ? 1 : 0) 41 | end 42 | 43 | # Append multiple integers 44 | def append_ints(*ints) 45 | ints.each { |int| append_int(int) } 46 | end 47 | 48 | def append_64bit_ints(*ints) 49 | ints.each { |int| append_64bit_int(int) } 50 | end 51 | 52 | # Append multiple floats 53 | def append_floats(*floats) 54 | floats.each { |float| append_float(float) } 55 | end 56 | 57 | # Append an array of strings - first appends the length of the array, 58 | # then each item's length and value. 59 | def append_array(array) 60 | append_int(array.length) 61 | 62 | array.each { |item| append_string(item) } 63 | end 64 | 65 | # Returns the entire message 66 | def to_s 67 | @message.string 68 | end 69 | end 70 | end 71 | end 72 | -------------------------------------------------------------------------------- /lib/riddle/client/response.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Riddle 4 | class Client 5 | # Used to interrogate responses from the Sphinx daemon. Keep in mind none 6 | # of the methods here check whether the data they're grabbing are what the 7 | # user expects - it just assumes the user knows what the data stream is 8 | # made up of. 9 | class Response 10 | # Create with the data to interpret 11 | def initialize(str) 12 | @str = str 13 | @marker = 0 14 | end 15 | 16 | # Return the next string value in the stream 17 | def next 18 | len = next_int 19 | result = @str[@marker, len] 20 | @marker += len 21 | 22 | Riddle.encode(result) 23 | end 24 | 25 | # Return the next integer value from the stream 26 | def next_int 27 | int = @str[@marker, 4].unpack('N*').first 28 | @marker += 4 29 | 30 | int 31 | end 32 | 33 | def next_64bit_int 34 | high, low = @str[@marker, 8].unpack('N*N*')[0..1] 35 | @marker += 8 36 | 37 | (high << 32) + low 38 | end 39 | 40 | # Return the next float value from the stream 41 | def next_float 42 | float = @str[@marker, 4].unpack('N*').pack('L').unpack('f*').first 43 | @marker += 4 44 | 45 | float 46 | end 47 | 48 | # Returns an array of string items 49 | def next_array 50 | count = next_int 51 | items = [] 52 | count.times do 53 | items << self.next 54 | end 55 | 56 | items 57 | end 58 | 59 | # Returns an array of int items 60 | def next_int_array 61 | count = next_int 62 | items = [] 63 | count.times do 64 | items << self.next_int 65 | end 66 | 67 | items 68 | end 69 | 70 | def next_float_array 71 | count = next_int 72 | items = [] 73 | count.times do 74 | items << self.next_float 75 | end 76 | 77 | items 78 | end 79 | 80 | def next_64bit_int_array 81 | byte_count = next_int 82 | items = [] 83 | (byte_count / 2).times do 84 | items << self.next_64bit_int 85 | end 86 | 87 | items 88 | end 89 | 90 | # Returns the length of the streamed data 91 | def length 92 | @str.length 93 | end 94 | end 95 | end 96 | end 97 | -------------------------------------------------------------------------------- /lib/riddle/command_failed_error.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Riddle::CommandFailedError < StandardError 4 | attr_accessor :command_result 5 | end 6 | -------------------------------------------------------------------------------- /lib/riddle/command_result.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Riddle::CommandResult 4 | attr_reader :command, :status, :output 5 | attr_accessor :successful 6 | 7 | def initialize(command, status, output = nil, successful = nil) 8 | @command, @status, @output = command, status, output 9 | 10 | if successful.nil? 11 | @successful = (@status == 0) 12 | else 13 | @successful = successful 14 | end 15 | end 16 | end 17 | -------------------------------------------------------------------------------- /lib/riddle/configuration.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'riddle/configuration/section' 4 | require 'riddle/configuration/index_settings' 5 | 6 | require 'riddle/configuration/common' 7 | require 'riddle/configuration/distributed_index' 8 | require 'riddle/configuration/index' 9 | require 'riddle/configuration/indexer' 10 | require 'riddle/configuration/realtime_index' 11 | require 'riddle/configuration/remote_index' 12 | require 'riddle/configuration/searchd' 13 | require 'riddle/configuration/source' 14 | require 'riddle/configuration/sql_source' 15 | require 'riddle/configuration/template_index' 16 | require 'riddle/configuration/tsv_source' 17 | require 'riddle/configuration/xml_source' 18 | 19 | require 'riddle/configuration/parser' 20 | 21 | module Riddle 22 | class Configuration 23 | class ConfigurationError < StandardError #:nodoc: 24 | end 25 | 26 | attr_reader :common, :indices, :searchd, :sources 27 | attr_accessor :indexer 28 | 29 | def self.parse!(input) 30 | Riddle::Configuration::Parser.new(input).parse! 31 | end 32 | 33 | def initialize 34 | @common = Riddle::Configuration::Common.new 35 | @indexer = Riddle::Configuration::Indexer.new 36 | @searchd = Riddle::Configuration::Searchd.new 37 | @indices = [] 38 | @sources = [] 39 | end 40 | 41 | def render 42 | ( 43 | [@common.render, @indexer.render, @searchd.render] + 44 | @sources.collect { |source| source.render } + 45 | @indices.collect { |index| index.render } 46 | ).join("\n") 47 | end 48 | end 49 | end 50 | -------------------------------------------------------------------------------- /lib/riddle/configuration/common.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Riddle 4 | class Configuration 5 | class Common < Riddle::Configuration::Section 6 | def self.settings 7 | [ 8 | :lemmatizer_base, :json_autoconv_numbers, :json_autoconv_keynames, 9 | :on_json_attr_error, :rlp_root, :rlp_environment, :rlp_max_batch_size, 10 | :rlp_max_batch_docs, :plugin_dir 11 | ] 12 | end 13 | 14 | attr_accessor :common_sphinx_configuration, *settings 15 | 16 | def render 17 | return unless common_sphinx_configuration 18 | raise ConfigurationError unless valid? 19 | 20 | ( 21 | ["common", "{"] + 22 | settings_body + 23 | ["}", ""] 24 | ).join("\n") 25 | end 26 | end 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /lib/riddle/configuration/distributed_index.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Riddle 4 | class Configuration 5 | class DistributedIndex < Riddle::Configuration::Section 6 | def self.settings 7 | [ 8 | :type, :local, :agent, :agent_blackhole, 9 | :agent_connect_timeout, :agent_query_timeout 10 | ] 11 | end 12 | 13 | attr_accessor :name, :local_indices, :remote_indices, :agent_blackhole, 14 | :agent_connect_timeout, :agent_query_timeout 15 | 16 | def initialize(name) 17 | @name = name 18 | @local_indices = [] 19 | @remote_indices = [] 20 | @agent_blackhole = [] 21 | end 22 | 23 | def type 24 | "distributed" 25 | end 26 | 27 | def local 28 | self.local_indices 29 | end 30 | 31 | def agent 32 | agents = remote_indices.collect { |index| index.remote }.uniq 33 | agents.collect { |agent| 34 | agent + ":" + remote_indices.select { |index| 35 | index.remote == agent 36 | }.collect { |index| index.name }.join(",") 37 | } 38 | end 39 | 40 | def render 41 | raise ConfigurationError unless valid? 42 | 43 | ( 44 | ["index #{name}", "{"] + 45 | settings_body + 46 | ["}", ""] 47 | ).join("\n") 48 | end 49 | 50 | def valid? 51 | @local_indices.length > 0 || @remote_indices.length > 0 52 | end 53 | end 54 | end 55 | end 56 | -------------------------------------------------------------------------------- /lib/riddle/configuration/index.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Riddle 4 | class Configuration 5 | class Index < Riddle::Configuration::Section 6 | include Riddle::Configuration::IndexSettings 7 | 8 | def self.settings 9 | Riddle::Configuration::IndexSettings.settings + [:source] 10 | end 11 | 12 | attr_accessor :parent, :sources 13 | 14 | def initialize(name, *sources) 15 | @name = name 16 | @sources = sources 17 | 18 | initialize_settings 19 | end 20 | 21 | def source 22 | @sources.collect { |s| s.name } 23 | end 24 | 25 | def render 26 | raise ConfigurationError, "#{@name} #{@sources.inspect} #{@path} #{@parent}" unless valid? 27 | 28 | inherited_name = parent ? "#{name} : #{parent}" : "#{name}" 29 | ( 30 | @sources.collect { |s| s.render } + 31 | ["index #{inherited_name}", "{"] + 32 | settings_body + 33 | ["}", ""] 34 | ).join("\n") 35 | end 36 | 37 | def valid? 38 | (!@name.nil?) && (!( @sources.length == 0 || @path.nil? ) || !@parent.nil?) 39 | end 40 | end 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /lib/riddle/configuration/index_settings.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Riddle 4 | class Configuration 5 | module IndexSettings 6 | def self.settings 7 | [ 8 | :type, :path, :docinfo, :mlock, :morphology, 9 | :dict, :index_sp, :index_zones, :min_stemming_len, :stopwords, 10 | :wordforms, :exceptions, :min_word_len, :charset_dictpath, 11 | :charset_type, :charset_table, :ignore_chars, :min_prefix_len, 12 | :min_infix_len, :prefix_fields, :infix_fields, :enable_star, 13 | :expand_keywords, :ngram_len, :ngram_chars, :phrase_boundary, 14 | :phrase_boundary_step, :blend_chars, :blend_mode, :html_strip, 15 | :html_index_attrs, :html_remove_elements, :preopen, :ondisk_dict, 16 | :inplace_enable, :inplace_hit_gap, :inplace_docinfo_gap, 17 | :inplace_reloc_factor, :inplace_write_factor, :index_exact_words, 18 | :overshort_step, :stopword_step, :hitless_words, :ha_strategy, 19 | :bigram_freq_words, :bigram_index, :index_field_lengths, 20 | :regexp_filter, :stopwords_unstemmed, :global_idf, :rlp_context, 21 | :ondisk_attrs 22 | ] 23 | end 24 | 25 | attr_accessor :name, :type, :path, :docinfo, :mlock, 26 | :morphologies, :dict, :index_sp, :index_zones, :min_stemming_len, 27 | :stopword_files, :wordform_files, :exception_files, :min_word_len, 28 | :charset_dictpath, :charset_type, :charset_table, :ignore_characters, 29 | :min_prefix_len, :min_infix_len, :prefix_field_names, 30 | :infix_field_names, :enable_star, :expand_keywords, :ngram_len, 31 | :ngram_characters, :phrase_boundaries, :phrase_boundary_step, 32 | :blend_chars, :blend_mode, :html_strip, :html_index_attrs, 33 | :html_remove_element_tags, :preopen, :ondisk_dict, :inplace_enable, 34 | :inplace_hit_gap, :inplace_docinfo_gap, :inplace_reloc_factor, 35 | :inplace_write_factor, :index_exact_words, :overshort_step, 36 | :stopword_step, :hitless_words, :ha_strategy, :bigram_freq_words, 37 | :bigram_index, :index_field_lengths, :regexp_filter, 38 | :stopwords_unstemmed, :global_idf, :rlp_context, :ondisk_attrs 39 | 40 | def initialize_settings 41 | @morphologies = [] 42 | @stopword_files = [] 43 | @wordform_files = [] 44 | @exception_files = [] 45 | @ignore_characters = [] 46 | @prefix_field_names = [] 47 | @infix_field_names = [] 48 | @ngram_characters = [] 49 | @phrase_boundaries = [] 50 | @html_remove_element_tags = [] 51 | @regexp_filter = [] 52 | end 53 | 54 | def morphology 55 | nil_join @morphologies, ", " 56 | end 57 | 58 | def morphology=(morphology) 59 | @morphologies = nil_split morphology, /,\s?/ 60 | end 61 | 62 | def stopwords 63 | nil_join @stopword_files, " " 64 | end 65 | 66 | def stopwords=(stopwords) 67 | @stopword_files = nil_split stopwords, ' ' 68 | end 69 | 70 | def wordforms 71 | nil_join @wordform_files, " " 72 | end 73 | 74 | def wordforms=(wordforms) 75 | @wordform_files = nil_split wordforms, ' ' 76 | end 77 | 78 | def exceptions 79 | nil_join @exception_files, " " 80 | end 81 | 82 | def exceptions=(exceptions) 83 | @exception_files = nil_split exceptions, ' ' 84 | end 85 | 86 | def ignore_chars 87 | nil_join @ignore_characters, ", " 88 | end 89 | 90 | def ignore_chars=(ignore_chars) 91 | @ignore_characters = nil_split ignore_chars, /,\s?/ 92 | end 93 | 94 | def prefix_fields 95 | nil_join @prefix_field_names, ", " 96 | end 97 | 98 | def prefix_fields=(fields) 99 | if fields.is_a?(Array) 100 | @prefix_field_names = fields 101 | else 102 | @prefix_field_names = fields.split(/,\s*/) 103 | end 104 | end 105 | 106 | def infix_fields 107 | nil_join @infix_field_names, ", " 108 | end 109 | 110 | def infix_fields=(fields) 111 | if fields.is_a?(Array) 112 | @infix_field_names = fields 113 | else 114 | @infix_field_names = fields.split(/,\s*/) 115 | end 116 | end 117 | 118 | def ngram_chars 119 | nil_join @ngram_characters, ", " 120 | end 121 | 122 | def ngram_chars=(ngram_chars) 123 | @ngram_characters = nil_split ngram_chars, /,\s?/ 124 | end 125 | 126 | def phrase_boundary 127 | nil_join @phrase_boundaries, ", " 128 | end 129 | 130 | def phrase_boundary=(phrase_boundary) 131 | @phrase_boundaries = nil_split phrase_boundary, /,\s?/ 132 | end 133 | 134 | def html_remove_elements 135 | nil_join @html_remove_element_tags, ", " 136 | end 137 | 138 | def html_remove_elements=(html_remove_elements) 139 | @html_remove_element_tags = nil_split html_remove_elements, /,\s?/ 140 | end 141 | 142 | private 143 | 144 | def nil_split(string, pattern) 145 | (string || "").split(pattern) 146 | end 147 | 148 | def nil_join(array, delimiter) 149 | if array.length == 0 150 | nil 151 | else 152 | array.join(delimiter) 153 | end 154 | end 155 | end 156 | end 157 | end 158 | -------------------------------------------------------------------------------- /lib/riddle/configuration/indexer.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Riddle 4 | class Configuration 5 | class Indexer < Riddle::Configuration::Section 6 | def self.settings 7 | [ 8 | :mem_limit, :max_iops, :max_iosize, :max_xmlpipe2_field, 9 | :write_buffer, :max_file_field_buffer, :on_file_field_error, 10 | :lemmatizer_cache 11 | ] + shared_settings 12 | end 13 | 14 | def self.shared_settings 15 | [ 16 | :lemmatizer_base, :json_autoconv_numbers, :json_autoconv_keynames, 17 | :on_json_attr_error, :rlp_root, :rlp_environment, :rlp_max_batch_size, 18 | :rlp_max_batch_docs 19 | ] 20 | end 21 | 22 | attr_accessor :common_sphinx_configuration, *settings 23 | 24 | def render 25 | raise ConfigurationError unless valid? 26 | 27 | ( 28 | ["indexer", "{"] + 29 | settings_body + 30 | ["}", ""] 31 | ).join("\n") 32 | end 33 | 34 | 35 | private 36 | 37 | def settings 38 | settings = self.class.settings 39 | settings -= self.class.shared_settings if common_sphinx_configuration 40 | settings 41 | end 42 | end 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /lib/riddle/configuration/parser.rb: -------------------------------------------------------------------------------- 1 | # encoding: UTF-8 2 | # frozen_string_literal: true 3 | 4 | require 'stringio' 5 | 6 | class Riddle::Configuration::Parser 7 | SOURCE_CLASSES = { 8 | 'mysql' => Riddle::Configuration::SQLSource, 9 | 'pgsql' => Riddle::Configuration::SQLSource, 10 | 'mssql' => Riddle::Configuration::SQLSource, 11 | 'xmlpipe' => Riddle::Configuration::XMLSource, 12 | 'xmlpipe2' => Riddle::Configuration::XMLSource, 13 | 'odbc' => Riddle::Configuration::SQLSource, 14 | 'tsvpipe' => Riddle::Configuration::TSVSource 15 | } 16 | 17 | INDEX_CLASSES = { 18 | 'plain' => Riddle::Configuration::Index, 19 | 'distributed' => Riddle::Configuration::DistributedIndex, 20 | 'rt' => Riddle::Configuration::RealtimeIndex, 21 | 'template' => Riddle::Configuration::TemplateIndex 22 | } 23 | 24 | def initialize(input) 25 | @input = input 26 | end 27 | 28 | def parse! 29 | set_common 30 | set_indexer 31 | set_searchd 32 | set_sources 33 | set_indices 34 | 35 | add_orphan_sources 36 | 37 | configuration 38 | end 39 | 40 | private 41 | 42 | def add_orphan_sources 43 | all_names = sources.keys 44 | attached_names = configuration.indices.collect { |index| 45 | index.respond_to?(:sources) ? index.sources.collect(&:name) : [] 46 | }.flatten 47 | 48 | (all_names - attached_names).each do |name| 49 | configuration.sources << sources[name] 50 | end 51 | end 52 | 53 | def inner 54 | @inner ||= InnerParser.new(@input).parse! 55 | end 56 | 57 | def configuration 58 | @configuration ||= Riddle::Configuration.new 59 | end 60 | 61 | def sources 62 | @sources ||= {} 63 | end 64 | 65 | def each_with_prefix(prefix) 66 | inner.keys.select { |key| key[/^#{prefix}\s+/] }.each do |key| 67 | yield key.gsub(/^#{prefix}\s+/, '').gsub(/\s*\{$/, ''), inner[key] 68 | end 69 | end 70 | 71 | def set_common 72 | if inner['common'] && inner['common'].values.compact.any? 73 | configuration.common.common_sphinx_configuration = true 74 | end 75 | 76 | set_settings configuration.common, inner['common'] || {} 77 | end 78 | 79 | def set_indexer 80 | set_settings configuration.indexer, inner['indexer'] || {} 81 | end 82 | 83 | def set_searchd 84 | set_settings configuration.searchd, inner['searchd'] || {} 85 | end 86 | 87 | def set_sources 88 | each_with_prefix 'source' do |name, settings| 89 | names = name.split(/\s*:\s*/) 90 | types = settings.delete('type') 91 | parent = names.length > 1 ? names.last : nil 92 | types ||= [sources[parent].type] if parent 93 | type = types.first 94 | 95 | source = SOURCE_CLASSES[type].new names.first, type 96 | source.parent = parent 97 | 98 | set_settings source, settings 99 | 100 | sources[source.name] = source 101 | end 102 | end 103 | 104 | def set_indices 105 | each_with_prefix 'index' do |name, settings| 106 | names = name.split(/\s*:\s*/) 107 | type = (settings.delete('type') || ['plain']).first 108 | index = INDEX_CLASSES[type].new names.first 109 | index.parent = names.last if names.length > 1 110 | 111 | (settings.delete('source') || []).each do |source_name| 112 | index.sources << sources[source_name] 113 | end 114 | 115 | set_settings index, settings 116 | 117 | configuration.indices << index 118 | end 119 | end 120 | 121 | def set_settings(object, hash) 122 | hash.each do |key, values| 123 | values.each do |value| 124 | set_setting object, key, value 125 | end 126 | end 127 | end 128 | 129 | def set_setting(object, key, value) 130 | if object.send(key).is_a?(Array) 131 | object.send(key) << value 132 | else 133 | object.send "#{key}=", value 134 | end 135 | end 136 | 137 | class InnerParser 138 | SETTING_PATTERN = /^(\w+)\s*=\s*(.*)$/ 139 | 140 | EndOfFileError = Class.new StandardError 141 | 142 | def initialize(input) 143 | @stream = StringIO.new(input.gsub("\\\n", '')) 144 | @sections = {} 145 | end 146 | 147 | def parse! 148 | while label = next_line do 149 | @sections[label] = next_settings 150 | end 151 | 152 | @sections 153 | rescue EndOfFileError 154 | @sections 155 | end 156 | 157 | private 158 | 159 | def next_line 160 | line = @stream.gets 161 | raise EndOfFileError if line.nil? 162 | 163 | line = line.strip 164 | (line.empty? || line[/^#/]) ? next_line : line 165 | end 166 | 167 | def next_settings 168 | settings = Hash.new { |hash, key| hash[key] = [] } 169 | line = '' 170 | while line.empty? || line == '{' do 171 | line = next_line 172 | end 173 | 174 | while line != '}' do 175 | begin 176 | match = SETTING_PATTERN.match(line) 177 | unless match.nil? 178 | key, value = *match.captures 179 | settings[key] << value 180 | while value[/\\$/] do 181 | value = next_line 182 | settings[key].last << "\n" << value 183 | end 184 | end 185 | rescue => error 186 | raise error, "Error handling line '#{line}': #{error.message}", 187 | error.backtrace 188 | end 189 | 190 | line = next_line 191 | end 192 | 193 | settings 194 | end 195 | end 196 | end 197 | -------------------------------------------------------------------------------- /lib/riddle/configuration/realtime_index.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Riddle 4 | class Configuration 5 | class RealtimeIndex < Riddle::Configuration::Section 6 | include Riddle::Configuration::IndexSettings 7 | 8 | def self.settings 9 | Riddle::Configuration::IndexSettings.settings + [ 10 | :rt_mem_limit, :rt_field, :rt_attr_uint, :rt_attr_bigint, 11 | :rt_attr_float, :rt_attr_timestamp, :rt_attr_string, :rt_attr_multi, 12 | :rt_attr_multi_64, :rt_attr_bool, :rt_attr_json 13 | ] 14 | end 15 | 16 | attr_accessor :rt_mem_limit, :rt_field, :rt_attr_uint, :rt_attr_bigint, 17 | :rt_attr_float, :rt_attr_timestamp, :rt_attr_string, :rt_attr_multi, 18 | :rt_attr_multi_64, :rt_attr_bool, :rt_attr_json 19 | 20 | def initialize(name) 21 | @name = name 22 | @rt_field = [] 23 | @rt_attr_uint = [] 24 | @rt_attr_bigint = [] 25 | @rt_attr_float = [] 26 | @rt_attr_timestamp = [] 27 | @rt_attr_string = [] 28 | @rt_attr_multi = [] 29 | @rt_attr_multi_64 = [] 30 | @rt_attr_bool = [] 31 | @rt_attr_json = [] 32 | 33 | initialize_settings 34 | end 35 | 36 | def type 37 | "rt" 38 | end 39 | 40 | def valid? 41 | !(@name.nil? || @path.nil?) 42 | end 43 | 44 | def render 45 | raise ConfigurationError unless valid? 46 | 47 | ( 48 | ["index #{name}", "{"] + 49 | settings_body + 50 | ["}", ""] 51 | ).join("\n") 52 | end 53 | end 54 | end 55 | end 56 | -------------------------------------------------------------------------------- /lib/riddle/configuration/remote_index.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Riddle 4 | class Configuration 5 | class RemoteIndex 6 | attr_accessor :address, :port, :name 7 | 8 | def initialize(address, port, name) 9 | @address = address 10 | @port = port 11 | @name = name 12 | end 13 | 14 | def remote 15 | "#{address}:#{port}" 16 | end 17 | end 18 | end 19 | end -------------------------------------------------------------------------------- /lib/riddle/configuration/searchd.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Riddle 4 | class Configuration 5 | class Searchd < Riddle::Configuration::Section 6 | def self.settings 7 | [ 8 | :listen, :address, :port, :log, :query_log, 9 | :query_log_format, :read_timeout, :client_timeout, :max_children, 10 | :pid_file, :max_matches, :seamless_rotate, :preopen_indexes, 11 | :unlink_old, :attr_flush_period, :ondisk_dict_default, 12 | :max_packet_size, :mva_updates_pool, :crash_log_path, :max_filters, 13 | :max_filter_values, :listen_backlog, :read_buffer, :read_unhinted, 14 | :max_batch_queries, :subtree_docs_cache, :subtree_hits_cache, 15 | :workers, :dist_threads, :binlog_path, :binlog_flush, 16 | :binlog_max_log_size, :snippets_file_prefix, :collation_server, 17 | :collation_libc_locale, :mysql_version_string, 18 | :rt_flush_period, :thread_stack, :expansion_limit, 19 | :compat_sphinxql_magics, :watchdog, :prefork_rotation_throttle, 20 | :sphinxql_state, :ha_ping_interval, :ha_period_karma, 21 | :persistent_connections_limit, :rt_merge_iops, :rt_merge_maxiosize, 22 | :predicted_time_costs, :snippets_file_prefix, :shutdown_timeout, 23 | :ondisk_attrs_default, :query_log_min_msec, :agent_connect_timeout, 24 | :agent_query_timeout, :agent_retry_count, :agenty_retry_delay, 25 | :client_key 26 | ] 27 | end 28 | 29 | attr_accessor *self.settings 30 | attr_accessor :mysql41, :socket 31 | 32 | def render 33 | raise ConfigurationError unless valid? 34 | 35 | ( 36 | ["searchd", "{"] + 37 | settings_body + 38 | ["}", ""] 39 | ).join("\n") 40 | end 41 | 42 | def valid? 43 | !( @port.nil? || @pid_file.nil? ) 44 | end 45 | end 46 | end 47 | end 48 | -------------------------------------------------------------------------------- /lib/riddle/configuration/section.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Riddle 4 | class Configuration 5 | class Section 6 | def self.settings 7 | [] 8 | end 9 | 10 | def valid? 11 | true 12 | end 13 | 14 | private 15 | 16 | def settings_body 17 | settings.select { |setting| 18 | !send(setting).nil? 19 | }.collect { |setting| 20 | if send(setting) == "" 21 | conf = " #{setting} = " 22 | else 23 | conf = setting_to_array(setting).collect { |set| 24 | " #{setting} = #{rendered_setting set}" 25 | } 26 | end 27 | conf.length == 0 ? nil : conf 28 | }.flatten.compact 29 | end 30 | 31 | def setting_to_array(setting) 32 | value = send(setting) 33 | case value 34 | when Array then value 35 | when TrueClass then [1] 36 | when FalseClass then [0] 37 | else 38 | [value] 39 | end 40 | end 41 | 42 | def rendered_setting(setting) 43 | return setting unless setting.is_a?(String) 44 | 45 | index = 8100 46 | output = String.new(setting) 47 | 48 | while index < output.length 49 | output.insert(index, "\\\n") 50 | index += 8100 51 | end 52 | 53 | output 54 | end 55 | 56 | def settings 57 | self.class.settings 58 | end 59 | end 60 | end 61 | end 62 | -------------------------------------------------------------------------------- /lib/riddle/configuration/source.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Riddle 4 | class Configuration 5 | class Source < Riddle::Configuration::Section 6 | attr_accessor :name, :parent, :type 7 | 8 | def render 9 | raise ConfigurationError unless valid? 10 | 11 | inherited_name = "#{name}" 12 | inherited_name += " : #{parent}" if parent 13 | ( 14 | ["source #{inherited_name}", "{"] + 15 | settings_body + 16 | ["}", ""] 17 | ).join("\n") 18 | end 19 | 20 | def valid? 21 | !( @name.nil? || @type.nil? ) 22 | end 23 | end 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /lib/riddle/configuration/sql_source.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Riddle 4 | class Configuration 5 | class SQLSource < Riddle::Configuration::Source 6 | def self.settings 7 | [ 8 | :type, :sql_host, :sql_user, :sql_pass, :sql_db, 9 | :sql_port, :sql_sock, :mysql_connect_flags, :mysql_ssl_cert, 10 | :mysql_ssl_key, :mysql_ssl_ca, :odbc_dsn, :sql_query_pre, :sql_query, 11 | :sql_joined_field, :sql_file_field, :sql_query_range, 12 | :sql_range_step, :sql_query_killlist, :sql_attr_uint, :sql_attr_bool, 13 | :sql_attr_bigint, :sql_attr_timestamp, :sql_attr_str2ordinal, 14 | :sql_attr_float, :sql_attr_multi, :sql_attr_string, 15 | :sql_attr_str2wordcount, :sql_attr_json, 16 | :sql_column_buffers, :sql_field_string, :sql_field_str2wordcount, 17 | :sql_query_post, :sql_query_post_index, :sql_ranged_throttle, 18 | :sql_query_info, :mssql_winauth, :mssql_unicode, :unpack_zlib, 19 | :unpack_mysqlcompress, :unpack_mysqlcompress_maxsize 20 | ] 21 | end 22 | 23 | attr_accessor *self.settings 24 | 25 | def initialize(name, type) 26 | @name = name 27 | @type = type 28 | 29 | @sql_query_pre = [] 30 | @sql_joined_field = [] 31 | @sql_file_field = [] 32 | @sql_attr_uint = [] 33 | @sql_attr_bool = [] 34 | @sql_attr_bigint = [] 35 | @sql_attr_timestamp = [] 36 | @sql_attr_str2ordinal = [] 37 | @sql_attr_float = [] 38 | @sql_attr_multi = [] 39 | @sql_attr_string = [] 40 | @sql_attr_str2wordcount = [] 41 | @sql_attr_json = [] 42 | @sql_field_string = [] 43 | @sql_field_str2wordcount = [] 44 | @sql_query_post = [] 45 | @sql_query_post_index = [] 46 | @unpack_zlib = [] 47 | @unpack_mysqlcompress = [] 48 | end 49 | 50 | def valid? 51 | super && (!( @sql_host.nil? || @sql_user.nil? || @sql_db.nil? || 52 | @sql_query.nil? ) || !@parent.nil?) 53 | end 54 | end 55 | end 56 | end 57 | -------------------------------------------------------------------------------- /lib/riddle/configuration/template_index.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Riddle 4 | class Configuration 5 | class TemplateIndex < Riddle::Configuration::Section 6 | include Riddle::Configuration::IndexSettings 7 | 8 | def self.settings 9 | Riddle::Configuration::IndexSettings.settings 10 | end 11 | 12 | attr_accessor :parent 13 | 14 | def initialize(name) 15 | @name = name 16 | @type = 'template' 17 | 18 | initialize_settings 19 | end 20 | 21 | def render 22 | raise ConfigurationError, "#{@name} #{@parent}" unless valid? 23 | 24 | inherited_name = "#{name}" 25 | inherited_name << " : #{parent}" if parent 26 | ( 27 | ["index #{inherited_name}", "{"] + 28 | settings_body + 29 | ["}", ""] 30 | ).join("\n") 31 | end 32 | 33 | def valid? 34 | @name 35 | end 36 | end 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /lib/riddle/configuration/tsv_source.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Riddle 4 | class Configuration 5 | class TSVSource < Riddle::Configuration::Source 6 | def self.settings 7 | [:type, :tsvpipe_command, :tsvpipe_attr_field, :tsvpipe_attr_multi] 8 | end 9 | 10 | attr_accessor *self.settings 11 | 12 | def initialize(name, type = 'tsvpipe') 13 | @name, @type = name, type 14 | end 15 | 16 | def valid? 17 | super && (@tsvpipe_command || @parent) 18 | end 19 | end 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /lib/riddle/configuration/xml_source.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Riddle 4 | class Configuration 5 | class XMLSource < Riddle::Configuration::Source 6 | def self.settings 7 | [ 8 | :type, :xmlpipe_command, :xmlpipe_field, 9 | :xmlpipe_attr_uint, :xmlpipe_attr_bool, :xmlpipe_attr_timestamp, 10 | :xmlpipe_attr_str2ordinal, :xmlpipe_attr_float, :xmlpipe_attr_multi, 11 | :xmlpipe_fixup_utf8 12 | ] 13 | end 14 | 15 | attr_accessor *self.settings 16 | 17 | def initialize(name, type) 18 | @name = name 19 | @type = type 20 | 21 | @xmlpipe_field = [] 22 | @xmlpipe_attr_uint = [] 23 | @xmlpipe_attr_bool = [] 24 | @xmlpipe_attr_timestamp = [] 25 | @xmlpipe_attr_str2ordinal = [] 26 | @xmlpipe_attr_float = [] 27 | @xmlpipe_attr_multi = [] 28 | end 29 | 30 | def valid? 31 | super && ( !@xmlpipe_command.nil? || !parent.nil? ) 32 | end 33 | end 34 | end 35 | end 36 | -------------------------------------------------------------------------------- /lib/riddle/controller.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Riddle 4 | NoConfigurationFileError = Class.new StandardError 5 | 6 | class Controller 7 | DEFAULT_MERGE_OPTIONS = {:filters => {}}.freeze 8 | 9 | attr_accessor :path, :bin_path, :searchd_binary_name, :indexer_binary_name 10 | 11 | def initialize(configuration, path) 12 | @configuration = configuration 13 | @path = path 14 | 15 | @bin_path = '' 16 | @searchd_binary_name = 'searchd' 17 | @indexer_binary_name = 'indexer' 18 | end 19 | 20 | def sphinx_version 21 | `#{indexer} 2>&1`[/(Sphinx|Manticore) (\d+\.\d+(\.\d+|(?:-dev|(\-id64)?\-beta)))/, 2] 22 | rescue 23 | nil 24 | end 25 | 26 | def index(*indices) 27 | options = indices.last.is_a?(Hash) ? indices.pop : {} 28 | indices << '--all' if indices.empty? 29 | 30 | command = "#{indexer} --config \"#{@path}\" #{indices.join(' ')}" 31 | command = "#{command} --rotate" if running? 32 | 33 | Riddle::ExecuteCommand.call command, options[:verbose] 34 | end 35 | 36 | def merge(destination, source, options = {}) 37 | options = DEFAULT_MERGE_OPTIONS.merge options 38 | 39 | command = "#{indexer} --config \"#{@path}\"".dup 40 | command << " --merge #{destination} #{source}" 41 | options[:filters].each do |attribute, value| 42 | value = value..value unless value.is_a?(Range) 43 | command << " --merge-dst-range #{attribute} #{value.min} #{value.max}" 44 | end 45 | command << " --rotate" if running? 46 | 47 | Riddle::ExecuteCommand.call command, options[:verbose] 48 | end 49 | 50 | def start(options = {}) 51 | return if running? 52 | check_for_configuration_file 53 | 54 | command = "#{searchd} --pidfile --config \"#{@path}\"" 55 | command = "#{command} --nodetach" if options[:nodetach] 56 | 57 | exec(command) if options[:nodetach] 58 | 59 | # Code does not get here if nodetach is true. 60 | Riddle::ExecuteCommand.call command, options[:verbose] 61 | end 62 | 63 | def stop(options = {}) 64 | return true unless running? 65 | check_for_configuration_file 66 | 67 | stop_flag = 'stopwait' 68 | stop_flag = 'stop' if Riddle.loaded_version.split('.').first == '0' 69 | command = %(#{searchd} --pidfile --config "#{@path}" --#{stop_flag}) 70 | 71 | result = Riddle::ExecuteCommand.call command, options[:verbose] 72 | result.successful = !running? 73 | result 74 | end 75 | 76 | def pid 77 | if File.exist?(configuration.searchd.pid_file) 78 | File.read(configuration.searchd.pid_file)[/\d+/] 79 | else 80 | nil 81 | end 82 | end 83 | 84 | def rotate 85 | pid && Process.kill(:HUP, pid.to_i) 86 | end 87 | 88 | def running? 89 | !!pid && !!Process.kill(0, pid.to_i) 90 | rescue 91 | false 92 | end 93 | 94 | private 95 | 96 | attr_reader :configuration 97 | 98 | def indexer 99 | "#{bin_path}#{indexer_binary_name}" 100 | end 101 | 102 | def searchd 103 | "#{bin_path}#{searchd_binary_name}" 104 | end 105 | 106 | def check_for_configuration_file 107 | return if File.exist?(@path) 108 | 109 | raise Riddle::NoConfigurationFileError, "'#{@path}' does not exist" 110 | end 111 | end 112 | end 113 | -------------------------------------------------------------------------------- /lib/riddle/execute_command.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Riddle::ExecuteCommand 4 | WINDOWS = (RUBY_PLATFORM =~ /mswin|mingw/) 5 | 6 | def self.call(command, verbose = true) 7 | new(command, verbose).call 8 | end 9 | 10 | def initialize(command, verbose) 11 | @command, @verbose = command, verbose 12 | 13 | return unless WINDOWS 14 | 15 | @command = "start /B #{@command} 1> NUL 2>&1" 16 | @verbose = true 17 | end 18 | 19 | def call 20 | result = verbose? ? result_from_system : result_from_backticks 21 | return result if result.status == 0 22 | 23 | error = Riddle::CommandFailedError.new "Sphinx command failed to execute" 24 | error.command_result = result 25 | raise error 26 | end 27 | 28 | private 29 | 30 | attr_reader :command, :verbose 31 | 32 | def result_from_backticks 33 | begin 34 | output = `#{command}` 35 | rescue SystemCallError => error 36 | output = error.message 37 | end 38 | 39 | Riddle::CommandResult.new command, $?.exitstatus, output 40 | end 41 | 42 | def result_from_system 43 | system command 44 | 45 | Riddle::CommandResult.new command, $?.exitstatus 46 | end 47 | 48 | def verbose? 49 | verbose 50 | end 51 | end 52 | -------------------------------------------------------------------------------- /lib/riddle/query.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Riddle::Query 4 | ESCAPE_CHARACTERS = /[\(\)\|\-!@~\/"\/\^\$\\><&=\?]/ 5 | # http://sphinxsearch.com/docs/current/extended-syntax.html 6 | ESCAPE_WORDS = /\b(?:MAYBE|NEAR|PARAGRAPH|SENTENCE|ZONE|ZONESPAN)\b/ 7 | MYSQL2_ESCAPE = defined?(Mysql2) && defined?(Mysql::Client) 8 | 9 | def self.connection(address = '127.0.0.1', port = 9312) 10 | require 'mysql2' 11 | 12 | # If you use localhost, MySQL insists on a socket connection, but Sphinx 13 | # requires a TCP connection. Using 127.0.0.1 fixes that. 14 | address = '127.0.0.1' if address == 'localhost' 15 | 16 | Mysql2::Client.new( 17 | :host => address, 18 | :port => port 19 | ) 20 | end 21 | 22 | def self.meta 23 | 'SHOW META' 24 | end 25 | 26 | def self.warnings 27 | 'SHOW WARNINGS' 28 | end 29 | 30 | def self.status 31 | 'SHOW STATUS' 32 | end 33 | 34 | def self.tables 35 | 'SHOW TABLES' 36 | end 37 | 38 | def self.variables 39 | 'SHOW VARIABLES' 40 | end 41 | 42 | def self.collation 43 | 'SHOW COLLATION' 44 | end 45 | 46 | def self.describe(index) 47 | "DESCRIBE #{index}" 48 | end 49 | 50 | def self.begin 51 | 'BEGIN' 52 | end 53 | 54 | def self.commit 55 | 'COMMIT' 56 | end 57 | 58 | def self.rollback 59 | 'ROLLBACK' 60 | end 61 | 62 | def self.set(variable, values, global = true) 63 | values = "(#{values.join(', ')})" if values.is_a?(Array) 64 | "SET#{ ' GLOBAL' if global } #{variable} = #{values}" 65 | end 66 | 67 | def self.snippets(data, index, query, options = nil) 68 | data, index, query = quote(data), quote(index), quote(query) 69 | 70 | options = ', ' + options.keys.collect { |key| 71 | value = translate_value options[key] 72 | value = quote value if value.is_a?(String) 73 | 74 | "#{value} AS #{key}" 75 | }.join(', ') unless options.nil? 76 | 77 | "CALL SNIPPETS(#{data}, #{index}, #{query}#{options})" 78 | end 79 | 80 | def self.create_function(name, type, file) 81 | type = type.to_s.upcase 82 | "CREATE FUNCTION #{name} RETURNS #{type} SONAME #{quote file}" 83 | end 84 | 85 | def self.drop_function(name) 86 | "DROP FUNCTION #{name}" 87 | end 88 | 89 | def self.update(index, id, values = {}) 90 | values = values.keys.collect { |key| 91 | "#{key} = #{translate_value values[key]}" 92 | }.join(', ') 93 | 94 | "UPDATE #{index} SET #{values} WHERE id = #{id}" 95 | end 96 | 97 | def self.translate_value(value) 98 | case value 99 | when TrueClass 100 | 1 101 | when FalseClass 102 | 0 103 | else 104 | value 105 | end 106 | end 107 | 108 | def self.escape(string) 109 | string.gsub(ESCAPE_CHARACTERS) { |match| "\\#{match}" } 110 | .gsub(ESCAPE_WORDS) { |word| "\\#{word}" } 111 | end 112 | 113 | def self.quote(string) 114 | "'#{sql_escape string}'" 115 | end 116 | 117 | def self.sql_escape(string) 118 | return Mysql2::Client.escape(string) if MYSQL2_ESCAPE 119 | 120 | string.gsub(/['"\\]/) { |character| "\\#{character}" } 121 | end 122 | end 123 | 124 | require 'riddle/query/delete' 125 | require 'riddle/query/insert' 126 | require 'riddle/query/select' 127 | -------------------------------------------------------------------------------- /lib/riddle/query/delete.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Riddle::Query::Delete 4 | def initialize(index, *ids) 5 | @index = index 6 | @ids = ids.flatten 7 | end 8 | 9 | def to_sql 10 | if @ids.length > 1 11 | "DELETE FROM #{@index} WHERE id IN (#{@ids.join(', ')})" 12 | else 13 | "DELETE FROM #{@index} WHERE id = #{@ids.first}" 14 | end 15 | end 16 | end 17 | -------------------------------------------------------------------------------- /lib/riddle/query/insert.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Riddle::Query::Insert 4 | attr_reader :columns, :values 5 | 6 | def initialize(index, columns = [], values = []) 7 | @index = index 8 | @columns = columns 9 | @values = values.first.is_a?(Array) ? values : [values] 10 | @replace = false 11 | end 12 | 13 | def replace! 14 | @replace = true 15 | self 16 | end 17 | 18 | def to_sql 19 | "#{command} INTO #{@index} (#{columns_to_s}) VALUES (#{values_to_s})" 20 | end 21 | 22 | private 23 | 24 | def command 25 | @replace ? 'REPLACE' : 'INSERT' 26 | end 27 | 28 | def columns_to_s 29 | columns.collect { |column| 30 | column.to_s == 'id' ? 'id' : "`#{column}`" 31 | }.join(', ') 32 | end 33 | 34 | def values_to_s 35 | values.collect { |value_set| 36 | value_set.collect { |value| 37 | translated_value(value) 38 | }.join(', ') 39 | }.join('), (') 40 | end 41 | 42 | def translated_value(value) 43 | case value 44 | when String 45 | "'#{Riddle::Query.sql_escape(value).gsub(/\s+/, ' ')}'" 46 | when TrueClass, FalseClass 47 | value ? 1 : 0 48 | when Time 49 | value.to_i 50 | when Date 51 | value.to_time.to_i 52 | when Array 53 | "(#{value.join(',')})" 54 | else 55 | value 56 | end 57 | end 58 | end 59 | -------------------------------------------------------------------------------- /lib/riddle/query/select.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class Riddle::Query::Select 4 | def initialize 5 | @values = [] 6 | @indices = [] 7 | @matching = nil 8 | @wheres = {} 9 | @where_alls = {} 10 | @where_nots = {} 11 | @where_not_alls = {} 12 | @group_by = nil 13 | @group_best = nil 14 | @having = [] 15 | @order_by = nil 16 | @order_within_group_by = nil 17 | @offset = nil 18 | @limit = nil 19 | @options = {} 20 | end 21 | 22 | def values(*values) 23 | @values += values 24 | self 25 | end 26 | 27 | def prepend_values(*values) 28 | @values.insert 0, *values 29 | self 30 | end 31 | 32 | def from(*indices) 33 | @indices += indices 34 | self 35 | end 36 | 37 | def matching(match) 38 | @matching = match 39 | self 40 | end 41 | 42 | def where(filters = {}) 43 | @wheres.merge!(filters) 44 | self 45 | end 46 | 47 | def where_all(filters = {}) 48 | @where_alls.merge!(filters) 49 | self 50 | end 51 | 52 | def where_not(filters = {}) 53 | @where_nots.merge!(filters) 54 | self 55 | end 56 | 57 | def where_not_all(filters = {}) 58 | @where_not_alls.merge!(filters) 59 | self 60 | end 61 | 62 | def group_by(attribute) 63 | @group_by = attribute 64 | self 65 | end 66 | 67 | def group_best(count) 68 | @group_best = count 69 | self 70 | end 71 | 72 | def having(*conditions) 73 | @having += conditions 74 | self 75 | end 76 | 77 | def order_by(order) 78 | @order_by = order 79 | self 80 | end 81 | 82 | def order_within_group_by(order) 83 | @order_within_group_by = order 84 | self 85 | end 86 | 87 | def limit(limit) 88 | @limit = limit 89 | self 90 | end 91 | 92 | def offset(offset) 93 | @offset = offset 94 | self 95 | end 96 | 97 | def with_options(options = {}) 98 | @options.merge! options 99 | self 100 | end 101 | 102 | def to_sql 103 | sql = StringIO.new String.new(""), "w" 104 | sql << "SELECT #{ extended_values } FROM #{ @indices.join(', ') }" 105 | sql << " WHERE #{ combined_wheres }" if wheres? 106 | sql << " #{group_prefix} #{escape_columns(@group_by)}" if !@group_by.nil? 107 | unless @order_within_group_by.nil? 108 | sql << " WITHIN GROUP ORDER BY #{escape_columns(@order_within_group_by)}" 109 | end 110 | sql << " HAVING #{@having.join(' AND ')}" unless @having.empty? 111 | sql << " ORDER BY #{escape_columns(@order_by)}" if !@order_by.nil? 112 | sql << " #{limit_clause}" unless @limit.nil? && @offset.nil? 113 | sql << " #{options_clause}" unless @options.empty? 114 | 115 | sql.string 116 | end 117 | 118 | private 119 | 120 | def extended_values 121 | @values.empty? ? '*' : @values.join(', ') 122 | end 123 | 124 | def group_prefix 125 | ['GROUP', @group_best, 'BY'].compact.join(' ') 126 | end 127 | 128 | def wheres? 129 | !(@wheres.empty? && @where_alls.empty? && @where_nots.empty? && @where_not_alls.empty? && @matching.nil?) 130 | end 131 | 132 | def combined_wheres 133 | wheres = wheres_to_s 134 | 135 | if @matching.nil? 136 | wheres 137 | elsif wheres.empty? 138 | "MATCH(#{Riddle::Query.quote @matching})" 139 | else 140 | "MATCH(#{Riddle::Query.quote @matching}) AND #{wheres}" 141 | end 142 | end 143 | 144 | def wheres_to_s 145 | ( 146 | @wheres.keys.collect { |key| 147 | filter_comparison_and_value key, @wheres[key] 148 | } + 149 | @where_alls.collect { |key, values| 150 | values.collect { |value| 151 | filter_comparison_and_value key, value 152 | } 153 | } + 154 | @where_nots.keys.collect { |key| 155 | exclusive_filter_comparison_and_value key, @where_nots[key] 156 | } + 157 | @where_not_alls.collect { |key, values| 158 | '(' + values.collect { |value| 159 | exclusive_filter_comparison_and_value key, value 160 | }.join(' OR ') + ')' 161 | } 162 | ).flatten.compact.join(' AND ') 163 | end 164 | 165 | def filter_comparison_and_value(attribute, value) 166 | case value 167 | when Array 168 | if !value.flatten.empty? 169 | "#{escape_column(attribute)} IN (#{value.collect { |val| filter_value(val) }.join(', ')})" 170 | end 171 | when Range 172 | "#{escape_column(attribute)} BETWEEN #{filter_value(value.first)} AND #{filter_value(value.last)}" 173 | else 174 | "#{escape_column(attribute)} = #{filter_value(value)}" 175 | end 176 | end 177 | 178 | def exclusive_filter_comparison_and_value(attribute, value) 179 | case value 180 | when Array 181 | if !value.flatten.empty? 182 | "#{escape_column(attribute)} NOT IN (#{value.collect { |val| filter_value(val) }.join(', ')})" 183 | end 184 | when Range 185 | "#{escape_column(attribute)} < #{filter_value(value.first)} OR #{attribute} > #{filter_value(value.last)}" 186 | else 187 | "#{escape_column(attribute)} <> #{filter_value(value)}" 188 | end 189 | end 190 | 191 | def filter_value(value) 192 | case value 193 | when TrueClass 194 | 1 195 | when FalseClass 196 | 0 197 | when Time 198 | value.to_i 199 | when Date 200 | Time.utc(value.year, value.month, value.day).to_i 201 | when String 202 | "'#{value.gsub("'", "\\'")}'" 203 | else 204 | value 205 | end 206 | end 207 | 208 | def limit_clause 209 | if @offset.nil? 210 | "LIMIT #{@limit}" 211 | else 212 | "LIMIT #{@offset}, #{@limit || 20}" 213 | end 214 | end 215 | 216 | def options_clause 217 | 'OPTION ' + @options.keys.collect { |key| 218 | "#{key}=#{option_value @options[key]}" 219 | }.join(', ') 220 | end 221 | 222 | def option_value(value) 223 | case value 224 | when Hash 225 | '(' + value.collect { |key, value| "#{key}=#{value}" }.join(', ') + ')' 226 | else 227 | value 228 | end 229 | end 230 | 231 | def escape_column(column) 232 | if column.to_s[/\A[`@]/] || column.to_s[/\A\w+\(/] || column.to_s[/\A\w+[.\[]/] 233 | column 234 | else 235 | column_name, *extra = column.to_s.split(' ') 236 | extra.unshift("`#{column_name}`").compact.join(' ') 237 | end 238 | end 239 | 240 | def escape_columns(columns) 241 | columns.to_s.split(/,\s*/).collect { |column| 242 | escape_column(column) 243 | }.join(', ') 244 | end 245 | end 246 | -------------------------------------------------------------------------------- /riddle.gemspec: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # -*- encoding: utf-8 -*- 4 | $:.push File.expand_path('../lib', __FILE__) 5 | 6 | Gem::Specification.new do |s| 7 | s.name = 'riddle' 8 | s.version = '2.4.3' 9 | s.platform = Gem::Platform::RUBY 10 | s.authors = ['Pat Allan'] 11 | s.email = ['pat@freelancing-gods.com'] 12 | s.homepage = 'http://pat.github.io/riddle/' 13 | s.summary = %q{An API for Sphinx, written in and for Ruby.} 14 | s.description = %q{A Ruby API and configuration helper for the Sphinx search service.} 15 | s.license = 'MIT' 16 | 17 | s.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } 18 | s.require_paths = ['lib'] 19 | 20 | s.add_development_dependency 'rake', '>= 0.9.2' 21 | s.add_development_dependency 'rspec', '>= 2.5.0' 22 | s.add_development_dependency 'yard', '>= 0.7.2' 23 | end 24 | -------------------------------------------------------------------------------- /spec/fixtures/.gitignore: -------------------------------------------------------------------------------- 1 | sql/conf.yml 2 | sphinx 3 | -------------------------------------------------------------------------------- /spec/fixtures/data/0.9.9/anchor.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/0.9.9/anchor.bin -------------------------------------------------------------------------------- /spec/fixtures/data/0.9.9/any.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/0.9.9/any.bin -------------------------------------------------------------------------------- /spec/fixtures/data/0.9.9/boolean.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/0.9.9/boolean.bin -------------------------------------------------------------------------------- /spec/fixtures/data/0.9.9/comment.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/0.9.9/comment.bin -------------------------------------------------------------------------------- /spec/fixtures/data/0.9.9/distinct.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/0.9.9/distinct.bin -------------------------------------------------------------------------------- /spec/fixtures/data/0.9.9/field_weights.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/0.9.9/field_weights.bin -------------------------------------------------------------------------------- /spec/fixtures/data/0.9.9/filter.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/0.9.9/filter.bin -------------------------------------------------------------------------------- /spec/fixtures/data/0.9.9/filter_array.bin: -------------------------------------------------------------------------------- 1 | field -------------------------------------------------------------------------------- /spec/fixtures/data/0.9.9/filter_array_exclude.bin: -------------------------------------------------------------------------------- 1 | field -------------------------------------------------------------------------------- /spec/fixtures/data/0.9.9/filter_boolean.bin: -------------------------------------------------------------------------------- 1 | field -------------------------------------------------------------------------------- /spec/fixtures/data/0.9.9/filter_floats.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/0.9.9/filter_floats.bin -------------------------------------------------------------------------------- /spec/fixtures/data/0.9.9/filter_floats_exclude.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/0.9.9/filter_floats_exclude.bin -------------------------------------------------------------------------------- /spec/fixtures/data/0.9.9/filter_range.bin: -------------------------------------------------------------------------------- 1 | field -------------------------------------------------------------------------------- /spec/fixtures/data/0.9.9/filter_range_exclude.bin: -------------------------------------------------------------------------------- 1 | field -------------------------------------------------------------------------------- /spec/fixtures/data/0.9.9/group.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/0.9.9/group.bin -------------------------------------------------------------------------------- /spec/fixtures/data/0.9.9/index.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/0.9.9/index.bin -------------------------------------------------------------------------------- /spec/fixtures/data/0.9.9/index_weights.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/0.9.9/index_weights.bin -------------------------------------------------------------------------------- /spec/fixtures/data/0.9.9/keywords_with_hits.bin: -------------------------------------------------------------------------------- 1 | patpeople -------------------------------------------------------------------------------- /spec/fixtures/data/0.9.9/keywords_without_hits.bin: -------------------------------------------------------------------------------- 1 | patpeople -------------------------------------------------------------------------------- /spec/fixtures/data/0.9.9/overrides.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/0.9.9/overrides.bin -------------------------------------------------------------------------------- /spec/fixtures/data/0.9.9/phrase.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/0.9.9/phrase.bin -------------------------------------------------------------------------------- /spec/fixtures/data/0.9.9/rank_mode.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/0.9.9/rank_mode.bin -------------------------------------------------------------------------------- /spec/fixtures/data/0.9.9/select.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/0.9.9/select.bin -------------------------------------------------------------------------------- /spec/fixtures/data/0.9.9/simple.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/0.9.9/simple.bin -------------------------------------------------------------------------------- /spec/fixtures/data/0.9.9/sort.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/0.9.9/sort.bin -------------------------------------------------------------------------------- /spec/fixtures/data/0.9.9/update_simple.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/0.9.9/update_simple.bin -------------------------------------------------------------------------------- /spec/fixtures/data/0.9.9/weights.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/0.9.9/weights.bin -------------------------------------------------------------------------------- /spec/fixtures/data/1.10/anchor.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/1.10/anchor.bin -------------------------------------------------------------------------------- /spec/fixtures/data/1.10/any.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/1.10/any.bin -------------------------------------------------------------------------------- /spec/fixtures/data/1.10/boolean.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/1.10/boolean.bin -------------------------------------------------------------------------------- /spec/fixtures/data/1.10/comment.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/1.10/comment.bin -------------------------------------------------------------------------------- /spec/fixtures/data/1.10/distinct.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/1.10/distinct.bin -------------------------------------------------------------------------------- /spec/fixtures/data/1.10/field_weights.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/1.10/field_weights.bin -------------------------------------------------------------------------------- /spec/fixtures/data/1.10/filter.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/1.10/filter.bin -------------------------------------------------------------------------------- /spec/fixtures/data/1.10/filter_array.bin: -------------------------------------------------------------------------------- 1 | field -------------------------------------------------------------------------------- /spec/fixtures/data/1.10/filter_array_exclude.bin: -------------------------------------------------------------------------------- 1 | field -------------------------------------------------------------------------------- /spec/fixtures/data/1.10/filter_boolean.bin: -------------------------------------------------------------------------------- 1 | field -------------------------------------------------------------------------------- /spec/fixtures/data/1.10/filter_floats.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/1.10/filter_floats.bin -------------------------------------------------------------------------------- /spec/fixtures/data/1.10/filter_floats_exclude.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/1.10/filter_floats_exclude.bin -------------------------------------------------------------------------------- /spec/fixtures/data/1.10/filter_range.bin: -------------------------------------------------------------------------------- 1 | field -------------------------------------------------------------------------------- /spec/fixtures/data/1.10/filter_range_exclude.bin: -------------------------------------------------------------------------------- 1 | field -------------------------------------------------------------------------------- /spec/fixtures/data/1.10/group.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/1.10/group.bin -------------------------------------------------------------------------------- /spec/fixtures/data/1.10/index.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/1.10/index.bin -------------------------------------------------------------------------------- /spec/fixtures/data/1.10/index_weights.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/1.10/index_weights.bin -------------------------------------------------------------------------------- /spec/fixtures/data/1.10/keywords_with_hits.bin: -------------------------------------------------------------------------------- 1 | patpeople -------------------------------------------------------------------------------- /spec/fixtures/data/1.10/keywords_without_hits.bin: -------------------------------------------------------------------------------- 1 | patpeople -------------------------------------------------------------------------------- /spec/fixtures/data/1.10/overrides.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/1.10/overrides.bin -------------------------------------------------------------------------------- /spec/fixtures/data/1.10/phrase.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/1.10/phrase.bin -------------------------------------------------------------------------------- /spec/fixtures/data/1.10/rank_mode.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/1.10/rank_mode.bin -------------------------------------------------------------------------------- /spec/fixtures/data/1.10/select.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/1.10/select.bin -------------------------------------------------------------------------------- /spec/fixtures/data/1.10/simple.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/1.10/simple.bin -------------------------------------------------------------------------------- /spec/fixtures/data/1.10/sort.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/1.10/sort.bin -------------------------------------------------------------------------------- /spec/fixtures/data/1.10/update_simple.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/1.10/update_simple.bin -------------------------------------------------------------------------------- /spec/fixtures/data/1.10/weights.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/1.10/weights.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.0.1/anchor.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.0.1/anchor.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.0.1/any.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.0.1/any.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.0.1/boolean.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.0.1/boolean.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.0.1/comment.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.0.1/comment.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.0.1/distinct.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.0.1/distinct.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.0.1/field_weights.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.0.1/field_weights.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.0.1/filter.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.0.1/filter.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.0.1/filter_array.bin: -------------------------------------------------------------------------------- 1 | field -------------------------------------------------------------------------------- /spec/fixtures/data/2.0.1/filter_array_exclude.bin: -------------------------------------------------------------------------------- 1 | field -------------------------------------------------------------------------------- /spec/fixtures/data/2.0.1/filter_boolean.bin: -------------------------------------------------------------------------------- 1 | field -------------------------------------------------------------------------------- /spec/fixtures/data/2.0.1/filter_floats.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.0.1/filter_floats.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.0.1/filter_floats_exclude.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.0.1/filter_floats_exclude.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.0.1/filter_range.bin: -------------------------------------------------------------------------------- 1 | field -------------------------------------------------------------------------------- /spec/fixtures/data/2.0.1/filter_range_exclude.bin: -------------------------------------------------------------------------------- 1 | field -------------------------------------------------------------------------------- /spec/fixtures/data/2.0.1/group.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.0.1/group.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.0.1/index.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.0.1/index.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.0.1/index_weights.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.0.1/index_weights.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.0.1/keywords_with_hits.bin: -------------------------------------------------------------------------------- 1 | patpeople -------------------------------------------------------------------------------- /spec/fixtures/data/2.0.1/keywords_without_hits.bin: -------------------------------------------------------------------------------- 1 | patpeople -------------------------------------------------------------------------------- /spec/fixtures/data/2.0.1/overrides.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.0.1/overrides.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.0.1/phrase.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.0.1/phrase.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.0.1/rank_mode.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.0.1/rank_mode.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.0.1/select.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.0.1/select.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.0.1/simple.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.0.1/simple.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.0.1/sort.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.0.1/sort.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.0.1/update_simple.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.0.1/update_simple.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.0.1/weights.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.0.1/weights.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.1.0/anchor.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.1.0/anchor.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.1.0/any.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.1.0/any.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.1.0/boolean.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.1.0/boolean.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.1.0/comment.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.1.0/comment.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.1.0/distinct.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.1.0/distinct.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.1.0/field_weights.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.1.0/field_weights.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.1.0/filter.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.1.0/filter.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.1.0/filter_array.bin: -------------------------------------------------------------------------------- 1 | field -------------------------------------------------------------------------------- /spec/fixtures/data/2.1.0/filter_array_exclude.bin: -------------------------------------------------------------------------------- 1 | field -------------------------------------------------------------------------------- /spec/fixtures/data/2.1.0/filter_boolean.bin: -------------------------------------------------------------------------------- 1 | field -------------------------------------------------------------------------------- /spec/fixtures/data/2.1.0/filter_floats.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.1.0/filter_floats.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.1.0/filter_floats_exclude.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.1.0/filter_floats_exclude.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.1.0/filter_range.bin: -------------------------------------------------------------------------------- 1 | field -------------------------------------------------------------------------------- /spec/fixtures/data/2.1.0/filter_range_exclude.bin: -------------------------------------------------------------------------------- 1 | field -------------------------------------------------------------------------------- /spec/fixtures/data/2.1.0/group.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.1.0/group.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.1.0/index.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.1.0/index.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.1.0/index_weights.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.1.0/index_weights.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.1.0/keywords_with_hits.bin: -------------------------------------------------------------------------------- 1 | patpeople -------------------------------------------------------------------------------- /spec/fixtures/data/2.1.0/keywords_without_hits.bin: -------------------------------------------------------------------------------- 1 | patpeople -------------------------------------------------------------------------------- /spec/fixtures/data/2.1.0/overrides.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.1.0/overrides.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.1.0/phrase.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.1.0/phrase.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.1.0/rank_mode.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.1.0/rank_mode.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.1.0/select.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.1.0/select.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.1.0/simple.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.1.0/simple.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.1.0/sort.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.1.0/sort.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.1.0/update_simple.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.1.0/update_simple.bin -------------------------------------------------------------------------------- /spec/fixtures/data/2.1.0/weights.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pat/riddle/748ca04d14a9c04b7a5f2d64dbcd4897c1ab3f43/spec/fixtures/data/2.1.0/weights.bin -------------------------------------------------------------------------------- /spec/fixtures/sphinx/configuration.erb: -------------------------------------------------------------------------------- 1 | indexer 2 | { 3 | mem_limit = 64M 4 | } 5 | 6 | searchd 7 | { 8 | <% if Riddle.loaded_version.to_f < 0.9 %> 9 | port = 9313 10 | <% else %> 11 | listen = 9313:sphinx 12 | listen = 9306:mysql41 13 | <% end %> 14 | log = <%= fixtures_path %>/sphinx/searchd.log 15 | query_log = <%= fixtures_path %>/sphinx/searchd.query.log 16 | read_timeout = 5 17 | max_children = 30 18 | workers = threads 19 | pid_file = <%= fixtures_path %>/sphinx/searchd.pid 20 | binlog_path = <%= fixtures_path %>/sphinx/binlog/ 21 | } 22 | 23 | source peoples 24 | { 25 | type = mysql 26 | sql_host = <%= host %> 27 | sql_user = <%= username %> 28 | sql_pass = <%= password %> 29 | sql_port = <%= port %> 30 | sql_db = riddle 31 | 32 | sql_query = SELECT id, first_name, middle_initial, last_name, gender, street_address, city, state, postcode, email, UNIX_TIMESTAMP(birthday) AS birthday FROM people WHERE id >= $start AND id <= $end 33 | sql_query_range = SELECT MIN(id), MAX(id) FROM people 34 | <% if ENV["SPHINX_VERSION"].to_i < 3 %> 35 | sql_query_info = SELECT * FROM people WHERE id = $id 36 | sql_attr_timestamp = birthday 37 | <% else %> 38 | sql_attr_uint = birthday 39 | <% end %> 40 | } 41 | 42 | index people 43 | { 44 | source = peoples 45 | path = <%= fixtures_path %>/sphinx/people 46 | <% if ENV["SPHINX_VERSION"].to_i < 3 %> 47 | morphology = stem_en 48 | charset_type = utf-8 49 | enable_star = 1 50 | <% end %> 51 | min_prefix_len = 1 52 | } 53 | 54 | source article_core_source 55 | { 56 | type = mysql 57 | sql_host = <%= host %> 58 | sql_user = <%= username %> 59 | sql_pass = <%= password %> 60 | sql_port = <%= port %> 61 | sql_db = riddle 62 | 63 | sql_query = SELECT id, title, 0 AS deleted FROM articles WHERE id >= $start AND id <= $end AND delta = 0 64 | sql_query_range = SELECT MIN(id), MAX(id) FROM articles 65 | <% if ENV["SPHINX_VERSION"].to_i < 3 %> 66 | sql_attr_timestamp = deleted 67 | <% else %> 68 | sql_attr_uint = deleted 69 | <% end %> 70 | } 71 | 72 | index article_core 73 | { 74 | source = article_core_source 75 | path = <%= fixtures_path %>/sphinx/article_core 76 | <% if ENV["SPHINX_VERSION"].to_i < 3 %> 77 | charset_type = utf-8 78 | <% end %> 79 | } 80 | 81 | source article_delta_source 82 | { 83 | type = mysql 84 | sql_host = <%= host %> 85 | sql_user = <%= username %> 86 | sql_pass = <%= password %> 87 | sql_port = <%= port %> 88 | sql_db = riddle 89 | 90 | sql_query = SELECT id, title, 0 AS deleted FROM articles WHERE id >= $start AND id <= $end AND delta = 1 91 | sql_query_range = SELECT MIN(id), MAX(id) FROM articles 92 | <% if ENV["SPHINX_VERSION"].to_i < 3 %> 93 | sql_attr_timestamp = deleted 94 | <% else %> 95 | sql_attr_uint = deleted 96 | <% end %> 97 | } 98 | 99 | index article_delta 100 | { 101 | source = article_delta_source 102 | path = <%= fixtures_path %>/sphinx/article_delta 103 | <% if ENV["SPHINX_VERSION"].to_i < 3 %> 104 | charset_type = utf-8 105 | <% end %> 106 | } 107 | -------------------------------------------------------------------------------- /spec/fixtures/sql/conf.example.yml: -------------------------------------------------------------------------------- 1 | host: "localhost" 2 | username: "anonymous" 3 | password: "" -------------------------------------------------------------------------------- /spec/fixtures/sql/structure.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS `people`; 2 | 3 | CREATE TABLE `people` ( 4 | `id` int(11) NOT NULL auto_increment, 5 | `first_name` varchar(50) NOT NULL, 6 | `middle_initial` varchar(10) NOT NULL, 7 | `last_name` varchar(50) NOT NULL, 8 | `gender` varchar(10) NOT NULL, 9 | `street_address` varchar(200) NOT NULL, 10 | `city` varchar(100) NOT NULL, 11 | `state` varchar(100) NOT NULL, 12 | `postcode` varchar(10) NOT NULL, 13 | `email` varchar(100) NOT NULL, 14 | `birthday` datetime NOT NULL, 15 | PRIMARY KEY (`id`) 16 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8; 17 | 18 | DROP TABLE IF EXISTS `articles`; 19 | 20 | CREATE TABLE `articles` ( 21 | `id` int(11) NOT NULL auto_increment, 22 | `title` varchar(255) NOT NULL, 23 | `delta` int(11) NOT NULL DEFAULT 1, 24 | PRIMARY KEY (`id`) 25 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8 26 | -------------------------------------------------------------------------------- /spec/functional/connection_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | class RiddleSpecConnectionProcError < StandardError; end 6 | 7 | describe 'Sphinx Client', :live => true do 8 | let(:client) { Riddle::Client.new 'localhost', 9313 } 9 | 10 | after :each do 11 | Riddle::Client.connection = nil 12 | end 13 | 14 | describe '.connection' do 15 | it "should use the given block" do 16 | Riddle::Client.connection = lambda { |client| 17 | TCPSocket.new(client.server, client.port) 18 | } 19 | client.query('smith').should be_kind_of(Hash) 20 | end 21 | 22 | it "should fail with errors from the given block" do 23 | Riddle::Client.connection = lambda { |client| 24 | raise RiddleSpecConnectionProcError 25 | } 26 | lambda { client.query('smith') }. 27 | should raise_error(Riddle::ResponseError) 28 | end 29 | end 30 | 31 | describe '#connection' do 32 | it "use the given block" do 33 | client.connection = lambda { |client| 34 | TCPSocket.new(client.server, client.port) 35 | } 36 | client.query('smith').should be_kind_of(Hash) 37 | end 38 | 39 | it "should fail with errors from the given block" do 40 | client.connection = lambda { |client| 41 | raise RiddleSpecConnectionProcError 42 | } 43 | lambda { client.query('smith') }. 44 | should raise_error(Riddle::ResponseError) 45 | end 46 | 47 | it "should not override OutOfBoundsError instances" do 48 | client.connection = lambda { |client| 49 | raise Riddle::OutOfBoundsError 50 | } 51 | lambda { client.query('smith') }. 52 | should raise_error(Riddle::OutOfBoundsError) 53 | end 54 | 55 | it "should prioritise instance over class connection" do 56 | Riddle::Client.connection = lambda { |client| 57 | raise RiddleSpecConnectionProcError 58 | } 59 | client.connection = lambda { |client| 60 | TCPSocket.new(client.server, client.port) 61 | } 62 | 63 | lambda { client.query('smith') }.should_not raise_error 64 | end 65 | end 66 | end 67 | -------------------------------------------------------------------------------- /spec/functional/escaping_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe 'SphinxQL escaping', :live => true do 6 | let(:connection) { Mysql2::Client.new :host => '127.0.0.1', :port => 9306 } 7 | 8 | def sphinxql_matching(string) 9 | select = Riddle::Query::Select.new 10 | select.from 'people' 11 | select.matching string 12 | select.to_sql 13 | end 14 | 15 | ['@', "'", '"', '\\"', "\\'", "?"].each do |string| 16 | it "escapes #{string}" do 17 | lambda { 18 | connection.query sphinxql_matching(Riddle::Query.escape(string)) 19 | }.should_not raise_error 20 | end 21 | end 22 | 23 | context 'on snippets' do 24 | def snippets_for(text, words = '', options = nil) 25 | snippets_query = Riddle::Query.snippets(text, 'people', words, options) 26 | connection.query(snippets_query).first['snippet'] 27 | end 28 | 29 | it 'preserves original text with special SphinxQL escape characters' do 30 | text = 'email: john@example.com (yay!)' 31 | snippets_for(text).should == text 32 | end 33 | 34 | it 'preserves original text with special MySQL escape characters' do 35 | text = "'Dear' Susie\nAlways use {\\LaTeX}" 36 | snippets_for(text).should == text 37 | end 38 | 39 | it 'escapes match delimiters with special SphinxQL escape characters' do 40 | snippets = snippets_for('hello world', 'world', 41 | :before_match => '()|-!', :after_match => '@~"/^$') 42 | snippets.should == 'hello ()|-!world@~"/^$' 43 | end 44 | 45 | it 'escapes match delimiters with special MySQL escape characters' do 46 | snippets = snippets_for('hello world', 'world', 47 | :before_match => "'\"", :after_match => "\n\t\\") 48 | snippets.should == "hello '\"world\n\t\\" 49 | end 50 | end 51 | end unless RUBY_PLATFORM == 'java' || Riddle.loaded_version.to_i < 2 52 | -------------------------------------------------------------------------------- /spec/functional/excerpt_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe "Sphinx Excepts", :live => true do 6 | let(:client) { Riddle::Client.new "localhost", 9313 } 7 | let(:controller) do 8 | controller = Riddle::Controller.new nil, '' 9 | controller.bin_path = Sphinx.new.bin_path 10 | controller 11 | end 12 | 13 | it "should highlight a single word multiple times in a document" do 14 | excerpts = client.excerpts( 15 | :index => "people", 16 | :words => "Mary", 17 | :docs => ["Mary, Mary, quite contrary."] 18 | ) 19 | 20 | if controller.sphinx_version.to_i >= 3 21 | excerpts.should == [ 22 | 'Mary, Mary, quite contrary.' 23 | ] 24 | else 25 | excerpts.should == [ 26 | 'Mary, Mary, quite contrary.' 27 | ] 28 | end 29 | end 30 | 31 | it "should use specified word markers" do 32 | excerpts = client.excerpts( 33 | :index => "people", 34 | :words => "Mary", 35 | :docs => ["Mary, Mary, quite contrary."], 36 | :before_match => "", 37 | :after_match => "" 38 | ) 39 | 40 | if controller.sphinx_version.to_i >= 3 41 | excerpts.should == [ 42 | "Mary, Mary, quite contrary." 43 | ] 44 | else 45 | excerpts.should == [ 46 | "Mary, Mary, quite contrary." 47 | ] 48 | end 49 | end 50 | 51 | it "should separate matches that are far apart by an ellipsis by default" do 52 | excerpts = client.excerpts( 53 | :index => "people", 54 | :words => "Pat", 55 | :docs => [ 56 | <<-SENTENCE 57 | This is a really long sentence written by Pat. It has to be over 256 58 | characters long, between keywords. But what is the keyword? Well, I 59 | can't tell you just yet... wait patiently until we've hit the 256 mark. 60 | It'll take a bit longer than you think. We're probably just hitting the 61 | 200 mark at this point. But I think we've now arrived - so I can tell 62 | you what the keyword is. I bet you're really interested in finding out, 63 | yeah? Excerpts are particularly riveting. This keyword, however, is 64 | not. It's just my name: Pat. 65 | SENTENCE 66 | ], 67 | :before_match => "", 68 | :after_match => "" 69 | ) 70 | 71 | case Riddle.loaded_version 72 | when '0.9.9' 73 | excerpts.should == [ 74 | <<-SENTENCE 75 | This is a really long sentence written by Pat. It has to be over 256 76 | characters long, between keywords. But what is the … 're really interested in finding out, 77 | yeah? Excerpts are particularly riveting. This keyword, however, is 78 | not. It's just my name: Pat. 79 | SENTENCE 80 | ] 81 | when '1.10' 82 | excerpts.should == [" … really long sentence written by Pat. It has to be over … . This keyword, however, is\nnot. It's just my name: Pat … "] 83 | when '2.0.1', '2.1.0' 84 | excerpts.should == [" … really long sentence written by Pat. It has to be over … . It's just my name: Pat.\n"] 85 | else 86 | excerpts.should == [ 87 | <<-SENTENCE 88 | This is a really long sentence written by Pat. It has to be over 256 89 | characters long, between keywords. But what is the keyword? … interested in finding out, 90 | yeah? Excerpts are particularly riveting. This keyword, however, is 91 | not. It's just my name: Pat. 92 | SENTENCE 93 | ] 94 | end 95 | end 96 | 97 | it "should use the provided separator" do 98 | excerpts = client.excerpts( 99 | :index => "people", 100 | :words => "Pat", 101 | :docs => [ 102 | <<-SENTENCE 103 | This is a really long sentence written by Pat. It has to be over 256 104 | characters long, between keywords. But what is the keyword? Well, I 105 | can't tell you just yet... wait patiently until we've hit the 256 mark. 106 | It'll take a bit longer than you think. We're probably just hitting the 107 | 200 mark at this point. But I think we've now arrived - so I can tell 108 | you what the keyword is. I bet you're really interested in finding out, 109 | yeah? Excerpts are particularly riveting. This keyword, however, is 110 | not. It's just my name: Pat. 111 | SENTENCE 112 | ], 113 | :before_match => "", 114 | :after_match => "", 115 | :chunk_separator => " --- " 116 | ) 117 | 118 | case Riddle.loaded_version 119 | when '0.9.9' 120 | excerpts.should == [ 121 | <<-SENTENCE 122 | This is a really long sentence written by Pat. It has to be over 256 123 | characters long, between keywords. But what is the --- 're really interested in finding out, 124 | yeah? Excerpts are particularly riveting. This keyword, however, is 125 | not. It's just my name: Pat. 126 | SENTENCE 127 | ] 128 | when '1.10' 129 | excerpts.should == [" --- really long sentence written by Pat. It has to be over --- . This keyword, however, is\nnot. It's just my name: Pat --- "] 130 | when '2.0.1', '2.1.0' 131 | excerpts.should == [" --- really long sentence written by Pat. It has to be over --- . It's just my name: Pat.\n"] 132 | else 133 | excerpts.should == [ 134 | <<-SENTENCE 135 | This is a really long sentence written by Pat. It has to be over 256 136 | characters long, between keywords. But what is the keyword? --- interested in finding out, 137 | yeah? Excerpts are particularly riveting. This keyword, however, is 138 | not. It's just my name: Pat. 139 | SENTENCE 140 | ] 141 | end 142 | end 143 | 144 | it "should return multiple results for multiple documents" do 145 | excerpts = client.excerpts( 146 | :index => "people", 147 | :words => "Mary", 148 | :docs => [ 149 | "Mary, Mary, quite contrary.", 150 | "The epithet \"Bloody Mary\" is associated with a number of historical and fictional women, most notably Queen Mary I of England" 151 | ], 152 | :before_match => "", 153 | :after_match => "" 154 | ) 155 | 156 | if controller.sphinx_version.to_f >= 3 157 | excerpts.should == [ 158 | "Mary, Mary, quite contrary.", 159 | "The epithet \"Bloody Mary\" is associated with a number of historical and fictional women, most notably Queen Mary I of England" 160 | ] 161 | else 162 | excerpts.should == [ 163 | "Mary, Mary, quite contrary.", 164 | "The epithet \"Bloody Mary\" is associated with a number of historical and fictional women, most notably Queen Mary I of England" 165 | ] 166 | end 167 | end 168 | end 169 | -------------------------------------------------------------------------------- /spec/functional/keywords_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe "Sphinx Keywords", :live => true do 6 | before :each do 7 | @client = Riddle::Client.new("localhost", 9313) 8 | end 9 | 10 | it "should return an array of hashes" do 11 | results = @client.keywords("pat", "people") 12 | results.should be_kind_of(Array) 13 | 14 | results.each do |result| 15 | result.should be_kind_of(Hash) 16 | end 17 | end 18 | 19 | it "should have keys for normalised and tokenised versions of the keywords" do 20 | results = @client.keywords("pat", "people") 21 | results.each do |result| 22 | result.keys.should include(:normalised) 23 | result.keys.should include(:tokenised) 24 | end 25 | end 26 | 27 | it "shouldn't have docs or hits keys if not requested" do 28 | results = @client.keywords("pat", "people") 29 | results.each do |result| 30 | result.keys.should_not include(:docs) 31 | result.keys.should_not include(:hits) 32 | end 33 | end 34 | 35 | it "should have docs and hits keys if requested" do 36 | results = @client.keywords("pat", "people", true) 37 | results.each do |result| 38 | result.keys.should include(:docs) 39 | result.keys.should include(:hits) 40 | end 41 | end 42 | end -------------------------------------------------------------------------------- /spec/functional/merging_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe "Merging indices", :live => true do 6 | let(:connection) { Mysql2::Client.new :host => '127.0.0.1', :port => 9306 } 7 | let(:path) { "spec/fixtures/sphinx/spec.conf" } 8 | let(:configuration) do 9 | Riddle::Configuration::Parser.new(File.read(path)).parse! 10 | end 11 | let(:controller) { Riddle::Controller.new configuration, path } 12 | let(:sphinx) { Sphinx.new } 13 | 14 | def record_matches?(index, string) 15 | select = Riddle::Query::Select.new 16 | select.from index 17 | select.matching string 18 | select.to_sql 19 | 20 | !!connection.query(select.to_sql).first 21 | end 22 | 23 | before :each do 24 | controller.bin_path = sphinx.bin_path 25 | 26 | sphinx.mysql_client.execute "USE riddle" 27 | sphinx.mysql_client.execute "DELETE FROM articles" 28 | end 29 | 30 | it "merges in new records" do 31 | controller.index 32 | 33 | sphinx.mysql_client.execute <<-SQL 34 | INSERT INTO articles (title, delta) VALUES ('pancakes', 1) 35 | SQL 36 | controller.index "article_delta" 37 | 38 | sleep 1.5 39 | 40 | expect(record_matches?("article_delta", "pancakes")).to eq(true) 41 | expect(record_matches?("article_core", "pancakes")).to eq(false) 42 | 43 | controller.merge "article_core", "article_delta" 44 | 45 | sleep 1.5 46 | 47 | expect(record_matches?("article_core", "pancakes")).to eq(true) 48 | end 49 | 50 | it "merges in existing records" do 51 | sphinx.mysql_client.execute <<-SQL 52 | INSERT INTO articles (title, delta) VALUES ('pancakes', 0) 53 | SQL 54 | controller.index 55 | 56 | sleep 1.5 57 | 58 | expect(record_matches?("article_core", "pancakes")).to eq(true) 59 | expect(record_matches?("article_delta", "pancakes")).to eq(false) 60 | 61 | sphinx.mysql_client.execute <<-SQL 62 | UPDATE articles SET title = 'waffles', delta = 1 WHERE title = 'pancakes' 63 | SQL 64 | controller.index "article_delta" 65 | 66 | sleep 1.5 67 | 68 | expect(record_matches?("article_delta", "waffles")).to eq(true) 69 | expect(record_matches?("article_core", "waffles")).to eq(false) 70 | expect(record_matches?("article_core", "pancakes")).to eq(true) 71 | 72 | id = connection.query("SELECT id FROM article_core").first["id"] 73 | connection.query "UPDATE article_core SET deleted = 1 WHERE id = #{id}" 74 | expect( 75 | connection.query("SELECT id FROM article_core WHERE deleted = 1").to_a 76 | ).to_not be_empty 77 | 78 | controller.merge "article_core", "article_delta", 79 | :filters => {:deleted => 0} 80 | 81 | sleep 1.5 82 | 83 | expect(record_matches?("article_core", "pancakes")).to eq(false) 84 | expect(record_matches?("article_core", "waffles")).to eq(true) 85 | expect( 86 | connection.query("SELECT id FROM article_core WHERE deleted = 1").to_a 87 | ).to be_empty 88 | end 89 | end unless RUBY_PLATFORM == 'java' 90 | -------------------------------------------------------------------------------- /spec/functional/parsing_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "spec_helper" 4 | 5 | RSpec.describe "Parsing" do 6 | it "handles invalid configuration files" do 7 | Riddle::Configuration.parse!(<<-DOC) 8 | latex_documents = [ 9 | # 10 | ] 11 | DOC 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /spec/functional/persistance_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe "Sphinx Persistance Connection", :live => true do 6 | before :each do 7 | @client = Riddle::Client.new("localhost", 9313) 8 | end 9 | 10 | it "should raise errors once already opened" do 11 | @client.open 12 | lambda { @client.open }.should raise_error 13 | @client.close 14 | end 15 | 16 | it "should raise errors if closing when already closed" do 17 | lambda { @client.close }.should raise_error 18 | end 19 | end -------------------------------------------------------------------------------- /spec/functional/search_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe "Sphinx Searches", :live => true do 6 | let(:client) { Riddle::Client.new 'localhost', 9313 } 7 | 8 | it "should return a single hash if a single query" do 9 | client.query("smith", "people").should be_kind_of(Hash) 10 | end 11 | 12 | it "should return an array of hashs if multiple queries are run" do 13 | client.append_query "smith", "people" 14 | client.append_query "jones", "people" 15 | results = client.run 16 | results.should be_kind_of(Array) 17 | results.each { |result| result.should be_kind_of(Hash) } 18 | end 19 | 20 | it "should return an array of matches" do 21 | matches = client.query("smith", "people")[:matches] 22 | matches.should be_kind_of(Array) 23 | matches.each { |match| match.should be_kind_of(Hash) } 24 | end 25 | 26 | it "should return an array of string fields" do 27 | fields = client.query("smith", "people")[:fields] 28 | fields.should be_kind_of(Array) 29 | fields.each { |field| field.should be_kind_of(String) } 30 | end 31 | 32 | it "should return an array of attribute names" do 33 | attributes = client.query("smith", "people")[:attribute_names] 34 | attributes.should be_kind_of(Array) 35 | attributes.each { |a| a.should be_kind_of(String) } 36 | end 37 | 38 | it "should return a hash of attributes" do 39 | attributes = client.query("smith", "people")[:attributes] 40 | attributes.should be_kind_of(Hash) 41 | attributes.each do |key,value| 42 | key.should be_kind_of(String) 43 | value.should be_kind_of(Integer) 44 | end 45 | end 46 | 47 | it "should return the total number of results returned" do 48 | client.query("smith", "people")[:total].should be_kind_of(Integer) 49 | end 50 | 51 | it "should return the total number of results available" do 52 | client.query("smith", "people")[:total_found].should be_kind_of(Integer) 53 | end 54 | 55 | it "should return the time taken for the query as a float" do 56 | client.query("smith", "people")[:time].should be_kind_of(Float) 57 | end 58 | 59 | it "should return a hash of the words from the query, with the number of documents and the number of hits" do 60 | words = client.query("smith victoria", "people")[:words] 61 | words.should be_kind_of(Hash) 62 | words.each do |word,hash| 63 | word.should be_kind_of(String) 64 | hash.should be_kind_of(Hash) 65 | hash[:docs].should be_kind_of(Integer) 66 | hash[:hits].should be_kind_of(Integer) 67 | end 68 | end 69 | end 70 | -------------------------------------------------------------------------------- /spec/functional/status_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | if Riddle.loaded_version == '0.9.9' || Riddle.loaded_version == '1.10' 6 | describe "Sphinx Status", :live => true do 7 | before :each do 8 | @client = Riddle::Client.new("localhost", 9313) 9 | @status = @client.status 10 | end 11 | 12 | it "should return a hash" do 13 | @status.should be_a(Hash) 14 | end 15 | 16 | it "should include the uptime, connections, and command_search keys" do 17 | # Not checking all values, but ensuring keys are being set correctly 18 | @status[:uptime].should_not be_nil 19 | @status[:connections].should_not be_nil 20 | @status[:command_search].should_not be_nil 21 | end 22 | end 23 | end -------------------------------------------------------------------------------- /spec/functional/update_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe "Sphinx Updates", :live => true do 6 | before :each do 7 | @client = Riddle::Client.new("localhost", 9313) 8 | end 9 | 10 | it "should update a single record appropriately" do 11 | # check existing birthday 12 | result = @client.query("Ellie K Ford", "people") 13 | result[:matches].should_not be_empty 14 | result[:matches].length.should == 1 15 | ellie = result[:matches].first 16 | ellie[:attributes]["birthday"].should == Time.local(1970, 1, 23).to_i 17 | 18 | # make Ellie younger by 6 years 19 | @client.update("people", ["birthday"], {ellie[:doc] => [Time.local(1976, 1, 23).to_i]}) 20 | 21 | # check attribute's value 22 | result = @client.query("Ellie K Ford", "people") 23 | result[:matches].should_not be_empty 24 | result[:matches].length.should == 1 25 | ellie = result[:matches].first 26 | ellie[:attributes]["birthday"].should == Time.local(1976, 1, 23).to_i 27 | end 28 | 29 | it "should update multiple records appropriately" do 30 | result = @client.query("Steele", "people") 31 | pairs = {} 32 | result[:matches].each do |match| 33 | pairs[match[:doc]] = [match[:attributes]["birthday"] + (365*24*60*60)] 34 | end 35 | 36 | @client.update "people", ["birthday"], pairs 37 | 38 | result = @client.query("Steele", "people") 39 | result[:matches].each do |match| 40 | match[:attributes]["birthday"].should == pairs[match[:doc]].first 41 | end 42 | end 43 | end 44 | -------------------------------------------------------------------------------- /spec/riddle/auto_version_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe Riddle::AutoVersion do 6 | describe '.configure' do 7 | before :each do 8 | @controller = Riddle::Controller.new stub('configuration'), 'sphinx.conf' 9 | Riddle::Controller.stub(:new => @controller) 10 | 11 | unless ENV['SPHINX_VERSION'].nil? 12 | @env_version, ENV['SPHINX_VERSION'] = ENV['SPHINX_VERSION'].dup, nil 13 | end 14 | end 15 | 16 | after :each do 17 | ENV['SPHINX_VERSION'] = @env_version unless @env_version.nil? 18 | end 19 | 20 | it "should require 0.9.8 if that is the known version" do 21 | Riddle::AutoVersion.should_receive(:require).with('riddle/0.9.8') 22 | 23 | @controller.stub(:sphinx_version => '0.9.8') 24 | Riddle::AutoVersion.configure 25 | end 26 | 27 | it "should require 0.9.9 if that is the known version" do 28 | Riddle::AutoVersion.should_receive(:require).with('riddle/0.9.9') 29 | 30 | @controller.stub(:sphinx_version => '0.9.9') 31 | Riddle::AutoVersion.configure 32 | end 33 | 34 | it "should require 1.10 if that is the known version" do 35 | Riddle::AutoVersion.should_receive(:require).with('riddle/1.10') 36 | 37 | @controller.stub(:sphinx_version => '1.10-beta') 38 | Riddle::AutoVersion.configure 39 | end 40 | 41 | it "should require 1.10 if using 1.10 with 64 bit IDs" do 42 | Riddle::AutoVersion.should_receive(:require).with('riddle/1.10') 43 | 44 | @controller.stub(:sphinx_version => '1.10-id64-beta') 45 | Riddle::AutoVersion.configure 46 | end 47 | 48 | it "should require 2.0.1 if that is the known version" do 49 | Riddle::AutoVersion.should_receive(:require).with('riddle/2.0.1') 50 | 51 | @controller.stub(:sphinx_version => '2.0.1-beta') 52 | Riddle::AutoVersion.configure 53 | end 54 | 55 | it "should require 2.0.1 if 2.0.2-dev is being used" do 56 | Riddle::AutoVersion.should_receive(:require).with('riddle/2.0.1') 57 | 58 | @controller.stub(:sphinx_version => '2.0.2-dev') 59 | Riddle::AutoVersion.configure 60 | end 61 | 62 | it "should require 2.1.0 if 2.0.3 is being used" do 63 | Riddle::AutoVersion.should_receive(:require).with('riddle/2.1.0') 64 | 65 | @controller.stub(:sphinx_version => '2.0.3-release') 66 | Riddle::AutoVersion.configure 67 | end 68 | 69 | it "should require 2.1.0 if 2.0.4 is being used" do 70 | Riddle::AutoVersion.should_receive(:require).with('riddle/2.1.0') 71 | 72 | @controller.stub(:sphinx_version => '2.0.4-release') 73 | Riddle::AutoVersion.configure 74 | end 75 | 76 | it "should require 2.1.0 if 2.0.5 is being used" do 77 | Riddle::AutoVersion.should_receive(:require).with('riddle/2.1.0') 78 | 79 | @controller.stub(:sphinx_version => '2.0.5-release') 80 | Riddle::AutoVersion.configure 81 | end 82 | 83 | it "should require 2.1.0 if 2.2.1 is being used" do 84 | Riddle::AutoVersion.should_receive(:require).with('riddle/2.1.0') 85 | 86 | @controller.stub(:sphinx_version => '2.2.1-beta') 87 | Riddle::AutoVersion.configure 88 | end 89 | 90 | it "should require 2.1.0 if that is the known version" do 91 | Riddle::AutoVersion.should_receive(:require).with('riddle/2.1.0') 92 | 93 | @controller.stub(:sphinx_version => '2.1.0-dev') 94 | Riddle::AutoVersion.configure 95 | end 96 | end 97 | end 98 | -------------------------------------------------------------------------------- /spec/riddle/client_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe Riddle::Client do 6 | describe '#initialize' do 7 | it "should check the loaded Sphinx version" do 8 | Riddle.should_receive(:version_warning) 9 | 10 | Riddle::Client.new 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /spec/riddle/configuration_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe Riddle::Configuration do 6 | describe Riddle::Configuration::Index do 7 | describe '#settings' do 8 | it 'should return array with all settings of index' do 9 | Riddle::Configuration::Index.settings.should_not be_empty 10 | end 11 | 12 | it 'should return array which contains a docinfo' do 13 | Riddle::Configuration::Index.settings.should be_include :docinfo 14 | end 15 | end 16 | end 17 | 18 | describe 'class inherited from Riddle::Configuration::Index' do 19 | before :all do 20 | class TestIndex < Riddle::Configuration::Index; end 21 | end 22 | 23 | describe '#settings' do 24 | it 'should has same settings as Riddle::Configuration::Index' do 25 | TestIndex.settings.should == Riddle::Configuration::Index.settings 26 | end 27 | end 28 | 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /spec/riddle/controller_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe Riddle::Controller do 6 | let(:controller) do 7 | Riddle::Controller.new double('configuration'), 'sphinx.conf' 8 | end 9 | 10 | describe '#sphinx_version' do 11 | it "should return 1.10 if using 1.10-beta" do 12 | controller.stub(:` => 'Sphinx 1.10-beta (r2420)') 13 | controller.sphinx_version.should == '1.10-beta' 14 | end 15 | 16 | it "should return 0.9.9 if using 0.9.9" do 17 | controller.stub(:` => 'Sphinx 0.9.9-release (r2117)') 18 | controller.sphinx_version.should == '0.9.9' 19 | end 20 | 21 | it "should return 0.9.9 if using 0.9.9 rc2" do 22 | controller.stub(:` => 'Sphinx 0.9.9-rc2 (r1785)') 23 | controller.sphinx_version.should == '0.9.9' 24 | end 25 | 26 | it "should return 0.9.9 if using 0.9.9 rc1" do 27 | controller.stub(:` => 'Sphinx 0.9.9-rc1 (r1566)') 28 | controller.sphinx_version.should == '0.9.9' 29 | end 30 | 31 | it "should return 0.9.8 if using 0.9.8.1" do 32 | controller.stub(:` => 'Sphinx 0.9.8.1-release (r1533)') 33 | controller.sphinx_version.should == '0.9.8' 34 | end 35 | 36 | it "should return 0.9.8 if using 0.9.8" do 37 | controller.stub(:` => 'Sphinx 0.9.8-release (r1371)') 38 | controller.sphinx_version.should == '0.9.8' 39 | end 40 | end 41 | 42 | describe "#merge" do 43 | before :each do 44 | allow(Riddle::ExecuteCommand).to receive(:call) 45 | allow(controller).to receive(:running?).and_return(false) 46 | end 47 | 48 | it "generates the command" do 49 | expect(Riddle::ExecuteCommand).to receive(:call).with( 50 | "indexer --config \"sphinx.conf\" --merge foo bar", nil 51 | ) 52 | 53 | controller.merge "foo", "bar" 54 | end 55 | 56 | it "passes through the verbose option" do 57 | expect(Riddle::ExecuteCommand).to receive(:call).with( 58 | "indexer --config \"sphinx.conf\" --merge foo bar", true 59 | ) 60 | 61 | controller.merge "foo", "bar", :verbose => true 62 | end 63 | 64 | it "adds filters with range values" do 65 | expect(Riddle::ExecuteCommand).to receive(:call).with( 66 | "indexer --config \"sphinx.conf\" --merge foo bar --merge-dst-range flagged 0 1", nil 67 | ) 68 | 69 | controller.merge "foo", "bar", :filters => {:flagged => 0..1} 70 | end 71 | 72 | it "adds filters with single values" do 73 | expect(Riddle::ExecuteCommand).to receive(:call).with( 74 | "indexer --config \"sphinx.conf\" --merge foo bar --merge-dst-range flagged 0 0", nil 75 | ) 76 | 77 | controller.merge "foo", "bar", :filters => {:flagged => 0} 78 | end 79 | 80 | it "rotates if Sphinx is running" do 81 | allow(controller).to receive(:running?).and_return(true) 82 | 83 | expect(Riddle::ExecuteCommand).to receive(:call).with( 84 | "indexer --config \"sphinx.conf\" --merge foo bar --rotate", nil 85 | ) 86 | 87 | controller.merge "foo", "bar" 88 | end 89 | end 90 | end 91 | -------------------------------------------------------------------------------- /spec/riddle/query/delete_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe Riddle::Query::Delete do 6 | it 'handles a single id' do 7 | query = Riddle::Query::Delete.new 'foo_core', 5 8 | query.to_sql.should == 'DELETE FROM foo_core WHERE id = 5' 9 | end 10 | 11 | it 'handles multiple ids' do 12 | query = Riddle::Query::Delete.new 'foo_core', 5, 6, 7 13 | query.to_sql.should == 'DELETE FROM foo_core WHERE id IN (5, 6, 7)' 14 | end 15 | 16 | it 'handles multiple ids in an explicit array' do 17 | query = Riddle::Query::Delete.new 'foo_core', [5, 6, 7] 18 | query.to_sql.should == 'DELETE FROM foo_core WHERE id IN (5, 6, 7)' 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /spec/riddle/query/insert_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe Riddle::Query::Insert do 6 | it 'handles inserts' do 7 | query = Riddle::Query::Insert.new('foo_core', [:id, :deleted], [4, false]) 8 | query.to_sql.should == 'INSERT INTO foo_core (id, `deleted`) VALUES (4, 0)' 9 | end 10 | 11 | it 'handles replaces' do 12 | query = Riddle::Query::Insert.new('foo_core', [:id, :deleted], [4, false]) 13 | query.replace! 14 | query.to_sql.should == 'REPLACE INTO foo_core (id, `deleted`) VALUES (4, 0)' 15 | end 16 | 17 | it 'encloses strings in single quotes' do 18 | query = Riddle::Query::Insert.new('foo_core', [:id, :name], [4, 'bar']) 19 | query.to_sql.should == "INSERT INTO foo_core (id, `name`) VALUES (4, 'bar')" 20 | end 21 | 22 | it 'handles inserts with more than one set of values' do 23 | query = Riddle::Query::Insert.new 'foo_core', [:id, :name], [[4, 'bar'], [5, 'baz']] 24 | query.to_sql. 25 | should == "INSERT INTO foo_core (id, `name`) VALUES (4, 'bar'), (5, 'baz')" 26 | end 27 | 28 | it 'handles values with single quotes' do 29 | query = Riddle::Query::Insert.new('foo_core', [:id, :name], [4, "bar's"]) 30 | query.to_sql. 31 | should == "INSERT INTO foo_core (id, `name`) VALUES (4, 'bar\\'s')" 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /spec/riddle/query/select_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | require 'date' 5 | require 'time' 6 | 7 | describe Riddle::Query::Select do 8 | let(:query) { Riddle::Query::Select.new } 9 | 10 | it 'handles basic queries on a specific index' do 11 | query.from('foo_core').to_sql.should == 'SELECT * FROM foo_core' 12 | end 13 | 14 | it 'handles queries on multiple indices' do 15 | query.from('foo_core').from('foo_delta').to_sql. 16 | should == 'SELECT * FROM foo_core, foo_delta' 17 | end 18 | 19 | it 'accepts multiple arguments for indices' do 20 | query.from('foo_core', 'foo_delta').to_sql. 21 | should == 'SELECT * FROM foo_core, foo_delta' 22 | end 23 | 24 | it "handles custom select values" do 25 | query.values('@weight').from('foo_core').to_sql. 26 | should == 'SELECT @weight FROM foo_core' 27 | end 28 | 29 | it 'handles JSON as a select value using dot notation' do 30 | query.values('key1.key2.key3').from('foo_core').to_sql. 31 | should == 'SELECT key1.key2.key3 FROM foo_core' 32 | end 33 | 34 | it 'handles JSON as a select value using bracket notation 2' do 35 | query.values("key1['key2']['key3']").from('foo_core').to_sql. 36 | should == "SELECT key1['key2']['key3'] FROM foo_core" 37 | end 38 | 39 | it "can prepend select values" do 40 | query.values('@weight').prepend_values('foo').from('foo_core').to_sql. 41 | should == 'SELECT foo, @weight FROM foo_core' 42 | end 43 | 44 | it 'handles basic queries with a search term' do 45 | query.from('foo_core').matching('foo').to_sql. 46 | should == "SELECT * FROM foo_core WHERE MATCH('foo')" 47 | end 48 | 49 | it "escapes single quotes in the search terms" do 50 | query.from('foo_core').matching("fo'o").to_sql. 51 | should == "SELECT * FROM foo_core WHERE MATCH('fo\\'o')" 52 | end 53 | 54 | it 'handles filters with integers' do 55 | query.from('foo_core').matching('foo').where(:bar_id => 10).to_sql. 56 | should == "SELECT * FROM foo_core WHERE MATCH('foo') AND `bar_id` = 10" 57 | end 58 | 59 | it "handles exclusive filters with integers" do 60 | query.from('foo_core').matching('foo').where_not(:bar_id => 10).to_sql. 61 | should == "SELECT * FROM foo_core WHERE MATCH('foo') AND `bar_id` <> 10" 62 | end 63 | 64 | it "handles filters with true" do 65 | query.from('foo_core').matching('foo').where(:bar => true).to_sql. 66 | should == "SELECT * FROM foo_core WHERE MATCH('foo') AND `bar` = 1" 67 | end 68 | 69 | it "handles exclusive filters with true" do 70 | query.from('foo_core').matching('foo').where_not(:bar => true).to_sql. 71 | should == "SELECT * FROM foo_core WHERE MATCH('foo') AND `bar` <> 1" 72 | end 73 | 74 | it "handles filters with false" do 75 | query.from('foo_core').matching('foo').where(:bar => false).to_sql. 76 | should == "SELECT * FROM foo_core WHERE MATCH('foo') AND `bar` = 0" 77 | end 78 | 79 | it "handles exclusive filters with false" do 80 | query.from('foo_core').matching('foo').where_not(:bar => false).to_sql. 81 | should == "SELECT * FROM foo_core WHERE MATCH('foo') AND `bar` <> 0" 82 | end 83 | 84 | it "handles filters with arrays" do 85 | query.from('foo_core').matching('foo').where(:bars => [1, 2]).to_sql. 86 | should == "SELECT * FROM foo_core WHERE MATCH('foo') AND `bars` IN (1, 2)" 87 | end 88 | 89 | it "ignores filters with empty arrays" do 90 | query.from('foo_core').matching('foo').where(:bars => []).to_sql. 91 | should == "SELECT * FROM foo_core WHERE MATCH('foo')" 92 | end 93 | 94 | it "handles exclusive filters with arrays" do 95 | query.from('foo_core').matching('foo').where_not(:bars => [1, 2]).to_sql. 96 | should == "SELECT * FROM foo_core WHERE MATCH('foo') AND `bars` NOT IN (1, 2)" 97 | end 98 | 99 | it "ignores exclusive filters with empty arrays" do 100 | query.from('foo_core').matching('foo').where_not(:bars => []).to_sql. 101 | should == "SELECT * FROM foo_core WHERE MATCH('foo')" 102 | end 103 | 104 | it "handles filters with timestamps" do 105 | time = Time.now 106 | query.from('foo_core').matching('foo').where(:created_at => time).to_sql. 107 | should == "SELECT * FROM foo_core WHERE MATCH('foo') AND `created_at` = #{time.to_i}" 108 | end 109 | 110 | it "handles filters with dates" do 111 | date = Date.new 2014, 1, 1 112 | query.from('foo_core').matching('foo').where(:created_at => date).to_sql. 113 | should == "SELECT * FROM foo_core WHERE MATCH('foo') AND `created_at` = #{Time.utc(2014, 1, 1).to_i}" 114 | end 115 | 116 | it "handles exclusive filters with timestamps" do 117 | time = Time.now 118 | query.from('foo_core').matching('foo').where_not(:created_at => time). 119 | to_sql.should == "SELECT * FROM foo_core WHERE MATCH('foo') AND `created_at` <> #{time.to_i}" 120 | end 121 | 122 | it "handles filters with ranges" do 123 | query.from('foo_core').matching('foo').where(:bar => 1..5).to_sql. 124 | should == "SELECT * FROM foo_core WHERE MATCH('foo') AND `bar` BETWEEN 1 AND 5" 125 | end 126 | 127 | it "handles filters with strings" do 128 | query.from('foo_core').where(:bar => 'baz').to_sql. 129 | should == "SELECT * FROM foo_core WHERE `bar` = 'baz'" 130 | end 131 | 132 | it "handles filters expecting matches on all values" do 133 | query.from('foo_core').where_all(:bars => [1, 2]).to_sql. 134 | should == "SELECT * FROM foo_core WHERE `bars` = 1 AND `bars` = 2" 135 | end 136 | 137 | it "handles filters expecting matches on all combinations of values" do 138 | query.from('foo_core').where_all(:bars => [[1,2], 3]).to_sql. 139 | should == "SELECT * FROM foo_core WHERE `bars` IN (1, 2) AND `bars` = 3" 140 | end 141 | 142 | it "handles exclusive filters expecting matches on none of the values" do 143 | query.from('foo_core').where_not_all(:bars => [1, 2]).to_sql. 144 | should == "SELECT * FROM foo_core WHERE (`bars` <> 1 OR `bars` <> 2)" 145 | end 146 | 147 | it 'handles filters on JSON with dot syntax' do 148 | query.from('foo_core').where('key1.key2.key3' => 10).to_sql. 149 | should == "SELECT * FROM foo_core WHERE key1.key2.key3 = 10" 150 | end 151 | 152 | it 'handles filters on JSON with bracket syntax' do 153 | query.from('foo_core').where("key1['key2']['key3']" => 10).to_sql. 154 | should == "SELECT * FROM foo_core WHERE key1['key2']['key3'] = 10" 155 | end 156 | 157 | it 'handles grouping' do 158 | query.from('foo_core').group_by('bar_id').to_sql. 159 | should == "SELECT * FROM foo_core GROUP BY `bar_id`" 160 | end 161 | 162 | it "handles grouping n-best results" do 163 | query.from('foo_core').group_by('bar_id').group_best(3).to_sql. 164 | should == "SELECT * FROM foo_core GROUP 3 BY `bar_id`" 165 | end 166 | 167 | it 'handles having conditions' do 168 | query.from('foo_core').group_by('bar_id').having('bar_id > 10').to_sql. 169 | should == "SELECT * FROM foo_core GROUP BY `bar_id` HAVING bar_id > 10" 170 | end 171 | 172 | it 'handles ordering' do 173 | query.from('foo_core').order_by('bar_id ASC').to_sql. 174 | should == 'SELECT * FROM foo_core ORDER BY `bar_id` ASC' 175 | end 176 | 177 | it 'handles ordering when an already escaped column is passed in' do 178 | query.from('foo_core').order_by('`bar_id` ASC').to_sql. 179 | should == 'SELECT * FROM foo_core ORDER BY `bar_id` ASC' 180 | end 181 | 182 | it 'handles ordering when just a symbol is passed in' do 183 | query.from('foo_core').order_by(:bar_id).to_sql. 184 | should == 'SELECT * FROM foo_core ORDER BY `bar_id`' 185 | end 186 | 187 | it 'handles ordering when a computed sphinx variable is passed in' do 188 | query.from('foo_core').order_by('@weight DESC').to_sql. 189 | should == 'SELECT * FROM foo_core ORDER BY @weight DESC' 190 | end 191 | 192 | it "handles ordering when a sphinx function is passed in" do 193 | query.from('foo_core').order_by('weight() DESC').to_sql. 194 | should == 'SELECT * FROM foo_core ORDER BY weight() DESC' 195 | end 196 | 197 | it 'handles group ordering' do 198 | query.from('foo_core').order_within_group_by('bar_id ASC').to_sql. 199 | should == 'SELECT * FROM foo_core WITHIN GROUP ORDER BY `bar_id` ASC' 200 | end 201 | 202 | it 'handles a limit' do 203 | query.from('foo_core').limit(10).to_sql. 204 | should == 'SELECT * FROM foo_core LIMIT 10' 205 | end 206 | 207 | it 'handles an offset' do 208 | query.from('foo_core').offset(20).to_sql. 209 | should == 'SELECT * FROM foo_core LIMIT 20, 20' 210 | end 211 | 212 | it 'handles an option' do 213 | query.from('foo_core').with_options(:bar => :baz).to_sql. 214 | should == 'SELECT * FROM foo_core OPTION bar=baz' 215 | end 216 | 217 | it 'handles multiple options' do 218 | sql = query.from('foo_core').with_options(:bar => :baz, :qux => :quux). 219 | to_sql 220 | sql.should match(/OPTION .*bar=baz/) 221 | sql.should match(/OPTION .*qux=quux/) 222 | end 223 | 224 | it "handles options of hashes" do 225 | query.from('foo_core').with_options(:weights => {:foo => 5}).to_sql. 226 | should == 'SELECT * FROM foo_core OPTION weights=(foo=5)' 227 | end 228 | end 229 | -------------------------------------------------------------------------------- /spec/riddle/query_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe Riddle::Query, :live => true do 6 | describe '.connection' do 7 | let(:connection) { Riddle::Query.connection 'localhost', 9306 } 8 | 9 | it "returns a MySQL Client" do 10 | connection.should be_a(Mysql2::Client) 11 | end 12 | 13 | it "should handle search requests" do 14 | connection.query(Riddle::Query.tables).to_a.should match_array([ 15 | {'Index' => 'people', 'Type' => 'local'}, 16 | {'Index' => 'article_core', 'Type' => 'local'}, 17 | {'Index' => 'article_delta', 'Type' => 'local'} 18 | ]) 19 | end 20 | end 21 | end unless RUBY_PLATFORM == 'java' || Riddle.loaded_version.to_i < 2 22 | 23 | describe Riddle::Query do 24 | describe '.set' do 25 | it 'handles a single value' do 26 | Riddle::Query.set('foo', 'bar').should == 'SET GLOBAL foo = bar' 27 | end 28 | 29 | it 'handles multiple values' do 30 | Riddle::Query.set('foo', [1, 2, 3]).should == 'SET GLOBAL foo = (1, 2, 3)' 31 | end 32 | 33 | it 'handles non-global settings' do 34 | Riddle::Query.set('foo', 'bar', false).should == 'SET foo = bar' 35 | end 36 | end 37 | 38 | describe '.snippets' do 39 | it 'handles a basic request' do 40 | Riddle::Query.snippets('foo bar baz', 'foo_core', 'foo'). 41 | should == "CALL SNIPPETS('foo bar baz', 'foo_core', 'foo')" 42 | end 43 | 44 | it 'handles a request with options' do 45 | Riddle::Query.snippets('foo bar baz', 'foo_core', 'foo', :around => 5). 46 | should == "CALL SNIPPETS('foo bar baz', 'foo_core', 'foo', 5 AS around)" 47 | end 48 | 49 | it 'handles string options' do 50 | Riddle::Query.snippets('foo bar baz', 'foo_core', 'foo', 51 | :before_match => '').should == "CALL SNIPPETS('foo bar baz', 'foo_core', 'foo', '' AS before_match)" 52 | end 53 | 54 | it "handles boolean options" do 55 | Riddle::Query.snippets('foo bar baz', 'foo_core', 'foo', 56 | :exact_phrase => true).should == "CALL SNIPPETS('foo bar baz', 'foo_core', 'foo', 1 AS exact_phrase)" 57 | end 58 | 59 | it "escapes quotes in the text data" do 60 | Riddle::Query.snippets("foo bar 'baz", 'foo_core', 'foo'). 61 | should == "CALL SNIPPETS('foo bar \\'baz', 'foo_core', 'foo')" 62 | end 63 | 64 | it "escapes quotes in the query data" do 65 | Riddle::Query.snippets("foo bar baz", 'foo_core', "foo'"). 66 | should == "CALL SNIPPETS('foo bar baz', 'foo_core', 'foo\\'')" 67 | end 68 | end 69 | 70 | describe '.create_function' do 71 | it 'handles a basic create request' do 72 | Riddle::Query.create_function('foo', :bigint, 'foo.sh'). 73 | should == "CREATE FUNCTION foo RETURNS BIGINT SONAME 'foo.sh'" 74 | end 75 | end 76 | 77 | describe '.update' do 78 | it 'handles a basic update request' do 79 | Riddle::Query.update('foo_core', 5, :deleted => 1). 80 | should == 'UPDATE foo_core SET deleted = 1 WHERE id = 5' 81 | end 82 | end 83 | 84 | describe '.escape' do 85 | %w(( ) | - ! @ ~ / ^ $ " > < ?).each do |reserved| 86 | it "escapes #{reserved}" do 87 | Riddle::Query.escape(reserved).should == "\\#{reserved}" 88 | end 89 | end 90 | 91 | it "escapes word-operators correctly" do 92 | operators = ['MAYBE', 'NEAR', 'PARAGRAPH', 'SENTENCE', 'ZONE', 'ZONESPAN'] 93 | operators.each do |operator| 94 | base = "string with #{operator} operator" 95 | Riddle::Query.escape(base).should == base.gsub(operator, "\\#{operator}") 96 | end 97 | 98 | Riddle::Query.escape("FIND THE ZONES").should == "FIND THE ZONES" 99 | end 100 | end 101 | end 102 | -------------------------------------------------------------------------------- /spec/riddle_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe Riddle do 6 | describe '.version_warning' do 7 | before :each do 8 | @existing_version = Riddle.loaded_version 9 | end 10 | 11 | after :each do 12 | Riddle.loaded_version = @existing_version 13 | end 14 | 15 | it "should do nothing if there is a Sphinx version loaded" do 16 | STDERR.should_not_receive(:puts) 17 | 18 | Riddle.loaded_version = '0.9.8' 19 | Riddle.version_warning 20 | end 21 | 22 | it "should output a warning if no version is loaded" do 23 | STDERR.should_receive(:puts) 24 | 25 | Riddle.loaded_version = nil 26 | Riddle.version_warning 27 | end 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'rubygems' 4 | require 'bundler' 5 | 6 | $:.unshift File.dirname(__FILE__) + '/../lib' 7 | $:.unshift File.dirname(__FILE__) + '/..' 8 | 9 | Dir['spec/support/**/*.rb'].each { |f| require f } 10 | 11 | Bundler.require :default, :development 12 | 13 | require 'riddle' 14 | 15 | RSpec.configure do |config| 16 | config.include BinaryFixtures 17 | 18 | sphinx = Sphinx.new 19 | sphinx.setup_mysql 20 | sphinx.generate_configuration 21 | sphinx.index 22 | 23 | config.before :all do |group| 24 | sphinx.start if group.class.metadata[:live] 25 | end 26 | 27 | config.after :all do |group| 28 | sphinx.stop if group.class.metadata[:live] 29 | end 30 | 31 | # enable filtering for examples 32 | config.filter_run :wip => true 33 | config.run_all_when_everything_filtered = true 34 | end 35 | -------------------------------------------------------------------------------- /spec/support/binary_fixtures.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module BinaryFixtures 4 | def query_contents(key) 5 | path = "spec/fixtures/data/#{Riddle.loaded_version}/#{key}.bin" 6 | contents = open(path) { |f| f.read } 7 | contents.respond_to?(:encoding) ? 8 | contents.force_encoding('ASCII-8BIT') : contents 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /spec/support/jruby_client.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class JRubyClient 4 | def initialize(host, username, password, port) 5 | address = "jdbc:mysql://#{host}" 6 | properties = Java::JavaUtil::Properties.new 7 | properties.setProperty "user", username if username 8 | properties.setProperty "password", password if password 9 | properties.setProperty "port", port if port 10 | properties.setProperty "useSSL", "false" 11 | properties.setProperty "allowLoadLocalInfile", "true" 12 | 13 | @client = Java::ComMysqlJdbc::Driver.new.connect address, properties 14 | end 15 | 16 | def query(statement) 17 | set = client.createStatement.executeQuery(statement) 18 | results = [] 19 | results << set.getString(1) while set.next 20 | results 21 | end 22 | 23 | def execute(statement) 24 | client.createStatement.execute statement 25 | end 26 | 27 | def close 28 | @client.close 29 | end 30 | 31 | private 32 | 33 | attr_reader :client 34 | end 35 | -------------------------------------------------------------------------------- /spec/support/mri_client.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class MRIClient 4 | def initialize(host, username, password, port) 5 | @client = Mysql2::Client.new( 6 | :host => host, 7 | :username => username, 8 | :password => password, 9 | :port => port, 10 | :local_infile => true 11 | ) 12 | end 13 | 14 | def query(statement) 15 | client.query(statement, :as => :array).to_a.flatten 16 | end 17 | 18 | def execute(statement) 19 | client.query statement 20 | end 21 | 22 | def close 23 | @client.close 24 | end 25 | 26 | private 27 | 28 | attr_reader :client 29 | end 30 | -------------------------------------------------------------------------------- /spec/support/sphinx.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'erb' 4 | require 'yaml' 5 | require 'tempfile' 6 | 7 | if RUBY_PLATFORM == 'java' 8 | require 'java' 9 | require 'jdbc/mysql' 10 | Jdbc::MySQL.load_driver 11 | end 12 | 13 | class Sphinx 14 | attr_accessor :host, :username, :password, :port 15 | 16 | def initialize 17 | self.host = ENV['MYSQL_HOST'] || 'localhost' 18 | self.username = ENV['MYSQL_USER'] || 'root' 19 | self.password = ENV['MYSQL_PASSWORD'] || '' 20 | self.port = ENV['MYSQL_PORT'] || nil 21 | 22 | if File.exist?('spec/fixtures/sql/conf.yml') 23 | config = YAML.load(File.open('spec/fixtures/sql/conf.yml')) 24 | self.host = config['host'] 25 | self.username = config['username'] 26 | self.password = config['password'] 27 | self.port = config['port'] 28 | end 29 | end 30 | 31 | def bin_path 32 | ENV.fetch 'SPHINX_BIN', '' 33 | end 34 | 35 | def setup_mysql 36 | databases = mysql_client.query "SHOW DATABASES" 37 | unless databases.include?("riddle") 38 | mysql_client.execute "CREATE DATABASE riddle" 39 | end 40 | mysql_client.execute 'USE riddle' 41 | 42 | structure = File.open('spec/fixtures/sql/structure.sql') { |f| f.read } 43 | structure.split(/;/).each { |sql| mysql_client.execute sql } 44 | 45 | mysql_client.execute <<-SQL 46 | LOAD DATA LOCAL INFILE '#{fixtures_path}/sql/data.tsv' INTO TABLE 47 | `riddle`.`people` FIELDS TERMINATED BY ',' ENCLOSED BY "'" (gender, 48 | first_name, middle_initial, last_name, street_address, city, state, 49 | postcode, email, birthday) 50 | SQL 51 | 52 | mysql_client.close 53 | end 54 | 55 | def mysql_client 56 | @mysql_client ||= if RUBY_PLATFORM == 'java' 57 | JRubyClient.new host, username, password, port 58 | else 59 | MRIClient.new host, username, password, port 60 | end 61 | end 62 | 63 | def generate_configuration 64 | template = File.open('spec/fixtures/sphinx/configuration.erb') { |f| f.read } 65 | File.open('spec/fixtures/sphinx/spec.conf', 'w') { |f| 66 | f.puts ERB.new(template).result(binding) 67 | } 68 | 69 | FileUtils.mkdir_p "spec/fixtures/sphinx/binlog" 70 | end 71 | 72 | def index 73 | cmd = "#{bin_path}indexer --config #{fixtures_path}/sphinx/spec.conf --all" 74 | cmd << ' --rotate' if running? 75 | `#{cmd}` 76 | end 77 | 78 | def start 79 | return if running? 80 | 81 | `#{bin_path}searchd --config #{fixtures_path}/sphinx/spec.conf` 82 | 83 | sleep(1) 84 | 85 | unless running? 86 | puts 'Failed to start searchd daemon. Check fixtures/sphinx/searchd.log.' 87 | end 88 | end 89 | 90 | def stop 91 | return unless running? 92 | 93 | stop_flag = '--stopwait' 94 | stop_flag = '--stop' if Riddle.loaded_version.to_i < 1 95 | `#{bin_path}searchd --config #{fixtures_path}/sphinx/spec.conf #{stop_flag}` 96 | end 97 | 98 | private 99 | 100 | def fixtures_path 101 | File.expand_path File.join(File.dirname(__FILE__), '..', 'fixtures') 102 | end 103 | 104 | def pid 105 | if File.exist?("#{fixtures_path}/sphinx/searchd.pid") 106 | `cat #{fixtures_path}/sphinx/searchd.pid`[/\d+/] 107 | else 108 | nil 109 | end 110 | end 111 | 112 | def running? 113 | pid && `ps #{pid} | wc -l`.to_i > 1 114 | end 115 | end 116 | -------------------------------------------------------------------------------- /spec/unit/client_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe Riddle::Client do 6 | it "should have the same keys for both commands and versions, except persist" do 7 | 8 | (Riddle::Client::Commands.keys - [:persist]).should == Riddle::Client::Versions.keys 9 | end 10 | 11 | it "should default to localhost as the server" do 12 | Riddle::Client.new.server.should == "localhost" 13 | end 14 | 15 | it "should default to port 9312" do 16 | Riddle::Client.new.port.should == 9312 17 | end 18 | 19 | it "should accept an array of servers" do 20 | servers = ["1.1.1.1", "2.2.2.2", "3.3.3.3"] 21 | client = Riddle::Client.new(servers) 22 | client.servers.should == servers 23 | end 24 | 25 | it "should translate anchor arguments correctly" do 26 | client = Riddle::Client.new 27 | client.set_anchor "latitude", 10.0, "longitude", 95.0 28 | client.anchor.should == { 29 | :latitude_attribute => "latitude", 30 | :latitude => 10.0, 31 | :longitude_attribute => "longitude", 32 | :longitude => 95.0 33 | } 34 | end 35 | 36 | it "should add queries to the queue" do 37 | client = Riddle::Client.new 38 | client.queue.should be_empty 39 | client.append_query "spec" 40 | client.queue.should_not be_empty 41 | end 42 | 43 | describe 'query contents' do 44 | it "should build a basic search message correctly" do 45 | client = Riddle::Client.new 46 | client.append_query "test " 47 | client.queue.first.should == query_contents(:simple) 48 | end 49 | 50 | it "should build a message with a specified index correctly" do 51 | client = Riddle::Client.new 52 | client.append_query "test ", "edition" 53 | client.queue.first.should == query_contents(:index) 54 | end 55 | 56 | it "should build a message using match mode :any correctly" do 57 | client = Riddle::Client.new 58 | client.match_mode = :any 59 | client.append_query "test this " 60 | client.queue.first.should == query_contents(:any) 61 | end 62 | 63 | it "should build a message using sort by correctly" do 64 | client = Riddle::Client.new 65 | client.sort_by = 'id' 66 | client.sort_mode = :extended 67 | client.append_query "testing " 68 | client.queue.first.should == query_contents(:sort) 69 | end 70 | 71 | it "should build a message using match mode :boolean correctly" do 72 | client = Riddle::Client.new 73 | client.match_mode = :boolean 74 | client.append_query "test " 75 | client.queue.first.should == query_contents(:boolean) 76 | end 77 | 78 | it "should build a message using match mode :phrase correctly" do 79 | client = Riddle::Client.new 80 | client.match_mode = :phrase 81 | client.append_query "testing this " 82 | client.queue.first.should == query_contents(:phrase) 83 | end 84 | 85 | it "should build a message with a filter correctly" do 86 | client = Riddle::Client.new 87 | client.filters << Riddle::Client::Filter.new("id", [10, 100, 1000]) 88 | client.append_query "test " 89 | client.queue.first.should == query_contents(:filter) 90 | end 91 | 92 | it "should build a message with group values correctly" do 93 | client = Riddle::Client.new 94 | client.group_by = "id" 95 | client.group_function = :attr 96 | client.group_clause = "id" 97 | client.append_query "test " 98 | client.queue.first.should == query_contents(:group) 99 | end 100 | 101 | it "should build a message with group distinct value correctly" do 102 | client = Riddle::Client.new 103 | client.group_distinct = "id" 104 | client.append_query "test " 105 | client.queue.first.should == query_contents(:distinct) 106 | end 107 | 108 | it "should build a message with weights correctly" do 109 | client = Riddle::Client.new 110 | client.weights = [100, 1] 111 | client.append_query "test " 112 | client.queue.first.should == query_contents(:weights) 113 | end 114 | 115 | it "should build a message with an anchor correctly" do 116 | client = Riddle::Client.new 117 | client.set_anchor "latitude", 10.0, "longitude", 95.0 118 | client.append_query "test " 119 | client.queue.first.should == query_contents(:anchor) 120 | end 121 | 122 | it "should build a message with index weights correctly" do 123 | client = Riddle::Client.new 124 | client.index_weights = {"people" => 101} 125 | client.append_query "test " 126 | client.queue.first.should == query_contents(:index_weights) 127 | end 128 | 129 | it "should build a message with field weights correctly" do 130 | client = Riddle::Client.new 131 | client.field_weights = {"city" => 101} 132 | client.append_query "test " 133 | client.queue.first.should == query_contents(:field_weights) 134 | end 135 | 136 | it "should build a message with a comment correctly" do 137 | client = Riddle::Client.new 138 | client.append_query "test ", "*", "commenting" 139 | client.queue.first.should == query_contents(:comment) 140 | end 141 | 142 | if Riddle.loaded_version == '0.9.9' || Riddle.loaded_version == '1.10' 143 | it "should build a message with overrides correctly" do 144 | client = Riddle::Client.new 145 | client.add_override("rating", :float, {1 => 10.0}) 146 | client.append_query "test " 147 | client.queue.first.should == query_contents(:overrides) 148 | end 149 | 150 | it "should build a message with selects correctly" do 151 | client = Riddle::Client.new 152 | client.select = "selecting" 153 | client.append_query "test " 154 | client.queue.first.should == query_contents(:select) 155 | end 156 | end 157 | 158 | it "should keep multiple messages in the queue" do 159 | client = Riddle::Client.new 160 | client.weights = [100, 1] 161 | client.append_query "test " 162 | client.append_query "test " 163 | client.queue.length.should == 2 164 | client.queue.each { |item| item.should == query_contents(:weights) } 165 | end 166 | 167 | it "should keep multiple messages in the queue with different params" do 168 | client = Riddle::Client.new 169 | client.weights = [100, 1] 170 | client.append_query "test " 171 | client.weights = [] 172 | client.append_query "test ", "edition" 173 | client.queue.first.should == query_contents(:weights) 174 | client.queue.last.should == query_contents(:index) 175 | end 176 | 177 | it "should build a basic update message correctly" do 178 | client = Riddle::Client.new 179 | client.send( 180 | :update_message, 181 | "people", 182 | ["birthday"], 183 | {1 => [191163600]} 184 | ).should == query_contents(:update_simple) 185 | end 186 | 187 | it "should build a keywords request without hits correctly" do 188 | client = Riddle::Client.new 189 | client.send( 190 | :keywords_message, 191 | "pat", 192 | "people", 193 | false 194 | ).should == query_contents(:keywords_without_hits) 195 | end 196 | 197 | it "should build a keywords request with hits correctly" do 198 | client = Riddle::Client.new 199 | client.send( 200 | :keywords_message, 201 | "pat", 202 | "people", 203 | true 204 | ).should == query_contents(:keywords_with_hits) 205 | end 206 | end 207 | 208 | it "should timeout after a specified time" do 209 | client = Riddle::Client.new 210 | client.port = 9314 211 | client.timeout = 1 212 | 213 | server = TCPServer.new "localhost", 9314 214 | 215 | lambda { 216 | client.send(:connect) { |socket| } 217 | }.should raise_error(Riddle::ConnectionError) 218 | 219 | server.close 220 | end unless RUBY_PLATFORM == 'java' # JRuby doesn't like Timeout 221 | 222 | context "connection retrying" do 223 | it "should try fives time when connection refused" do 224 | client = Riddle::Client.new 225 | client.port = 3314 226 | 227 | TCPSocket.should_receive(:new).with('localhost', 3314).exactly(5).times. 228 | and_raise(Errno::ECONNREFUSED) 229 | 230 | lambda { 231 | client.send(:connect) { |socket| } 232 | }.should raise_error(Riddle::ConnectionError) 233 | end 234 | end 235 | 236 | context "connection fail over" do 237 | it "should try each of several server addresses after timeouts" do 238 | client = Riddle::Client.new 239 | client.port = 3314 240 | client.servers = %w[localhost 127.0.0.1 0.0.0.0] 241 | client.timeout = 1 242 | 243 | TCPSocket.should_receive(:new).with( 244 | an_instance_of(String), 3314 245 | ).exactly(3).and_raise Timeout::Error 246 | 247 | lambda { 248 | client.send(:connect) { |socket| } 249 | }.should raise_error(Riddle::ConnectionError) 250 | end unless RUBY_PLATFORM == 'java' # JRuby doesn't like Timeout 251 | 252 | it "should try each of several server addresses after a connection refused" do 253 | client = Riddle::Client.new 254 | client.port = 3314 255 | client.servers = %w[localhost 127.0.0.1 0.0.0.0] 256 | client.timeout = 1 257 | 258 | # initialise_socket will retry 5 times before failing, 259 | # these combined with the multiple server failover should result in 15 260 | # calls to TCPSocket.new 261 | TCPSocket.should_receive(:new).with( 262 | an_instance_of(String), 3314 263 | ).exactly(3 * 5).and_raise Errno::ECONNREFUSED 264 | 265 | lambda { 266 | client.send(:connect) { |socket| } 267 | }.should raise_error(Riddle::ConnectionError) 268 | end unless RUBY_PLATFORM == 'java' # JRuby doesn't like Timeout 269 | end 270 | 271 | it "should fail if the server has the wrong version" do 272 | client = Riddle::Client.new 273 | client.port = 9314 274 | client.timeout = 1 275 | 276 | server = TCPServer.new "localhost", 9314 277 | 278 | thread = Thread.new do 279 | client = server.accept 280 | client.send [0].pack("N"), 0 281 | client.close 282 | end 283 | 284 | lambda { 285 | client.send(:connect) { |socket| } 286 | }.should raise_error(Riddle::VersionError) 287 | 288 | thread.exit 289 | server.close 290 | end 291 | 292 | end 293 | -------------------------------------------------------------------------------- /spec/unit/configuration/common_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe Riddle::Configuration::Common do 6 | it "should always be valid" do 7 | common = Riddle::Configuration::Common.new 8 | common.should be_valid 9 | end 10 | 11 | it "should support Sphinx's common settings" do 12 | settings = %w( lemmatizer_base on_json_attr_error json_autoconv_numbers 13 | json_autoconv_keynames rlp_root rlp_environment rlp_max_batch_size 14 | rlp_max_batch_docs ) 15 | common = Riddle::Configuration::Common.new 16 | 17 | settings.each do |setting| 18 | common.should respond_to(setting.to_sym) 19 | common.should respond_to("#{setting}=".to_sym) 20 | end 21 | end 22 | 23 | it "should render a correct configuration" do 24 | common = Riddle::Configuration::Common.new 25 | common.common_sphinx_configuration = true 26 | 27 | common.render.should == <<-COMMON 28 | common 29 | { 30 | } 31 | COMMON 32 | 33 | common.lemmatizer_base = "/tmp" 34 | common.render.should == <<-COMMON 35 | common 36 | { 37 | lemmatizer_base = /tmp 38 | } 39 | COMMON 40 | end 41 | 42 | it "should not be present when common_sphinx_configuration is not set" do 43 | common = Riddle::Configuration::Common.new 44 | common.render.should be_nil 45 | end 46 | 47 | it "should not be present when common_sphinx_configuration is false" do 48 | common = Riddle::Configuration::Common.new 49 | common.common_sphinx_configuration = false 50 | common.render.should be_nil 51 | end 52 | 53 | it "should render when common_sphinx_configuration is true" do 54 | common = Riddle::Configuration::Common.new 55 | common.common_sphinx_configuration = true 56 | common.render.should == <<-COMMON 57 | common 58 | { 59 | } 60 | COMMON 61 | end 62 | end -------------------------------------------------------------------------------- /spec/unit/configuration/distributed_index_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe Riddle::Configuration::DistributedIndex do 6 | it "should not be valid without any indices" do 7 | index = Riddle::Configuration::DistributedIndex.new("dist1") 8 | index.should_not be_valid 9 | end 10 | 11 | it "should be valid with just local indices" do 12 | index = Riddle::Configuration::DistributedIndex.new("dist1") 13 | index.local_indices << "local_one" 14 | index.should be_valid 15 | end 16 | 17 | it "should be valid with just remote indices" do 18 | index = Riddle::Configuration::DistributedIndex.new("dist1") 19 | index.remote_indices << Riddle::Configuration::RemoteIndex.new("local", 3312, "remote_one") 20 | index.should be_valid 21 | end 22 | 23 | it "should be of type 'distributed'" do 24 | index = Riddle::Configuration::DistributedIndex.new("dist1") 25 | index.type.should == 'distributed' 26 | end 27 | 28 | it "should raise a ConfigurationError if rendering when not valid" do 29 | index = Riddle::Configuration::DistributedIndex.new("dist1") 30 | lambda { index.render }.should raise_error(Riddle::Configuration::ConfigurationError) 31 | end 32 | 33 | it "should render correctly if supplied settings are valid" do 34 | index = Riddle::Configuration::DistributedIndex.new("dist1") 35 | 36 | index.local_indices << "test1" << "test1stemmed" 37 | index.remote_indices << 38 | Riddle::Configuration::RemoteIndex.new("localhost", 3313, "remote1") << 39 | Riddle::Configuration::RemoteIndex.new("localhost", 3314, "remote2") << 40 | Riddle::Configuration::RemoteIndex.new("localhost", 3314, "remote3") 41 | index.agent_blackhole << "testbox:3312:testindex1,testindex2" 42 | 43 | index.agent_connect_timeout = 1000 44 | index.agent_query_timeout = 3000 45 | 46 | index.render.should == <<-DISTINDEX 47 | index dist1 48 | { 49 | type = distributed 50 | local = test1 51 | local = test1stemmed 52 | agent = localhost:3313:remote1 53 | agent = localhost:3314:remote2,remote3 54 | agent_blackhole = testbox:3312:testindex1,testindex2 55 | agent_connect_timeout = 1000 56 | agent_query_timeout = 3000 57 | } 58 | DISTINDEX 59 | end 60 | end -------------------------------------------------------------------------------- /spec/unit/configuration/index_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe Riddle::Configuration::DistributedIndex do 6 | it "should be invalid without a name, sources or path if there's no parent" do 7 | index = Riddle::Configuration::Index.new(nil) 8 | index.should_not be_valid 9 | 10 | index.name = "test1" 11 | index.should_not be_valid 12 | 13 | index.sources << Riddle::Configuration::SQLSource.new("source", "mysql") 14 | index.should_not be_valid 15 | 16 | index.path = "a/path" 17 | index.should be_valid 18 | 19 | index.name = nil 20 | index.should_not be_valid 21 | 22 | index.name = "test1" 23 | index.sources.clear 24 | index.should_not be_valid 25 | end 26 | 27 | it "should be invalid without a name but not sources or path if it has a parent" do 28 | index = Riddle::Configuration::Index.new(nil) 29 | index.should_not be_valid 30 | 31 | index.name = "test1stemmed" 32 | index.should_not be_valid 33 | 34 | index.parent = "test1" 35 | index.should be_valid 36 | end 37 | 38 | it "should raise a ConfigurationError if rendering when not valid" do 39 | index = Riddle::Configuration::Index.new("test1") 40 | lambda { index.render }.should raise_error(Riddle::Configuration::ConfigurationError) 41 | end 42 | 43 | it "should render correctly if supplied settings are valid" do 44 | source = Riddle::Configuration::XMLSource.new("src1", "xmlpipe") 45 | source.xmlpipe_command = "ls /dev/null" 46 | 47 | index = Riddle::Configuration::Index.new("test1", source) 48 | index.path = "/var/data/test1" 49 | index.docinfo = "extern" 50 | index.mlock = 0 51 | index.morphologies << "stem_en" << "stem_ru" << "soundex" 52 | index.min_stemming_len = 1 53 | index.stopword_files << "/var/data/stopwords.txt" << "/var/data/stopwords2.txt" 54 | index.wordform_files << "/var/data/wordforms.txt" 55 | index.exception_files << "/var/data/exceptions.txt" 56 | index.min_word_len = 1 57 | index.charset_type = "utf-8" 58 | index.charset_table = "0..9, A..Z->a..z, _, a..z, U+410..U+42F->U+430..U+44F, U+430..U+44F" 59 | index.ignore_characters << "U+00AD" 60 | index.min_prefix_len = 0 61 | index.min_infix_len = 0 62 | index.prefix_field_names << "filename" 63 | index.infix_field_names << "url" << "domain" 64 | index.enable_star = true 65 | index.ngram_len = 1 66 | index.ngram_characters << "U+3000..U+2FA1F" 67 | index.phrase_boundaries << "." << "?" << "!" << "U+2026" 68 | index.phrase_boundary_step = 100 69 | index.html_strip = 0 70 | index.html_index_attrs = "img=alt,title; a=title" 71 | index.html_remove_element_tags << "style" << "script" 72 | index.preopen = 1 73 | index.ondisk_dict = 1 74 | index.inplace_enable = 1 75 | index.inplace_hit_gap = 0 76 | index.inplace_docinfo_gap = 0 77 | index.inplace_reloc_factor = 0.1 78 | index.inplace_write_factor = 0.1 79 | index.index_exact_words = 1 80 | 81 | index.render.should == <<-INDEX 82 | source src1 83 | { 84 | type = xmlpipe 85 | xmlpipe_command = ls /dev/null 86 | } 87 | 88 | index test1 89 | { 90 | path = /var/data/test1 91 | docinfo = extern 92 | mlock = 0 93 | morphology = stem_en, stem_ru, soundex 94 | min_stemming_len = 1 95 | stopwords = /var/data/stopwords.txt /var/data/stopwords2.txt 96 | wordforms = /var/data/wordforms.txt 97 | exceptions = /var/data/exceptions.txt 98 | min_word_len = 1 99 | charset_type = utf-8 100 | charset_table = 0..9, A..Z->a..z, _, a..z, U+410..U+42F->U+430..U+44F, U+430..U+44F 101 | ignore_chars = U+00AD 102 | min_prefix_len = 0 103 | min_infix_len = 0 104 | prefix_fields = filename 105 | infix_fields = url, domain 106 | enable_star = 1 107 | ngram_len = 1 108 | ngram_chars = U+3000..U+2FA1F 109 | phrase_boundary = ., ?, !, U+2026 110 | phrase_boundary_step = 100 111 | html_strip = 0 112 | html_index_attrs = img=alt,title; a=title 113 | html_remove_elements = style, script 114 | preopen = 1 115 | ondisk_dict = 1 116 | inplace_enable = 1 117 | inplace_hit_gap = 0 118 | inplace_docinfo_gap = 0 119 | inplace_reloc_factor = 0.1 120 | inplace_write_factor = 0.1 121 | index_exact_words = 1 122 | source = src1 123 | } 124 | INDEX 125 | end 126 | end -------------------------------------------------------------------------------- /spec/unit/configuration/indexer_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe Riddle::Configuration::Indexer do 6 | it "should always be valid" do 7 | indexer = Riddle::Configuration::Indexer.new 8 | indexer.should be_valid 9 | end 10 | 11 | it "should support Sphinx's indexer settings" do 12 | settings = %w( mem_limit max_iops max_iosize ) 13 | indexer = Riddle::Configuration::Indexer.new 14 | 15 | settings.each do |setting| 16 | indexer.should respond_to(setting.to_sym) 17 | indexer.should respond_to("#{setting}=".to_sym) 18 | end 19 | end 20 | 21 | it "should render a correct configuration" do 22 | indexer = Riddle::Configuration::Indexer.new 23 | 24 | indexer.render.should == <<-INDEXER 25 | indexer 26 | { 27 | } 28 | INDEXER 29 | 30 | indexer.mem_limit = "32M" 31 | indexer.render.should == <<-INDEXER 32 | indexer 33 | { 34 | mem_limit = 32M 35 | } 36 | INDEXER 37 | end 38 | 39 | it "should render shared settings when common_sphinx_configuration is not set" do 40 | indexer = Riddle::Configuration::Indexer.new 41 | indexer.rlp_root = '/tmp' 42 | 43 | indexer.render.should == <<-INDEXER 44 | indexer 45 | { 46 | rlp_root = /tmp 47 | } 48 | INDEXER 49 | end 50 | 51 | it "should render shared settings when common_sphinx_configuration is false" do 52 | indexer = Riddle::Configuration::Indexer.new 53 | indexer.common_sphinx_configuration = false 54 | indexer.rlp_root = '/tmp' 55 | 56 | indexer.render.should == <<-INDEXER 57 | indexer 58 | { 59 | rlp_root = /tmp 60 | } 61 | INDEXER 62 | end 63 | 64 | it "should not render shared settings when common_sphinx_configuration is true" do 65 | indexer = Riddle::Configuration::Indexer.new 66 | indexer.common_sphinx_configuration = true 67 | indexer.rlp_root = '/tmp' 68 | 69 | indexer.render.should == <<-INDEXER 70 | indexer 71 | { 72 | } 73 | INDEXER 74 | end 75 | end -------------------------------------------------------------------------------- /spec/unit/configuration/realtime_index_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe Riddle::Configuration::RealtimeIndex do 6 | let(:index) { Riddle::Configuration::RealtimeIndex.new('rt1') } 7 | 8 | describe '#valid?' do 9 | it "should not be valid without a name" do 10 | index.name = nil 11 | index.path = 'foo' 12 | index.should_not be_valid 13 | end 14 | 15 | it "should not be valid without a path" do 16 | index.path = nil 17 | index.should_not be_valid 18 | end 19 | 20 | it "should be valid with a name and path" do 21 | index.path = 'foo' 22 | index.should be_valid 23 | end 24 | end 25 | 26 | describe '#type' do 27 | it "should be 'rt'" do 28 | index.type.should == 'rt' 29 | end 30 | end 31 | 32 | describe '#render' do 33 | it "should raise a ConfigurationError if rendering when not valid" do 34 | lambda { 35 | index.render 36 | }.should raise_error(Riddle::Configuration::ConfigurationError) 37 | end 38 | 39 | it "should render correctly if supplied settings are valid" do 40 | index.path = '/var/data/rt' 41 | index.rt_mem_limit = '512M' 42 | index.rt_field << 'title' << 'content' 43 | 44 | index.rt_attr_uint << 'gid' 45 | index.rt_attr_bigint << 'guid' 46 | index.rt_attr_float << 'gpa' 47 | index.rt_attr_timestamp << 'ts_added' 48 | index.rt_attr_string << 'author' 49 | 50 | index.render.should == <<-RTINDEX 51 | index rt1 52 | { 53 | type = rt 54 | path = /var/data/rt 55 | rt_mem_limit = 512M 56 | rt_field = title 57 | rt_field = content 58 | rt_attr_uint = gid 59 | rt_attr_bigint = guid 60 | rt_attr_float = gpa 61 | rt_attr_timestamp = ts_added 62 | rt_attr_string = author 63 | } 64 | RTINDEX 65 | end 66 | end 67 | end 68 | -------------------------------------------------------------------------------- /spec/unit/configuration/source_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe Riddle::Configuration::Source do 6 | # 7 | end -------------------------------------------------------------------------------- /spec/unit/configuration/template_index_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe Riddle::Configuration::TemplateIndex do 6 | it "should be invalid without a name" do 7 | index = Riddle::Configuration::TemplateIndex.new(nil) 8 | index.should_not be_valid 9 | 10 | index.name = "test1" 11 | index.should be_valid 12 | end 13 | 14 | it "should raise a ConfigurationError if rendering when not valid" do 15 | index = Riddle::Configuration::TemplateIndex.new(nil) 16 | lambda { 17 | index.render 18 | }.should raise_error(Riddle::Configuration::ConfigurationError) 19 | end 20 | 21 | it "should render correctly if supplied settings are valid" do 22 | index = Riddle::Configuration::TemplateIndex.new("test1") 23 | index.docinfo = "extern" 24 | index.mlock = 0 25 | index.morphologies << "stem_en" << "stem_ru" << "soundex" 26 | index.min_stemming_len = 1 27 | index.stopword_files << "/var/data/stopwords.txt" << "/var/data/stopwords2.txt" 28 | index.wordform_files << "/var/data/wordforms.txt" 29 | index.exception_files << "/var/data/exceptions.txt" 30 | index.min_word_len = 1 31 | index.charset_type = "utf-8" 32 | index.charset_table = "0..9, A..Z->a..z, _, a..z, U+410..U+42F->U+430..U+44F, U+430..U+44F" 33 | index.ignore_characters << "U+00AD" 34 | index.min_prefix_len = 0 35 | index.min_infix_len = 0 36 | index.prefix_field_names << "filename" 37 | index.infix_field_names << "url" << "domain" 38 | index.enable_star = true 39 | index.ngram_len = 1 40 | index.ngram_characters << "U+3000..U+2FA1F" 41 | index.phrase_boundaries << "." << "?" << "!" << "U+2026" 42 | index.phrase_boundary_step = 100 43 | index.html_strip = 0 44 | index.html_index_attrs = "img=alt,title; a=title" 45 | index.html_remove_element_tags << "style" << "script" 46 | index.preopen = 1 47 | index.ondisk_dict = 1 48 | index.inplace_enable = 1 49 | index.inplace_hit_gap = 0 50 | index.inplace_docinfo_gap = 0 51 | index.inplace_reloc_factor = 0.1 52 | index.inplace_write_factor = 0.1 53 | index.index_exact_words = 1 54 | 55 | index.render.should == <<-INDEX 56 | index test1 57 | { 58 | type = template 59 | docinfo = extern 60 | mlock = 0 61 | morphology = stem_en, stem_ru, soundex 62 | min_stemming_len = 1 63 | stopwords = /var/data/stopwords.txt /var/data/stopwords2.txt 64 | wordforms = /var/data/wordforms.txt 65 | exceptions = /var/data/exceptions.txt 66 | min_word_len = 1 67 | charset_type = utf-8 68 | charset_table = 0..9, A..Z->a..z, _, a..z, U+410..U+42F->U+430..U+44F, U+430..U+44F 69 | ignore_chars = U+00AD 70 | min_prefix_len = 0 71 | min_infix_len = 0 72 | prefix_fields = filename 73 | infix_fields = url, domain 74 | enable_star = 1 75 | ngram_len = 1 76 | ngram_chars = U+3000..U+2FA1F 77 | phrase_boundary = ., ?, !, U+2026 78 | phrase_boundary_step = 100 79 | html_strip = 0 80 | html_index_attrs = img=alt,title; a=title 81 | html_remove_elements = style, script 82 | preopen = 1 83 | ondisk_dict = 1 84 | inplace_enable = 1 85 | inplace_hit_gap = 0 86 | inplace_docinfo_gap = 0 87 | inplace_reloc_factor = 0.1 88 | inplace_write_factor = 0.1 89 | index_exact_words = 1 90 | } 91 | INDEX 92 | end 93 | end 94 | -------------------------------------------------------------------------------- /spec/unit/configuration/tsv_source_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe Riddle::Configuration::TSVSource do 6 | it "should be invalid without an tsvpipe command, name and type if there's no parent" do 7 | source = Riddle::Configuration::TSVSource.new("tsv1") 8 | source.should_not be_valid 9 | 10 | source.tsvpipe_command = "ls /var/null" 11 | source.should be_valid 12 | 13 | source.name = nil 14 | source.should_not be_valid 15 | 16 | source.name = "tsv1" 17 | source.type = nil 18 | source.should_not be_valid 19 | end 20 | 21 | it "should be invalid without only a name and type if there is a parent" do 22 | source = Riddle::Configuration::TSVSource.new("tsv1") 23 | source.should_not be_valid 24 | 25 | source.parent = "tsvparent" 26 | source.should be_valid 27 | 28 | source.name = nil 29 | source.should_not be_valid 30 | 31 | source.name = "tsv1" 32 | source.type = nil 33 | source.should_not be_valid 34 | end 35 | 36 | it "should raise a ConfigurationError if rendering when not valid" do 37 | source = Riddle::Configuration::TSVSource.new("tsv1") 38 | lambda { 39 | source.render 40 | }.should raise_error(Riddle::Configuration::ConfigurationError) 41 | end 42 | 43 | it "should render correctly when valid" do 44 | source = Riddle::Configuration::TSVSource.new("tsv1") 45 | source.tsvpipe_command = "ls /var/null" 46 | 47 | source.render.should == <<-TSVSOURCE 48 | source tsv1 49 | { 50 | type = tsvpipe 51 | tsvpipe_command = ls /var/null 52 | } 53 | TSVSOURCE 54 | end 55 | end 56 | -------------------------------------------------------------------------------- /spec/unit/configuration/xml_source_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe Riddle::Configuration::XMLSource do 6 | it "should be invalid without an xmlpipe command, name and type if there's no parent" do 7 | source = Riddle::Configuration::XMLSource.new("xml1", "xmlpipe") 8 | source.should_not be_valid 9 | 10 | source.xmlpipe_command = "ls /var/null" 11 | source.should be_valid 12 | 13 | source.name = nil 14 | source.should_not be_valid 15 | 16 | source.name = "xml1" 17 | source.type = nil 18 | source.should_not be_valid 19 | end 20 | 21 | it "should be invalid without only a name and type if there is a parent" do 22 | source = Riddle::Configuration::XMLSource.new("xml1", "xmlpipe") 23 | source.should_not be_valid 24 | 25 | source.parent = "xmlparent" 26 | source.should be_valid 27 | 28 | source.name = nil 29 | source.should_not be_valid 30 | 31 | source.name = "xml1" 32 | source.type = nil 33 | source.should_not be_valid 34 | end 35 | 36 | it "should raise a ConfigurationError if rendering when not valid" do 37 | source = Riddle::Configuration::XMLSource.new("xml1", "xmlpipe") 38 | lambda { source.render }.should raise_error(Riddle::Configuration::ConfigurationError) 39 | end 40 | 41 | it "should render correctly when valid" do 42 | source = Riddle::Configuration::XMLSource.new("xml1", "xmlpipe") 43 | source.xmlpipe_command = "ls /var/null" 44 | 45 | source.render.should == <<-XMLSOURCE 46 | source xml1 47 | { 48 | type = xmlpipe 49 | xmlpipe_command = ls /var/null 50 | } 51 | XMLSOURCE 52 | end 53 | end -------------------------------------------------------------------------------- /spec/unit/configuration_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe Riddle::Configuration do 6 | it "should render all given indexes and sources, plus the indexer and search sections" do 7 | config = Riddle::Configuration.new 8 | 9 | config.searchd.port = 3312 10 | config.searchd.pid_file = "file.pid" 11 | 12 | source = Riddle::Configuration::XMLSource.new("src1", "xmlpipe") 13 | source.xmlpipe_command = "ls /dev/null" 14 | 15 | index = Riddle::Configuration::Index.new("index1") 16 | index.path = "/path/to/index1" 17 | index.sources << source 18 | 19 | config.indices << index 20 | generated_conf = config.render 21 | 22 | generated_conf.should match(/index index1/) 23 | generated_conf.should match(/source src1/) 24 | generated_conf.should match(/indexer/) 25 | generated_conf.should match(/searchd/) 26 | end 27 | end -------------------------------------------------------------------------------- /spec/unit/filter_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe Riddle::Client::Filter do 6 | it "should render a filter that uses an array of ints correctly" do 7 | filter = Riddle::Client::Filter.new("field", [1, 2, 3]) 8 | filter.query_message.should == query_contents(:filter_array) 9 | end 10 | 11 | it "should render a filter that has exclude set correctly" do 12 | filter = Riddle::Client::Filter.new("field", [1, 2, 3], true) 13 | filter.query_message.should == query_contents(:filter_array_exclude) 14 | end 15 | 16 | it "should render a filter that is a range of ints correctly" do 17 | filter = Riddle::Client::Filter.new("field", 1..3) 18 | filter.query_message.should == query_contents(:filter_range) 19 | end 20 | 21 | it "should render a filter that is a range of ints as exclude correctly" do 22 | filter = Riddle::Client::Filter.new("field", 1..3, true) 23 | filter.query_message.should == query_contents(:filter_range_exclude) 24 | end 25 | 26 | it "should render a filter that is a range of floats correctly" do 27 | filter = Riddle::Client::Filter.new("field", 5.4..13.5) 28 | filter.query_message.should == query_contents(:filter_floats) 29 | end 30 | 31 | it "should render a filter that is a range of floats as exclude correctly" do 32 | filter = Riddle::Client::Filter.new("field", 5.4..13.5, true) 33 | filter.query_message.should == query_contents(:filter_floats_exclude) 34 | end 35 | 36 | it "should render a filter that is an array of boolean values correctly" do 37 | filter = Riddle::Client::Filter.new("field", [false, true]) 38 | filter.query_message.should == query_contents(:filter_boolean) 39 | end 40 | end -------------------------------------------------------------------------------- /spec/unit/message_spec.rb: -------------------------------------------------------------------------------- 1 | # encoding: BINARY 2 | # frozen_string_literal: true 3 | 4 | require 'spec_helper' 5 | 6 | describe Riddle::Client::Message do 7 | it "should start with an empty string" do 8 | Riddle::Client::Message.new.to_s.should == "" 9 | end 10 | 11 | it "should append raw data correctly" do 12 | data = [1, 2, 3].pack('NNN') 13 | message = Riddle::Client::Message.new 14 | message.append data 15 | message.to_s.should == data 16 | end 17 | 18 | it "should append strings correctly - with length first" do 19 | str = "something to test with" 20 | message = Riddle::Client::Message.new 21 | message.append_string str 22 | message.to_s.should == [str.length].pack('N') + str 23 | end 24 | 25 | it "should append integers correctly - packed with N" do 26 | message = Riddle::Client::Message.new 27 | message.append_int 234 28 | message.to_s.should == "\x00\x00\x00\xEA" 29 | end 30 | 31 | it "should append integers as strings correctly - packed with N" do 32 | message = Riddle::Client::Message.new 33 | message.append_int "234" 34 | message.to_s.should == "\x00\x00\x00\xEA" 35 | end 36 | 37 | it "should append 64bit integers correctly" do 38 | message = Riddle::Client::Message.new 39 | message.append_64bit_int 234 40 | message.to_s.should == "\x00\x00\x00\x00\x00\x00\x00\xEA" 41 | end 42 | 43 | it "should append 64bit integers that use exactly 32bits correctly" do 44 | message = Riddle::Client::Message.new 45 | message.append_64bit_int 4294967295 46 | message.to_s.should == "\x00\x00\x00\x00\xFF\xFF\xFF\xFF" 47 | end 48 | 49 | it "should append 64bit integers that use more than 32 bits correctly" do 50 | message = Riddle::Client::Message.new 51 | message.append_64bit_int 4294967296 52 | message.to_s.should == "\x00\x00\x00\x01\x00\x00\x00\x00" 53 | end 54 | 55 | it "should append 64bit integers as strings correctly" do 56 | message = Riddle::Client::Message.new 57 | message.append_64bit_int "234" 58 | message.to_s.should == "\x00\x00\x00\x00\x00\x00\x00\xEA" 59 | end 60 | 61 | it "should append floats correctly - packed with f" do 62 | message = Riddle::Client::Message.new 63 | message.append_float 1.4 64 | message.to_s.should == [1.4].pack('f').unpack('L*').pack('N') 65 | end 66 | 67 | it "should append a collection of integers correctly" do 68 | message = Riddle::Client::Message.new 69 | message.append_ints 1, 2, 3, 4 70 | message.to_s.should == [1, 2, 3, 4].pack('NNNN') 71 | end 72 | 73 | it "should append a collection of floats correctly" do 74 | message = Riddle::Client::Message.new 75 | message.append_floats 1.0, 1.1, 1.2, 1.3 76 | message.to_s.should == [1.0, 1.1, 1.2, 1.3].pack('ffff').unpack('L*L*L*L*').pack('NNNN') 77 | end 78 | 79 | it "should append an array of strings correctly" do 80 | arr = ["a", "bb", "ccc"] 81 | message = Riddle::Client::Message.new 82 | message.append_array arr 83 | message.to_s.should == [3, 1].pack('NN') + "a" + [2].pack('N') + "bb" + 84 | [3].pack('N') + "ccc" 85 | end 86 | 87 | it "should append a variety of objects correctly" do 88 | message = Riddle::Client::Message.new 89 | message.append_int 4 90 | message.append_string "test" 91 | message.append_array ["one", "two"] 92 | message.append_floats 1.5, 1.7 93 | message.to_s.should == [4, 4].pack('NN') + "test" + [2, 3].pack('NN') + 94 | "one" + [3].pack('N') + "two" + [1.5, 1.7].pack('ff').unpack('L*L*').pack('NN') 95 | end 96 | end 97 | -------------------------------------------------------------------------------- /spec/unit/response_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe Riddle::Client::Response do 6 | it "should interpret an integer correctly" do 7 | Riddle::Client::Response.new([42].pack('N')).next_int.should == 42 8 | end 9 | 10 | it "should interpret a string correctly" do 11 | str = "this is a string" 12 | Riddle::Client::Response.new( 13 | [str.length].pack('N') + str 14 | ).next.should == str 15 | end 16 | 17 | # Comparing floats with decimal places doesn't seem to be exact 18 | it "should interpret a float correctly" do 19 | Riddle::Client::Response.new([1.0].pack('f').unpack('L*').pack('N')).next_float.should == 1.0 20 | end 21 | 22 | it "should interpret an array of strings correctly" do 23 | arr = ["a", "b", "c", "d"] 24 | Riddle::Client::Response.new( 25 | [arr.length].pack('N') + arr.collect { |str| 26 | [str.length].pack('N') + str 27 | }.join("") 28 | ).next_array.should == arr 29 | end 30 | 31 | it "should interpret an array of ints correctly" do 32 | arr = [1, 2, 3, 4] 33 | Riddle::Client::Response.new( 34 | [arr.length].pack('N') + arr.collect { |int| 35 | [int].pack('N') 36 | }.join("") 37 | ).next_int_array.should == arr 38 | end 39 | 40 | it "should reflect the length of the incoming data correctly" do 41 | data = [1, 2, 3, 4].pack('NNNN') 42 | Riddle::Client::Response.new(data).length.should == data.length 43 | end 44 | 45 | it "should handle a combination of strings and ints correctly" do 46 | data = [1, 3, 5, 1].pack('NNNN') + 'a' + [2, 4].pack('NN') + 'test' 47 | response = Riddle::Client::Response.new(data) 48 | response.next_int.should == 1 49 | response.next_int.should == 3 50 | response.next_int.should == 5 51 | response.next.should == 'a' 52 | response.next_int.should == 2 53 | response.next.should == 'test' 54 | end 55 | 56 | it "should handle a combination of strings, ints, floats and string arrays correctly" do 57 | data = [1, 2, 2].pack('NNN') + 'aa' + [2].pack('N') + 'bb' + [4].pack('N') + 58 | "word" + [7].pack('f').unpack('L*').pack('N') + [3, 2, 2, 2].pack('NNNN') 59 | response = Riddle::Client::Response.new(data) 60 | response.next_int.should == 1 61 | response.next_array.should == ['aa', 'bb'] 62 | response.next.should == "word" 63 | response.next_float.should == 7 64 | response.next_int_array.should == [2, 2, 2] 65 | end 66 | end -------------------------------------------------------------------------------- /spec/unit/riddle_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe "Riddle" do 6 | it "should escape characters correctly" do 7 | invalid_chars = ['(', ')', '|', '-', '!', '@', '~', '"', '/'] 8 | invalid_chars.each do |char| 9 | base = "string with '#{char}' character" 10 | Riddle.escape(base).should == base.gsub(char, "\\#{char}") 11 | end 12 | 13 | # Not sure why this doesn't work within the loop... 14 | Riddle.escape("string with & character").should == "string with \\& character" 15 | 16 | all_chars = invalid_chars.join('') + '&' 17 | Riddle.escape(all_chars).should == "\\(\\)\\|\\-\\!\\@\\~\\\"\\/\\&" 18 | end 19 | end --------------------------------------------------------------------------------