├── .gitignore
├── install.rb
├── uninstall.rb
├── init.rb
├── test
├── fixtures
│ ├── example_model.rb
│ ├── schema.rb
│ └── example_models.yml
├── test_helper.rb
└── dystopian_index_test.rb
├── tasks
└── dystopian_index_tasks.rake
├── Rakefile
├── MIT-LICENSE
├── benchmarks
└── indexing.rb
├── README.textile
└── lib
└── dystopian_index.rb
/.gitignore:
--------------------------------------------------------------------------------
1 | *.swp
2 | *.swo
3 |
--------------------------------------------------------------------------------
/install.rb:
--------------------------------------------------------------------------------
1 | # Install hook code here
2 |
--------------------------------------------------------------------------------
/uninstall.rb:
--------------------------------------------------------------------------------
1 | # Uninstall hook code here
2 |
--------------------------------------------------------------------------------
/init.rb:
--------------------------------------------------------------------------------
1 | $:.unshift "#{File.dirname(__FILE__)}/lib"
2 | require 'rufus/tokyo/dystopia'
3 | require 'dystopian_index'
4 |
5 | DystopianIndex.enable
6 |
--------------------------------------------------------------------------------
/test/fixtures/example_model.rb:
--------------------------------------------------------------------------------
1 | class ExampleModel < ActiveRecord::Base
2 | dystopian_index do
3 | indexes :content
4 | indexes :name
5 | order_by :created_at
6 | end
7 | end
8 |
--------------------------------------------------------------------------------
/test/fixtures/schema.rb:
--------------------------------------------------------------------------------
1 | unless ExampleModel.table_exists?
2 | ActiveRecord::Schema.define do
3 | create_table 'example_models', :force => true do |t|
4 | t.column 'name', :text
5 | t.column 'content', :text
6 | t.timestamps
7 | end
8 | end
9 | end
10 |
--------------------------------------------------------------------------------
/tasks/dystopian_index_tasks.rake:
--------------------------------------------------------------------------------
1 | require File.join(RAILS_ROOT, 'config', 'environment')
2 |
3 | namespace :dystopia do
4 | desc "Indexes all models"
5 | task :index do
6 | DystopianIndex.index_all
7 | end
8 |
9 | desc "Runs benchmarks"
10 | task :benchmarks do
11 | Dir["#{File.dirname(__FILE__)}/../benchmarks/*.rb"].each do |file|
12 | load file
13 | end
14 | end
15 | end
16 |
--------------------------------------------------------------------------------
/Rakefile:
--------------------------------------------------------------------------------
1 | require 'rake'
2 | require 'rake/testtask'
3 | require 'rake/rdoctask'
4 |
5 | desc 'Default: run unit tests.'
6 | task :default => :test
7 |
8 | desc 'Test the dystopian_index plugin.'
9 | Rake::TestTask.new(:test) do |t|
10 | t.libs << 'lib'
11 | t.libs << 'test'
12 | t.pattern = 'test/**/*_test.rb'
13 | t.verbose = true
14 | end
15 |
16 | desc 'Generate documentation for the dystopian_index plugin.'
17 | Rake::RDocTask.new(:rdoc) do |rdoc|
18 | rdoc.rdoc_dir = 'rdoc'
19 | rdoc.title = 'DystopianIndex'
20 | rdoc.options << '--line-numbers' << '--inline-source'
21 | rdoc.rdoc_files.include('README')
22 | rdoc.rdoc_files.include('lib/**/*.rb')
23 | end
24 |
--------------------------------------------------------------------------------
/test/fixtures/example_models.yml:
--------------------------------------------------------------------------------
1 | alex:
2 | name: Alex Young
3 | content: Software engineer based in London, England.
4 | created_at: 2009-07-01 18:33:50
5 | yuka:
6 | name: Yuka Young
7 | content: Bilingual videogames enthusiast based in London.
8 | created_at: 2009-07-02 18:33:50
9 | rowland:
10 | name: Rowland Watkins
11 | content: Programming and business consultant based in Hong Kong.
12 | created_at: 2009-06-06 18:33:50
13 | kev:
14 | name: Kevin Ford
15 | content: Systems administrator currently based in Swindon, Wilts.
16 | created_at: 2009-07-20 18:33:50
17 | simon:
18 | name: Simon Starr
19 | content: Rails developer based in Yorkshire.
20 | created_at: 2009-07-30 18:33:50
21 |
--------------------------------------------------------------------------------
/test/test_helper.rb:
--------------------------------------------------------------------------------
1 | require 'test/unit'
2 | require 'rubygems'
3 |
4 | # Include the application's test_helper
5 | unless defined?(ActiveRecord)
6 | begin
7 | require '../../../test/test_helper'
8 | rescue LoadError
9 | require 'test_helper'
10 | end
11 | end
12 |
13 | DYSTOPIA_PLUGIN_PATH = File.join(RAILS_ROOT, 'vendor', 'plugins', 'dystopian_index')
14 | fixtures_path = File.join(DYSTOPIA_PLUGIN_PATH, 'test', 'fixtures')
15 |
16 | DystopianIndex.config.db_path = File.join(DYSTOPIA_PLUGIN_PATH, 'test', 'fixtures', 'indexes')
17 | DystopianIndex.config.model_directories = [File.join(DYSTOPIA_PLUGIN_PATH, 'test', 'fixtures')]
18 |
19 | require File.join(fixtures_path, 'example_model')
20 | require File.join(fixtures_path, 'schema')
21 |
22 | Fixtures.create_fixtures(fixtures_path, ActiveRecord::Base.connection.tables)
23 |
--------------------------------------------------------------------------------
/test/dystopian_index_test.rb:
--------------------------------------------------------------------------------
1 | require File.join(File.dirname(__FILE__), 'test_helper')
2 |
3 | class DystopianIndexTest < ActiveSupport::TestCase
4 | def setup
5 | ExampleModel.clear_dystopian_index!
6 | ExampleModel.index_all
7 | end
8 |
9 | test "search" do
10 | assert_equal 1, ExampleModel.search('alex').size
11 | end
12 |
13 | test "pagination" do
14 | assert_equal 2, ExampleModel.search('based', :per_page => 2, :page => 1).size
15 | end
16 |
17 | # The example model has order_by enabled
18 | test "sorting" do
19 | results = ExampleModel.search('based')
20 | assert results.first.created_at < results.last.created_at
21 | end
22 |
23 | test "deletion removes records from index" do
24 | assert !ExampleModel.search('alex').empty?
25 | ExampleModel.search('alex').each { |m| m.destroy }
26 | assert ExampleModel.search('alex').empty?
27 | end
28 |
29 | test "searching for nil should return nil" do
30 | assert_equal ExampleModel.search(nil), nil
31 | end
32 | end
33 |
--------------------------------------------------------------------------------
/MIT-LICENSE:
--------------------------------------------------------------------------------
1 | Copyright (c) 2009 Alex R. Young, for Helicoid Limited
2 |
3 | Permission is hereby granted, free of charge, to any person obtaining
4 | a copy of this software and associated documentation files (the
5 | "Software"), to deal in the Software without restriction, including
6 | without limitation the rights to use, copy, modify, merge, publish,
7 | distribute, sublicense, and/or sell copies of the Software, and to
8 | permit persons to whom the Software is furnished to do so, subject to
9 | the following conditions:
10 |
11 | The above copyright notice and this permission notice shall be
12 | included in all copies or substantial portions of the Software.
13 |
14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
21 |
--------------------------------------------------------------------------------
/benchmarks/indexing.rb:
--------------------------------------------------------------------------------
1 | require 'test_helper'
2 | require File.join(File.dirname(__FILE__), '..', 'test', 'test_helper')
3 | require 'benchmark'
4 | require 'faker'
5 |
6 | def clear!
7 | ExampleModel.clear_dystopian_index!
8 | ExampleModel.delete_all
9 | DystopianIndex.config.disabled = false
10 | end
11 |
12 | def fake_record
13 | ExampleModel.create :name => Faker::Name.name, :content => Faker::Lorem.paragraph
14 | end
15 |
16 | count = 500
17 |
18 | puts "Running benchmarks. Please make a cup of tea or coffee."
19 | clear!
20 |
21 | #
22 | # Benchmark #1
23 | #
24 |
25 | bm = Benchmark.measure do
26 | count.times do
27 | fake_record
28 | end
29 | end
30 |
31 | puts "[with indexing enabled] #{count} records created in: #{bm}"
32 | clear!
33 |
34 | #
35 | # Benchmark #2
36 | #
37 |
38 | DystopianIndex.config.disabled = true
39 |
40 | bm = Benchmark.measure do
41 | count.times do
42 | fake_record
43 | end
44 | end
45 |
46 | puts "[with indexing disabled] #{count} records created in: #{bm}"
47 | clear!
48 |
49 | #
50 | # Benchmark #3
51 | #
52 |
53 | DystopianIndex.config.disabled = true
54 |
55 | db = Rufus::Tokyo::Dystopia::Core.new ExampleModel.dystopian_config[:db]
56 |
57 | bm = Benchmark.measure do
58 | count.times do
59 | model = fake_record
60 | db.store model.id, model.dystopian_payload
61 | end
62 | end
63 |
64 | db.close
65 |
66 | puts "[with a single dystopia connection] #{count} records created in: #{bm}"
67 |
--------------------------------------------------------------------------------
/README.textile:
--------------------------------------------------------------------------------
1 | h2. DystopianIndex
2 |
3 | This is a Rails plugin that uses Tokyo Dystopia to index models. It's ideal for small projects that need a fast indexer because it's easy to setup and understand, and doesn't use too many resources or daemon processes.
4 |
5 | h3. Requirements
6 |
7 | Gems:
8 |
9 | * faker
10 | * rufus-tokyo
11 |
12 | h3. Installation
13 |
14 | Build and compile Tokyo Cabinet and Tokyo Dystopia from "tokyocabinet.sourceforge.net":http://tokyocabinet.sourceforge.net/ -- they're clean and simple C projects and should build on your Mac or Linux machine.
15 |
16 | Then install the "rufus-tokyo":http://github.com/jmettraux/rufus-tokyo gem.
17 |
18 | h3. Usage
19 |
20 | To use the plugin with a model, define an index like this:
21 |
22 |
23 | dystopian_index do
24 | indexes :content, :name
25 | order_by :created_at
26 | end
27 |
28 |
29 | h3. Rake Tasks
30 |
31 |
32 | rake dystopia:benchmarks # Runs benchmarks
33 | rake dystopia:index # Indexes all models
34 |
35 |
36 | Database files are stored in db/indexes. You can change the path where the database is stored by setting DystopianIndex.config.db_path.
37 |
38 | h3. Plugin Design
39 |
40 | * Each model has its own Dystopia index
41 | * Fields are concatenated into one big index record -- this means you can't currently search according to field
42 | * When an update is performed, the index is opened and closed on demand to help prevent data loss. I've indexed this scheme (see the benchmarks) and it's not too slow
43 | * Data information is stored in the first few bytes of each index record to help sort values
44 | * Indexes are stored in db/indexes/ -- the plugin will make this directory if required
45 |
46 | h3. TODO
47 |
48 | * Tests only run from within a Rails app, and by executing each test file with ruby
49 | * It might be better to create separate indexes for each field.
50 | * order_by needs desc
51 | * Sorting fetches every matching ID from dystopia and sorts according to date integers. There might be a better way to do this, but it's still pretty fast
52 |
53 | h3. Acknowledgements
54 |
55 | The Rake tasks need to load all models. This code was based on "ThinkingSphinx's":http://freelancing-god.github.com/ts/en/ code that does the same thing (which is a nice piece of work by the way). I hope to refactor this out later.
56 |
57 | Copyright (c) 2009 Alex R. Young, released under the MIT license
58 |
--------------------------------------------------------------------------------
/lib/dystopian_index.rb:
--------------------------------------------------------------------------------
1 | require 'singleton'
2 |
3 | module DystopianIndex
4 | def self.debug ; true ; end
5 |
6 | def self.included(base)
7 | base.extend ActiveRecordHook
8 | end
9 |
10 | def self.enable
11 | config.reset
12 |
13 | if Object.const_defined? 'ActiveRecord'
14 | ActiveRecord::Base.class_eval { include DystopianIndex }
15 | end
16 | end
17 |
18 | def self.load_models
19 | config.model_directories.each do |base|
20 | Dir["#{base}**/*.rb"].each do |file|
21 | model_name = file.gsub(/^#{base}([\w_\/\\]+)\.rb/, '\1')
22 |
23 | next if model_name.nil?
24 | next if ::ActiveRecord::Base.send(:subclasses).detect { |model|
25 | model.name == model_name
26 | }
27 |
28 | begin
29 | model_name.camelize.constantize
30 | rescue LoadError
31 | model_name.gsub!(/.*[\/\\]/, '').nil? ? next : retry
32 | rescue NameError
33 | next
34 | end
35 | end
36 | end
37 | end
38 |
39 | def self.index_all
40 | load_models
41 | DystopianIndex.config.models.each do |table_name, settings|
42 | settings[:klass].clear_dystopian_index!
43 | settings[:klass].index_all
44 | end
45 | end
46 |
47 | def self.config
48 | DystopianIndex::Configuration.instance
49 | end
50 |
51 | class Builder
52 | def initialize(model)
53 | @model = model
54 | @fields = []
55 | @order_by = nil
56 |
57 | Dir.mkdir(db_path) unless File.exists?(db_path)
58 | @db_name = File.join db_path, "#{model.table_name}.dys"
59 | end
60 |
61 | def db_path
62 | DystopianIndex.config.db_path
63 | end
64 |
65 | def order_by(field)
66 | @order_by = field
67 | end
68 |
69 | def indexes(*fields)
70 | @fields += fields
71 | end
72 |
73 | def apply!
74 | apply_methods!
75 | add_settings
76 | end
77 |
78 | private
79 |
80 | def apply_methods!
81 | @model.class_eval <<-RUBY
82 | include DystopianIndex::ModelMethods
83 | after_save :update_dystopian_index
84 | before_destroy :remove_from_index
85 | RUBY
86 |
87 | @model.extend DystopianIndex::ModelClassMethods
88 | end
89 |
90 | def add_settings
91 | DystopianIndex.config.models[@model.table_name] = {
92 | :fields => @fields,
93 | :db => @db_name,
94 | :klass => @model,
95 | :order_by => @order_by
96 | }
97 | end
98 | end
99 |
100 | class Configuration
101 | include Singleton
102 |
103 | attr_accessor :models, :app_root, :model_directories, :disabled, :db_path
104 |
105 | def reset
106 | self.app_root = RAILS_ROOT
107 | self.model_directories = ["#{app_root}/app/models/"]
108 | self.disabled = false
109 | self.db_path = File.join(self.app_root, 'db', 'indexes')
110 | end
111 | end
112 |
113 | module ModelClassMethods
114 | def dystopian_config
115 | DystopianIndex.config.models[table_name]
116 | end
117 |
118 | # == Examples
119 | #
120 | # Simple text search:
121 | #
122 | # User.search "name"
123 | #
124 | # With pagination:
125 | #
126 | # User.search "name", :page => (params[:page]), :per_page => 10
127 | #
128 | def search(*args)
129 | query = args.first
130 | return if query.nil?
131 |
132 | args = args.last.is_a?(Hash) ? args.last : {}
133 | args[:order] = dystopian_config[:order_by]
134 | paginate_results search_ids(query, args), args
135 | end
136 |
137 | def search_ids(query, args = {})
138 | sort_results(with_dystopian_db { |db| db.search(query) })
139 | end
140 |
141 | def clear_dystopian_index!
142 | with_dystopian_db do |db|
143 | db.clear
144 | end
145 | end
146 |
147 | def index_all
148 | find(:all).each do |model|
149 | model.update_dystopian_index
150 | end
151 | end
152 |
153 | def indexer_uses_timestamps?
154 | return false unless dystopian_config[:order_by]
155 | order_by = dystopian_config[:order_by].to_s
156 | if columns_hash[order_by]
157 | columns_hash[order_by].type == :datetime
158 | end
159 | end
160 |
161 | # This will optionally use the date integer (specified by order_by) to sort results
162 | def sort_results(ids, args = {})
163 | if indexer_uses_timestamps?
164 | records = []
165 | with_dystopian_db do |db|
166 | records = ids.collect { |id| [id, db.fetch(id)[0..12].to_i] }
167 | end
168 | records.sort! { |a, b| a[1] <=> b[1] }
169 | records.collect { |a| a[0] }
170 | else
171 | ids
172 | end
173 | end
174 |
175 | def paginate_results(ids, args)
176 | if args[:page] and defined?(WillPaginate)
177 | args[:page] = args[:page].to_i
178 | args[:per_page] = args[:per_page].to_i
179 |
180 | WillPaginate::Collection.create(args[:page], args[:per_page], ids.size) do |pager|
181 | start = (args[:page] - 1) * args[:per_page]
182 | pager.replace(find ids[start, args[:per_page]], :order => dystopian_config[:order_by])
183 | end
184 | else
185 | find ids, args
186 | end
187 | end
188 |
189 | def with_dystopian_db
190 | db = Rufus::Tokyo::Dystopia::Core.new dystopian_config[:db]
191 | results = yield(db)
192 | ensure
193 | db.close if db.respond_to?(:close)
194 | results
195 | end
196 | end
197 |
198 | module ModelMethods
199 | def dystopian_fields
200 | self.class.dystopian_config[:fields]
201 | end
202 |
203 | def with_dystopian_db(&block)
204 | self.class.with_dystopian_db &block
205 | end
206 |
207 | def update_dystopian_index
208 | return if DystopianIndex.config.disabled
209 | with_dystopian_db do |db|
210 | db.store id, dystopian_payload
211 | end
212 | end
213 |
214 | def remove_from_index
215 | return if DystopianIndex.config.disabled
216 | with_dystopian_db do |db|
217 | db.delete id
218 | end
219 | end
220 |
221 | def dystopian_data
222 | dystopian_fields.collect { |field| read_attribute field }.join("\n")
223 | end
224 |
225 | def dystopian_timestamps
226 | if self.class.indexer_uses_timestamps?
227 | # to_f.to_i in case the value is DateTime
228 | read_attribute(self.class.dystopian_config[:order_by]).to_f.to_i.to_s.ljust(13)
229 | else
230 | ''
231 | end
232 | end
233 |
234 | def dystopian_payload
235 | "#{dystopian_timestamps}#{dystopian_data}"
236 | end
237 | end
238 |
239 | module ActiveRecordHook
240 | def dystopian_index(&block)
241 | DystopianIndex.config.models ||= {}
242 |
243 | builder = DystopianIndex::Builder.new(self)
244 | builder.instance_eval(&block)
245 | builder.apply!
246 | end
247 | end
248 | end
249 |
--------------------------------------------------------------------------------