├── test ├── files │ ├── have-null.txt │ ├── japanese.txt │ ├── one-page.odt │ ├── one-page.pdf │ └── encrypted.zip ├── application_system_test_case.rb ├── functional │ └── fts_query_expansions_controller_test.rb ├── unit │ └── full_text_search │ │ ├── issue_query_any_searchable_test.rb │ │ ├── project_test.rb │ │ ├── changeset_test.rb │ │ ├── journal_test.rb │ │ ├── plugin_wiki_extensions_tag_searchable_test.rb │ │ ├── query_expansion_synchronizer_test.rb │ │ ├── issue_test.rb │ │ ├── change_git_test.rb │ │ ├── similar_search_issue_test.rb │ │ └── change_subversion_test.rb ├── system │ └── full_text_search │ │ └── search_test.rb └── test_helper.rb ├── app ├── views │ ├── fts_query_expand │ │ └── index.html.erb │ ├── fts_query_expansions │ │ ├── _form.html.erb │ │ ├── edit.html.erb │ │ ├── new.html.erb │ │ ├── show.html.erb │ │ └── index.html.erb │ ├── search │ │ └── index.api.rsb │ ├── issues │ │ └── full_text_search │ │ │ └── _view_issues_show_description_bottom.html.erb │ └── settings │ │ └── _full_text_search.html.erb ├── models │ ├── full_text_search │ │ ├── issue_content.rb │ │ ├── query_expansion_request.rb │ │ ├── tag_type.rb │ │ ├── type.rb │ │ ├── tag.rb │ │ └── target.rb │ └── fts_query_expansion.rb ├── jobs │ └── full_text_search │ │ ├── extract_text_job.rb │ │ ├── upsert_target_job.rb │ │ └── update_issue_content_job.rb ├── controllers │ ├── fts_query_expand_controller.rb │ └── fts_query_expansions_controller.rb └── types │ └── full_text_search │ └── mroonga_integer_array_type.rb ├── assets ├── stylesheets │ ├── similar_issues.css │ ├── fontawesome-free-5.8.2-web │ │ ├── webfonts │ │ │ ├── fa-solid-900.eot │ │ │ ├── fa-solid-900.ttf │ │ │ ├── fa-brands-400.eot │ │ │ ├── fa-brands-400.ttf │ │ │ ├── fa-brands-400.woff │ │ │ ├── fa-regular-400.eot │ │ │ ├── fa-regular-400.ttf │ │ │ ├── fa-solid-900.woff │ │ │ ├── fa-solid-900.woff2 │ │ │ ├── fa-brands-400.woff2 │ │ │ ├── fa-regular-400.woff │ │ │ └── fa-regular-400.woff2 │ │ ├── css │ │ │ ├── brands.min.css │ │ │ ├── solid.min.css │ │ │ ├── regular.min.css │ │ │ ├── brands.css │ │ │ ├── solid.css │ │ │ ├── regular.css │ │ │ └── svg-with-js.min.css │ │ └── LICENSE.txt │ ├── score.css │ ├── query_expansions.css │ └── search.css └── javascripts │ └── query_expansions.js ├── db └── migrate │ ├── 20190226093842_extend_content_size.rb │ ├── 20170810052427_add_index_to_issue_contents.rb │ ├── 20170630075027_load_comments_from_changesets.rb │ ├── 20170630075028_add_index_to_searcher_records.rb │ ├── 20170630063757_copy_records_to_searcher_records.rb │ ├── 20190226093843_add_missing_indexes_to_searcher_records.rb │ ├── 20190603060948_create_fts_types.rb │ ├── 20190603061110_create_fts_tag_types.rb │ ├── 20170630063557_enable_pgroonga.rb │ ├── 20190728022920_mroonga_add_source_type_id_index_to_fts_targets.rb │ ├── 20190603061445_create_fts_tags.rb │ ├── 20190925074645_add_name_index_to_fts_tags.rb │ ├── 20190807085000_create_fts_query_expansions.rb │ ├── 20170810045914_create_issue_contents.rb │ ├── 20170630063657_create_searcher_records.rb │ ├── 20190603061606_create_fts_targets.rb │ ├── 20250225000050_remove_contents_from_issue_contents.rb │ ├── 20190603054615_drop_searcher_records.rb │ └── 20231210061330_add_registered_at_to_fts_targets_with_index.rb ├── .github ├── dependabot.yml └── workflows │ └── release.yml ├── config ├── initializers │ ├── chupa_text.rb │ └── schema_format.rb ├── routes.rb ├── database.yml.example.5.1.postgresql ├── database.yml.example.6.0.postgresql ├── database.yml.example.6.1.postgresql ├── database.yml.example.master.postgresql ├── database.yml.example.5.1.mysql ├── database.yml.example.6.0.mysql ├── database.yml.example.6.1.mysql ├── database.yml.example.master.mysql └── locales │ ├── ja.yml │ └── en.yml ├── bin ├── analize-log ├── report-statistics ├── filter-similar-words └── dump-mysql ├── dev ├── run-test.sh ├── run-postgresql.sh ├── run-mysql.sh ├── setup.sh └── initialize-redmine.sh ├── Gemfile ├── lib ├── full_text_search.rb ├── full_text_search │ ├── condition_builder.rb │ ├── hooks │ │ ├── issue_query_any_searchable.rb │ │ ├── issues_show_description_bottom_hook.rb │ │ ├── search_index_options_content_bottom_hook.rb │ │ ├── settings_helper.rb │ │ ├── search_helper.rb │ │ ├── similar_issues_helper.rb │ │ └── controller_search_index.rb │ ├── issue_synchronizable.rb │ ├── journal_synchronizable.rb │ ├── attachment_synchronizable.rb │ ├── custom_field_callbacks.rb │ ├── plugin_wiki_extensions_tag_searchable.rb │ ├── migration.rb │ ├── document_mapper.rb │ ├── news_mapper.rb │ ├── scm_adapter_cat_io.rb │ ├── project_mapper.rb │ ├── resolver.rb │ ├── similar_searcher.rb │ ├── message_mapper.rb │ ├── text_extractor.rb │ ├── repository_entry.rb │ ├── wiki_page_mapper.rb │ ├── changeset_mapper.rb │ ├── tracer.rb │ ├── similar_searcher │ │ ├── pgroonga.rb │ │ └── mroonga.rb │ ├── markup_parser.rb │ ├── issue_mapper.rb │ ├── custom_value_mapper.rb │ ├── query_expansion_synchronizer.rb │ ├── settings.rb │ ├── journal_mapper.rb │ ├── similar_words_filter.rb │ ├── mroonga.rb │ ├── pgroonga.rb │ ├── scm_adapter_all_file_entries.rb │ └── attachment_mapper.rb └── tasks │ └── full_text_search.rake ├── .dir-locals.el ├── .gitignore ├── LICENSE └── init.rb /test/files/have-null.txt: -------------------------------------------------------------------------------- 1 | AB 2 | -------------------------------------------------------------------------------- /test/files/japanese.txt: -------------------------------------------------------------------------------- 1 | こんにちは 2 | -------------------------------------------------------------------------------- /app/views/fts_query_expand/index.html.erb: -------------------------------------------------------------------------------- 1 |

<%= @expanded %>

2 | -------------------------------------------------------------------------------- /assets/stylesheets/similar_issues.css: -------------------------------------------------------------------------------- 1 | .similar-issues table.list { 2 | border: 0; 3 | } 4 | -------------------------------------------------------------------------------- /test/files/one-page.odt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clear-code/redmine_full_text_search/HEAD/test/files/one-page.odt -------------------------------------------------------------------------------- /test/files/one-page.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clear-code/redmine_full_text_search/HEAD/test/files/one-page.pdf -------------------------------------------------------------------------------- /test/files/encrypted.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clear-code/redmine_full_text_search/HEAD/test/files/encrypted.zip -------------------------------------------------------------------------------- /app/models/full_text_search/issue_content.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | class IssueContent < ApplicationRecord 3 | end 4 | end 5 | -------------------------------------------------------------------------------- /db/migrate/20190226093842_extend_content_size.rb: -------------------------------------------------------------------------------- 1 | class ExtendContentSize < ActiveRecord::Migration[5.2] 2 | def change 3 | end 4 | end 5 | -------------------------------------------------------------------------------- /db/migrate/20170810052427_add_index_to_issue_contents.rb: -------------------------------------------------------------------------------- 1 | class AddIndexToIssueContents < ActiveRecord::Migration[4.2] 2 | def change 3 | end 4 | end 5 | -------------------------------------------------------------------------------- /db/migrate/20170630075027_load_comments_from_changesets.rb: -------------------------------------------------------------------------------- 1 | class LoadCommentsFromChangesets < ActiveRecord::Migration[4.2] 2 | def change 3 | end 4 | end 5 | -------------------------------------------------------------------------------- /db/migrate/20170630075028_add_index_to_searcher_records.rb: -------------------------------------------------------------------------------- 1 | class AddIndexToSearcherRecords < ActiveRecord::Migration[4.2] 2 | def change 3 | end 4 | end 5 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "github-actions" 4 | directory: "/" 5 | schedule: 6 | interval: "weekly" 7 | -------------------------------------------------------------------------------- /db/migrate/20170630063757_copy_records_to_searcher_records.rb: -------------------------------------------------------------------------------- 1 | class CopyRecordsToSearcherRecords < ActiveRecord::Migration[4.2] 2 | def change 3 | end 4 | end 5 | -------------------------------------------------------------------------------- /config/initializers/chupa_text.rb: -------------------------------------------------------------------------------- 1 | ChupaText.logger = Rails.logger 2 | ChupaText::Decomposers.enable_all_gems 3 | ChupaText::Decomposers.load 4 | ChupaText::Configuration.default 5 | -------------------------------------------------------------------------------- /db/migrate/20190226093843_add_missing_indexes_to_searcher_records.rb: -------------------------------------------------------------------------------- 1 | class AddMissingIndexesToSearcherRecords < ActiveRecord::Migration[5.2] 2 | def change 3 | end 4 | end 5 | -------------------------------------------------------------------------------- /test/application_system_test_case.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path('../test_helper', __FILE__) 2 | require File.expand_path('../../../../test/application_system_test_case', __FILE__) 3 | -------------------------------------------------------------------------------- /bin/analize-log: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require_relative "../lib/full_text_search/log_analyzer" 4 | 5 | analyzer = FullTextSearch::LogAnalyzer.new 6 | exit(analyzer.run(ARGV)) 7 | -------------------------------------------------------------------------------- /dev/run-test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -exu 4 | 5 | env \ 6 | PSQLRC=/tmp/nonexistent \ 7 | RAILS_ENV=test \ 8 | ${RUBY:-ruby} bin/rails test "plugins/*/test/**/*_test.rb" "$@" 9 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | gem "chupa-text", ">= 1.3.3" 4 | gem "groonga-client", ">= 0.6.1" 5 | gem "tty-progressbar" 6 | 7 | group :test do 8 | gem "webrick" 9 | end 10 | -------------------------------------------------------------------------------- /assets/stylesheets/fontawesome-free-5.8.2-web/webfonts/fa-solid-900.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clear-code/redmine_full_text_search/HEAD/assets/stylesheets/fontawesome-free-5.8.2-web/webfonts/fa-solid-900.eot -------------------------------------------------------------------------------- /assets/stylesheets/fontawesome-free-5.8.2-web/webfonts/fa-solid-900.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clear-code/redmine_full_text_search/HEAD/assets/stylesheets/fontawesome-free-5.8.2-web/webfonts/fa-solid-900.ttf -------------------------------------------------------------------------------- /assets/stylesheets/fontawesome-free-5.8.2-web/webfonts/fa-brands-400.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clear-code/redmine_full_text_search/HEAD/assets/stylesheets/fontawesome-free-5.8.2-web/webfonts/fa-brands-400.eot -------------------------------------------------------------------------------- /assets/stylesheets/fontawesome-free-5.8.2-web/webfonts/fa-brands-400.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clear-code/redmine_full_text_search/HEAD/assets/stylesheets/fontawesome-free-5.8.2-web/webfonts/fa-brands-400.ttf -------------------------------------------------------------------------------- /assets/stylesheets/fontawesome-free-5.8.2-web/webfonts/fa-brands-400.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clear-code/redmine_full_text_search/HEAD/assets/stylesheets/fontawesome-free-5.8.2-web/webfonts/fa-brands-400.woff -------------------------------------------------------------------------------- /assets/stylesheets/fontawesome-free-5.8.2-web/webfonts/fa-regular-400.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clear-code/redmine_full_text_search/HEAD/assets/stylesheets/fontawesome-free-5.8.2-web/webfonts/fa-regular-400.eot -------------------------------------------------------------------------------- /assets/stylesheets/fontawesome-free-5.8.2-web/webfonts/fa-regular-400.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clear-code/redmine_full_text_search/HEAD/assets/stylesheets/fontawesome-free-5.8.2-web/webfonts/fa-regular-400.ttf -------------------------------------------------------------------------------- /assets/stylesheets/fontawesome-free-5.8.2-web/webfonts/fa-solid-900.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clear-code/redmine_full_text_search/HEAD/assets/stylesheets/fontawesome-free-5.8.2-web/webfonts/fa-solid-900.woff -------------------------------------------------------------------------------- /assets/stylesheets/fontawesome-free-5.8.2-web/webfonts/fa-solid-900.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clear-code/redmine_full_text_search/HEAD/assets/stylesheets/fontawesome-free-5.8.2-web/webfonts/fa-solid-900.woff2 -------------------------------------------------------------------------------- /config/routes.rb: -------------------------------------------------------------------------------- 1 | scope :full_text_search do 2 | resources :fts_query_expansions, path: "query_expansions" 3 | get "query_expand", 4 | to: "fts_query_expand#index", 5 | as: :fts_query_expand 6 | end 7 | -------------------------------------------------------------------------------- /assets/stylesheets/fontawesome-free-5.8.2-web/webfonts/fa-brands-400.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clear-code/redmine_full_text_search/HEAD/assets/stylesheets/fontawesome-free-5.8.2-web/webfonts/fa-brands-400.woff2 -------------------------------------------------------------------------------- /assets/stylesheets/fontawesome-free-5.8.2-web/webfonts/fa-regular-400.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clear-code/redmine_full_text_search/HEAD/assets/stylesheets/fontawesome-free-5.8.2-web/webfonts/fa-regular-400.woff -------------------------------------------------------------------------------- /assets/stylesheets/fontawesome-free-5.8.2-web/webfonts/fa-regular-400.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clear-code/redmine_full_text_search/HEAD/assets/stylesheets/fontawesome-free-5.8.2-web/webfonts/fa-regular-400.woff2 -------------------------------------------------------------------------------- /lib/full_text_search.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | class << self 3 | def resolver 4 | @resolver ||= Resolver.new 5 | end 6 | 7 | def attach 8 | resolver 9 | end 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /lib/full_text_search/condition_builder.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | module ConditionBuilder 3 | def build_condition(operator, *conditions) 4 | operator = " #{operator} " 5 | "(#{conditions.compact.join(operator)})" 6 | end 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /app/views/fts_query_expansions/_form.html.erb: -------------------------------------------------------------------------------- 1 | <%= error_messages_for "query_expansion" %> 2 | 3 |
4 |

<%= f.text_field :source, :required => true, :size => 60 %>

5 |

<%= f.text_field :destination, :required => true, :size => 60 %>

6 |
7 | -------------------------------------------------------------------------------- /.dir-locals.el: -------------------------------------------------------------------------------- 1 | ((sh-mode . ((indent-tabs-mode . nil) 2 | (sh-indentation . 2) 3 | (sh-basic-offset . 2))) 4 | (css-mode . ((indent-tabs-mode . nil) 5 | (css-indent-offset . 2))) 6 | (js-mode . ((indent-tabs-mode . nil) 7 | (js-indent-level . 2)))) 8 | -------------------------------------------------------------------------------- /db/migrate/20190603060948_create_fts_types.rb: -------------------------------------------------------------------------------- 1 | class CreateFtsTypes < ActiveRecord::Migration[5.2] 2 | def change 3 | return if reverting? and !table_exists?(:fts_types) 4 | 5 | create_table :fts_types do |t| 6 | t.string :name, null: false 7 | t.index :name, unique: true 8 | end 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /config/initializers/schema_format.rb: -------------------------------------------------------------------------------- 1 | # This is just for running test. 2 | # This isn't used normal use case. 3 | # Redmine doesn't use plugins/*/config/initializers/*.rb 4 | if ActiveRecord.respond_to?(:schema_format=) 5 | # For Rails >= 7 6 | ActiveRecord.schema_format = :sql 7 | else 8 | ActiveRecord::Base.schema_format = :sql 9 | end 10 | -------------------------------------------------------------------------------- /db/migrate/20190603061110_create_fts_tag_types.rb: -------------------------------------------------------------------------------- 1 | class CreateFtsTagTypes < ActiveRecord::Migration[5.2] 2 | def change 3 | return if reverting? and !table_exists?(:fts_tag_types) 4 | 5 | create_table :fts_tag_types do |t| 6 | t.string :name, null: false 7 | t.index :name, unique: true 8 | end 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /dev/run-postgresql.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -exu 4 | 5 | options=(--rm -p5432:5432 -ePOSTGRES_HOST_AUTH_METHOD=trust) 6 | if [ $# -ge 1 ]; then 7 | db_dir=$1 8 | rm -rf ${db_dir} 9 | mkdir -p ${db_dir} 10 | options+=("-v${db_dir}:/var/lib/postgresql") 11 | fi 12 | 13 | docker run "${options[@]}" groonga/pgroonga:latest-debian-15 14 | -------------------------------------------------------------------------------- /db/migrate/20170630063557_enable_pgroonga.rb: -------------------------------------------------------------------------------- 1 | class EnablePgroonga < ActiveRecord::Migration[4.2] 2 | def change 3 | reversible do |d| 4 | d.up do 5 | if Redmine::Database.postgresql? 6 | enable_extension("pgroonga") unless extension_enabled?("pgroonga") 7 | end 8 | end 9 | end 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /lib/full_text_search/hooks/issue_query_any_searchable.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | module Hooks 3 | module IssueQueryAnySearchable 4 | def sql_for_any_searchable_field(field, operator, value) 5 | # TODO: Implement AND searches across multiple fields. 6 | super(field, operator, value) 7 | end 8 | end 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /app/models/full_text_search/query_expansion_request.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | class QueryExpansionRequest 3 | include ActiveModel::Model 4 | extend ActiveModel::Naming 5 | 6 | attr_writer :query 7 | 8 | def to_params 9 | { 10 | "query" => query, 11 | } 12 | end 13 | 14 | def query 15 | (@query || "").strip 16 | end 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /db/migrate/20190728022920_mroonga_add_source_type_id_index_to_fts_targets.rb: -------------------------------------------------------------------------------- 1 | class MroongaAddSourceTypeIdIndexToFtsTargets < ActiveRecord::Migration[5.2] 2 | def change 3 | return if reverting? and !table_exists?(:fts_targets) 4 | return unless Redmine::Database.mysql? 5 | return if index_exists?(:fts_targets, :source_type_id) 6 | add_index :fts_targets, :source_type_id 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /lib/full_text_search/hooks/issues_show_description_bottom_hook.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | module Hooks 3 | class IssuesShowDescriptionBottomHook < Redmine::Hook::ViewListener 4 | include Redmine::I18n 5 | 6 | render_on(:view_issues_show_description_bottom, 7 | partial: "issues/full_text_search/view_issues_show_description_bottom") 8 | end 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /app/jobs/full_text_search/extract_text_job.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | class ExtractTextJob < ActiveJob::Base 3 | queue_as :full_text_search 4 | queue_with_priority 15 5 | 6 | discard_on ActiveRecord::RecordNotFound 7 | 8 | def perform(id) 9 | target = Target.find(id) 10 | mapper = target.mapper.redmine_mapper 11 | mapper.extract_text 12 | end 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /app/controllers/fts_query_expand_controller.rb: -------------------------------------------------------------------------------- 1 | class FtsQueryExpandController < ApplicationController 2 | def index 3 | @request = FullTextSearch::QueryExpansionRequest.new(request_params) 4 | @expanded = FtsQueryExpansion.expand_query(@request.query) 5 | render layout: false if request.xhr? 6 | end 7 | 8 | private 9 | def request_params 10 | params.permit(:query) 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /lib/full_text_search/hooks/search_index_options_content_bottom_hook.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | module Hooks 3 | class SearchIndexOptionsContentBottomHook < Redmine::Hook::ViewListener 4 | include Redmine::I18n 5 | 6 | render_on(:view_search_index_options_content_bottom, 7 | partial: "search/full_text_search/view_search_index_options_content_bottom") 8 | end 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /app/views/fts_query_expansions/edit.html.erb: -------------------------------------------------------------------------------- 1 |
2 | <%= link_to(t("fts_query_expansions.index.title"), 3 | fts_query_expansions_path, 4 | class: "icon icon-list") %> 5 |
6 | 7 | <%= title t(".title") %> 8 | 9 | <%= labelled_form_for @query_expansion do |f| %> 10 | <%= render :partial => 'form', locals: {f: f} %> 11 | <%= submit_tag l(:button_save) %> 12 | <% end %> 13 | -------------------------------------------------------------------------------- /assets/stylesheets/score.css: -------------------------------------------------------------------------------- 1 | #search-results dt a::after { 2 | content: " (" attr(data-rank) ")"; 3 | color: #bbbbbb; 4 | font-size: x-small; 5 | } 6 | 7 | .similar-issues td.subject::after { 8 | content: " (" attr(data-rank) ")"; 9 | color: #bbbbbb; 10 | font-size: x-small; 11 | } 12 | 13 | #search-elapsed { 14 | color: #bbbbbb; 15 | font-size: x-small; 16 | } 17 | 18 | .similar-issues .elapsed { 19 | color: #bbbbbb; 20 | font-size: x-small; 21 | } 22 | -------------------------------------------------------------------------------- /config/database.yml.example.5.1.postgresql: -------------------------------------------------------------------------------- 1 | production: 2 | adapter: postgresql 3 | database: redmine_51 4 | host: 127.0.0.1 5 | username: postgres 6 | password: "" 7 | 8 | development: 9 | adapter: postgresql 10 | database: redmine_51_development 11 | host: 127.0.0.1 12 | username: postgres 13 | password: "" 14 | 15 | test: 16 | adapter: postgresql 17 | database: redmine_51_test 18 | host: 127.0.0.1 19 | username: postgres 20 | password: "" 21 | -------------------------------------------------------------------------------- /config/database.yml.example.6.0.postgresql: -------------------------------------------------------------------------------- 1 | production: 2 | adapter: postgresql 3 | database: redmine_60 4 | host: 127.0.0.1 5 | username: postgres 6 | password: "" 7 | 8 | development: 9 | adapter: postgresql 10 | database: redmine_60_development 11 | host: 127.0.0.1 12 | username: postgres 13 | password: "" 14 | 15 | test: 16 | adapter: postgresql 17 | database: redmine_60_test 18 | host: 127.0.0.1 19 | username: postgres 20 | password: "" 21 | -------------------------------------------------------------------------------- /config/database.yml.example.6.1.postgresql: -------------------------------------------------------------------------------- 1 | production: 2 | adapter: postgresql 3 | database: redmine_61 4 | host: 127.0.0.1 5 | username: postgres 6 | password: "" 7 | 8 | development: 9 | adapter: postgresql 10 | database: redmine_61_development 11 | host: 127.0.0.1 12 | username: postgres 13 | password: "" 14 | 15 | test: 16 | adapter: postgresql 17 | database: redmine_61_test 18 | host: 127.0.0.1 19 | username: postgres 20 | password: "" 21 | -------------------------------------------------------------------------------- /assets/javascripts/query_expansions.js: -------------------------------------------------------------------------------- 1 | function observeIncrementalSearch(input, onChange) { 2 | var $input = $(input); 3 | var previousValue = $input.val(); 4 | $input.keyup(function() { 5 | var currentValue = $input.val(); 6 | if (currentValue === previousValue) { 7 | return; 8 | } 9 | previousValue = currentValue; 10 | onChange($input, currentValue); 11 | }); 12 | if (previousValue !== "") { 13 | onChange($input, previousValue); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /config/database.yml.example.master.postgresql: -------------------------------------------------------------------------------- 1 | production: 2 | adapter: postgresql 3 | database: redmine_master 4 | host: 127.0.0.1 5 | username: postgres 6 | password: "" 7 | 8 | development: 9 | adapter: postgresql 10 | database: redmine_master_development 11 | host: 127.0.0.1 12 | username: postgres 13 | password: "" 14 | 15 | test: 16 | adapter: postgresql 17 | database: redmine_master_test 18 | host: 127.0.0.1 19 | username: postgres 20 | password: "" 21 | -------------------------------------------------------------------------------- /app/views/fts_query_expansions/new.html.erb: -------------------------------------------------------------------------------- 1 |
2 | <%= link_to(t("fts_query_expansions.index.title"), 3 | fts_query_expansions_path, 4 | class: "icon icon-list") %> 5 |
6 | 7 | <%= title t(".title") %> 8 | 9 | <%= labelled_form_for @query_expansion do |f| %> 10 | <%= render :partial => 'form', locals: {f: f} %> 11 | <%= submit_tag l(:button_create) %> 12 | <%= submit_tag l(:button_create_and_continue), name: "continue" %> 13 | <% end %> 14 | -------------------------------------------------------------------------------- /lib/full_text_search/hooks/settings_helper.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | module Hooks 3 | module SettingsHelper 4 | def fts_display_score? 5 | Setting.plugin_full_text_search.display_score? 6 | end 7 | 8 | def fts_display_similar_issues? 9 | Setting.plugin_full_text_search.display_similar_issues? 10 | end 11 | 12 | def fts_enable_tracking? 13 | Setting.plugin_full_text_search.enable_tracking? 14 | end 15 | end 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /test/functional/fts_query_expansions_controller_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path("../../test_helper", __FILE__) 2 | 3 | class FtsQueryExpansionsControllerTest < ActionController::TestCase 4 | include PrettyInspectable 5 | 6 | fixtures :users 7 | 8 | def setup 9 | User.current = nil 10 | @request.session[:user_id] = 1 # admin 11 | end 12 | 13 | def test_require_admin 14 | @request.session[:user_id] = nil 15 | get :index 16 | assert_response 302 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /config/database.yml.example.5.1.mysql: -------------------------------------------------------------------------------- 1 | production: 2 | adapter: mysql2 3 | database: redmine_51 4 | host: 127.0.0.1 5 | username: root 6 | password: "" 7 | encoding: utf8mb4 8 | 9 | development: 10 | adapter: mysql2 11 | database: redmine_51_development 12 | host: 127.0.0.1 13 | username: root 14 | password: "" 15 | encoding: utf8mb4 16 | 17 | test: 18 | adapter: mysql2 19 | database: redmine_51_test 20 | host: 127.0.0.1 21 | username: root 22 | password: "" 23 | encoding: utf8mb4 24 | -------------------------------------------------------------------------------- /config/database.yml.example.6.0.mysql: -------------------------------------------------------------------------------- 1 | production: 2 | adapter: mysql2 3 | database: redmine_60 4 | host: 127.0.0.1 5 | username: root 6 | password: "" 7 | encoding: utf8mb4 8 | 9 | development: 10 | adapter: mysql2 11 | database: redmine_60_development 12 | host: 127.0.0.1 13 | username: root 14 | password: "" 15 | encoding: utf8mb4 16 | 17 | test: 18 | adapter: mysql2 19 | database: redmine_60_test 20 | host: 127.0.0.1 21 | username: root 22 | password: "" 23 | encoding: utf8mb4 24 | -------------------------------------------------------------------------------- /config/database.yml.example.6.1.mysql: -------------------------------------------------------------------------------- 1 | production: 2 | adapter: mysql2 3 | database: redmine_61 4 | host: 127.0.0.1 5 | username: root 6 | password: "" 7 | encoding: utf8mb4 8 | 9 | development: 10 | adapter: mysql2 11 | database: redmine_61_development 12 | host: 127.0.0.1 13 | username: root 14 | password: "" 15 | encoding: utf8mb4 16 | 17 | test: 18 | adapter: mysql2 19 | database: redmine_61_test 20 | host: 127.0.0.1 21 | username: root 22 | password: "" 23 | encoding: utf8mb4 24 | -------------------------------------------------------------------------------- /db/migrate/20190603061445_create_fts_tags.rb: -------------------------------------------------------------------------------- 1 | class CreateFtsTags < ActiveRecord::Migration[5.2] 2 | def change 3 | return if reverting? and !table_exists?(:fts_tags) 4 | 5 | if Redmine::Database.mysql? 6 | options = "ENGINE=Mroonga" 7 | else 8 | options = nil 9 | end 10 | create_table :fts_tags, options: options do |t| 11 | t.integer :type_id, null: false 12 | t.string :name, null: false 13 | t.index [:type_id, :name], unique: true 14 | end 15 | end 16 | end 17 | -------------------------------------------------------------------------------- /app/jobs/full_text_search/upsert_target_job.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | class UpsertTargetJob < ActiveJob::Base 3 | queue_as :full_text_search 4 | queue_with_priority 15 5 | 6 | discard_on ActiveRecord::RecordNotFound 7 | 8 | def perform(mapper_class_name, source_id) 9 | mapper_class = mapper_class_name.constantize 10 | source = mapper_class.redmine_class.find(source_id) 11 | mapper = mapper_class.redmine_mapper(source) 12 | mapper.upsert_fts_target 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /config/database.yml.example.master.mysql: -------------------------------------------------------------------------------- 1 | production: 2 | adapter: mysql2 3 | database: redmine_master 4 | host: 127.0.0.1 5 | username: root 6 | password: "" 7 | encoding: utf8mb4 8 | 9 | development: 10 | adapter: mysql2 11 | database: redmine_master_development 12 | host: 127.0.0.1 13 | username: root 14 | password: "" 15 | encoding: utf8mb4 16 | 17 | test: 18 | adapter: mysql2 19 | database: redmine_master_test 20 | host: 127.0.0.1 21 | username: root 22 | password: "" 23 | encoding: utf8mb4 24 | -------------------------------------------------------------------------------- /bin/report-statistics: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | config_dir = File.expand_path("../../../../config", __FILE__) 4 | APP_PATH = File.join(config_dir, "application") 5 | require APP_PATH 6 | Rails.application.require_environment! 7 | 8 | puts("N characters:") 9 | counts = FullTextSearch::Target 10 | .group(:source_type_id) 11 | .sum("CHAR_LENGTH(title) + CHAR_LENGTH(content)") 12 | counts.each do |id, n_characters| 13 | puts("#{FullTextSearch::Type.find(id).name}: #{n_characters}") 14 | end 15 | puts("Total: #{counts.values.sum}") 16 | -------------------------------------------------------------------------------- /lib/full_text_search/hooks/search_helper.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | module Hooks 3 | module SearchHelper 4 | include FullTextSearch::Hooks::SettingsHelper 5 | 6 | def search_result_entry_url(e, i) 7 | if fts_enable_tracking? 8 | search_parameters = { 9 | "search_id" => @search_request.search_id, 10 | "search_n" => i + @search_request.offset, 11 | } 12 | else 13 | search_parameters = {} 14 | end 15 | e.event_url.merge(search_parameters) 16 | end 17 | end 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /assets/stylesheets/query_expansions.css: -------------------------------------------------------------------------------- 1 | .query-expansion-input-box { 2 | display: flex; 3 | justify-content: center; 4 | align-items: center; 5 | align-content: center; 6 | } 7 | 8 | .query-expansion-input-box input, 9 | .query-expansion-input-box button { 10 | border: 1px solid #bbbbbb; 11 | height: 32px; 12 | padding: 8px; 13 | margin: 0px; 14 | } 15 | 16 | .query-expansion-input-box input { 17 | flex-basis: 100%; 18 | } 19 | 20 | .query-expansion-input-box button { 21 | border-left-width: 0; 22 | cursor: pointer; 23 | } 24 | 25 | .query-expansion-input-box button:hover { 26 | background: #169; 27 | } 28 | -------------------------------------------------------------------------------- /db/migrate/20190925074645_add_name_index_to_fts_tags.rb: -------------------------------------------------------------------------------- 1 | # For auto load 2 | FullTextSearch::Migration 3 | 4 | class AddNameIndexToFtsTags < ActiveRecord::Migration[5.2] 5 | def change 6 | return if reverting? and !table_exists?(:fts_tags) 7 | 8 | if Redmine::Database.mysql? 9 | add_index :fts_tags, 10 | :name, 11 | type: "fulltext", 12 | comment: "NORMALIZER 'NormalizerNFKC121'" 13 | else 14 | add_index :fts_tags, 15 | :name, 16 | using: "PGroonga", 17 | with: "normalizer = 'NormalizerNFKC121'" 18 | end 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /lib/full_text_search/issue_synchronizable.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | module IssueSynchronizable 3 | extend ActiveSupport::Concern 4 | 5 | included do 6 | after_commit :queue_sync_on_commit 7 | after_destroy :queue_sync_on_destroy 8 | end 9 | 10 | private 11 | 12 | def queue_sync(action) 13 | FullTextSearch::UpdateIssueContentJob.perform_later( 14 | self.class.name, 15 | id, 16 | action 17 | ) 18 | end 19 | 20 | def queue_sync_on_commit 21 | queue_sync("commit") 22 | end 23 | 24 | def queue_sync_on_destroy 25 | queue_sync("destroy") 26 | end 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /assets/stylesheets/fontawesome-free-5.8.2-web/css/brands.min.css: -------------------------------------------------------------------------------- 1 | /*! 2 | * Font Awesome Free 5.8.2 by @fontawesome - https://fontawesome.com 3 | * License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) 4 | */ 5 | @font-face{font-family:"Font Awesome 5 Brands";font-style:normal;font-weight:normal;font-display:auto;src:url(../webfonts/fa-brands-400.eot);src:url(../webfonts/fa-brands-400.eot?#iefix) format("embedded-opentype"),url(../webfonts/fa-brands-400.woff2) format("woff2"),url(../webfonts/fa-brands-400.woff) format("woff"),url(../webfonts/fa-brands-400.ttf) format("truetype"),url(../webfonts/fa-brands-400.svg#fontawesome) format("svg")}.fab{font-family:"Font Awesome 5 Brands"} -------------------------------------------------------------------------------- /assets/stylesheets/fontawesome-free-5.8.2-web/css/solid.min.css: -------------------------------------------------------------------------------- 1 | /*! 2 | * Font Awesome Free 5.8.2 by @fontawesome - https://fontawesome.com 3 | * License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) 4 | */ 5 | @font-face{font-family:"Font Awesome 5 Free";font-style:normal;font-weight:900;font-display:auto;src:url(../webfonts/fa-solid-900.eot);src:url(../webfonts/fa-solid-900.eot?#iefix) format("embedded-opentype"),url(../webfonts/fa-solid-900.woff2) format("woff2"),url(../webfonts/fa-solid-900.woff) format("woff"),url(../webfonts/fa-solid-900.ttf) format("truetype"),url(../webfonts/fa-solid-900.svg#fontawesome) format("svg")}.fa,.fas{font-family:"Font Awesome 5 Free";font-weight:900} -------------------------------------------------------------------------------- /assets/stylesheets/fontawesome-free-5.8.2-web/css/regular.min.css: -------------------------------------------------------------------------------- 1 | /*! 2 | * Font Awesome Free 5.8.2 by @fontawesome - https://fontawesome.com 3 | * License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) 4 | */ 5 | @font-face{font-family:"Font Awesome 5 Free";font-style:normal;font-weight:400;font-display:auto;src:url(../webfonts/fa-regular-400.eot);src:url(../webfonts/fa-regular-400.eot?#iefix) format("embedded-opentype"),url(../webfonts/fa-regular-400.woff2) format("woff2"),url(../webfonts/fa-regular-400.woff) format("woff"),url(../webfonts/fa-regular-400.ttf) format("truetype"),url(../webfonts/fa-regular-400.svg#fontawesome) format("svg")}.far{font-family:"Font Awesome 5 Free";font-weight:400} -------------------------------------------------------------------------------- /lib/full_text_search/journal_synchronizable.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | module JournalSynchronizable 3 | extend ActiveSupport::Concern 4 | 5 | included do 6 | after_commit :queue_sync_on_commit 7 | after_destroy :queue_sync_on_destroy 8 | end 9 | 10 | private 11 | 12 | def queue_sync(action, options = {}) 13 | FullTextSearch::UpdateIssueContentJob.perform_later( 14 | self.class.name, 15 | id, 16 | action, 17 | options 18 | ) 19 | end 20 | 21 | def queue_sync_on_commit 22 | queue_sync("commit") 23 | end 24 | 25 | def queue_sync_on_destroy 26 | queue_sync("destroy", issue_id: journalized_id) 27 | end 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /lib/full_text_search/attachment_synchronizable.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | module AttachmentSynchronizable 3 | extend ActiveSupport::Concern 4 | 5 | included do 6 | after_commit :queue_sync_on_commit 7 | after_destroy :queue_sync_on_destroy 8 | end 9 | 10 | private 11 | 12 | def queue_sync(action, options = {}) 13 | FullTextSearch::UpdateIssueContentJob.perform_later( 14 | self.class.name, 15 | id, 16 | action, 17 | options 18 | ) 19 | end 20 | 21 | def queue_sync_on_commit 22 | queue_sync("commit") 23 | end 24 | 25 | def queue_sync_on_destroy 26 | return if container_type != Issue.name 27 | queue_sync("destroy", issue_id: container_id) 28 | end 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /lib/full_text_search/custom_field_callbacks.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | # We need this because CustomField uses `has_many :custom_values, 3 | # :dependent => :delete_all` relation. We need `after_destroy` 4 | # callback to synchronize `CustomFieldValue` and 5 | # `FullTextSearch::Target(source_type_id: Type.custom_value.id)` but 6 | # `after_destroy` callback doesn't exist with `:dependent => 7 | # :delete_all`. 8 | class CustomFieldCallbacks 9 | class << self 10 | def attach 11 | CustomField.after_destroy(self) 12 | end 13 | 14 | def after_destroy(record) 15 | Target 16 | .where(source_type_id: Type.custom_value.id, 17 | custom_field_id: record.id) 18 | .destroy_all 19 | end 20 | end 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /assets/stylesheets/fontawesome-free-5.8.2-web/css/brands.css: -------------------------------------------------------------------------------- 1 | /*! 2 | * Font Awesome Free 5.8.2 by @fontawesome - https://fontawesome.com 3 | * License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) 4 | */ 5 | @font-face { 6 | font-family: 'Font Awesome 5 Brands'; 7 | font-style: normal; 8 | font-weight: normal; 9 | font-display: auto; 10 | src: url("../webfonts/fa-brands-400.eot"); 11 | src: url("../webfonts/fa-brands-400.eot?#iefix") format("embedded-opentype"), url("../webfonts/fa-brands-400.woff2") format("woff2"), url("../webfonts/fa-brands-400.woff") format("woff"), url("../webfonts/fa-brands-400.ttf") format("truetype"), url("../webfonts/fa-brands-400.svg#fontawesome") format("svg"); } 12 | 13 | .fab { 14 | font-family: 'Font Awesome 5 Brands'; } 15 | -------------------------------------------------------------------------------- /assets/stylesheets/fontawesome-free-5.8.2-web/css/solid.css: -------------------------------------------------------------------------------- 1 | /*! 2 | * Font Awesome Free 5.8.2 by @fontawesome - https://fontawesome.com 3 | * License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) 4 | */ 5 | @font-face { 6 | font-family: 'Font Awesome 5 Free'; 7 | font-style: normal; 8 | font-weight: 900; 9 | font-display: auto; 10 | src: url("../webfonts/fa-solid-900.eot"); 11 | src: url("../webfonts/fa-solid-900.eot?#iefix") format("embedded-opentype"), url("../webfonts/fa-solid-900.woff2") format("woff2"), url("../webfonts/fa-solid-900.woff") format("woff"), url("../webfonts/fa-solid-900.ttf") format("truetype"), url("../webfonts/fa-solid-900.svg#fontawesome") format("svg"); } 12 | 13 | .fa, 14 | .fas { 15 | font-family: 'Font Awesome 5 Free'; 16 | font-weight: 900; } 17 | -------------------------------------------------------------------------------- /assets/stylesheets/fontawesome-free-5.8.2-web/css/regular.css: -------------------------------------------------------------------------------- 1 | /*! 2 | * Font Awesome Free 5.8.2 by @fontawesome - https://fontawesome.com 3 | * License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) 4 | */ 5 | @font-face { 6 | font-family: 'Font Awesome 5 Free'; 7 | font-style: normal; 8 | font-weight: 400; 9 | font-display: auto; 10 | src: url("../webfonts/fa-regular-400.eot"); 11 | src: url("../webfonts/fa-regular-400.eot?#iefix") format("embedded-opentype"), url("../webfonts/fa-regular-400.woff2") format("woff2"), url("../webfonts/fa-regular-400.woff") format("woff"), url("../webfonts/fa-regular-400.ttf") format("truetype"), url("../webfonts/fa-regular-400.svg#fontawesome") format("svg"); } 12 | 13 | .far { 14 | font-family: 'Font Awesome 5 Free'; 15 | font-weight: 400; } 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.rbc 2 | capybara-*.html 3 | .rspec 4 | /env 5 | /log 6 | /tmp 7 | /db/*.sqlite3 8 | /db/*.sqlite3-journal 9 | /public/system 10 | /coverage/ 11 | /spec/tmp 12 | **.orig 13 | rerun.txt 14 | pickle-email-*.html 15 | 16 | # TODO Comment out these rules if you are OK with secrets being uploaded to the repo 17 | config/initializers/secret_token.rb 18 | config/secrets.yml 19 | 20 | ## Environment normalization: 21 | /.bundle 22 | /vendor/bundle 23 | 24 | # these should all be checked in to normalize the environment: 25 | # Gemfile.lock, .ruby-version, .ruby-gemset 26 | 27 | # unless supporting rvm < 1.11.0 or doing something fancy, ignore this: 28 | .rvmrc 29 | 30 | # if using bower-rails ignore default bower_components path bower.json files 31 | /vendor/assets/bower_components 32 | *.bowerrc 33 | bower.json 34 | 35 | # Ignore pow environment settings 36 | .powenv 37 | -------------------------------------------------------------------------------- /app/models/fts_query_expansion.rb: -------------------------------------------------------------------------------- 1 | class FtsQueryExpansion < ApplicationRecord 2 | case connection_db_config.adapter 3 | when "postgresql" 4 | include FullTextSearch::Pgroonga 5 | when "mysql2" 6 | include FullTextSearch::Mroonga 7 | end 8 | 9 | class << self 10 | def source_column_name 11 | "source" 12 | end 13 | 14 | def destination_column_name 15 | "destination" 16 | end 17 | 18 | def expand_query(query) 19 | sql_part = build_expand_query_sql_part(query) 20 | placeholder = sql_part[0] 21 | arguments = sql_part[1] 22 | sql_template = <<-SELECT 23 | SELECT #{placeholder} 24 | SELECT 25 | sql = sanitize_sql([sql_template, *arguments]) 26 | connection.select_value(sql) 27 | end 28 | end 29 | 30 | validates_presence_of :source 31 | validates_presence_of :destination 32 | end 33 | -------------------------------------------------------------------------------- /dev/run-mysql.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -exu 4 | 5 | options=(--rm -p3306:3306) 6 | options+=(-eMYSQL_ALLOW_EMPTY_PASSWORD=yes) 7 | if [ $# -ge 1 ]; then 8 | db_base_dir=$1 9 | if [ -e ${db_base_dir} ]; then 10 | rm -rf ${db_base_dir} 11 | if [ -e ${db_base_dir} ]; then 12 | sudo -H rm -rf ${db_base_dir} 13 | fi 14 | fi 15 | mkdir -p ${db_base_dir}/mysql 16 | mkdir -p ${db_base_dir}/log/mysql 17 | chmod -R go+wx ${db_base_dir}/log 18 | options+=("-v${db_base_dir}/mysql:/var/lib/mysql") 19 | options+=("-v${db_base_dir}/log:/var/log") 20 | fi 21 | 22 | db_conf_dir=/tmp/redmine-full-text-search/mysql/conf.d 23 | mkdir -p ${db_conf_dir} 24 | cat < ${db_conf_dir}/local.cnf 25 | [mysqld] 26 | max_allowed_packet = 256M 27 | MY_CNF 28 | options+=("-v${db_conf_dir}:/etc/mysql/conf.d") 29 | 30 | docker run "${options[@]}" groonga/mroonga:mysql-8.0-latest 31 | -------------------------------------------------------------------------------- /app/models/full_text_search/tag_type.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | class TagType < ApplicationRecord 3 | self.table_name = :fts_tag_types 4 | has_many :tags, foreign_key: "type_id" 5 | 6 | class << self 7 | def extension 8 | find_or_create_by(name: "extension") 9 | end 10 | 11 | def identifier 12 | find_or_create_by(name: "identifier") 13 | end 14 | 15 | def issue_status 16 | find_or_create_by(name: "issue-status") 17 | end 18 | 19 | def text_extraction 20 | find_or_create_by(name: "text-extraction") 21 | end 22 | 23 | def tracker 24 | find_or_create_by(name: "tracker") 25 | end 26 | 27 | def user 28 | find_or_create_by(name: "user") 29 | end 30 | 31 | def label 32 | find_or_create_by(name: "label") 33 | end 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /bin/filter-similar-words: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require "optparse" 4 | require "json" 5 | 6 | config_dir = File.expand_path("../../../../config", __FILE__) 7 | APP_PATH = File.join(config_dir, "application") 8 | require APP_PATH 9 | Rails.application.require_environment! 10 | 11 | plugin = Redmine::Plugin.find(:full_text_search) 12 | 13 | filter = FullTextSearch::SimilarWordsFilter.new 14 | 15 | parser = OptionParser.new 16 | parser.version = plugin.version 17 | parser.on("--cosign-threshold=THRESHOLD", Float, 18 | "Use THRESHOLD for cosign similarity", 19 | "(#{filter.cosine_threshold})") do |threshold| 20 | filter.cosine_threshold = threshold 21 | end 22 | parser.on("--engine=ENGINE", 23 | "Use only records that is generated by ENGINE") do |engine| 24 | filter.engine = engine 25 | end 26 | 27 | filtered_records = filter.run(JSON.parse(ARGF.read)) 28 | puts(JSON.pretty_generate(filtered_records)) 29 | -------------------------------------------------------------------------------- /dev/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -exu 4 | 5 | full_text_search_plugin_dir="$(dirname $(dirname $0))" 6 | 7 | if [ ! -e plugins/full_text_search ]; then 8 | (cd plugins && \ 9 | ln -fs \ 10 | "../${full_text_search_plugin_dir}" \ 11 | full_text_search) 12 | fi 13 | 14 | redmine_version=$(basename $PWD | cut -d- -f2) 15 | case $(basename $PWD | cut -d- -f3) in 16 | mroonga) 17 | rdbms=mysql 18 | ;; 19 | pgroonga) 20 | rdbms=postgresql 21 | ;; 22 | esac 23 | 24 | if [ ! -e config/database.yml ]; then 25 | (cd config && \ 26 | ln -fs \ 27 | ../plugins/full_text_search/config/database.yml.example.${redmine_version}.${rdbms} \ 28 | database.yml) 29 | fi 30 | 31 | if [ ! -e config/initializers/schema_format.rb ]; then 32 | (cd config/initializers && \ 33 | ln -fs \ 34 | ../../plugins/full_text_search/config/initializers/schema_format.rb \ 35 | ./) 36 | fi 37 | -------------------------------------------------------------------------------- /app/views/search/index.api.rsb: -------------------------------------------------------------------------------- 1 | api.array :results, api_meta(:total_count => @result_set.n_hits, 2 | :offset => @search_request.offset, 3 | :limit => @search_request.limit) do 4 | @result_set.each do |record| 5 | api.result do 6 | api.id record.event_id 7 | api.title record.event_highlighted_title 8 | api.type record.event_type 9 | api.url url_for(record.event_url(:only_path => false)) 10 | api.description record.event_content_snippets.join("\n") 11 | # 'datetime' is deprecated but maintained for backward compatibility. 12 | # Use 'last_modified_at' or 'registered_at' explicitly. 13 | api.datetime record.event_datetime 14 | api.last_modified_at record.last_modified_at 15 | api.registered_at record.registered_at 16 | api.rank record.rank 17 | end 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /app/types/full_text_search/mroonga_integer_array_type.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | class MroongaIntegerArrayType < ActiveModel::Type::Value 3 | def initialize(vector_load_is_supported, *args, &block) 4 | @vector_load_is_supported = vector_load_is_supported 5 | super(*args, &block) 6 | end 7 | 8 | def type 9 | :mroonga_integer_array 10 | end 11 | 12 | def deserialize(value) 13 | return nil if value.nil? 14 | return [] if value.empty? 15 | case value 16 | when Array 17 | value 18 | else 19 | if @vector_load_is_supported and 20 | value.start_with?("[") and 21 | value.end_with?("]") 22 | return JSON.parse(value) 23 | end 24 | value.unpack("l*") 25 | end 26 | end 27 | 28 | def serialize(value) 29 | return nil if value.nil? 30 | if @vector_load_is_supported 31 | value.to_json 32 | else 33 | "" 34 | end 35 | end 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | on: 3 | push: 4 | tags: 5 | - "*" 6 | jobs: 7 | github: 8 | name: GitHub 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v6 12 | - name: Extract release note 13 | run: | 14 | ruby \ 15 | -e 'print("## Full text search plugin "); \ 16 | puts(ARGF.read.split(/^## /)[1]. \ 17 | gsub(/ {.+?}/, ""). \ 18 | gsub(/\[(.+?)\]\[.+?\]/) {$1})' \ 19 | NEWS.md > release-note.md 20 | - name: Upload to release 21 | run: | 22 | title=$(head -n1 release-note.md | sed -e 's/^## //') 23 | tail -n +2 release-note.md > release-note-without-version.md 24 | gh release create ${GITHUB_REF_NAME} \ 25 | --discussion-category Announcements \ 26 | --notes-file release-note-without-version.md \ 27 | --title "${title}" 28 | env: 29 | GH_TOKEN: ${{ github.token }} 30 | -------------------------------------------------------------------------------- /lib/full_text_search/plugin_wiki_extensions_tag_searchable.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | module PluginWikiExtensionsTagSearchable 3 | # Wiki Extensions tags 4 | # https://github.com/haru/redmine_wiki_extensions 5 | 6 | extend ActiveSupport::Concern 7 | 8 | included do 9 | after_create :fts_after_create 10 | after_destroy :fts_after_destroy 11 | end 12 | 13 | private 14 | 15 | def find_fts_target 16 | Target.find_by( 17 | source_id: wiki_page_id, 18 | source_type_id: Type.wiki_page.id) 19 | end 20 | 21 | def find_fts_tag_label_id 22 | Tag.label(tag.name).id 23 | end 24 | 25 | def fts_after_create 26 | fts_target = find_fts_target 27 | return unless fts_target 28 | fts_target.tag_ids |= [find_fts_tag_label_id] 29 | fts_target.save! 30 | end 31 | 32 | def fts_after_destroy 33 | fts_target = find_fts_target 34 | return unless fts_target 35 | fts_target.tag_ids -= [find_fts_tag_label_id] 36 | fts_target.save! 37 | end 38 | end 39 | end 40 | 41 | -------------------------------------------------------------------------------- /app/views/issues/full_text_search/_view_issues_show_description_bottom.html.erb: -------------------------------------------------------------------------------- 1 | <% if fts_display_similar_issues? %> 2 | <% if fts_display_score? %> 3 | <% 4 | duration = nil 5 | ActiveSupport::Notifications.subscribe("groonga.similar.search") do |*args| 6 | groonga_similar_search_event = ActiveSupport::Notifications::Event.new(*args) 7 | duration = "%.1f" % [groonga_similar_search_event.duration] 8 | end 9 | %> 10 | <% content_for :header_tags do %> 11 | <%= stylesheet_link_tag 'score', :plugin => 'full_text_search' %> 12 | <% end %> 13 | <% end %> 14 | <% content_for :header_tags do %> 15 | <%= stylesheet_link_tag 'similar_issues', :plugin => 'full_text_search' %> 16 | <% end %> 17 | 18 |
19 |
20 | <% similar_issues = render_similar_issues(issue) %> 21 |

22 | <%= l(:label_similar_issues) %> 23 | <% if fts_display_score? %> 24 | (<%= duration %>ms) 25 | <% end %> 26 |

27 | <%= similar_issues %> 28 |
29 | <% end %> 30 | -------------------------------------------------------------------------------- /app/views/fts_query_expansions/show.html.erb: -------------------------------------------------------------------------------- 1 | <% content_for :header_tags do %> 2 | <%= stylesheet_link_tag "query_expansions", :plugin => "full_text_search" %> 3 | <% fontawesome_prefix = "fontawesome-free-5.8.2-web" %> 4 | <%= stylesheet_link_tag "#{fontawesome_prefix}/css/all.css", 5 | :plugin => "full_text_search" %> 6 | <% end %> 7 | 8 |
9 | <%= link_to(t("fts_query_expansions.index.title"), 10 | fts_query_expansions_path, 11 | class: "icon icon-list") %> 12 | <%= link_to(l(:button_edit), 13 | edit_fts_query_expansion_path(@query_expansion), 14 | class: "icon icon-edit", 15 | accesskey: accesskey(:edit)) %> 16 | <%= delete_link(@query_expansion) %> 17 |
18 | 19 | <%= title("#{t(".title")} \##{@query_expansion.id}") %> 20 | 21 |
22 |
<%= FtsQueryExpansion.human_attribute_name("from") %>
23 |
<%= @query_expansion.from %>
24 |
<%= FtsQueryExpansion.human_attribute_name("to") %>
25 |
<%= @query_expansion.to %>
26 |
27 | -------------------------------------------------------------------------------- /lib/full_text_search/migration.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | module Migration 3 | if Redmine::Database.postgresql? 4 | # TODO: Send a patch for WITH support to Active Record. 5 | 6 | module IndexDefinitionWithSupport 7 | attr_accessor :with 8 | end 9 | ::ActiveRecord::ConnectionAdapters::IndexDefinition.prepend(IndexDefinitionWithSupport) 10 | 11 | module SchemaCreationWithSupport 12 | private 13 | def visit_CreateIndexDefinition(o) 14 | sql = super 15 | with = o.index.with 16 | sql << " WITH (#{with})" if with 17 | sql 18 | end 19 | end 20 | ::ActiveRecord::ConnectionAdapters::SchemaCreation.prepend(SchemaCreationWithSupport) 21 | 22 | module PostgreSQLAdapterWithSupport 23 | def add_index_options(table_name, column_name, with: nil, **options) 24 | result = super(table_name, column_name, **options) 25 | result[0].with = with 26 | result 27 | end 28 | end 29 | ::ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.prepend(PostgreSQLAdapterWithSupport) 30 | end 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /lib/full_text_search/hooks/similar_issues_helper.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | module Hooks 3 | module SimilarIssuesHelper 4 | include FullTextSearch::Hooks::SettingsHelper 5 | 6 | def render_similar_issues(issue) 7 | s = '' 8 | issue.similar_issues.each do |similar_issue| 9 | css = "list issue issue-#{similar_issue.id} #{similar_issue.css_classes}" 10 | s << content_tag( 11 | "tr", 12 | content_tag("td", link_to_issue(similar_issue, project: (issue.project_id != similar_issue.project_id)), class: "subject", style: "width: 50%", data: { rank: similar_issue.similarity_score }) + 13 | content_tag("td", h(similar_issue.status), class: "status") + 14 | content_tag("td", link_to_user(similar_issue.assigned_to), class: "assigned_to") + 15 | content_tag("td", similar_issue.disabled_core_fields.include?("done_ratio") ? "" : progress_bar(similar_issue.done_ratio), class: "done_ratio"), 16 | class: css 17 | ) 18 | end 19 | s << '
' 20 | s.html_safe 21 | end 22 | end 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /lib/full_text_search/document_mapper.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | class DocumentMapper < Mapper 3 | class << self 4 | def redmine_mapper_class 5 | RedmineDocumentMapper 6 | end 7 | 8 | def fts_mapper_class 9 | FtsDocumentMapper 10 | end 11 | end 12 | end 13 | resolver.register(Document, DocumentMapper) 14 | 15 | class RedmineDocumentMapper < RedmineMapper 16 | def upsert_fts_target(options={}) 17 | fts_target = find_fts_target 18 | fts_target.source_id = @record.id 19 | fts_target.source_type_id = Type[@record.class].id 20 | fts_target.project_id = @record.project_id 21 | fts_target.title = @record.title 22 | fts_target.content = @record.description 23 | fts_target.last_modified_at = @record.created_on 24 | fts_target.registered_at = @record.created_on 25 | fts_target.save! 26 | end 27 | end 28 | 29 | class FtsDocumentMapper < FtsMapper 30 | def title_prefix 31 | "#{l(:label_document)}: " 32 | end 33 | 34 | def url 35 | { 36 | controller: "documents", 37 | action: "show", 38 | id: @record.source_id, 39 | } 40 | end 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /dev/initialize-redmine.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -exu 4 | 5 | test_svn_repository="tmp/test/subversion_repository" 6 | if [ ! -d "${test_svn_repository}" ]; then 7 | svnadmin create "${test_svn_repository}" 8 | zcat test/fixtures/repositories/subversion_repository.dump.gz | \ 9 | svnadmin load "${test_svn_repository}" 10 | fi 11 | 12 | test_git_repository="tmp/test/git_repository" 13 | if [ ! -d "${test_git_repository}" ]; then 14 | tar xf test/fixtures/repositories/git_repository.tar.gz \ 15 | -C "$(dirname ${test_git_repository})" 16 | fi 17 | 18 | ${RUBY:-ruby} bin/rails db:drop || true 19 | ${RUBY:-ruby} bin/rails generate_secret_token 20 | ${RUBY:-ruby} bin/rails db:create 21 | rails_version_major=$(grep "^gem 'rails'" Gemfile | grep -o '[0-9]*' | head -n1) 22 | if [ ${rails_version_major} -ge 8 ]; then 23 | ${RUBY:-ruby} bin/rails db:migrate:reset 24 | else 25 | ${RUBY:-ruby} bin/rails db:migrate 26 | fi 27 | ${RUBY:-ruby} bin/rails redmine:load_default_data REDMINE_LANG=en 28 | ${RUBY:-ruby} bin/rails redmine:plugins:migrate 29 | 30 | ${RUBY:-ruby} bin/rails db:schema:dump 31 | 32 | ${RUBY:-ruby} bin/rails runner ' 33 | u = User.find(1) 34 | u.password = u.password_confirmation = "adminadmin" 35 | u.must_change_passwd = false 36 | u.save! 37 | ' 38 | 39 | -------------------------------------------------------------------------------- /db/migrate/20190807085000_create_fts_query_expansions.rb: -------------------------------------------------------------------------------- 1 | class CreateFtsQueryExpansions < ActiveRecord::Migration[5.2] 2 | def change 3 | return if reverting? and !table_exists?(:fts_query_expansions) 4 | 5 | if Redmine::Database.mysql? 6 | options = "ENGINE=Mroonga" 7 | else 8 | options = nil 9 | end 10 | create_table :fts_query_expansions, options: options do |t| 11 | if Redmine::Database.mysql? 12 | t.string :source, null: false 13 | t.string :destination, null: false 14 | else 15 | t.text :source, null: false 16 | t.text :destination, null: false 17 | end 18 | t.timestamps 19 | if Redmine::Database.mysql? 20 | t.index [:source], comment: "NORMALIZER 'NormalizerNFKC121'" 21 | t.index [:destination], comment: "NORMALIZER 'NormalizerNFKC121'" 22 | else 23 | t.index [ 24 | "source pgroonga_text_term_search_ops_v2", 25 | "destination pgroonga_text_term_search_ops_v2", 26 | ].join(", "), 27 | using: "PGroonga", 28 | with: "normalizer = 'NormalizerNFKC121'", 29 | name: "fts_query_expansions_index_pgroonga" 30 | end 31 | t.index :updated_at 32 | end 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016-2018 okkez 4 | Copyright (c) 2017-2019 Kouhei Sutou 5 | Copyright (c) 2019 Shimadzu Corporation 6 | Copyright (c) 2024 Abe Tomoaki 7 | 8 | Permission is hereby granted, free of charge, to any person obtaining a copy 9 | of this software and associated documentation files (the "Software"), to deal 10 | in the Software without restriction, including without limitation the rights 11 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 12 | copies of the Software, and to permit persons to whom the Software is 13 | furnished to do so, subject to the following conditions: 14 | 15 | The above copyright notice and this permission notice shall be included in all 16 | copies or substantial portions of the Software. 17 | 18 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 23 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 24 | SOFTWARE. 25 | -------------------------------------------------------------------------------- /lib/full_text_search/news_mapper.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | class NewsMapper < Mapper 3 | class << self 4 | def redmine_mapper_class 5 | RedmineNewsMapper 6 | end 7 | 8 | def fts_mapper_class 9 | FtsNewsMapper 10 | end 11 | end 12 | end 13 | resolver.register(News, NewsMapper) 14 | 15 | class RedmineNewsMapper < RedmineMapper 16 | class << self 17 | def with_project(redmine_class) 18 | redmine_class.joins(:project) 19 | end 20 | end 21 | 22 | def upsert_fts_target(options={}) 23 | fts_target = find_fts_target 24 | fts_target.source_id = @record.id 25 | fts_target.source_type_id = Type[@record.class].id 26 | fts_target.project_id = @record.project_id 27 | fts_target.title = @record.title 28 | fts_target.content = [ 29 | @record.summary.presence, 30 | @record.description.presence, 31 | ].compact.join("\n") 32 | fts_target.last_modified_at = @record.created_on 33 | fts_target.registered_at = @record.created_on 34 | fts_target.save! 35 | end 36 | end 37 | 38 | class FtsNewsMapper < FtsMapper 39 | def url 40 | { 41 | controller: "news", 42 | action: "show", 43 | id: @record.source_id, 44 | } 45 | end 46 | end 47 | end 48 | -------------------------------------------------------------------------------- /lib/full_text_search/scm_adapter_cat_io.rb: -------------------------------------------------------------------------------- 1 | # For auto load 2 | Redmine::Scm::Adapters::GitAdapter 3 | Redmine::Scm::Adapters::SubversionAdapter 4 | 5 | # TODO: Submit a patch to Redmine 6 | 7 | module Redmine 8 | module Scm 9 | module Adapters 10 | class GitAdapter 11 | def cat_io(path, identifier=nil) 12 | if identifier.nil? 13 | identifier = 'HEAD' 14 | end 15 | cmd_args = %w|show --no-color| 16 | cmd_args << "#{identifier}:#{scm_iconv(@path_encoding, 'UTF-8', path)}" 17 | cat = nil 18 | git_cmd(cmd_args) do |io| 19 | io.binmode 20 | yield(io) 21 | end 22 | rescue ScmCommandAborted 23 | end 24 | end 25 | 26 | class SubversionAdapter 27 | def cat_io(path, identifier=nil) 28 | identifier = (identifier and identifier.to_i > 0) ? identifier.to_i : "HEAD" 29 | cmd = "#{self.class.sq_bin} cat #{target(path)}@#{identifier}" 30 | cmd << credentials_string 31 | cat = nil 32 | shellout(cmd) do |io| 33 | io.binmode 34 | yield(io) 35 | end 36 | end 37 | end 38 | end 39 | end 40 | end 41 | 42 | # For auto load 43 | module FullTextSearch 44 | module ScmAdapterCatIo 45 | end 46 | end 47 | -------------------------------------------------------------------------------- /test/unit/full_text_search/issue_query_any_searchable_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path("../../../test_helper", __FILE__) 2 | 3 | module FullTextSearch 4 | class IssueQueryAnySearchableTest < ActiveSupport::TestCase 5 | setup do 6 | unless IssueQuery.method_defined?(:sql_for_any_searchable_field) 7 | skip("Required feature 'sql_for_any_searchable_field' does not exist.") 8 | end 9 | end 10 | 11 | def test_or_one_word 12 | Issue.destroy_all 13 | subject_groonga = Issue.generate!(subject: "Groonga") 14 | description_groonga = Issue.generate!(description: "Groonga") 15 | without_groonga = Issue.generate!(subject: "no-keyword", 16 | description: "no-keyword") 17 | journal_groonga = Issue.generate!.journals.create!(notes: "Groonga") 18 | query = IssueQuery.new( 19 | :name => "_", 20 | :filters => { 21 | "any_searchable" => { 22 | :operator => "~", 23 | :values => ["Groonga"] 24 | } 25 | }, 26 | :sort_criteria => [["id", "asc"]] 27 | ) 28 | expected_issues = [ 29 | subject_groonga, 30 | description_groonga, 31 | journal_groonga.issue 32 | ] 33 | assert_equal(expected_issues, query.issues) 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /db/migrate/20170810045914_create_issue_contents.rb: -------------------------------------------------------------------------------- 1 | # For auto load 2 | FullTextSearch::Migration 3 | 4 | class CreateIssueContents < ActiveRecord::Migration[4.2] 5 | def change 6 | return if reverting? and !table_exists?(:issue_contents) 7 | 8 | options = nil 9 | contents_limit = nil 10 | if Redmine::Database.mysql? 11 | options = "ENGINE=Mroonga DEFAULT CHARSET=utf8mb4" 12 | contents_limit = 16.megabytes 13 | end 14 | create_table :issue_contents, options: options do |t| 15 | t.integer :project_id 16 | t.integer :issue_id, unique: true, null: false 17 | t.text :subject 18 | t.text :contents, limit: contents_limit 19 | t.integer :status_id 20 | t.boolean :is_private 21 | 22 | if Redmine::Database.mysql? 23 | t.index :contents, 24 | type: "fulltext", 25 | comment: "TOKENIZER 'TokenMecab'" 26 | else 27 | t.index [:id, 28 | :project_id, 29 | :issue_id, 30 | :subject, 31 | :contents, 32 | :status_id, 33 | :is_private], 34 | name: "index_issue_contents_pgroonga", 35 | using: "PGroonga", 36 | with: [ 37 | "tokenizer = 'TokenMecab'", 38 | "normalizer = 'NormalizerNFKC121'", 39 | ].join(", ") 40 | end 41 | end 42 | end 43 | end 44 | -------------------------------------------------------------------------------- /test/unit/full_text_search/project_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path("../../../test_helper", __FILE__) 2 | 3 | module FullTextSearch 4 | class ProjectTest < ActiveSupport::TestCase 5 | include PrettyInspectable 6 | include NullValues 7 | 8 | fixtures :custom_fields 9 | fixtures :custom_fields_projects 10 | fixtures :custom_fields_trackers 11 | fixtures :custom_values 12 | fixtures :enumerations 13 | fixtures :issue_statuses 14 | fixtures :projects 15 | fixtures :projects_trackers 16 | fixtures :trackers 17 | fixtures :users 18 | 19 | def test_destroy 20 | custom_field = ProjectCustomField.generate!(searchable: true) 21 | project = Project.generate! do |p| 22 | p.custom_fields = [ 23 | { 24 | "id" => custom_field.id.to_s, 25 | "value" => "Hello", 26 | }, 27 | ] 28 | end 29 | project_targets = Target.where(source_id: project.id, 30 | source_type_id: Type.project.id) 31 | custom_value_targets = Target.where(container_id: project.id, 32 | source_type_id: Type.custom_value.id) 33 | assert_equal([1, 1], 34 | [project_targets.size, custom_value_targets.size]) 35 | project.destroy! 36 | assert_equal([[], []], 37 | [project_targets.to_a, custom_value_targets.to_a]) 38 | end 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /lib/full_text_search/project_mapper.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | class ProjectMapper < Mapper 3 | class << self 4 | def redmine_mapper_class 5 | RedmineProjectMapper 6 | end 7 | 8 | def fts_mapper_class 9 | FtsProjectMapper 10 | end 11 | end 12 | end 13 | resolver.register(Project, ProjectMapper) 14 | 15 | class RedmineProjectMapper < RedmineMapper 16 | class << self 17 | def with_project(redmine_class) 18 | redmine_class 19 | end 20 | end 21 | 22 | def upsert_fts_target(options={}) 23 | fts_target = find_fts_target 24 | fts_target.source_id = @record.id 25 | fts_target.source_type_id = Type[@record.class].id 26 | fts_target.project_id = @record.id 27 | fts_target.title = @record.name 28 | fts_target.content = @record.description 29 | fts_target.last_modified_at = @record.updated_on 30 | fts_target.registered_at = @record.created_on 31 | tag_ids = [] 32 | if @record.identifier 33 | tag_ids << Tag.identifier(@record.identifier).id 34 | end 35 | fts_target.tag_ids = tag_ids 36 | fts_target.save! 37 | end 38 | end 39 | 40 | class FtsProjectMapper < FtsMapper 41 | def title_prefix 42 | "#{l(:label_project)}: " 43 | end 44 | 45 | def url 46 | { 47 | controller: "projects", 48 | action: "show", 49 | id: @record.source_id 50 | } 51 | end 52 | end 53 | end 54 | -------------------------------------------------------------------------------- /lib/full_text_search/resolver.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | class Resolver 3 | include Enumerable 4 | 5 | def initialize 6 | @redmine_to_mapper = {} 7 | @name_to_mapper = {} 8 | @mapper_to_redmine = {} 9 | end 10 | 11 | def register(redmine_class, mapper_class) 12 | @redmine_to_mapper[redmine_class] = mapper_class 13 | @name_to_mapper[normalize_name(redmine_class.name)] = mapper_class 14 | @mapper_to_redmine[mapper_class] = redmine_class 15 | end 16 | 17 | def resolve(key) 18 | case key 19 | when Target 20 | name = Type.find(key.source_type_id).name 21 | mapper = @name_to_mapper[normalize_name(name)] 22 | mapper.fts_mapper(key) 23 | when Class 24 | if key <= Mapper 25 | @mapper_to_redmine[key] 26 | else 27 | @redmine_to_mapper[key] 28 | end 29 | when ActiveRecord::Base 30 | mapper = @redmine_to_mapper[key] 31 | mapper.redmine_mapper(key) 32 | when String 33 | @name_to_mapper[normalize_name(key)] 34 | else 35 | message = "must be FullTextSearch::Target, Redmine model class, " 36 | message << "Redmine model instance, String or " 37 | message << "FullTextSearch::Mapper: #{key.inspect}" 38 | raise ArgumentError, message 39 | end 40 | end 41 | 42 | def each(&block) 43 | @redmine_to_mapper.each(&block) 44 | end 45 | 46 | private 47 | def normalize_name(name) 48 | name.downcase 49 | end 50 | end 51 | end 52 | -------------------------------------------------------------------------------- /lib/full_text_search/similar_searcher.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | module SimilarSearcher 3 | module Model 4 | def self.included(base) 5 | base.include(InstanceMethods) 6 | case base.connection_db_config.adapter 7 | when "postgresql" 8 | base.include(FullTextSearch::SimilarSearcher::Pgroonga) 9 | when "mysql2" 10 | base.include(FullTextSearch::SimilarSearcher::Mroonga) 11 | end 12 | end 13 | 14 | module InstanceMethods 15 | def filter_condition(user = User.current, project_ids = []) 16 | conditions = [] 17 | target_ids = Project.allowed_to(user, :view_issues).pluck(:id) 18 | target_ids &= project_ids if project_ids.present? 19 | if target_ids.present? 20 | conditions << ("in_values(project_id, #{target_ids.join(',')})") 21 | end 22 | if conditions.empty? 23 | "1==1" 24 | else 25 | build_condition("||", conditions) 26 | end 27 | end 28 | 29 | def similar_content 30 | contents = [subject, description] 31 | notes = journals.sort_by(&:id).map(&:notes) 32 | contents.concat(notes) 33 | attachments.order(:id).each do |attachment| 34 | contents << attachment.filename if attachment.filename.present? 35 | contents << attachment.description if attachment.description.present? 36 | end 37 | contents.join("\n") 38 | end 39 | end 40 | end 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /config/locales/ja.yml: -------------------------------------------------------------------------------- 1 | ja: 2 | label_result_all: すべて 3 | label_full_text_search_result_order: 検索結果の並び順 4 | label_full_text_search_result_order_target: 並び替え対象 5 | label_full_text_search_result_order_type: 並び順 6 | label_full_text_search_order_target_score: スコア 7 | label_full_text_search_order_target_last_modified_time: 更新日時 8 | label_full_text_search_order_target_registered_time: 作成日時 9 | label_full_text_search_order_type_asc: 昇順 10 | label_full_text_search_order_type_desc: 降順 11 | label_full_text_search_display_score: スコアを表示 12 | label_full_text_search_display_similar_issues: 類似チケットを表示 13 | label_full_text_search_attachment_max_text_size: 添付ファイルから抽出する最大テキストサイズ 14 | label_full_text_search_text_extraction_timeout: テキスト抽出処理のタイムアウト時間（秒） 15 | label_full_text_search_external_command_max_memory: 外部コマンドの最大使用可能メモリー 16 | label_full_text_search_server_url: ChupaTextサーバーのURL 17 | label_full_text_search_enable_tracking: トラッキングを有効化 18 | label_similar_issues: 類似チケット 19 | label_extension: 拡張子 20 | label_text_extraction: テキスト抽出 21 | label_identifier: ID 22 | label_label: ラベル 23 | label: 24 | full_text_search: 25 | drilldown: 26 | deselect: 解除 27 | menu: 28 | query_expansions: 29 | plural: クエリー展開一覧 30 | 31 | fts_query_expansions: 32 | index: 33 | title: クエリー展開一覧 34 | result: クエリー展開結果 35 | show: 36 | title: クエリー展開 37 | new: 38 | title: 新しいクエリー展開 39 | 40 | activerecord: 41 | attributes: 42 | fts_query_expansion: 43 | source: 展開元 44 | destination: 展開先 45 | -------------------------------------------------------------------------------- /lib/full_text_search/message_mapper.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | class MessageMapper < Mapper 3 | class << self 4 | def redmine_mapper_class 5 | RedmineMessageMapper 6 | end 7 | 8 | def fts_mapper_class 9 | FtsMessageMapper 10 | end 11 | end 12 | end 13 | resolver.register(Message, MessageMapper) 14 | 15 | class RedmineMessageMapper < RedmineMapper 16 | class << self 17 | def with_project(redmine_class) 18 | redmine_class.joins(board: :project) 19 | end 20 | end 21 | 22 | def upsert_fts_target(options={}) 23 | fts_target = find_fts_target 24 | fts_target.source_id = @record.id 25 | fts_target.source_type_id = Type[@record.class].id 26 | fts_target.project_id = @record.board.project_id 27 | fts_target.title = @record.subject 28 | fts_target.content = @record.content 29 | fts_target.last_modified_at = @record.updated_on 30 | fts_target.registered_at = @record.created_on 31 | fts_target.save! 32 | end 33 | end 34 | 35 | class FtsMessageMapper < FtsMapper 36 | def type 37 | message = redmine_record 38 | if message.parent_id.nil? 39 | "message" 40 | else 41 | "reply" 42 | end 43 | end 44 | 45 | def title_prefix 46 | message = redmine_record 47 | "#{message.board.name}: " 48 | end 49 | 50 | def url 51 | message = redmine_record 52 | { 53 | controller: "messages", 54 | action: "show", 55 | board_id: message.board.id, 56 | id: message.id, 57 | } 58 | end 59 | end 60 | end 61 | -------------------------------------------------------------------------------- /assets/stylesheets/fontawesome-free-5.8.2-web/LICENSE.txt: -------------------------------------------------------------------------------- 1 | Font Awesome Free License 2 | ------------------------- 3 | 4 | Font Awesome Free is free, open source, and GPL friendly. You can use it for 5 | commercial projects, open source projects, or really almost whatever you want. 6 | Full Font Awesome Free license: https://fontawesome.com/license/free. 7 | 8 | # Icons: CC BY 4.0 License (https://creativecommons.org/licenses/by/4.0/) 9 | In the Font Awesome Free download, the CC BY 4.0 license applies to all icons 10 | packaged as SVG and JS file types. 11 | 12 | # Fonts: SIL OFL 1.1 License (https://scripts.sil.org/OFL) 13 | In the Font Awesome Free download, the SIL OFL license applies to all icons 14 | packaged as web and desktop font files. 15 | 16 | # Code: MIT License (https://opensource.org/licenses/MIT) 17 | In the Font Awesome Free download, the MIT license applies to all non-font and 18 | non-icon files. 19 | 20 | # Attribution 21 | Attribution is required by MIT, SIL OFL, and CC BY licenses. Downloaded Font 22 | Awesome Free files already contain embedded comments with sufficient 23 | attribution, so you shouldn't need to do anything additional when using these 24 | files normally. 25 | 26 | We've kept attribution comments terse, so we ask that you do not actively work 27 | to remove them from files, especially code. They're a great way for folks to 28 | learn about Font Awesome. 29 | 30 | # Brand Icons 31 | All brand icons are trademarks of their respective owners. The use of these 32 | trademarks does not indicate endorsement of the trademark holder by Font 33 | Awesome, nor vice versa. **Please do not use brand logos for any purpose except 34 | to represent the company, product, or service to which they refer.** 35 | -------------------------------------------------------------------------------- /assets/stylesheets/search.css: -------------------------------------------------------------------------------- 1 | .search-input-box { 2 | display: flex; 3 | justify-content: center; 4 | align-content: center; 5 | } 6 | 7 | .search-input-box input, 8 | .search-input-box button { 9 | border: 1px solid #bbbbbb; 10 | height: 32px; 11 | padding: 8px; 12 | margin: 0px; 13 | } 14 | 15 | .search-input-box input { 16 | flex-basis: 100%; 17 | } 18 | 19 | .search-input-box button { 20 | border-left-width: 0; 21 | cursor: pointer; 22 | } 23 | 24 | .search-input-box button:hover { 25 | background: #169; 26 | } 27 | 28 | #search-result { 29 | display: grid; 30 | grid-template-columns: 10em calc(100% - 10em); 31 | } 32 | 33 | /* For IE11: start */ 34 | #search-result { 35 | display: -ms-grid; 36 | -ms-grid-columns: 10em calc(100% - 10em); 37 | } 38 | 39 | #search-result-content { 40 | -ms-grid-column: 2; 41 | } 42 | /* For IE11: end */ 43 | 44 | #search-result h3 { 45 | margin-top: 0.5em; 46 | } 47 | 48 | #search-result-metadata { 49 | border-right: 1px solid #bbbbbb; 50 | padding-right: 3px; 51 | } 52 | 53 | .search-order ul { 54 | padding: 0; 55 | margin: 0; 56 | list-style-type: none; 57 | } 58 | 59 | .search-order li { 60 | padding: 0; 61 | margin: 0; 62 | } 63 | 64 | .search-drilldown ul { 65 | padding: 0; 66 | padding-left: 1em; 67 | margin: 0; 68 | } 69 | 70 | .search-drilldown ul li { 71 | padding: 0; 72 | margin: 0; 73 | } 74 | 75 | ol.search-snippets { 76 | padding: 0.25em 0; 77 | margin: 0; 78 | list-style-type: none; 79 | } 80 | 81 | ol.search-snippets li { 82 | padding: 0.25em 0; 83 | } 84 | 85 | #search-results .last-modified::after { 86 | content: '|'; 87 | padding: 0 0.25em; 88 | } 89 | 90 | .keyword { 91 | background-color: #FCFD8D; 92 | } 93 | -------------------------------------------------------------------------------- /config/locales/en.yml: -------------------------------------------------------------------------------- 1 | en: 2 | label_result_all: All 3 | label_full_text_search_result_order: Search result order 4 | label_full_text_search_result_order_target: Sort by 5 | label_full_text_search_result_order_type: Sort order 6 | label_full_text_search_order_target_score: score 7 | label_full_text_search_order_target_last_modified_time: last modified 8 | label_full_text_search_order_target_registered_time: registered 9 | label_full_text_search_order_type_asc: asc 10 | label_full_text_search_order_type_desc: desc 11 | label_full_text_search_display_score: Display score 12 | label_full_text_search_display_similar_issues: Display similar issues 13 | label_full_text_search_attachment_max_text_size: Max text size extracted from attachments 14 | label_full_text_search_text_extraction_timeout: Timeout in seconds for text extraction 15 | label_full_text_search_external_command_max_memory: Max memory usage for executing an external command 16 | label_full_text_search_server_url: ChupaText server URL 17 | label_full_text_search_enable_tracking: Enable tracking 18 | label_similar_issues: Similar issues 19 | label_extension: Extension 20 | label_text_extraction: Text extraction 21 | label_identifier: ID 22 | label_label: Label 23 | label: 24 | full_text_search: 25 | drilldown: 26 | deselect: Deselect 27 | menu: 28 | query_expansions: 29 | plural: Query expansions 30 | 31 | fts_query_expansions: 32 | index: 33 | title: Query expansions 34 | result: Expanded query 35 | show: 36 | title: Query expansion 37 | new: 38 | title: New query expansion 39 | 40 | activerecord: 41 | attributes: 42 | fts_query_expansion: 43 | source: Source 44 | destination: Destination 45 | -------------------------------------------------------------------------------- /bin/dump-mysql: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | config_dir = File.expand_path("../../../../config", __FILE__) 4 | APP_PATH = File.join(config_dir, "application") 5 | require APP_PATH 6 | Rails.application.require_environment! 7 | 8 | db_config = ActiveRecord::Base.connection_config 9 | host = db_config[:host] 10 | port = db_config[:port] || 3306 11 | user = db_config[:username] 12 | password = db_config[:password] 13 | database = db_config[:database] 14 | 15 | have_delayed_job = true 16 | begin 17 | Redmine::Plugin.find(:delayed_job) 18 | rescue Redmine::PluginNotFound 19 | have_delayed_job = false 20 | end 21 | 22 | 23 | mysqldump = ENV["MYSQLDUMP"] || "mysqldump" 24 | 25 | schema_only_dump_tables = [ 26 | "fts_tags", 27 | "fts_targets", 28 | "issue_contents", 29 | ] 30 | 31 | ignore_tables = schema_only_dump_tables.dup 32 | if have_delayed_job 33 | ignore_tables << "delayed_jobs" 34 | end 35 | 36 | mysqldump_command_line = [ 37 | mysqldump, 38 | "--host=#{host}", 39 | "--port=#{port}", 40 | "--user=#{user}", 41 | "--password=#{password}", 42 | "--single-transaction", 43 | ] 44 | 45 | normal_dump_command_line = [] 46 | ignore_tables.each do |ignore_table| 47 | normal_dump_command_line << "--ignore-table=#{database}.#{ignore_table}" 48 | end 49 | normal_dump_command_line << database 50 | system(*mysqldump_command_line, *normal_dump_command_line) 51 | if have_delayed_job 52 | system(*mysqldump_command_line, 53 | "--where=queue <> 'full_text_search'", 54 | database, 55 | "delayed_jobs") 56 | end 57 | if have_delayed_job 58 | system(*mysqldump_command_line, 59 | "--where=queue <> 'full_text_search'", 60 | database, 61 | "delayed_jobs") 62 | end 63 | system(*mysqldump_command_line, 64 | "--no-data", 65 | database, 66 | *schema_only_dump_tables) 67 | -------------------------------------------------------------------------------- /app/models/full_text_search/type.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | class Type < ApplicationRecord 3 | self.table_name = :fts_types 4 | 5 | class << self 6 | private def normalize_key(key) 7 | case key 8 | when Class 9 | key.name.underscore 10 | when ActiveRecord::Base 11 | key.class.name.underscore 12 | when /\A[A-Z]/ 13 | key.underscore 14 | else 15 | key.singularize 16 | end 17 | end 18 | 19 | def [](key) 20 | __send__(normalize_key(key)) 21 | end 22 | 23 | def available?(name) 24 | respond_to?(normalize_key(name)) 25 | end 26 | 27 | def attachment 28 | find_or_create_by(name: "Attachment") 29 | end 30 | 31 | def change 32 | find_or_create_by(name: "Change") 33 | end 34 | 35 | def changeset 36 | find_or_create_by(name: "Changeset") 37 | end 38 | 39 | def custom_value 40 | find_or_create_by(name: "CustomValue") 41 | end 42 | 43 | def document 44 | find_or_create_by(name: "Document") 45 | end 46 | 47 | def file 48 | find_or_create_by(name: "File") 49 | end 50 | 51 | def issue 52 | find_or_create_by(name: "Issue") 53 | end 54 | 55 | def journal 56 | find_or_create_by(name: "Journal") 57 | end 58 | 59 | def message 60 | find_or_create_by(name: "Message") 61 | end 62 | 63 | def news 64 | find_or_create_by(name: "News") 65 | end 66 | 67 | def project 68 | find_or_create_by(name: "Project") 69 | end 70 | 71 | def repository 72 | find_or_create_by(name: "Repository") 73 | end 74 | 75 | def version 76 | find_or_create_by(name: "Version") 77 | end 78 | 79 | def wiki_page 80 | find_or_create_by(name: "WikiPage") 81 | end 82 | end 83 | end 84 | end 85 | -------------------------------------------------------------------------------- /lib/full_text_search/text_extractor.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | class TextExtractor 3 | class << self 4 | @@extractors = {} 5 | def extractor 6 | @@extractors[Thread.current.object_id] ||= build_extractor 7 | end 8 | 9 | def build_extractor 10 | extractor = ChupaText::Extractor.new 11 | extractor.apply_configuration(ChupaText::Configuration.default) 12 | extractor 13 | end 14 | end 15 | 16 | def extract(path, input, content_type) 17 | apply_settings 18 | if input 19 | data = ChupaText::VirtualFileData.new(path, input) 20 | else 21 | data = ChupaText::InputData.new(path) 22 | end 23 | text = +"" 24 | begin 25 | data.need_screenshot = false 26 | data.mime_type = content_type 27 | data.timeout = @timeout 28 | data.max_body_size = @max_size 29 | self.class.extractor.extract(data) do |extracted| 30 | body = +extracted.body 31 | extracted.release 32 | body.scrub!("") 33 | body.gsub!("\u0000", "") 34 | next if body.empty? 35 | text << "\n" unless text.empty? 36 | text << body 37 | if text.bytesize >= @max_size 38 | text = text.byteslice(0, @max_size) 39 | text.scrub!("") 40 | break 41 | end 42 | end 43 | ensure 44 | data.release 45 | end 46 | text 47 | end 48 | 49 | private 50 | def apply_settings 51 | settings = Setting.plugin_full_text_search 52 | @timeout = settings.text_extraction_timeout 53 | ChupaText::ExternalCommand.default_timeout = @timeout 54 | ChupaText::ExternalCommand.default_limit_cpu = @timeout 55 | ChupaText::ExternalCommand.default_limit_as = 56 | settings.external_command_max_memory 57 | ChupaText::Decomposers::HTTPServer.default_url = 58 | settings.server_url 59 | @max_size = settings.attachment_max_text_size 60 | end 61 | end 62 | end 63 | -------------------------------------------------------------------------------- /lib/full_text_search/repository_entry.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | class RepositoryEntry 3 | attr_reader :path 4 | def initialize(repository, path, identifier) 5 | @repository = repository 6 | @path = path 7 | @identifier = identifier 8 | @entry = fetch_entry 9 | end 10 | 11 | def exist? 12 | not @entry.nil? 13 | end 14 | 15 | def file? 16 | @entry and @entry.is_file? 17 | end 18 | 19 | def directory? 20 | @entry and @entry.is_dir? 21 | end 22 | 23 | def cat(&block) 24 | trace("Fetched content") do 25 | @repository.scm.cat_io(@path, @identifier, &block) 26 | end 27 | end 28 | 29 | private 30 | def fetch_entry 31 | trace("Fetched an entry") do 32 | relative_path = @repository.relative_path(@path) 33 | parts = relative_path.to_s.split(%r{[\/\\]}).select {|n| !n.blank?} 34 | search_path = parts[0..-2].join('/') 35 | search_name = parts[-1] 36 | if search_path.blank? and search_name.blank? 37 | @repository.entry(relative_path, @identifier) 38 | else 39 | entries = fetch_entries(search_path) 40 | entries&.detect {|entry| entry.name == search_name} 41 | end 42 | end 43 | end 44 | 45 | @@cache_mutex = Mutex.new 46 | @@cached_entries = {} 47 | def fetch_entries(path) 48 | trace("Fetched entries") do 49 | cache_key = [@repository.id, path, @identifier] 50 | @@cache_mutex.synchronize do 51 | @@cached_entries[cache_key] ||= 52 | @repository.scm.entries(path, @identifier) 53 | end 54 | end 55 | end 56 | 57 | def trace(label) 58 | tracer = Tracer.new("[repository-entry]") 59 | tracer_data = [ 60 | ["Repository", @repository.id], 61 | ["path", @path], 62 | ["identifier", @identifier], 63 | ] 64 | begin 65 | yield 66 | ensure 67 | tracer.trace(:info, label, tracer_data) 68 | end 69 | end 70 | end 71 | end 72 | -------------------------------------------------------------------------------- /lib/full_text_search/wiki_page_mapper.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | class WikiPageMapper < Mapper 3 | class << self 4 | def redmine_mapper_class 5 | RedmineWikiPageMapper 6 | end 7 | 8 | def fts_mapper_class 9 | FtsWikiPageMapper 10 | end 11 | end 12 | end 13 | resolver.register(WikiPage, WikiPageMapper) 14 | 15 | class RedmineWikiPageMapper < RedmineMapper 16 | class << self 17 | def with_project(redmine_class) 18 | redmine_class.joins(wiki: :project) 19 | end 20 | end 21 | 22 | def upsert_fts_target(options={}) 23 | fts_target = find_fts_target 24 | fts_target.source_id = @record.id 25 | fts_target.source_type_id = Type[@record.class].id 26 | fts_target.project_id = @record.wiki.project_id 27 | fts_target.title = @record.title 28 | tag_ids = [] 29 | content = @record.content 30 | if content 31 | parser = MarkupParser.new(@record.wiki.project) 32 | content_text, content_tag_ids = parser.parse(content, :text) 33 | fts_target.content = content_text 34 | tag_ids.concat(content_tag_ids) 35 | else 36 | fts_target.content = nil 37 | end 38 | fts_target.tag_ids = tag_ids | plugin_wiki_extensions_tag_ids 39 | fts_target.last_modified_at = @record.updated_on 40 | fts_target.registered_at = @record.created_on 41 | fts_target.save! 42 | end 43 | 44 | private 45 | def plugin_wiki_extensions_tag_ids 46 | return [] unless @record.respond_to?(:wiki_ext_tags) 47 | @record.wiki_ext_tags.collect do |tag| 48 | Tag.label(tag.name).id 49 | end 50 | end 51 | end 52 | 53 | class FtsWikiPageMapper < FtsMapper 54 | def type 55 | "wiki-page" 56 | end 57 | 58 | def title_prefix 59 | "#{l(:label_wiki)}: " 60 | end 61 | 62 | def url 63 | { 64 | controller: "wiki", 65 | action: "show", 66 | project_id: @record.project_id, 67 | id: @record.title, 68 | } 69 | end 70 | end 71 | end 72 | -------------------------------------------------------------------------------- /lib/full_text_search/changeset_mapper.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | class ChangesetMapper < Mapper 3 | class << self 4 | def redmine_mapper_class 5 | RedmineChangesetMapper 6 | end 7 | 8 | def fts_mapper_class 9 | FtsChangesetMapper 10 | end 11 | end 12 | end 13 | resolver.register(Changeset, ChangesetMapper) 14 | 15 | class RedmineChangesetMapper < RedmineMapper 16 | class << self 17 | def with_project(redmine_class) 18 | redmine_class.joins(repository: :project) 19 | end 20 | end 21 | 22 | def upsert_fts_target(options={}) 23 | repository = @record.repository 24 | fts_target = find_fts_target 25 | if repository.nil? 26 | fts_target.destroy! if fts_target.persisted? 27 | return 28 | end 29 | fts_target.source_id = @record.id 30 | fts_target.source_type_id = Type[@record.class].id 31 | fts_target.project_id = repository.project_id 32 | if @record.user 33 | fts_target.tag_ids = [Tag.user(@record.user.id).id] 34 | end 35 | fts_target.title = normalize_text(@record.short_comments&.strip) 36 | fts_target.content = normalize_text(@record.long_comments&.strip) 37 | fts_target.last_modified_at = @record.committed_on 38 | fts_target.registered_at = @record.committed_on 39 | fts_target.save! 40 | end 41 | end 42 | 43 | class FtsChangesetMapper < FtsMapper 44 | def title_prefix 45 | changeset = redmine_record 46 | repository = changeset.repository 47 | if repository and repository.identifier.present? 48 | repository = " (#{repository.identifier})" 49 | else 50 | repository = "" 51 | end 52 | "#{l(:label_revision)} #{changeset.format_identifier}#{repository}: " 53 | end 54 | 55 | def url 56 | changeset = redmine_record 57 | { 58 | controller: "repositories", 59 | action: "revision", 60 | id: @record.project_id, 61 | repository_id: changeset.repository.identifier_param, 62 | rev: changeset.identifier, 63 | } 64 | end 65 | end 66 | end 67 | -------------------------------------------------------------------------------- /app/controllers/fts_query_expansions_controller.rb: -------------------------------------------------------------------------------- 1 | class FtsQueryExpansionsController < ApplicationController 2 | layout "admin" 3 | self.main_menu = false 4 | before_action :require_admin 5 | 6 | helper :sort 7 | include SortHelper 8 | 9 | before_action :set_query_expansion, 10 | only: [:show, :edit, :update, :destroy] 11 | 12 | def index 13 | sort_init([["source", "asc"], ["destination", "asc"]]) 14 | sort_update(["source", "destination", "created_at", "updated_at"]) 15 | 16 | @request = FullTextSearch::QueryExpansionRequest.new(request_params) 17 | @n_expansions = FtsQueryExpansion.count 18 | @paginator = Paginator.new(@n_expansions, per_page_option, params["page"]) 19 | @expansions = 20 | FtsQueryExpansion 21 | .order(sort_clause) 22 | .offset(@paginator.offset) 23 | .limit(@paginator.per_page) 24 | end 25 | 26 | def show 27 | end 28 | 29 | def new 30 | @query_expansion = FtsQueryExpansion.new 31 | end 32 | 33 | def create 34 | @query_expansion = FtsQueryExpansion.new(query_expansion_params) 35 | if @query_expansion.save 36 | notice = l(:notice_successful_create) 37 | if params[:continue] 38 | redirect_to new_fts_query_expansion_path, 39 | notice: notice 40 | else 41 | redirect_to fts_query_expansions_path, 42 | notice: notice 43 | end 44 | else 45 | render :new 46 | end 47 | end 48 | 49 | def update 50 | if @query_expansion.update(query_expansion_params) 51 | redirect_to @query_expansion, notice: l(:notice_successful_update) 52 | else 53 | render :edit 54 | end 55 | end 56 | 57 | def destroy 58 | @query_expansion.destroy 59 | redirect_to fts_query_expansions_url, notice: l(:notice_successful_delete) 60 | end 61 | 62 | private 63 | def set_query_expansion 64 | @query_expansion = FtsQueryExpansion.find(params[:id]) 65 | end 66 | 67 | def request_params 68 | params.permit(:query) 69 | end 70 | 71 | def query_expansion_params 72 | params.require(:fts_query_expansion).permit(:source, :destination) 73 | end 74 | end 75 | -------------------------------------------------------------------------------- /app/views/settings/_full_text_search.html.erb: -------------------------------------------------------------------------------- 1 |

2 | <%= label_tag("settings[display_score]", 3 | l(:label_full_text_search_display_score)) %> 4 | <%= check_box("settings", 5 | "display_score", 6 | checked: settings.display_score?) %> 7 |

8 | 9 |

10 | <%= label_tag("settings[display_similar_issues]", 11 | l(:label_full_text_search_display_similar_issues)) %> 12 | <%= check_box("settings", 13 | "display_similar_issues", 14 | checked: settings.display_similar_issues?) %> 15 |

16 | 17 |

18 | <%= label_tag("settings[enable_tracking]", 19 | l(:label_full_text_search_enable_tracking)) %> 20 | <%= check_box("settings", 21 | "enable_tracking", 22 | checked: settings.enable_tracking?) %> 23 |

24 | 25 |

26 | <%= label_tag("settings[attachment_max_text_size_in_mb]", 27 | l(:label_full_text_search_attachment_max_text_size)) %> 28 | <%= text_field_tag("settings[attachment_max_text_size_in_mb]", 29 | settings.attachment_max_text_size_in_mb, 30 | size: 6) %> 31 | <%= l(:"number.human.storage_units.units.mb") %> 32 |

33 | 34 |

35 | <%= label_tag("settings[text_extraction_timeout]", 36 | l(:label_full_text_search_text_extraction_timeout)) %> 37 | <%= text_field_tag("settings[text_extraction_timeout]", 38 | settings.text_extraction_timeout, 39 | size: 6) %> 40 |

41 | 42 |

43 | <%= label_tag("settings[external_command_max_memory_in_mb]", 44 | l(:label_full_text_search_external_command_max_memory)) %> 45 | <%= text_field_tag("settings[external_command_max_memory_in_mb]", 46 | settings.external_command_max_memory_in_mb, 47 | size: 6) %> 48 | <%= l(:"number.human.storage_units.units.mb") %> 49 |

50 | 51 |

52 | <%= label_tag("settings[server_url]", 53 | l(:label_full_text_search_server_url)) %> 54 | <%= text_field_tag("settings[server_url]", 55 | settings.server_url) %> 56 |

57 | -------------------------------------------------------------------------------- /test/unit/full_text_search/changeset_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path("../../../test_helper", __FILE__) 2 | 3 | module FullTextSearch 4 | class ChangesetTest < ActiveSupport::TestCase 5 | include PrettyInspectable 6 | include NullValues 7 | include TimeValue 8 | 9 | fixtures :enabled_modules 10 | fixtures :projects 11 | fixtures :repositories 12 | fixtures :users 13 | fixtures :roles 14 | 15 | def test_save 16 | changeset = Changeset.generate! do |generating_changeset| 17 | generating_changeset.comments = "Fix a memory leak\n\nThis is critical." 18 | generating_changeset.committer = User.find(2).login 19 | end 20 | changeset.reload 21 | targets = Target.where(source_id: changeset.id, 22 | source_type_id: Type.changeset.id) 23 | assert_equal([ 24 | { 25 | "project_id" => changeset.repository.project_id, 26 | "source_id" => changeset.id, 27 | "source_type_id" => Type.changeset.id, 28 | "last_modified_at" => changeset.committed_on, 29 | "registered_at" => changeset.committed_on, 30 | "is_private" => null_boolean, 31 | "title" => "Fix a memory leak", 32 | "content" => "This is critical.", 33 | "custom_field_id" => null_number, 34 | "container_id" => null_number, 35 | "container_type_id" => null_number, 36 | "tag_ids" => [ 37 | Tag.user(changeset.user.id).id, 38 | ], 39 | }, 40 | ], 41 | targets.collect {|target| target.attributes.except("id")}) 42 | end 43 | 44 | def test_destroy 45 | changeset = Changeset.generate! 46 | targets = Target.where(source_id: changeset.id, 47 | source_type_id: Type.changeset.id) 48 | assert_equal(1, targets.size) 49 | changeset.destroy! 50 | assert_equal([], targets.reload.to_a) 51 | end 52 | end 53 | end 54 | -------------------------------------------------------------------------------- /test/unit/full_text_search/journal_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path("../../../test_helper", __FILE__) 2 | 3 | module FullTextSearch 4 | class JournalTest < ActiveSupport::TestCase 5 | include PrettyInspectable 6 | include NullValues 7 | 8 | fixtures :enumerations 9 | fixtures :issue_statuses 10 | fixtures :issues 11 | fixtures :projects 12 | fixtures :projects_trackers 13 | fixtures :trackers 14 | fixtures :users 15 | 16 | def test_save 17 | journal = Journal.generate!(notes: "Hello!") 18 | journal.reload 19 | targets = Target.where(source_id: journal.id, 20 | source_type_id: Type.journal.id) 21 | issue = journal.journalized 22 | last_modified_at = journal.updated_on 23 | assert_equal([ 24 | { 25 | "project_id" => issue.project_id, 26 | "source_id" => journal.id, 27 | "source_type_id" => Type.journal.id, 28 | "last_modified_at" => last_modified_at, 29 | "registered_at" => journal.created_on, 30 | "title" => null_string, 31 | "tag_ids" => [ 32 | Tag.user(journal.user_id).id, 33 | Tag.tracker(issue.tracker_id).id, 34 | Tag.issue_status(issue.status_id).id, 35 | ], 36 | "is_private" => journal.private_notes, 37 | "content" => journal.notes, 38 | "custom_field_id" => null_number, 39 | "container_id" => issue.id, 40 | "container_type_id" => Type.issue.id, 41 | } 42 | ], 43 | targets.collect {|target| target.attributes.except("id")}) 44 | end 45 | 46 | def test_destroy 47 | journal = Journal.generate! 48 | targets = Target.where(source_id: journal.id, 49 | source_type_id: Type.journal.id) 50 | assert_equal(1, targets.size) 51 | journal.destroy! 52 | assert_equal([], targets.reload.to_a) 53 | end 54 | end 55 | end 56 | -------------------------------------------------------------------------------- /lib/full_text_search/tracer.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | class Tracer 3 | def initialize(tag) 4 | @tag = tag 5 | @start_memory_usage = compute_memory_usage 6 | @start_time = Time.now 7 | end 8 | 9 | def trace(log_level, label, data, error: nil) 10 | Rails.logger.__send__(log_level) do 11 | format_message(label, data, error) 12 | end 13 | end 14 | 15 | private 16 | def compute_memory_usage 17 | status_path = "/proc/self/status" 18 | if File.exist?(status_path) 19 | File.open(status_path) do |status| 20 | status.each_line do |line| 21 | case line 22 | when /\AVmRSS:\s+(\d+) kB/ 23 | return Integer($1, 10) * 1024 24 | end 25 | end 26 | end 27 | end 28 | 0 29 | end 30 | 31 | def format_message(label, data, error) 32 | message = "[full-text-search]#{@tag} #{label}" 33 | data.each do |data_label, data_value| 34 | message << ": #{data_label}: <#{data_value}>" 35 | end 36 | elapsed_time = Time.now - @start_time 37 | message << ": elapsed time: <#{format_elapsed_time(elapsed_time)}>" 38 | memory_usage = compute_memory_usage 39 | if memory_usage > 0 40 | message << ": memory usage: <#{format_memory_usage(memory_usage)}>" 41 | memory_usage_diff = memory_usage - @start_memory_usage 42 | message << 43 | ": memory usage diff: <#{format_memory_usage(memory_usage_diff)}>" 44 | end 45 | if error 46 | message << ": #{error.class}: #{error.message}\n" 47 | message << error.backtrace.join("\n") 48 | end 49 | message 50 | end 51 | 52 | def format_elapsed_time(elapsed_time) 53 | if elapsed_time < 1 54 | "%.2fms" % (elapsed_time * 1000) 55 | elsif elapsed_time < 60 56 | "%.2fs" % elapsed_time 57 | elsif elapsed_time < (60 * 60) 58 | "%.2fm" % (elapsed_time / 60) 59 | else 60 | "%.2fh" % (elapsed_time / 60 / 60) 61 | end 62 | end 63 | 64 | def format_memory_usage(memory_usage) 65 | if memory_usage < (1024.0 * 1024.0 * 1024.0) 66 | "%.2fMiB" % (memory_usage / 1024.0 / 1024.0) 67 | else 68 | "%.2fGiB" % (memory_usage / 1024.0 / 1024.0 / 1024.0) 69 | end 70 | end 71 | end 72 | end 73 | -------------------------------------------------------------------------------- /lib/full_text_search/similar_searcher/pgroonga.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | module SimilarSearcher 3 | module Pgroonga 4 | def self.included(base) 5 | base.include(InstanceMethods) 6 | base.include(FullTextSearch::ConditionBuilder) 7 | base.class_eval do 8 | attr_accessor :similarity_score 9 | end 10 | end 11 | 12 | module InstanceMethods 13 | def similar_issues(user: User.current, project_ids: [], limit: 5) 14 | sql = <<-SQL.strip_heredoc 15 | select pgroonga_command( 16 | 'select', 17 | ARRAY[ 18 | 'table', pgroonga_table_name('#{similar_issues_index_name}'), 19 | 'output_columns', 'issue_id, _score', 20 | 'filter', '(content *S ' || pgroonga_escape(:content) || ') && issue_id != :id' || ' && #{filter_condition(user, project_ids)}', 21 | 'limit', ':limit', 22 | 'sort_keys', '-_score' 23 | ] 24 | )::json 25 | SQL 26 | response = nil 27 | ActiveSupport::Notifications.instrument("groonga.similar.search", sql: sql) do 28 | response = self.class.connection.select_value(ActiveRecord::Base.send(:sanitize_sql_array, [sql, content: similar_content, id: id, limit: limit])) 29 | end 30 | command = Groonga::Command.find("select").new("select", {}) 31 | r = Groonga::Client::Response.parse(command, response) 32 | if r.success? 33 | issue_scores = r.records.map do |row| 34 | [row["issue_id"], row["_score"]] 35 | end.to_h 36 | logger.debug(r.records) 37 | similar_issues = Issue.where(id: issue_scores.keys).all 38 | similar_issues.each do |s| 39 | s.similarity_score = issue_scores[s.id] 40 | end 41 | similar_issues.sort_by {|s| - s.similarity_score } 42 | else 43 | if Rails.env.production? 44 | logger.warn(r.message) 45 | [] 46 | else 47 | raise r.message 48 | end 49 | end 50 | end 51 | 52 | def similar_issues_index_name 53 | "index_issue_contents_pgroonga" 54 | end 55 | end 56 | end 57 | end 58 | end 59 | -------------------------------------------------------------------------------- /app/models/full_text_search/tag.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | class Tag < ApplicationRecord 3 | self.table_name = :fts_tags 4 | belongs_to :type, class_name: "FullTextSearch::TagType" 5 | 6 | case connection_db_config.adapter 7 | when "postgresql" 8 | include Pgroonga 9 | when "mysql2" 10 | include Mroonga 11 | end 12 | 13 | class << self 14 | def extension(ext) 15 | type = TagType.extension 16 | find_or_create_by(type_id: type.id, 17 | name: ext.downcase) 18 | end 19 | 20 | def identifier(id) 21 | type = TagType.identifier 22 | find_or_create_by(type_id: type.id, 23 | name: id) 24 | end 25 | 26 | def issue_status(issue_status_id) 27 | type = TagType.issue_status 28 | find_or_create_by(type_id: type.id, 29 | name: issue_status_id.to_s) 30 | end 31 | 32 | def text_extraction(status) 33 | type = TagType.text_extraction 34 | find_or_create_by(type_id: type.id, 35 | name: status) 36 | end 37 | 38 | def text_extraction_error 39 | text_extraction("error") 40 | end 41 | 42 | def text_extraction_yet 43 | text_extraction("yet") 44 | end 45 | 46 | def text_extraction_ids 47 | type = TagType.text_extraction 48 | where(type_id: type.id).select(:id).collect(&:id) 49 | end 50 | 51 | def tracker(tracker_id) 52 | type = TagType.tracker 53 | find_or_create_by(type_id: type.id, 54 | name: tracker_id.to_s) 55 | end 56 | 57 | def user(user_id) 58 | type = TagType.user 59 | find_or_create_by(type_id: type.id, 60 | name: user_id.to_s) 61 | end 62 | 63 | def label(label) 64 | type = TagType.label 65 | find_or_create_by(type_id: type.id, 66 | name: label) 67 | end 68 | end 69 | 70 | def value 71 | @value ||= compute_value 72 | end 73 | 74 | private 75 | def compute_value 76 | case type_id 77 | when TagType.issue_status.id 78 | IssueStatus.find(name.to_i) 79 | when TagType.tracker.id 80 | Tracker.find(name.to_i) 81 | when TagType.user.id 82 | User.find(name.to_i) 83 | else 84 | name 85 | end 86 | end 87 | end 88 | end 89 | -------------------------------------------------------------------------------- /app/jobs/full_text_search/update_issue_content_job.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | class UpdateIssueContentJob < ActiveJob::Base 3 | queue_as :full_text_search 4 | queue_with_priority 15 5 | 6 | discard_on ActiveRecord::RecordNotFound 7 | 8 | def perform(record_class_name, record_id, action, options={}) 9 | record_class = record_class_name.constantize 10 | 11 | case action 12 | when "commit" 13 | record = record_class.find(record_id) 14 | case record 15 | when Issue 16 | content = FullTextSearch::IssueContent 17 | .find_or_initialize_by(issue_id: record.id) 18 | content.project_id = record.project_id 19 | content.subject = record.subject 20 | content.content = create_content(record.id) 21 | content.status_id = record.status_id 22 | content.save! 23 | when Journal 24 | issue_id = record.journalized_id 25 | FullTextSearch::IssueContent 26 | .where(issue_id: issue_id) 27 | .update_all(content: create_content(issue_id)) 28 | end 29 | when "destroy" 30 | case record_class_name 31 | when "Issue" 32 | FullTextSearch::IssueContent.where(issue_id: record_id).destroy_all 33 | when "Journal" 34 | issue_id = options[:issue_id] 35 | FullTextSearch::IssueContent 36 | .where(issue_id: issue_id) 37 | .update_all(content: create_content(issue_id, excludes: [record_id])) 38 | when "Attachment" 39 | issue_id = options[:issue_id] 40 | FullTextSearch::IssueContent 41 | .where(issue_id: issue_id) 42 | .update_all(content: create_content(issue_id)) 43 | end 44 | end 45 | end 46 | 47 | private 48 | def create_content(issue_id, excludes: []) 49 | issue = Issue.eager_load(:journals).find(issue_id) 50 | content = [issue.subject, issue.description] 51 | notes = issue.journals 52 | .reject {|j| j.notes.blank? || excludes.include?(j.id) } 53 | .sort_by(&:id) 54 | .map(&:notes) 55 | content.concat(notes) 56 | issue.attachments.order(:id).each do |attachment| 57 | content << attachment.filename if attachment.filename.present? 58 | content << attachment.description if attachment.description.present? 59 | end 60 | content.join("\n") 61 | end 62 | end 63 | end 64 | -------------------------------------------------------------------------------- /test/unit/full_text_search/plugin_wiki_extensions_tag_searchable_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path("../../../test_helper", __FILE__) 2 | 3 | module FullTextSearch 4 | class PluginWikiExtensionsTagSearchableText < ActiveSupport::TestCase 5 | fixtures :projects 6 | fixtures :wikis 7 | 8 | def setup 9 | unless defined?(WikiExtensionsTagRelation) 10 | skip 'redmine_wiki_extensions is not installed' 11 | end 12 | 13 | project = Project.find(1) 14 | 3.times do |i| 15 | WikiExtensionsTag.create( 16 | name: "tag-#{i}", 17 | project_id: project.id, 18 | ) 19 | end 20 | 21 | @page = WikiPage.new( 22 | wiki: project.wiki, 23 | title: 'PluginWikiExtensionsTagRelationTest' 24 | ) 25 | content = WikiContent.new(page: @page) 26 | @page.save_with_content(content) 27 | end 28 | 29 | def set_tags(*names) 30 | wiki_ext_tags = WikiExtensionsTag. 31 | where(name: names). 32 | collect{ |tag| [tag.id, tag.name] }. 33 | to_h 34 | @page.set_tags(wiki_ext_tags) 35 | @page.reload 36 | end 37 | 38 | def test_set_tag 39 | set_tags('tag-0', 'tag-2') 40 | target = Target.find_by(source_id: @page.id, 41 | source_type_id: Type.wiki_page.id) 42 | assert_equal( 43 | [ 44 | Tag.label('tag-0').id, 45 | Tag.label('tag-2').id 46 | ].sort, 47 | target.tag_ids.sort 48 | ) 49 | end 50 | 51 | def test_add_tag 52 | # First set tag-0 and tag-2 53 | set_tags('tag-0', 'tag-2') 54 | # Add 'tag-1' 55 | set_tags('tag-0', 'tag-1', 'tag-2') 56 | 57 | target = Target.find_by(source_id: @page.id, 58 | source_type_id: Type.wiki_page.id) 59 | assert_equal( 60 | [ 61 | Tag.label('tag-0').id, 62 | Tag.label('tag-1').id, 63 | Tag.label('tag-2').id 64 | ].sort, 65 | target.tag_ids.sort 66 | ) 67 | end 68 | 69 | def test_remove_tag 70 | # First set tag-0 and tag-2 71 | set_tags('tag-0', 'tag-2') 72 | # Remove 'tag-0' 73 | set_tags('tag-2') 74 | 75 | target = Target.find_by(source_id: @page.id, 76 | source_type_id: Type.wiki_page.id) 77 | assert_equal( 78 | [Tag.label('tag-2').id], 79 | target.tag_ids 80 | ) 81 | end 82 | end 83 | end 84 | -------------------------------------------------------------------------------- /lib/full_text_search/markup_parser.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | class MarkupParser 3 | include ERB::Util 4 | include ActionView::Helpers 5 | include Rails.application.routes.url_helpers 6 | include ApplicationHelper 7 | 8 | def initialize(project) 9 | @project = project 10 | assign_controller(WelcomeController.new) 11 | controller.request = ActionDispatch::Request.new({}) 12 | end 13 | 14 | def parse(object, attribute, options={}) 15 | html = with_user(User.admin.first) do 16 | textilizable(object, attribute, options) 17 | end 18 | return ["", []] unless html.present? 19 | 20 | document = Document.new 21 | # We need .to_s for old Nokogiri. We can remove the .to_s after we drop support for Redmine < 6.0. 22 | # Redmine 6.0 requires Nokogiri 1.18.3 or later. 23 | parser = Nokogiri::HTML::SAX::Parser.new(document, html.encoding.to_s) 24 | parser.parse(html) 25 | [document.text.strip, document.tag_ids] 26 | end 27 | 28 | private 29 | def with_user(user) 30 | current_user = User.current 31 | begin 32 | User.current = user 33 | yield 34 | ensure 35 | User.current = current_user 36 | end 37 | end 38 | 39 | class Document < Nokogiri::XML::SAX::Document 40 | attr_reader :text 41 | attr_reader :tag_ids 42 | def initialize 43 | @text = +"" 44 | @tag_ids = [] 45 | 46 | @tag_stack = [] 47 | @attributes_stack =[] 48 | @in_body = false 49 | end 50 | 51 | def start_element(name, attributes=[]) 52 | @tag_stack.push(name) 53 | @attributes_stack.push(attributes) 54 | unless @in_body 55 | @in_body = (@tag_stack == ["html", "body"]) 56 | return 57 | end 58 | end 59 | 60 | def end_element(name) 61 | @attributes_stack.pop 62 | @tag_stack.pop 63 | return unless @in_body 64 | 65 | if name == "body" and @tag_stack == ["html"] 66 | @in_body = false 67 | return 68 | end 69 | end 70 | 71 | def characters(text) 72 | @text << text if in_target_text? 73 | end 74 | 75 | private 76 | def in_target_text? 77 | return false unless @in_body 78 | 79 | @attributes_stack.last.each do |name, value| 80 | case name 81 | when "class" 82 | return false if value.split(/\s+/).include?("wiki-anchor") 83 | end 84 | end 85 | true 86 | end 87 | end 88 | end 89 | end 90 | -------------------------------------------------------------------------------- /lib/full_text_search/hooks/controller_search_index.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | module Hooks 3 | module ControllerSearchIndex 4 | def index 5 | @search_request = Request.new(query_params) 6 | @search_request.user = User.current 7 | @search_request.project = @project 8 | 9 | page = (params[:page].presence || 1).to_i 10 | case params[:format] 11 | when 'xml', 'json' 12 | offset, limit = api_offset_and_limit 13 | else 14 | limit = Setting.search_results_per_page.to_i 15 | limit = 10 if limit == 0 16 | offset = (page - 1) * limit 17 | end 18 | @search_request.offset = offset 19 | @search_request.limit = limit 20 | 21 | # quick jump to an issue 22 | if (m = @search_request.query.match(/^#?(\d+)$/)) && 23 | (issue = Issue.visible.find_by_id(m[1].to_i)) 24 | redirect_to issue_path(issue) 25 | return 26 | end 27 | 28 | searcher = Searcher.new(@search_request) 29 | @result_set = searcher.search 30 | context = @search_request.to_params 31 | context = context.merge("user_id" => @search_request.user.id, 32 | "project_id" => @search_request.project&.id, 33 | "n_hits" => @result_set.n_hits, 34 | "total_n_hits" => @result_set.total_n_hits, 35 | "elapsed_time" => @result_set.elapsed_time, 36 | "timestamp" => Time.zone.now.iso8601) 37 | log = "[full-text-search][search] #{context.to_json}" 38 | Rails.logger.info(log) 39 | @result_pages = Redmine::Pagination::Paginator.new(@result_set.n_hits, 40 | @search_request.limit, 41 | params["page"]) 42 | 43 | respond_to do |format| 44 | format.html { render layout: false if request.xhr? } 45 | format.api { render layout: false } 46 | end 47 | end 48 | 49 | private 50 | def query_params 51 | permitted_names = [ 52 | :search_id, 53 | :q, 54 | :scope, 55 | :all_words, 56 | :titles_only, 57 | :attachments, 58 | :open_issues, 59 | :format, 60 | :order_target, 61 | :order_type, 62 | :options, 63 | ] 64 | Redmine::Search.available_search_types.each do |type| 65 | permitted_names << type.to_sym 66 | end 67 | params.permit(*permitted_names, 68 | tags: []) 69 | end 70 | end 71 | end 72 | end 73 | -------------------------------------------------------------------------------- /lib/full_text_search/issue_mapper.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | class IssueMapper < Mapper 3 | class << self 4 | def redmine_mapper_class 5 | RedmineIssueMapper 6 | end 7 | 8 | def fts_mapper_class 9 | FtsIssueMapper 10 | end 11 | end 12 | end 13 | resolver.register(Issue, IssueMapper) 14 | 15 | class RedmineIssueMapper < RedmineMapper 16 | def upsert_fts_target(options={}) 17 | fts_target = find_fts_target 18 | fts_target.source_id = @record.id 19 | fts_target.source_type_id = Type[@record.class].id 20 | fts_target.project_id = @record.project_id 21 | tag_ids = [] 22 | tag_ids << Tag.tracker(@record.tracker_id).id if @record.tracker_id 23 | fts_target.title = @record.subject 24 | parser = MarkupParser.new(@record.project) 25 | content_text, content_tag_ids = parser.parse(@record, :description) 26 | fts_target.content = content_text 27 | tag_ids.concat(content_tag_ids) 28 | tag_ids << Tag.user(@record.author_id).id if @record.author_id 29 | fts_target.is_private = @record.is_private 30 | tag_ids << Tag.issue_status(@record.status_id).id if @record.status_id 31 | fts_target.tag_ids = tag_ids 32 | fts_target.last_modified_at = @record.updated_on 33 | fts_target.registered_at = @record.created_on 34 | fts_target.save! 35 | 36 | @record.journals.each do |journal| 37 | redmine_mapper = JournalMapper.redmine_mapper(journal) 38 | # We don't insert a new FTS target here to avoid an unique 39 | # constraint error when a new journal is added. The following 40 | # jobs are enqueued when a new journal is added: 41 | # 42 | # 1. UpsertTargetJob(IssueMapper) 43 | # 2. UpsertTargetJob(JournalMapper) 44 | # 45 | # If both of jobs are executed in parallel, both of job 1. and 46 | # job 2. may try adding new FTS target for the same 47 | # journal. In the case, one of them is failed by an unique 48 | # constraint error. If we don't create a new FTS target in 49 | # here (job 1.), we can avoid the error. 50 | next unless redmine_mapper.find_fts_target.persisted? 51 | redmine_mapper.upsert_fts_target(options) 52 | end 53 | # @record.custom_values 54 | end 55 | end 56 | 57 | class FtsIssueMapper < FtsMapper 58 | def type 59 | issue = redmine_record 60 | if issue.closed? 61 | "issue-closed" 62 | else 63 | "issue" 64 | end 65 | end 66 | 67 | def title_prefix 68 | issue = redmine_record 69 | "#{issue.tracker.name} \##{@record.source_id} (#{issue.status}): " 70 | end 71 | 72 | def url 73 | { 74 | controller: "issues", 75 | action: "show", 76 | id: @record.source_id, 77 | } 78 | end 79 | end 80 | end 81 | -------------------------------------------------------------------------------- /test/unit/full_text_search/query_expansion_synchronizer_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path("../../../test_helper", __FILE__) 2 | 3 | module FullTextSearch 4 | class QueryExpansionSynchronizerTest < ActiveSupport::TestCase 5 | include PrettyInspectable 6 | 7 | def setup 8 | FtsQueryExpansion.destroy_all 9 | end 10 | 11 | def synchronize(input) 12 | synchronizer = QueryExpansionSynchronizer.new(input) 13 | synchronizer.synchronize 14 | end 15 | 16 | def test_json_array 17 | synchronize(StringIO.new(+<<-JSON)) 18 | [ 19 | ["Groonga", "Groonga"], 20 | ["Groonga", "Senna"] 21 | ] 22 | JSON 23 | assert_equal([ 24 | FtsQueryExpansion.find_by(source: "Groonga", 25 | destination: "Groonga"), 26 | FtsQueryExpansion.find_by(source: "Groonga", 27 | destination: "Senna"), 28 | ], 29 | FtsQueryExpansion.order(:id)) 30 | end 31 | 32 | def test_json_object 33 | synchronize(StringIO.new(+<<-JSON)) 34 | [ 35 | {"source": "Groonga", "destination": "Groonga"}, 36 | {"source": "Groonga", "destination": "Senna"} 37 | ] 38 | JSON 39 | assert_equal([ 40 | FtsQueryExpansion.find_by(source: "Groonga", 41 | destination: "Groonga"), 42 | FtsQueryExpansion.find_by(source: "Groonga", 43 | destination: "Senna"), 44 | ], 45 | FtsQueryExpansion.order(:id)) 46 | end 47 | 48 | def test_csv 49 | synchronize(StringIO.new(+<<-CSV)) 50 | Groonga,Groonga 51 | Groonga,Senna 52 | CSV 53 | assert_equal([ 54 | FtsQueryExpansion.find_by(source: "Groonga", 55 | destination: "Groonga"), 56 | FtsQueryExpansion.find_by(source: "Groonga", 57 | destination: "Senna"), 58 | ], 59 | FtsQueryExpansion.order(:id)) 60 | end 61 | 62 | def test_remove_untouched_entries 63 | FtsQueryExpansion.create!(source: "Groonga", 64 | destination: "Groonga", 65 | updated_at: Time.current - 2) 66 | FtsQueryExpansion.create!(source: "Groonga", 67 | destination: "Senna", 68 | updated_at: Time.current - 2) 69 | synchronize(StringIO.new(+<<-CSV)) 70 | Groonga,Groonga 71 | CSV 72 | assert_equal([ 73 | FtsQueryExpansion.find_by(source: "Groonga", 74 | destination: "Groonga"), 75 | ], 76 | FtsQueryExpansion.all) 77 | end 78 | end 79 | end 80 | -------------------------------------------------------------------------------- /lib/full_text_search/custom_value_mapper.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | class CustomValueMapper < Mapper 3 | class << self 4 | def redmine_mapper_class 5 | RedmineCustomValueMapper 6 | end 7 | 8 | def fts_mapper_class 9 | FtsCustomValueMapper 10 | end 11 | end 12 | end 13 | resolver.register(CustomValue, CustomValueMapper) 14 | 15 | class RedmineCustomValueMapper < RedmineMapper 16 | class << self 17 | def with_project(redmine_class) 18 | redmine_class 19 | .joins(<<-JOIN) 20 | LEFT OUTER JOIN issues 21 | ON customized_type = 'Issue' AND issues.id = customized_id 22 | JOIN 23 | .joins(<<-JOIN) 24 | JOIN projects 25 | ON (customized_type = 'Project' AND projects.id = customized_id) OR 26 | (customized_type = 'Issue' AND projects.id = issues.project_id) 27 | JOIN 28 | end 29 | 30 | def not_mapped(redmine_class, options) 31 | super 32 | .joins(:custom_field) 33 | .where("custom_fields.searchable") 34 | end 35 | end 36 | 37 | def upsert_fts_target(options={}) 38 | fts_target = find_fts_target 39 | 40 | unless @record.custom_field.searchable? 41 | fts_target.destroy! if fts_target.persisted? 42 | return 43 | end 44 | 45 | customized = @record.customized 46 | unless customized 47 | fts_target.destroy! if fts_target.persisted? 48 | return 49 | end 50 | 51 | # searchable CustomValue belongs to issue or project 52 | fts_target.source_id = @record.id 53 | fts_target.source_type_id = Type[@record.class].id 54 | fts_target.content = @record.value 55 | fts_target.custom_field_id = @record.custom_field_id 56 | case @record.customized_type 57 | when "Issue" 58 | fts_target.project_id = customized.project_id 59 | fts_target.is_private = customized.is_private 60 | when "Project" 61 | fts_target.project_id = customized.id 62 | else 63 | fts_target.destroy! if fts_target.persisted? 64 | return 65 | end 66 | fts_target.container_id = customized.id 67 | fts_target.container_type_id = Type[customized].id 68 | # TODO: This may not be updated when issue or project is updated. 69 | fts_target.last_modified_at = customized.updated_on 70 | fts_target.registered_at = customized.created_on 71 | fts_target.save! 72 | end 73 | end 74 | 75 | class FtsCustomValueMapper < FtsMapper 76 | def type 77 | redmine_record.customized.event_type 78 | end 79 | 80 | def title 81 | redmine_record.customized.event_title 82 | end 83 | 84 | def description 85 | redmine_record.customized.event_description 86 | end 87 | 88 | def url 89 | redmine_record.customized.event_url 90 | end 91 | 92 | def id 93 | redmine_record.customized_id 94 | end 95 | end 96 | end 97 | -------------------------------------------------------------------------------- /lib/full_text_search/query_expansion_synchronizer.rb: -------------------------------------------------------------------------------- 1 | require "csv" 2 | require "json" 3 | 4 | module FullTextSearch 5 | class QueryExpansionSynchronizer 6 | def initialize(input) 7 | @input = input 8 | end 9 | 10 | def synchronize 11 | start = Time.current.change(usec: 0) 12 | each_record do |record| 13 | attributes = { 14 | source: record["source"], 15 | destination: record["destination"], 16 | } 17 | query_expansion = FtsQueryExpansion.find_or_initialize_by(attributes) 18 | query_expansion.updated_at = Time.current 19 | unless query_expansion.save 20 | Rails.logger.warn("#{log_tag} failed to save:") 21 | query_expansion.errors.full_messages.each do |message| 22 | Rails.logger.warn("#{log_tag} #{message}") 23 | end 24 | end 25 | end 26 | not_updated = FtsQueryExpansion.where("updated_at < ?", start) 27 | not_updated.destroy_all 28 | end 29 | 30 | private 31 | BOM_CODE_POINT = 0xfeff 32 | def open_input 33 | if @input.respond_to?(:getc) 34 | yield(@input) 35 | else 36 | File.open(@input) do |input| 37 | yield(input) 38 | end 39 | end 40 | end 41 | 42 | def each_record 43 | open_input do |input| 44 | skip_bom(input) 45 | case detect_format(input) 46 | when :json 47 | JSON.parse(input.read).each do |row| 48 | if row.is_a?(Array) 49 | record = { 50 | "source" => row[0], 51 | "destination" => row[1], 52 | } 53 | else 54 | record = row 55 | end 56 | yield(record) 57 | end 58 | else 59 | csv = CSV.new(input) 60 | csv.each do |row| 61 | record = { 62 | "source" => row[0], 63 | "destination" => row[1], 64 | } 65 | yield(record) 66 | end 67 | end 68 | end 69 | end 70 | 71 | def skip_bom(input) 72 | first_character = input.getc 73 | unless first_character.codepoints[0] == BOM_CODE_POINT 74 | input.ungetc(first_character) 75 | end 76 | end 77 | 78 | def detect_format(input) 79 | if input.respond_to?(:path) 80 | case input.path 81 | when /\.json\z/i 82 | :json 83 | when /\.csv\z/i 84 | :csv 85 | else 86 | nil 87 | end 88 | else 89 | first_character = input.getc 90 | input.ungetc(first_character) 91 | case first_character 92 | when "[" 93 | :json 94 | else 95 | nil 96 | end 97 | end 98 | end 99 | 100 | def log_tag 101 | "[full-text-search][query-expansion][synchronize]" 102 | end 103 | end 104 | end 105 | -------------------------------------------------------------------------------- /lib/full_text_search/settings.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | module SettingsObjectize 3 | def plugin_full_text_search 4 | Settings.new(super) 5 | end 6 | end 7 | 8 | class Settings 9 | TRUE_VALUE = "1" 10 | FALSE_VALUE = "0" 11 | 12 | def initialize(raw) 13 | @raw = raw || {} 14 | end 15 | 16 | def display_score? 17 | @raw["display_score"] == TRUE_VALUE 18 | end 19 | 20 | def display_similar_issues? 21 | @raw["display_similar_issues"] == TRUE_VALUE 22 | end 23 | 24 | def enable_tracking? 25 | @raw["enable_tracking"] != FALSE_VALUE 26 | end 27 | 28 | DEFAULT_ATTACHMENT_MAX_TEXT_SIZE_IN_MB = 4 29 | def attachment_max_text_size_in_mb 30 | size = @raw.fetch("attachment_max_text_size_in_mb", 31 | DEFAULT_ATTACHMENT_MAX_TEXT_SIZE_IN_MB) 32 | begin 33 | Float(size) 34 | rescue ArgumentError 35 | DEFAULT_ATTACHMENT_MAX_TEXT_SIZE_IN_MB 36 | end 37 | end 38 | 39 | def attachment_max_text_size 40 | size = (attachment_max_text_size_in_mb * 1.megabytes).floor 41 | if Redmine::Database.mysql? 42 | connection = ActiveRecord::Base.connection 43 | result = connection.exec_query("SELECT @@max_allowed_packet", 44 | "max allowed package") 45 | max_allowed_packet = result[0]["@@max_allowed_packet"] 46 | [size, (max_allowed_packet * 0.7).floor].min 47 | else 48 | size 49 | end 50 | end 51 | 52 | DEFAULT_TEXT_EXTRACTION_TIMEOUT = 180 53 | def text_extraction_timeout 54 | timeout = @raw.fetch("text_extraction_timeout", 55 | DEFAULT_TEXT_EXTRACTION_TIMEOUT) 56 | begin 57 | Float(timeout) 58 | rescue ArgumentError 59 | DEFAULT_TEXT_EXTRACTION_TIMEOUT 60 | end 61 | end 62 | 63 | external_command_max_memory_in_mb = 1024 64 | if File.readable?("/proc/meminfo") 65 | File.open("/proc/meminfo") do |meminfo| 66 | meminfo.each_line do |line| 67 | case line 68 | when /\AMemTotal:\s+(\d+) kB/ 69 | total_memory = Integer($1, 10) 70 | external_command_max_memory_in_mb = 71 | ((total_memory / 4) / 1024.0).round 72 | break 73 | end 74 | end 75 | end 76 | end 77 | DEFAULT_EXTERNAL_COMMAND_MAX_MEMORY_IN_MB = external_command_max_memory_in_mb 78 | def external_command_max_memory_in_mb 79 | size = @raw.fetch("external_command_max_memory_in_mb", 80 | DEFAULT_EXTERNAL_COMMAND_MAX_MEMORY_IN_MB) 81 | begin 82 | Float(size) 83 | rescue ArgumentError 84 | DEFAULT_EXTERNAL_COMMAND_MAX_MEMORY_IN_MB 85 | end 86 | end 87 | 88 | def external_command_max_memory 89 | (external_command_max_memory_in_mb * 1.megabytes).floor 90 | end 91 | 92 | def server_url 93 | @raw["server_url"].presence 94 | end 95 | end 96 | end 97 | -------------------------------------------------------------------------------- /lib/full_text_search/journal_mapper.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | class JournalMapper < Mapper 3 | class << self 4 | def redmine_mapper_class 5 | RedmineJournalMapper 6 | end 7 | 8 | def fts_mapper_class 9 | FtsJournalMapper 10 | end 11 | end 12 | end 13 | resolver.register(Journal, JournalMapper) 14 | 15 | class RedmineJournalMapper < RedmineMapper 16 | class << self 17 | def with_project(redmine_class) 18 | redmine_class 19 | .joins(<<-JOIN) 20 | JOIN issues 21 | ON journalized_type = 'Issue' AND issues.id = journalized_id 22 | JOIN 23 | .joins(<<-JOIN) 24 | JOIN projects 25 | ON projects.id = issues.project_id 26 | JOIN 27 | end 28 | end 29 | 30 | def upsert_fts_target(options={}) 31 | # journal belongs to an issue for now. 32 | issue = @record.journalized 33 | return if issue.nil? 34 | return unless issue.is_a?(Issue) 35 | 36 | fts_target = find_fts_target 37 | fts_target.source_id = @record.id 38 | fts_target.source_type_id = Type[@record.class].id 39 | fts_target.project_id = issue.project_id 40 | fts_target.container_id = issue.id 41 | fts_target.container_type_id = Type.issue.id 42 | tag_ids = [] 43 | parser = MarkupParser.new(issue.project) 44 | content_text, content_tag_ids = parser.parse(@record, :notes) 45 | fts_target.content = content_text 46 | tag_ids.concat(content_tag_ids) 47 | tag_ids << Tag.user(@record.user_id).id if @record.user_id 48 | fts_target.is_private = (issue.is_private or @record.private_notes) 49 | tag_ids << Tag.tracker(issue.tracker_id).id if issue.tracker_id 50 | tag_ids << Tag.issue_status(issue.status_id).id if issue.status_id 51 | fts_target.tag_ids = tag_ids 52 | fts_target.last_modified_at = @record.updated_on 53 | fts_target.registered_at = @record.created_on 54 | fts_target.save! 55 | end 56 | end 57 | 58 | class FtsJournalMapper < FtsMapper 59 | def type 60 | journal = redmine_record 61 | new_status = journal.new_status 62 | if new_status 63 | if new_status.is_closed? 64 | "issue-closed" 65 | else 66 | "issue-edit" 67 | end 68 | else 69 | "issue-note" 70 | end 71 | end 72 | 73 | def title_prefix 74 | journal = redmine_record 75 | issue = journal.issue 76 | prefix = "#{issue.tracker.name} " 77 | prefix << "\##{issue.id}\#change-#{journal.id} " 78 | prefix << "(#{issue.status}): " 79 | prefix 80 | end 81 | 82 | def title 83 | journal = redmine_record 84 | issue = journal.issue 85 | "#{title_prefix}#{issue.subject}#{title_suffix}" 86 | end 87 | 88 | def url 89 | journal = redmine_record 90 | { 91 | controller: "issues", 92 | action: "show", 93 | id: journal.issue.id, 94 | anchor: "change-#{journal.id}", 95 | } 96 | end 97 | end 98 | end 99 | -------------------------------------------------------------------------------- /lib/full_text_search/similar_searcher/mroonga.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | module SimilarSearcher 3 | module Mroonga 4 | def self.included(base) 5 | base.include(InstanceMethods) 6 | base.include(FullTextSearch::ConditionBuilder) 7 | base.class_eval do 8 | attr_accessor :similarity_score 9 | end 10 | end 11 | 12 | module InstanceMethods 13 | def similar_issues(user: User.current, project_ids: [], limit: 5) 14 | # NOTE: The sanitize_sql_array method in the MySQL adapter is not 15 | # schema-aware. It quotes numeric parameters as strings to prevent 16 | # query manipulation attacks. However, if numeric parameters like :id 17 | # and :limit are not explicitly converted to strings beforehand, this 18 | # can lead to a syntax error. 19 | # 20 | # For example, without the explicit conversion, the following error is 21 | # caused 22 | # 23 | # ...'limit', ''5'', 'sor' at line 5 24 | # 25 | # To prevent such syntax errors, we explicitly convert numeric 26 | # parameters to strings before passing them to sanitize_sql_array. 27 | sql = <<-SQL.strip_heredoc 28 | select mroonga_command( 29 | 'select', 30 | 'table', 'issue_contents', 31 | 'output_columns', 'issue_id, _score', 32 | 'filter', CONCAT('(content *S "', mroonga_escape(:content), '") && issue_id != ', :id, ' && #{filter_condition(user, project_ids)}'), 33 | 'limit', :limit, 34 | 'sort_keys', '-_score' 35 | ) 36 | SQL 37 | r = nil 38 | ActiveSupport::Notifications.instrument("groonga.similar.search", sql: sql) do 39 | r = self.class.connection.select_value(ActiveRecord::Base.send(:sanitize_sql_array, [sql, content: similar_content, id: id.to_s, limit: limit.to_s])) 40 | end 41 | # NOTE: Hack to use Groonga::Client::Response.parse 42 | # Raise Mysql2::Error if error occurred 43 | body = JSON.parse(r) 44 | header = [0, 0, 0] 45 | response = [header, body].to_json 46 | command = Groonga::Command.find("select").new("select", {}) 47 | r = Groonga::Client::Response.parse(command, response) 48 | issue_scores = r.records.map do |row| 49 | [row["issue_id"], row["_score"]] 50 | end.to_h 51 | logger.debug(r.records) 52 | similar_issues = Issue.where(id: issue_scores.keys).all 53 | similar_issues.each do |s| 54 | s.similarity_score = issue_scores[s.id] 55 | end 56 | similar_issues.sort_by {|s| - s.similarity_score } 57 | rescue => ex 58 | if Rails.env.production? 59 | logger.warn(ex.class => ex.message) 60 | [] 61 | else 62 | raise 63 | end 64 | end 65 | end 66 | end 67 | end 68 | end 69 | -------------------------------------------------------------------------------- /lib/tasks/full_text_search.rake: -------------------------------------------------------------------------------- 1 | # -*- ruby -*- 2 | 3 | namespace :full_text_search do 4 | desc "Tag" 5 | task :tag => :environment do 6 | plugin = Redmine::Plugin.find(:full_text_search) 7 | version = plugin.version 8 | cd(plugin.directory) do 9 | sh("git", "tag", 10 | "-a", "v#{version}", 11 | "-m", "#{version} has been released!!!") 12 | sh("git", "push", "--tags") 13 | end 14 | end 15 | 16 | desc "Release" 17 | task :release => :tag 18 | 19 | desc "Truncate" 20 | task :truncate => :environment do 21 | FullTextSearch::Target.truncate 22 | end 23 | 24 | wait_queue = lambda do 25 | queue_adapter = ActiveJob::Base.queue_adapter 26 | case queue_adapter 27 | when ActiveJob::QueueAdapters::AsyncAdapter 28 | queue_adapter.shutdown 29 | end 30 | end 31 | 32 | run_batch = lambda do |&block| 33 | upsert = ENV["UPSERT"] || "immediate" 34 | extract_text = ENV["EXTRACT_TEXT"] || "immediate" 35 | project = ENV["PROJECT"] 36 | type = ENV["TYPE"] 37 | batch_runner = FullTextSearch::BatchRunner.new(show_progress: true) 38 | block.call(batch_runner, 39 | project: project, 40 | type: type, 41 | upsert: upsert.to_sym, 42 | extract_text: extract_text.to_sym) 43 | wait_queue.call 44 | end 45 | 46 | desc "Synchronize" 47 | task :synchronize => :environment do 48 | run_batch.call do |batch_runner, **options| 49 | batch_runner.synchronize(**options) 50 | end 51 | end 52 | 53 | namespace :repository do 54 | desc "Synchronize only repository data" 55 | task :synchronize => :environment do 56 | run_batch.call do |batch_runner, **options| 57 | batch_runner.synchronize_repositories(**options) 58 | end 59 | end 60 | end 61 | 62 | namespace :similar_issues do 63 | desc "Synchronize similar issues data" 64 | task :synchronize => :environment do 65 | run_batch.call do |batch_runner, **options| 66 | batch_runner.synchronize_similar_issues(**options) 67 | end 68 | end 69 | end 70 | 71 | namespace :target do 72 | desc "Reload targets" 73 | task :reload => :environment do 74 | run_batch.call do |batch_runner, **options| 75 | batch_runner.reload_fts_targets(**options) 76 | end 77 | end 78 | end 79 | 80 | namespace :text do 81 | desc "Extract texts" 82 | task :extract => :environment do 83 | options = {} 84 | id = ENV["ID"] 85 | options[:ids] = [Integer(id, 10)] if id.present? 86 | batch_runner = FullTextSearch::BatchRunner.new(show_progress: true) 87 | batch_runner.extract_text(**options) 88 | wait_queue.call 89 | end 90 | end 91 | 92 | namespace :query_expansion do 93 | desc "Synchronize query expansion data" 94 | task :synchronize => :environment do 95 | input = ENV["INPUT"] || $stdin 96 | synchronizer = FullTextSearch::QueryExpansionSynchronizer.new(input) 97 | synchronizer.synchronize 98 | end 99 | end 100 | end 101 | -------------------------------------------------------------------------------- /db/migrate/20170630063657_create_searcher_records.rb: -------------------------------------------------------------------------------- 1 | # For auto load 2 | FullTextSearch::Migration 3 | 4 | class CreateSearcherRecords < ActiveRecord::Migration[4.2] 5 | def change 6 | return if reverting? and !table_exists?(:searcher_records) 7 | 8 | if Redmine::Database.mysql? 9 | options = "ENGINE=Mroonga" 10 | else 11 | options = nil 12 | end 13 | create_table :searcher_records, options: options do |t| 14 | # common 15 | t.integer :project_id, null: false 16 | t.string :project_name, null: false # for searching by project_name 17 | t.integer :original_id, null: false 18 | t.string :original_type, null: false 19 | t.timestamp :original_created_on 20 | t.timestamp :original_updated_on 21 | 22 | # projects 23 | t.string :name 24 | t.text :description 25 | t.string :identifier 26 | t.integer :status 27 | 28 | # news 29 | t.string :title 30 | t.string :summary 31 | # t.text :description 32 | 33 | # issues 34 | t.string :subject 35 | # t.text :description 36 | t.integer :author_id 37 | t.boolean :is_private 38 | t.integer :status_id 39 | t.integer :tracker_id 40 | t.integer :issue_id 41 | 42 | # documents 43 | # t.string :title 44 | # t.text :description 45 | 46 | # changesets 47 | t.text :comments 48 | t.text :short_comments 49 | t.text :long_comments 50 | 51 | # messages 52 | # t.string :subject 53 | t.text :content 54 | 55 | # journals 56 | t.text :notes 57 | # t.integer :user_id # => author_id 58 | t.boolean :private_notes 59 | # t.integer :status_id 60 | 61 | # wiki_pages 62 | # t.string :title 63 | t.text :text # wiki_contents.text w/ latest version 64 | 65 | # custom_value 66 | t.text :value 67 | t.integer :custom_field_id 68 | 69 | # attachments 70 | t.integer :container_id 71 | t.string :container_type 72 | t.string :filename 73 | # t.text :description 74 | 75 | t.index([:original_id, :original_type], unique: true) 76 | 77 | columns = [ 78 | :original_type, 79 | :project_name, 80 | :name, 81 | :identifier, 82 | :description, 83 | :title, 84 | :summary, 85 | :subject, 86 | :comments, 87 | :content, 88 | :notes, 89 | :text, 90 | :value, 91 | :container_type, 92 | :filename, 93 | ] 94 | if Redmine::Database.mysql? 95 | columns.each do |column| 96 | t.index column, type: "fulltext" 97 | end 98 | t.index :original_type, 99 | name: "index_searcher_records_on_original_type_perfect_matching" 100 | t.index :project_id 101 | t.index :issue_id 102 | else 103 | t.index [:id] + columns, 104 | name: "index_searcher_records_pgroonga", 105 | using: "PGroonga" 106 | end 107 | end 108 | end 109 | end 110 | -------------------------------------------------------------------------------- /db/migrate/20190603061606_create_fts_targets.rb: -------------------------------------------------------------------------------- 1 | # For auto load 2 | FullTextSearch::Migration 3 | 4 | class CreateFtsTargets < ActiveRecord::Migration[5.2] 5 | def comparable_version(version) 6 | version.split(".").collect {|component| Integer(component, 10)} 7 | end 8 | 9 | def change 10 | return if reverting? and !table_exists?(:fts_targets) 11 | 12 | if Redmine::Database.mysql? 13 | mroonga_version = connection.select_rows(<<-SQL)[0][1] 14 | SHOW VARIABLES LIKE 'mroonga_version'; 15 | SQL 16 | required_mroonga_version = "9.03" 17 | if (comparable_version(mroonga_version) <=> 18 | comparable_version(required_mroonga_version)) < 0 19 | message = "Mroonga #{required_mroonga_version} or later is required: " + 20 | mroonga_version 21 | raise message 22 | end 23 | options = "ENGINE=Mroonga DEFAULT CHARSET=utf8mb4" 24 | else 25 | options = nil 26 | end 27 | create_table :fts_targets, options: options do |t| 28 | t.integer :source_id, null: false 29 | t.integer :source_type_id, null: false 30 | t.integer :project_id, null: false 31 | t.integer :container_id 32 | t.integer :container_type_id 33 | t.integer :custom_field_id 34 | t.boolean :is_private 35 | t.timestamp :last_modified_at 36 | t.text :title 37 | if Redmine::Database.mysql? 38 | t.text :content, 39 | limit: 16.megabytes, 40 | comment: "FLAGS 'COLUMN_SCALAR|COMPRESS_ZSTD'" 41 | else 42 | t.text :content 43 | end 44 | if Redmine::Database.mysql? 45 | t.text :tag_ids, 46 | comment: "FLAGS 'COLUMN_VECTOR', GROONGA_TYPE 'Int64'" 47 | else 48 | t.integer :tag_ids, array: true 49 | end 50 | t.index [:source_id, :source_type_id], unique: true 51 | if Redmine::Database.mysql? 52 | t.index :project_id 53 | t.index :container_id 54 | t.index :container_type_id 55 | t.index :custom_field_id 56 | t.index :is_private 57 | t.index :last_modified_at 58 | t.index :title, 59 | type: "fulltext", 60 | comment: "NORMALIZER 'NormalizerNFKC121'" 61 | t.index :content, 62 | type: "fulltext", 63 | comment: "NORMALIZER 'NormalizerNFKC121', INDEX_FLAGS 'WITH_POSITION|INDEX_LARGE'" 64 | t.index :tag_ids, 65 | type: "fulltext", 66 | comment: "LEXICON 'fts_tags', INDEX_FLAGS ''" 67 | t.index :source_type_id 68 | else 69 | t.index [:id, 70 | :source_id, 71 | :source_type_id, 72 | :project_id, 73 | :container_id, 74 | :container_type_id, 75 | :custom_field_id, 76 | :is_private, 77 | :last_modified_at, 78 | :title, 79 | :content, 80 | :tag_ids], 81 | using: "PGroonga", 82 | with: "normalizer = 'NormalizerNFKC121'", 83 | name: "fts_targets_index_pgroonga" 84 | end 85 | end 86 | end 87 | end 88 | -------------------------------------------------------------------------------- /lib/full_text_search/similar_words_filter.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | class SimilarWordsFilter 3 | attr_accessor :cosine_threshold 4 | attr_accessor :engine 5 | attr_accessor :sentence_piece_space 6 | def initialize 7 | @cosine_threshold = 0.99 8 | @engine = nil 9 | @sentence_piece_space = "▁" 10 | end 11 | 12 | def run(records) 13 | expansions = {} 14 | records.each do |record| 15 | source = normalize_word(record["source"]) 16 | destination = normalize_word(record["destination"]) 17 | next unless target?(source, destination, record) 18 | (expansions[source] ||= []) << destination 19 | (expansions[destination] ||= []) << source 20 | end 21 | generate_records(expansions) 22 | end 23 | 24 | private 25 | def normalize_word(word) 26 | word. 27 | gsub(@sentence_piece_space, " "). 28 | unicode_normalize(:nfkc). 29 | downcase. 30 | strip 31 | end 32 | 33 | def ignore_character_only?(word) 34 | /\A[\p{Number}\p{Punctuation}\p{Symbol}]*\z/.match?(word) 35 | end 36 | 37 | def multibyte_word?(word) 38 | not word.ascii_only? 39 | end 40 | 41 | def sub_word?(word1, word2) 42 | word1.include?(word2) or word2.include?(word1) 43 | end 44 | 45 | JAPANESE_PARTICLES = [ 46 | "が", 47 | "で", 48 | "と", 49 | "に", 50 | "の", 51 | "は", 52 | "も", 53 | "を", 54 | ] 55 | def with_japanese_particle?(word) 56 | JAPANESE_PARTICLES.any? do |particle| 57 | word.start_with?(particle) or word.end_with?(particle) 58 | end 59 | end 60 | 61 | def target?(source, destination, record) 62 | return false if source.include?(" ") 63 | return false if destination.include?(" ") 64 | return false if source.size == 1 65 | return false if destination.size == 1 66 | return false if ignore_character_only?(source) 67 | return false if ignore_character_only?(destination) 68 | if multibyte_word?(source) or multibyte_word?(destination) 69 | return false if sub_word?(source, destination) 70 | end 71 | 72 | # TODO: Very heuristic. Remove me. 73 | return false if with_japanese_particle?(source) 74 | return false if with_japanese_particle?(destination) 75 | 76 | cosine = record["cosine"] 77 | if cosine and cosine < @cosine_threshold 78 | return false 79 | end 80 | 81 | if @engine and record["engine"] != @engine 82 | return false 83 | end 84 | 85 | true 86 | end 87 | 88 | def generate_records(expansions) 89 | records = [] 90 | expansions.keys.sort.each do |source| 91 | destinations = expansions[source] 92 | destinations = ([source] + destinations).uniq.sort 93 | next if destinations.size == 1 94 | destinations.each do |destination| 95 | record = { 96 | "source" => source, 97 | "destination" => destination, 98 | } 99 | records << record 100 | end 101 | end 102 | records 103 | end 104 | end 105 | end 106 | -------------------------------------------------------------------------------- /test/system/full_text_search/search_test.rb: -------------------------------------------------------------------------------- 1 | begin 2 | require File.expand_path("../../../application_system_test_case", __FILE__) 3 | rescue => error 4 | if error.class.name == "Webdrivers::VersionError" 5 | puts("Webdrivers < 5.3.0 doesn't work. " + 6 | "See also: https://github.com/titusfortner/webdrivers/pull/251") 7 | puts("#{error.class}: #{error}") 8 | return 9 | else 10 | raise 11 | end 12 | end 13 | 14 | module FullTextSearch 15 | class SearchTest < ApplicationSystemTestCase 16 | include PrettyInspectable 17 | 18 | fixtures :attachments 19 | fixtures :boards 20 | fixtures :changesets 21 | fixtures :custom_fields 22 | fixtures :custom_fields_projects 23 | fixtures :custom_fields_trackers 24 | fixtures :custom_values 25 | fixtures :documents 26 | fixtures :enabled_modules 27 | fixtures :enumerations 28 | fixtures :issue_statuses 29 | fixtures :issues 30 | fixtures :journals 31 | fixtures :member_roles 32 | fixtures :messages 33 | fixtures :news 34 | fixtures :projects 35 | fixtures :projects_trackers 36 | fixtures :repositories 37 | fixtures :roles 38 | fixtures :trackers 39 | fixtures :users 40 | fixtures :wiki_contents 41 | fixtures :wiki_pages 42 | fixtures :wikis 43 | 44 | setup do 45 | if Object.const_defined?(:Webdrivers) 46 | if Gem::Version.new(Webdrivers::VERSION) < Gem::Version.new("5.3.0") 47 | skip("Webdrivers < 5.3.0 doesn't work. " + 48 | "See also: https://github.com/titusfortner/webdrivers/pull/251") 49 | end 50 | end 51 | 52 | Target.destroy_all 53 | batch_runner = BatchRunner.new(show_progress: false) 54 | batch_runner.synchronize 55 | log_user("jsmith", "jsmith") 56 | end 57 | 58 | def test_keep_search_target 59 | visit(search_url) 60 | click_on("search-target-wiki-pages") 61 | fill_in("search-input", with: "cookbook") 62 | click_on("search-submit") 63 | within("#search-result") do 64 | within("#search-result-content") do 65 | within("#search-source-types") do 66 | find_link("search-target-wiki-pages", 67 | class: "selected") 68 | end 69 | end 70 | end 71 | end 72 | 73 | def test_no_pagination 74 | subproject1 = Project.find("subproject1") 75 | visit(url_for(controller: "search", 76 | action: "index", 77 | id: subproject1.identifier)) 78 | click_on("search-target-issues") 79 | within("#search-results") do 80 | assert_equal(2, all("li").size) 81 | end 82 | within(".pagination") do 83 | assert_equal([], all("li").to_a) 84 | end 85 | end 86 | 87 | def test_pagination 88 | visit(search_url) 89 | click_on("search-target-issues") 90 | within("#search-results") do 91 | assert_equal(10, all("li").size) 92 | end 93 | within(".pagination") do 94 | assert_equal("1", find(".current").text) 95 | find(".next a").click 96 | end 97 | within("#search-results") do 98 | assert_equal(10, all("li").size) 99 | end 100 | end 101 | end 102 | end 103 | -------------------------------------------------------------------------------- /db/migrate/20250225000050_remove_contents_from_issue_contents.rb: -------------------------------------------------------------------------------- 1 | # For auto load 2 | FullTextSearch::Migration 3 | 4 | class RemoveContentsFromIssueContents < ActiveRecord::Migration[6.1] 5 | if Redmine::Database.mysql? 6 | include FullTextSearch::Mroonga 7 | else 8 | include FullTextSearch::Pgroonga 9 | end 10 | 11 | def up 12 | return unless table_exists?(:issue_contents) 13 | 14 | if Redmine::Database.mysql? 15 | remove_index :issue_contents, :contents 16 | else 17 | remove_index :issue_contents, name: "index_issue_contents_pgroonga" 18 | end 19 | remove_column :issue_contents, :contents, :text 20 | remove_column :issue_contents, :is_private, :boolean 21 | content_limit = Redmine::Database.mysql? ? 16.megabytes : nil 22 | add_column :issue_contents, :content, :text, limit: content_limit 23 | 24 | # TODO: Replace 'TokenMecab' with a multilingual morphological based tokenizer 25 | # when available. See also: groonga/groonga#1941. 26 | if Redmine::Database.mysql? 27 | add_index :issue_contents, 28 | :content, 29 | type: "fulltext", 30 | comment: "TOKENIZER 'TokenMecab'" 31 | else 32 | add_index :issue_contents, 33 | [:id, 34 | :project_id, 35 | :issue_id, 36 | :subject, 37 | :content, 38 | :status_id], 39 | name: "index_issue_contents_pgroonga", 40 | using: "PGroonga", 41 | with: [ 42 | "tokenizer = 'TokenMecab'", 43 | "normalizer = '#{normalizer}'", 44 | ].join(", ") 45 | end 46 | end 47 | 48 | def down 49 | return unless table_exists?(:issue_contents) 50 | 51 | if Redmine::Database.mysql? 52 | remove_index :issue_contents, :content 53 | else 54 | remove_index :issue_contents, name: "index_issue_contents_pgroonga" 55 | end 56 | remove_column :issue_contents, :content, :text 57 | contents_limit = Redmine::Database.mysql? ? 16.megabytes : nil 58 | add_column :issue_contents, :contents, :text, limit: contents_limit 59 | add_column :issue_contents, :is_private, :boolean 60 | if Redmine::Database.mysql? 61 | add_index :issue_contents, 62 | :contents, 63 | type: "fulltext", 64 | comment: "TOKENIZER 'TokenMecab'" 65 | else 66 | add_index :issue_contents, 67 | [:id, 68 | :project_id, 69 | :issue_id, 70 | :subject, 71 | :contents, 72 | :status_id, 73 | :is_private], 74 | name: "index_issue_contents_pgroonga", 75 | using: "PGroonga", 76 | with: [ 77 | "tokenizer = 'TokenMecab'", 78 | "normalizer = '#{normalizer}'", 79 | ].join(", ") 80 | end 81 | end 82 | 83 | private 84 | 85 | def normalizer 86 | version = Gem::Version.new(self.class.groonga_version) 87 | if version >= Gem::Version.new("14.1.3") 88 | "NormalizerNFKC" 89 | elsif version >= Gem::Version.new("13.0.0") 90 | "NormalizerNFKC150" 91 | elsif version >= Gem::Version.new("10.0.9") 92 | "NormalizerNFKC130" 93 | else 94 | "NormalizerNFKC121" 95 | end 96 | end 97 | end 98 | -------------------------------------------------------------------------------- /db/migrate/20190603054615_drop_searcher_records.rb: -------------------------------------------------------------------------------- 1 | class DropSearcherRecords < ActiveRecord::Migration[5.2] 2 | def change 3 | reversible do |d| 4 | d.up do 5 | drop_table :searcher_records 6 | end 7 | d.down do 8 | create_table :searcher_records do |t| 9 | if Redmine::Database.mysql? 10 | t.integer :project_id, null: false 11 | t.string :project_name, null: false # for searching by project_name 12 | t.integer :original_id, null: false 13 | t.string :original_type, null: false 14 | t.timestamp :original_created_on 15 | t.timestamp :original_updated_on 16 | 17 | # projects 18 | t.string :name 19 | t.text :description 20 | t.string :identifier 21 | t.integer :status 22 | 23 | # news 24 | t.string :title 25 | t.string :summary 26 | # t.text :description 27 | 28 | # issues 29 | t.string :subject 30 | # t.text :description 31 | t.integer :author_id 32 | t.boolean :is_private 33 | t.integer :status_id 34 | t.integer :tracker_id 35 | t.integer :issue_id 36 | 37 | # documents 38 | # t.string :title 39 | # t.text :description 40 | 41 | # changesets 42 | t.text :comments 43 | t.text :short_comments 44 | t.text :long_comments 45 | 46 | # messages 47 | # t.string :subject 48 | t.text :content 49 | 50 | # journals 51 | t.text :notes 52 | # t.integer :user_id # => author_id 53 | t.boolean :private_notes 54 | # t.integer :status_id 55 | 56 | # wiki_pages 57 | # t.string :title 58 | t.text :text # wiki_contents.text w/ latest version 59 | 60 | # custom_value 61 | t.text :value 62 | t.integer :custom_field_id 63 | 64 | # attachments 65 | t.integer :container_id 66 | t.string :container_type 67 | t.string :filename 68 | # t.text :description 69 | 70 | t.index :original_type, type: "fulltext" 71 | t.index :project_name, type: "fulltext" 72 | t.index :name, type: "fulltext" 73 | t.index :identifier, type: "fulltext" 74 | t.index :description, type: "fulltext" 75 | t.index :title, type: "fulltext" 76 | t.index :summary, type: "fulltext" 77 | t.index :subject, type: "fulltext" 78 | t.index :comments, type: "fulltext" 79 | t.index :content, type: "fulltext" 80 | t.index :notes, type: "fulltext" 81 | t.index :text, type: "fulltext" 82 | t.index :value, type: "fulltext" 83 | t.index :container_type, type: "fulltext" 84 | t.index :filename, type: "fulltext" 85 | t.index :original_type, name: "index_searcher_records_on_original_type_perfect_matching" 86 | t.index :project_id 87 | t.index :issue_id 88 | t.index :short_comments, type: "fulltext" 89 | t.index :long_comments, type: "fulltext" 90 | elsif Redmine::Database.postgresql? 91 | t.index :id, name: "index_searcher_records_pgroonga" 92 | end 93 | end 94 | end 95 | end 96 | end 97 | end 98 | -------------------------------------------------------------------------------- /lib/full_text_search/mroonga.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | module Mroonga 3 | extend ActiveSupport::Concern 4 | 5 | module ClassMethods 6 | def select(command) 7 | sql = build_sql(command) 8 | now = Time.zone.now.to_f 9 | raw_response = connection.select_value(sql) 10 | elapsed_time = Time.zone.now.to_f - now 11 | response_class = Groonga::Client::Response.find(command.command_name) 12 | header = [0, now, elapsed_time] 13 | body = JSON.parse(raw_response) 14 | response = response_class.new(command, header, body) 15 | response.raw = "[#{header.to_json}, #{raw_response}]" 16 | response 17 | end 18 | 19 | def full_text_search(column, query) 20 | where("MATCH (#{connection.quote_column_name(column)}) " + 21 | "AGAINST (? IN BOOLEAN MODE)", 22 | query) 23 | end 24 | 25 | def build_expand_query_sql_part(query) 26 | [ 27 | "mroonga_query_expand(?, ?, ?, ?) AS query", 28 | [ 29 | table_name, 30 | source_column_name, 31 | destination_column_name, 32 | query, 33 | ], 34 | ] 35 | end 36 | 37 | def time_offset 38 | @time_offset ||= compute_time_offset 39 | end 40 | 41 | def mroonga_version 42 | connection.select_rows(<<-SQL)[0][1] 43 | SHOW VARIABLES LIKE 'mroonga_version'; 44 | SQL 45 | end 46 | 47 | def groonga_version 48 | connection.select_rows(<<-SQL)[0][1] 49 | SHOW VARIABLES LIKE 'mroonga_libgroonga_version'; 50 | SQL 51 | end 52 | 53 | def mroonga_vector_load_is_supported? 54 | Gem::Version.new(groonga_version) >= Gem::Version.new("9.0.5") 55 | end 56 | 57 | def multiple_column_unique_key_update_is_supported? 58 | Gem::Version.new(mroonga_version) >= Gem::Version.new("9.05") 59 | end 60 | 61 | private 62 | def build_sql(command) 63 | arguments = [command.command_name] 64 | placeholders = ["?"] 65 | command["table"] = table_name 66 | command.arguments.each do |name, value| 67 | next if value.blank? 68 | placeholders << "?" 69 | arguments << name 70 | if name == :query 71 | expand_query_sql_part = 72 | FtsQueryExpansion.build_expand_query_sql_part(value) 73 | placeholders << expand_query_sql_part[0] 74 | arguments.concat(expand_query_sql_part[1]) 75 | else 76 | placeholders << "?" 77 | arguments << value 78 | end 79 | end 80 | sql_template = <<-SELECT 81 | SELECT mroonga_command(#{placeholders.join(", ")}) 82 | SELECT 83 | sanitize_sql([sql_template, *arguments]) 84 | end 85 | 86 | def compute_time_offset 87 | mysql_system_time_zone = connection.select_rows(<<-SQL)[0][1] 88 | SHOW VARIABLES LIKE 'system_time_zone' 89 | SQL 90 | mysql_system_utc_offset = 0 91 | TZInfo::Timezone.all.each do |timezone| 92 | period = timezone.current_period 93 | if period.abbreviation.to_s == mysql_system_time_zone 94 | mysql_system_utc_offset = period.utc_offset 95 | break 96 | end 97 | end 98 | mysql_system_utc_offset - Time.now.utc_offset 99 | end 100 | end 101 | end 102 | end 103 | -------------------------------------------------------------------------------- /lib/full_text_search/pgroonga.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | module Pgroonga 3 | extend ActiveSupport::Concern 4 | 5 | module ClassMethods 6 | def select(command) 7 | sql = build_sql(command) 8 | raw_response = connection.select_value(sql) 9 | Groonga::Client::Response.parse(command, raw_response) 10 | end 11 | 12 | def full_text_search(column, query) 13 | where("#{connection.quote_column_name(column)} &@~ ?", 14 | query) 15 | end 16 | 17 | def build_expand_query_sql_part(query) 18 | [ 19 | "pgroonga_query_expand(?, ?, ?, ?)", 20 | [ 21 | table_name, 22 | source_column_name, 23 | destination_column_name, 24 | query, 25 | ], 26 | ] 27 | end 28 | 29 | def time_offset 30 | @time_offset ||= compute_time_offset 31 | end 32 | 33 | def groonga_version 34 | # Ensure loading PGroonga 35 | connection.select_rows(<<-SQL) 36 | SELECT pgroonga_command('status'); 37 | SQL 38 | connection.select_rows(<<-SQL)[0][0] 39 | SHOW pgroonga.libgroonga_version; 40 | SQL 41 | end 42 | 43 | def multiple_column_unique_key_update_is_supported? 44 | true 45 | end 46 | 47 | private 48 | def build_sql(command) 49 | arguments = [] 50 | placeholders = [] 51 | command["table"] = "pgroonga_table_name('#{pgroonga_index_name}')" 52 | if command["filter"].present? 53 | command["filter"] += " && pgroonga_tuple_is_alive(ctid)" 54 | else 55 | command["filter"] = "pgroonga_tuple_is_alive(ctid)" 56 | end 57 | command.arguments.each do |name, value| 58 | next if value.blank? 59 | next if name == :table 60 | placeholders << "?" 61 | arguments << name 62 | if name == :query 63 | expand_query_sql_part = 64 | FtsQueryExpansion.build_expand_query_sql_part(value) 65 | placeholders << expand_query_sql_part[0] 66 | arguments.concat(expand_query_sql_part[1]) 67 | else 68 | placeholders << "?" 69 | arguments << value 70 | end 71 | end 72 | sql_template = <<-SELECT 73 | SELECT pgroonga_command(?, 74 | ARRAY[ 75 | 'table', #{command["table"]}, 76 | #{placeholders.join(", ")} 77 | ] 78 | ) 79 | SELECT 80 | sanitize_sql([sql_template, 81 | command.command_name, 82 | *arguments]) 83 | end 84 | 85 | def compute_time_offset 86 | utc_offset = connection.select_value(<<-SQL) 87 | SELECT utc_offset 88 | FROM pg_timezone_names 89 | WHERE name = current_setting('timezone') 90 | SQL 91 | case utc_offset 92 | when /\A(-)?(\d+):(\d+):(\d+)\z/ 93 | minus = $1 94 | hours = Integer($2, 10) 95 | minutes = Integer($3, 10) 96 | seconds = Integer($4, 10) 97 | offset = (hours * 60 * 60) + (minutes * 60) + seconds 98 | offset = -offset if minus == "-" 99 | offset - Time.now.utc_offset 100 | when /\A[-+]?PT/ 101 | duration = ActiveSupport::Duration.parse(utc_offset) 102 | duration.in_seconds - Time.now.utc_offset 103 | else 104 | raise "Invalid time offset value: #{utc_offset.inspect}" 105 | end 106 | end 107 | end 108 | end 109 | end 110 | -------------------------------------------------------------------------------- /db/migrate/20231210061330_add_registered_at_to_fts_targets_with_index.rb: -------------------------------------------------------------------------------- 1 | # For auto load 2 | FullTextSearch::Migration 3 | 4 | class AddRegisteredAtToFtsTargetsWithIndex < ActiveRecord::Migration[5.2] 5 | def up 6 | return if !table_exists?(:fts_targets) 7 | 8 | add_column :fts_targets, :registered_at, :timestamp 9 | 10 | ActiveRecord::Base.transaction do 11 | update_registered_at_using_last_modified_at 12 | update_registered_at_using_created_on 13 | end 14 | 15 | if Redmine::Database.mysql? 16 | add_index :fts_targets, :registered_at 17 | else 18 | remove_index :fts_targets, name: "fts_targets_index_pgroonga" 19 | add_index :fts_targets, 20 | [:id, 21 | :source_id, 22 | :source_type_id, 23 | :project_id, 24 | :container_id, 25 | :container_type_id, 26 | :custom_field_id, 27 | :is_private, 28 | :last_modified_at, 29 | :registered_at, 30 | :title, 31 | :content, 32 | :tag_ids], 33 | using: "PGroonga", 34 | with: "normalizer = 'NormalizerNFKC121'", 35 | name: "fts_targets_index_pgroonga" 36 | end 37 | end 38 | 39 | def down 40 | return if !table_exists?(:fts_targets) 41 | 42 | if Redmine::Database.mysql? 43 | remove_index :fts_targets, :registered_at 44 | else 45 | remove_index :fts_targets, name: "fts_targets_index_pgroonga" 46 | add_index :fts_targets, 47 | [:id, 48 | :source_id, 49 | :source_type_id, 50 | :project_id, 51 | :container_id, 52 | :container_type_id, 53 | :custom_field_id, 54 | :is_private, 55 | :last_modified_at, 56 | :title, 57 | :content, 58 | :tag_ids], 59 | using: "PGroonga", 60 | with: "normalizer = 'NormalizerNFKC121'", 61 | name: "fts_targets_index_pgroonga" 62 | end 63 | 64 | remove_column :fts_targets, :registered_at 65 | end 66 | 67 | private 68 | 69 | class FtsTarget < ActiveRecord::Base 70 | end 71 | 72 | class FtsType < ActiveRecord::Base 73 | end 74 | 75 | class Issue < ActiveRecord::Base 76 | end 77 | 78 | class Message < ActiveRecord::Base 79 | end 80 | 81 | class Project < ActiveRecord::Base 82 | end 83 | 84 | class WikiPage < ActiveRecord::Base 85 | end 86 | 87 | def update_registered_at_using_last_modified_at 88 | type_names = ['Attachment', 'Change', 'Changeset', 'CustomValue', 'Document', 'Journal', 'News'] 89 | 90 | FtsTarget.where(source_type_id: FtsType.where(name: type_names).select(:id)) 91 | .update_all(registered_at: FtsTarget.arel_table[:last_modified_at]) 92 | end 93 | 94 | def update_registered_at_using_created_on 95 | types = [Issue, Message, Project, WikiPage] 96 | types.each do |type| 97 | subquery_for_created_on = type.where(type.arel_table[:id].eq(FtsTarget.arel_table[:source_id])) 98 | .select(:created_on) 99 | .to_sql 100 | FtsTarget.where(source_type_id: FtsType.where(name: type.name.demodulize).select(:id)) 101 | .update_all(registered_at: Arel.sql("(#{subquery_for_created_on})")) 102 | end 103 | end 104 | end 105 | -------------------------------------------------------------------------------- /test/unit/full_text_search/issue_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path("../../../test_helper", __FILE__) 2 | 3 | module FullTextSearch 4 | class IssueTest < ActiveSupport::TestCase 5 | include PrettyInspectable 6 | include NullValues 7 | 8 | fixtures :custom_fields 9 | fixtures :custom_fields_projects 10 | fixtures :custom_fields_trackers 11 | fixtures :custom_values 12 | fixtures :enumerations 13 | fixtures :issue_statuses 14 | fixtures :projects 15 | fixtures :projects_trackers 16 | fixtures :trackers 17 | fixtures :users 18 | 19 | def test_save 20 | issue = Issue.generate! 21 | issue.reload 22 | targets = Target.where(source_id: issue.id, 23 | source_type_id: Type.issue.id) 24 | assert_equal([ 25 | { 26 | "project_id" => issue.project_id, 27 | "source_id" => issue.id, 28 | "source_type_id" => Type.issue.id, 29 | "last_modified_at" => issue.updated_on, 30 | "registered_at" => issue.created_on, 31 | "title" => issue.subject, 32 | "content" => issue.description || "", 33 | "tag_ids" => [ 34 | Tag.tracker(issue.tracker_id).id, 35 | Tag.user(issue.author_id).id, 36 | Tag.issue_status(issue.status_id).id, 37 | ], 38 | "is_private" => issue.is_private, 39 | "custom_field_id" => null_number, 40 | "container_id" => null_number, 41 | "container_type_id" => null_number, 42 | } 43 | ], 44 | targets.collect {|target| target.attributes.except("id")}) 45 | end 46 | 47 | def test_save_journal_status_and_tracker 48 | issue = Issue.generate!( 49 | status: IssueStatus.find_by_name("New"), 50 | tracker: Tracker.find_by_name("Bug") 51 | ) 52 | journal = issue.journals.create!(notes: "comment") 53 | issue.status = IssueStatus.find_by_name("Closed") 54 | issue.tracker = Tracker.find_by_name("Support request") 55 | issue.save! 56 | 57 | journal_targets = Target.where(source_id: journal.id, 58 | source_type_id: Type.journal.id) 59 | assert_equal([ 60 | [ 61 | Tag.user(journal.user_id), 62 | Tag.tracker(issue.tracker_id), 63 | Tag.issue_status(issue.status_id), 64 | ].sort_by(&:id), 65 | ], 66 | journal_targets.collect {|target| target.tags.sort_by(&:id)}) 67 | end 68 | 69 | def test_destroy 70 | searchable_custom_field = custom_fields(:custom_fields_002) 71 | issue = Issue.generate! do |i| 72 | i.custom_fields = [ 73 | { 74 | "id" => searchable_custom_field.id.to_s, 75 | "value" => "Hello", 76 | }, 77 | ] 78 | end 79 | issue_targets = Target.where(source_id: issue.id, 80 | source_type_id: Type.issue.id) 81 | custom_value_targets = Target.where(container_id: issue.id, 82 | source_type_id: Type.custom_value.id) 83 | assert_equal([1, 1], 84 | [issue_targets.size, custom_value_targets.size]) 85 | issue.destroy! 86 | assert_equal([[], []], 87 | [issue_targets.to_a, custom_value_targets.to_a]) 88 | end 89 | end 90 | end 91 | -------------------------------------------------------------------------------- /test/unit/full_text_search/change_git_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path("../../../test_helper", __FILE__) 2 | 3 | module FullTextSearch 4 | class ChangeGitTest < ActiveSupport::TestCase 5 | include PrettyInspectable 6 | include NullValues 7 | include TimeValue 8 | 9 | fixtures :enabled_modules 10 | fixtures :projects 11 | fixtures :repositories 12 | fixtures :users 13 | fixtures :roles 14 | 15 | def setup 16 | unless Target.multiple_column_unique_key_update_is_supported? 17 | skip("Need Mroonga 9.05 or later") 18 | end 19 | @project = Project.find(3) 20 | end 21 | 22 | def test_fetch_changesets 23 | url = self.class.repository_path("git") 24 | repository = Repository::Git.create(:project => @project, 25 | :url => url) 26 | repository.fetch_changesets 27 | records = Target. 28 | where(container_id: repository.id, 29 | container_type_id: Type.repository.id). 30 | order(source_id: :asc) 31 | first_change = Change.find_by!(path: "images/edit.png") 32 | last_change = Change.where(path: "issue-8857/test01.txt").last 33 | expected_titles = [ 34 | "images/edit.png", 35 | "copied_README", 36 | "renamed_test.txt", 37 | "sources/watchers_controller.rb", 38 | "this_is_a_really_long_and_verbose_directory_name/this_is_because_of_a_simple_reason/it_is_testing_the_ability_of_redmine_to_use_really_long_path_names/These_names_exceed_255_chars_in_total/That_is_the_single_reason_why_we_have_this_directory_here/But_there_might_also_be_additonal_reasons/And_then_there_is_not_even_somthing_funny_in_here.txt", 39 | "filemane with spaces.txt", 40 | " filename with a leading space.txt ", 41 | "new_file.txt", 42 | "latin-1/test00.txt", 43 | "README", 44 | "latin-1-dir/make-latin-1-file.rb", 45 | "issue-8857/test00.txt", 46 | "issue-8857/test01.txt", 47 | ] 48 | expected_first_change = { 49 | "project_id" => @project.id, 50 | "source_id" => first_change.id, 51 | "source_type_id" => Type.change.id, 52 | "last_modified_at" => parse_time("2007-12-14T09:24:01Z"), 53 | "registered_at" => parse_time("2007-12-14T09:24:01Z"), 54 | "container_id" => repository.id, 55 | "container_type_id" => Type.repository.id, 56 | "title" => "images/edit.png", 57 | "content" => "", 58 | "custom_field_id" => null_number, 59 | "is_private" => null_boolean, 60 | "tag_ids" => [Tag.extension("png").id], 61 | } 62 | expected_last_change = { 63 | "project_id" => @project.id, 64 | "source_id" => last_change.id, 65 | "source_type_id" => Type.change.id, 66 | "last_modified_at" => parse_time("2011-01-01T03:00:00Z"), 67 | "registered_at" => parse_time("2011-01-01T03:00:00Z"), 68 | "container_id" => repository.id, 69 | "container_type_id" => Type.repository.id, 70 | "custom_field_id" => null_number, 71 | "title" => "issue-8857/test01.txt", 72 | "content" => <<-CONTENT, 73 | test 74 | test 75 | CONTENT 76 | "is_private" => null_boolean, 77 | "tag_ids" => [Tag.extension("txt").id], 78 | } 79 | assert_equal([ 80 | expected_titles, 81 | expected_first_change, 82 | expected_last_change, 83 | ], 84 | [ 85 | records.collect(&:title), 86 | records.first.attributes.except("id"), 87 | records.last.attributes.except("id"), 88 | ]) 89 | end 90 | end 91 | end 92 | -------------------------------------------------------------------------------- /app/views/fts_query_expansions/index.html.erb: -------------------------------------------------------------------------------- 1 | <% content_for :header_tags do %> 2 | <%= javascript_include_tag "query_expansions", plugin: "full_text_search" %> 3 | <%= stylesheet_link_tag "query_expansions", plugin: "full_text_search" %> 4 | <% fontawesome_prefix = "fontawesome-free-5.8.2-web" %> 5 | <%= stylesheet_link_tag "#{fontawesome_prefix}/css/all.css", 6 | plugin: "full_text_search" %> 7 | <% end %> 8 | 9 |
10 | <%= link_to(t("fts_query_expansions.new.title"), 11 | new_fts_query_expansion_path, 12 | class: "icon icon-add") %> 13 |
14 | 15 | <%= title t(".title") %> 16 | 17 | <%= form_with(model: @request, 18 | method: :get, 19 | url: url_for, 20 | id: "query-expansion-form") do |form| %> 21 |
22 | <%= l(:label_filter_plural) %> 23 |

24 | <%= form.label "query-expansion-input", 25 | l(:description_search), 26 | :class => "hidden-for-sighted" %> 27 | <%= form.search_field "query", 28 | id: "query-expansion-input", 29 | class: "autocomplete", 30 | name: "query", 31 | autocomplete: "off" %> 32 | <%= form.button type: "submit", id: "query-expansion-submit" do %> 33 | 34 | <% end %> 35 |

36 |
37 | <% end %> 38 | 39 |

<%= t(".result") %>

40 |
41 |
42 |
43 | 44 | <%= javascript_tag do %> 45 | observeIncrementalSearch("#query-expansion-input", 46 | function($element, value) { 47 | $.ajax({ 48 | url: "<%= escape_javascript(fts_query_expand_path) %>", 49 | type: "get", 50 | data: {"query": value}, 51 | success: function(data) { 52 | $("#query-expansion-result").html(data); 53 | }, 54 | beforeSend: function() { 55 | $element.addClass("ajax-loading"); 56 | }, 57 | complete: function() { 58 | $element.removeClass("ajax-loading"); 59 | } 60 | }); 61 | }); 62 | <% end %> 63 | 64 | <% if @expansions.empty? %> 65 |

<%= l(:label_no_data) %>

66 | <% else %> 67 | 68 | 69 | 70 | <%= sort_header_tag("source", 71 | caption: FtsQueryExpansion.human_attribute_name("source"), 72 | default_order: "asc") %> 73 | <%= sort_header_tag("destination", 74 | caption: FtsQueryExpansion.human_attribute_name("destination"), 75 | default_order: "asc") %> 76 | <%= sort_header_tag("created_at", 77 | caption: l(:field_created_on), 78 | default_order: "desc") %> 79 | <%= sort_header_tag("updated_at", 80 | caption: l(:field_updated_on), 81 | default_order: "desc") %> 82 | 83 | 84 | 85 | 86 | <% @expansions.each do |expansion| %> 87 | 88 | 89 | 90 | 91 | 92 | 93 | 96 | 97 | 98 | <% end %> 99 | 100 |
<%= expansion.source %><%= expansion.destination %><%= format_time(expansion.created_at) %><%= format_time(expansion.updated_at) %><%= link_to(l(:button_view), expansion) %><%= link_to(l(:button_edit), 94 | edit_fts_query_expansion_path(expansion), 95 | class: "icon icon-edit") %><%= delete_link(expansion) %>
101 | <% end %> 102 | 103 | <%= pagination_links_full(@paginator, @n_expansions) %> 104 | -------------------------------------------------------------------------------- /test/test_helper.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path('../../../../test/test_helper', __FILE__) 2 | 3 | require "webrick" 4 | require "pp" 5 | 6 | module PrettyInspectable 7 | class << self 8 | def wrap(object) 9 | case object 10 | when Hash 11 | HashInspector.new(object) 12 | when Array 13 | ArrayInspector.new(object) 14 | else 15 | object 16 | end 17 | end 18 | end 19 | 20 | class HashInspector 21 | def initialize(hash) 22 | @hash = hash 23 | end 24 | 25 | def inspect 26 | @hash.inspect 27 | end 28 | 29 | def pretty_print(q) 30 | q.group(1, '{', '}') do 31 | q.seplist(self, nil, :each_pair) do |k, v| 32 | q.group do 33 | q.pp(k) 34 | q.text('=>') 35 | q.group(1) do 36 | q.breakable('') 37 | q.pp(v) 38 | end 39 | end 40 | end 41 | end 42 | end 43 | 44 | def pretty_print_cycle(q) 45 | @hash.pretty_print_cycle(q) 46 | end 47 | 48 | def each_pair 49 | keys = @hash.keys 50 | begin 51 | keys = keys.sort 52 | rescue ArgumentError 53 | end 54 | keys.each do |key| 55 | yield(key, PrettyInspectable.wrap(@hash[key])) 56 | end 57 | end 58 | end 59 | 60 | class ArrayInspector 61 | def initialize(array) 62 | @array = array 63 | end 64 | 65 | def inspect 66 | @array.inspect 67 | end 68 | 69 | def pretty_print(q) 70 | q.group(1, '[', ']') do 71 | q.seplist(self) do |v| 72 | q.pp(v) 73 | end 74 | end 75 | end 76 | 77 | def pretty_print_cycle(q) 78 | @array.pretty_print_cycle(q) 79 | end 80 | 81 | def each(&block) 82 | @array.each do |element| 83 | yield(PrettyInspectable.wrap(element)) 84 | end 85 | end 86 | end 87 | 88 | def mu_pp(obj) 89 | PrettyInspectable.wrap(obj).pretty_inspect 90 | end 91 | end 92 | 93 | module FullTextSearchBackend 94 | def mroonga? 95 | Redmine::Database.mysql? 96 | end 97 | 98 | def pgroonga? 99 | Redmine::Database.postgresql? 100 | end 101 | end 102 | 103 | module NullValues 104 | include FullTextSearchBackend 105 | 106 | def null_string 107 | if mroonga? 108 | "" 109 | else 110 | nil 111 | end 112 | end 113 | 114 | def null_number 115 | if mroonga? 116 | 0 117 | else 118 | nil 119 | end 120 | end 121 | 122 | def null_boolean 123 | if mroonga? 124 | false 125 | else 126 | nil 127 | end 128 | end 129 | 130 | def null_datetime 131 | nil 132 | end 133 | 134 | def null_number_array 135 | if mroonga? 136 | [] 137 | else 138 | nil 139 | end 140 | end 141 | end 142 | 143 | module TimeValue 144 | include FullTextSearchBackend 145 | 146 | def parse_time(string) 147 | time = Time.zone.parse(string) 148 | if mroonga? 149 | time.change(nsec: 0) 150 | else 151 | time 152 | end 153 | end 154 | end 155 | 156 | class TestLogger 157 | attr_reader :messages 158 | def initialize 159 | @messages = [] 160 | end 161 | 162 | def debug(message=nil) 163 | @messages << [:debug, message || yield] 164 | end 165 | 166 | def info(message=nil) 167 | @messages << [:info, message || yield] 168 | end 169 | 170 | def error(message=nil) 171 | @messages << [:error, message || yield] 172 | end 173 | end 174 | 175 | class RepositoryInfo 176 | def initialize(repository) 177 | @repository = repository 178 | end 179 | 180 | def files 181 | collect_files 182 | end 183 | 184 | def n_files 185 | files.size 186 | end 187 | 188 | private 189 | def collect_files(path=nil) 190 | files = [] 191 | @repository.entries(path).each do |entry| 192 | if entry.is_file? 193 | files << entry.path 194 | elsif entry.is_dir? 195 | files.concat(collect_files(entry.path)) 196 | end 197 | end 198 | files 199 | end 200 | end 201 | -------------------------------------------------------------------------------- /init.rb: -------------------------------------------------------------------------------- 1 | Redmine::Plugin.register :full_text_search do 2 | name 'Full Text Search plugin' 3 | author 'ClearCode Inc.' 4 | description 'This plugin provides full text search for Redmine' 5 | version '2.0.3' 6 | url 'https://github.com/clear-code/redmine_full_text_search' 7 | author_url 'https://github.com/clear-code' 8 | # We can't use __dir__ here because we ensure that plugin directory path is a path in Redmine directory 9 | # even when we use a symbolic link to place this plugin into redmine/plugins/. If we don't use a symbolic 10 | # link like the following, we can use __dir__ here: 11 | # 12 | # $ git clone https://github.com/clear-code/redmine_full_text_search.git redmine/plugins/full_text_search 13 | # 14 | # But __dir__ doesn't work when we use a symbolic link: 15 | # 16 | # $ git clone https://github.com/clear-code/redmine_full_text_search.git 17 | # $ cd redmine/plugins 18 | # $ ln -s ../../redmine_full_text_search full_text_search 19 | # 20 | # In this case, __dir__ and __FILE__ are the followings: 21 | # 22 | # __dir__: /tmp/redmine_full_text_search 23 | # __FILE__: ./full_text_search/init.rb 24 | # 25 | # Note that "/tmp/redmine_full_text_search" isn't a path in Redmine's plugin directory (redmine/plugins/). 26 | # Redmine assumes that this is a path in Redmine's plugin directory. So we need to compute this from 27 | # __FILE__ not __dir__. 28 | directory File.dirname(File.absolute_path(__FILE__)) 29 | settings partial: "settings/full_text_search" 30 | end 31 | 32 | # For backward compatibility with Redmine < 6. 33 | # ApplicationRecord is inherited from ActiveRecord Models instead of ActiveRecord::Base in Redmine >= 6. 34 | # ref: https://www.redmine.org/issues/38975 35 | unless defined?(ApplicationRecord) 36 | ApplicationRecord = ActiveRecord::Base 37 | end 38 | 39 | Redmine::Search.map do |search| 40 | search.register :changes 41 | end 42 | 43 | Redmine::MenuManager.map :admin_menu do |menu| 44 | menu.push :fts_query_expansion, 45 | {controller: "fts_query_expansions", action: "index"}, 46 | caption: :"label.full_text_search.menu.query_expansions.plural", 47 | html: {class: "icon icon-magnifier"} 48 | end 49 | 50 | require_relative "config/initializers/chupa_text" 51 | 52 | FullTextSearch::Settings 53 | FullTextSearch::Tracer 54 | FullTextSearch::Resolver 55 | FullTextSearch::TextExtractor 56 | FullTextSearch::MarkupParser 57 | FullTextSearch::BatchRunner 58 | FullTextSearch::RepositoryEntry 59 | 60 | FullTextSearch::ScmAdapterCatIo 61 | FullTextSearch::ScmAdapterAllFileEntries 62 | 63 | # Order by priority on synchronize 64 | FullTextSearch::JournalMapper 65 | FullTextSearch::IssueMapper 66 | FullTextSearch::WikiPageMapper 67 | FullTextSearch::CustomValueMapper 68 | FullTextSearch::ProjectMapper 69 | FullTextSearch::NewsMapper 70 | FullTextSearch::DocumentMapper 71 | FullTextSearch::MessageMapper 72 | FullTextSearch::AttachmentMapper 73 | FullTextSearch::ChangesetMapper 74 | FullTextSearch::ChangeMapper 75 | 76 | FullTextSearch::Hooks::SearchIndexOptionsContentBottomHook 77 | FullTextSearch::Hooks::IssuesShowDescriptionBottomHook 78 | FullTextSearch::Hooks::SimilarIssuesHelper 79 | 80 | FullTextSearch::Searcher 81 | FullTextSearch::SimilarSearcher 82 | 83 | class << Setting 84 | prepend FullTextSearch::SettingsObjectize 85 | end 86 | 87 | FullTextSearch.resolver.each do |redmine_class, mapper_class| 88 | mapper_class.attach(redmine_class) 89 | end 90 | FullTextSearch::CustomFieldCallbacks.attach 91 | Attachment.include(FullTextSearch::AttachmentSynchronizable) 92 | Issue.include(FullTextSearch::IssueSynchronizable) 93 | Issue.include(FullTextSearch::SimilarSearcher::Model) 94 | Journal.include(FullTextSearch::JournalSynchronizable) 95 | SearchController.helper(FullTextSearch::Hooks::SearchHelper) 96 | SearchController.prepend(FullTextSearch::Hooks::ControllerSearchIndex) 97 | IssuesController.helper(FullTextSearch::Hooks::SimilarIssuesHelper) 98 | 99 | FullTextSearch::Tag 100 | FullTextSearch::TagType 101 | FullTextSearch::Type 102 | 103 | if IssueQuery.method_defined?(:sql_for_any_searchable_field) 104 | IssueQuery.prepend(FullTextSearch::Hooks::IssueQueryAnySearchable) 105 | end 106 | 107 | # Support plugins 108 | if defined?(WikiExtensionsTagRelation) 109 | # Wiki Extensions tags 110 | # https://github.com/haru/redmine_wiki_extensions 111 | WikiExtensionsTagRelation.include(FullTextSearch::PluginWikiExtensionsTagSearchable) 112 | end 113 | -------------------------------------------------------------------------------- /assets/stylesheets/fontawesome-free-5.8.2-web/css/svg-with-js.min.css: -------------------------------------------------------------------------------- 1 | /*! 2 | * Font Awesome Free 5.8.2 by @fontawesome - https://fontawesome.com 3 | * License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) 4 | */ 5 | .svg-inline--fa,svg:not(:root).svg-inline--fa{overflow:visible}.svg-inline--fa{display:inline-block;font-size:inherit;height:1em;vertical-align:-.125em}.svg-inline--fa.fa-lg{vertical-align:-.225em}.svg-inline--fa.fa-w-1{width:.0625em}.svg-inline--fa.fa-w-2{width:.125em}.svg-inline--fa.fa-w-3{width:.1875em}.svg-inline--fa.fa-w-4{width:.25em}.svg-inline--fa.fa-w-5{width:.3125em}.svg-inline--fa.fa-w-6{width:.375em}.svg-inline--fa.fa-w-7{width:.4375em}.svg-inline--fa.fa-w-8{width:.5em}.svg-inline--fa.fa-w-9{width:.5625em}.svg-inline--fa.fa-w-10{width:.625em}.svg-inline--fa.fa-w-11{width:.6875em}.svg-inline--fa.fa-w-12{width:.75em}.svg-inline--fa.fa-w-13{width:.8125em}.svg-inline--fa.fa-w-14{width:.875em}.svg-inline--fa.fa-w-15{width:.9375em}.svg-inline--fa.fa-w-16{width:1em}.svg-inline--fa.fa-w-17{width:1.0625em}.svg-inline--fa.fa-w-18{width:1.125em}.svg-inline--fa.fa-w-19{width:1.1875em}.svg-inline--fa.fa-w-20{width:1.25em}.svg-inline--fa.fa-pull-left{margin-right:.3em;width:auto}.svg-inline--fa.fa-pull-right{margin-left:.3em;width:auto}.svg-inline--fa.fa-border{height:1.5em}.svg-inline--fa.fa-li{width:2em}.svg-inline--fa.fa-fw{width:1.25em}.fa-layers svg.svg-inline--fa{bottom:0;left:0;margin:auto;position:absolute;right:0;top:0}.fa-layers{display:inline-block;height:1em;position:relative;text-align:center;vertical-align:-.125em;width:1em}.fa-layers svg.svg-inline--fa{transform-origin:center center}.fa-layers-counter,.fa-layers-text{display:inline-block;position:absolute;text-align:center}.fa-layers-text{left:50%;top:50%;transform:translate(-50%,-50%);transform-origin:center center}.fa-layers-counter{background-color:#ff253a;border-radius:1em;box-sizing:border-box;color:#fff;height:1.5em;line-height:1;max-width:5em;min-width:1.5em;overflow:hidden;padding:.25em;right:0;text-overflow:ellipsis;top:0;transform:scale(.25);transform-origin:top right}.fa-layers-bottom-right{bottom:0;right:0;top:auto;transform:scale(.25);transform-origin:bottom right}.fa-layers-bottom-left{bottom:0;left:0;right:auto;top:auto;transform:scale(.25);transform-origin:bottom left}.fa-layers-top-right{right:0;top:0;transform:scale(.25);transform-origin:top right}.fa-layers-top-left{left:0;right:auto;top:0;transform:scale(.25);transform-origin:top left}.fa-lg{font-size:1.33333em;line-height:.75em;vertical-align:-.0667em}.fa-xs{font-size:.75em}.fa-sm{font-size:.875em}.fa-1x{font-size:1em}.fa-2x{font-size:2em}.fa-3x{font-size:3em}.fa-4x{font-size:4em}.fa-5x{font-size:5em}.fa-6x{font-size:6em}.fa-7x{font-size:7em}.fa-8x{font-size:8em}.fa-9x{font-size:9em}.fa-10x{font-size:10em}.fa-fw{text-align:center;width:1.25em}.fa-ul{list-style-type:none;margin-left:2.5em;padding-left:0}.fa-ul>li{position:relative}.fa-li{left:-2em;position:absolute;text-align:center;width:2em;line-height:inherit}.fa-border{border:.08em solid #eee;border-radius:.1em;padding:.2em .25em .15em}.fa-pull-left{float:left}.fa-pull-right{float:right}.fa.fa-pull-left,.fab.fa-pull-left,.fal.fa-pull-left,.far.fa-pull-left,.fas.fa-pull-left{margin-right:.3em}.fa.fa-pull-right,.fab.fa-pull-right,.fal.fa-pull-right,.far.fa-pull-right,.fas.fa-pull-right{margin-left:.3em}.fa-spin{animation:fa-spin 2s infinite linear}.fa-pulse{animation:fa-spin 1s infinite steps(8)}@keyframes fa-spin{0%{transform:rotate(0deg)}to{transform:rotate(1turn)}}.fa-rotate-90{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=1)";transform:rotate(90deg)}.fa-rotate-180{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=2)";transform:rotate(180deg)}.fa-rotate-270{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=3)";transform:rotate(270deg)}.fa-flip-horizontal{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=0, mirror=1)";transform:scaleX(-1)}.fa-flip-vertical{transform:scaleY(-1)}.fa-flip-both,.fa-flip-horizontal.fa-flip-vertical,.fa-flip-vertical{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=2, mirror=1)"}.fa-flip-both,.fa-flip-horizontal.fa-flip-vertical{transform:scale(-1)}:root .fa-flip-both,:root .fa-flip-horizontal,:root .fa-flip-vertical,:root .fa-rotate-90,:root .fa-rotate-180,:root .fa-rotate-270{filter:none}.fa-stack{display:inline-block;height:2em;position:relative;width:2.5em}.fa-stack-1x,.fa-stack-2x{bottom:0;left:0;margin:auto;position:absolute;right:0;top:0}.svg-inline--fa.fa-stack-1x{height:1em;width:1.25em}.svg-inline--fa.fa-stack-2x{height:2em;width:2.5em}.fa-inverse{color:#fff}.sr-only{border:0;clip:rect(0,0,0,0);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px}.sr-only-focusable:active,.sr-only-focusable:focus{clip:auto;height:auto;margin:0;overflow:visible;position:static;width:auto} -------------------------------------------------------------------------------- /test/unit/full_text_search/similar_search_issue_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path("../../../test_helper", __FILE__) 2 | 3 | module FullTextSearch 4 | class SimilarSearchIssueTest < ActiveSupport::TestCase 5 | include ActiveJob::TestHelper 6 | 7 | fixtures :enumerations 8 | fixtures :issue_statuses 9 | fixtures :roles 10 | fixtures :trackers 11 | fixtures :users 12 | 13 | def setup 14 | IssueContent.destroy_all 15 | User.current = User.find(1) 16 | @project = Project.generate! 17 | User.add_to_project(User.current, @project) 18 | end 19 | 20 | def test_same_structure_on_issue 21 | fts_engine_groonga_open_source = 22 | perform_enqueued_jobs(only: FullTextSearch::UpdateIssueContentJob) do 23 | Issue.generate!( 24 | project: @project, 25 | subject: "ぐるんが", 26 | description: "高速に検索できます。 オープンソースです。") 27 | end 28 | fts_engine_pgroonga_open_source = 29 | perform_enqueued_jobs(only: FullTextSearch::UpdateIssueContentJob) do 30 | Issue.generate!( 31 | project: @project, 32 | subject: "ぴーじーるんが", 33 | description: "PostgreSQLに組み込んで高速に検索できます。 オープンソースです。") 34 | end 35 | 36 | similar_issues = 37 | fts_engine_groonga_open_source.similar_issues( 38 | project_ids: [@project.id]) 39 | assert_equal([fts_engine_pgroonga_open_source], 40 | similar_issues) 41 | 42 | perform_enqueued_jobs(only: FullTextSearch::UpdateIssueContentJob) do 43 | fts_engine_groonga_open_source.update!(description: nil) 44 | end 45 | similar_issues = 46 | fts_engine_groonga_open_source.similar_issues( 47 | project_ids: [@project.id]) 48 | assert_equal([], similar_issues) 49 | end 50 | 51 | def test_same_structure_with_journal 52 | fts_engine_groonga_open_source = 53 | perform_enqueued_jobs(only: FullTextSearch::UpdateIssueContentJob) do 54 | Issue.generate!(project: @project, subject: "ぐるんが") 55 | .journals.create!(notes: "高速に検索できます。 オープンソースです。") 56 | end 57 | fts_engine_pgroonga_open_source = 58 | perform_enqueued_jobs(only: FullTextSearch::UpdateIssueContentJob) do 59 | Issue.generate!(project: @project, subject: "ぴーじーるんが") 60 | .journals.create!( 61 | notes: "PostgreSQLに組み込んで高速に検索できます。 オープンソースです。") 62 | end 63 | target_issue = fts_engine_groonga_open_source.issue 64 | similar_issues = target_issue.similar_issues(project_ids: [@project.id]) 65 | assert_equal([fts_engine_pgroonga_open_source.issue], 66 | similar_issues) 67 | 68 | perform_enqueued_jobs(only: FullTextSearch::UpdateIssueContentJob) do 69 | target_issue.journals.first.destroy 70 | end 71 | similar_issues = 72 | target_issue.reload.similar_issues(project_ids: [@project.id]) 73 | assert_equal([], similar_issues) 74 | end 75 | 76 | def test_same_structure_with_attachment 77 | set_tmp_attachments_directory 78 | fts_engine_groonga = 79 | Issue.generate!(project: @project, subject: "ぐるんが") 80 | fts_engine_groonga.save_attachments( 81 | [ 82 | { 83 | "file" => mock_file_with_options( 84 | :original_filename => "groonga-latest.txt"), 85 | "description" => "高速に検索 オープンソース! 最新情報" 86 | } 87 | ] 88 | ) 89 | perform_enqueued_jobs(only: FullTextSearch::UpdateIssueContentJob) do 90 | fts_engine_groonga.save! 91 | end 92 | fts_engine_pgroonga = 93 | Issue.generate!(project: @project, subject: "ぴーじーるんが") 94 | fts_engine_pgroonga.save_attachments( 95 | [ 96 | { 97 | "file" => mock_file_with_options( 98 | :original_filename => "pgroonga-latest.txt"), 99 | "description" => "組み込んで高速に検索 オープンソース! 最新情報" 100 | } 101 | ] 102 | ) 103 | perform_enqueued_jobs(only: FullTextSearch::UpdateIssueContentJob) do 104 | fts_engine_pgroonga.save! 105 | end 106 | similar_issues = 107 | fts_engine_groonga 108 | .similar_issues(project_ids: [@project.id]) 109 | assert_equal([fts_engine_pgroonga], 110 | similar_issues) 111 | 112 | perform_enqueued_jobs(only: FullTextSearch::UpdateIssueContentJob) do 113 | fts_engine_groonga.attachments.first.destroy 114 | end 115 | similar_issues = 116 | fts_engine_groonga 117 | .reload 118 | .similar_issues(project_ids: [@project.id]) 119 | assert_equal([], similar_issues) 120 | end 121 | end 122 | end 123 | -------------------------------------------------------------------------------- /test/unit/full_text_search/change_subversion_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path("../../../test_helper", __FILE__) 2 | 3 | module FullTextSearch 4 | class ChangeSubversionTest < ActiveSupport::TestCase 5 | include PrettyInspectable 6 | include NullValues 7 | include TimeValue 8 | 9 | fixtures :enabled_modules 10 | fixtures :projects 11 | fixtures :repositories 12 | fixtures :users 13 | fixtures :roles 14 | 15 | def setup 16 | @project = Project.find(3) 17 | end 18 | 19 | def test_fetch_changesets 20 | url = self.class.subversion_repository_url 21 | repository = Repository::Subversion.create(:project => @project, 22 | :url => url) 23 | repository.fetch_changesets 24 | repository_info = RepositoryInfo.new(repository) 25 | files = repository_info.files.collect do |file| 26 | "/#{file}" 27 | end 28 | records = Target. 29 | where(container_id: repository.id, 30 | container_type_id: Type.repository.id). 31 | order(source_id: :asc) 32 | first_change = Change.find_by!(path: "/subversion_test/.project") 33 | assert_equal([ 34 | files.sort, 35 | { 36 | "project_id" => @project.id, 37 | "source_id" => first_change.id, 38 | "source_type_id" => Type.change.id, 39 | "last_modified_at" => parse_time("2007-09-10T16:54:52.203Z"), 40 | "registered_at" => parse_time("2007-09-10T16:54:52.203Z"), 41 | "container_id" => repository.id, 42 | "container_type_id" => Type.repository.id, 43 | "title" => "/subversion_test/.project", 44 | "content" => <<-PROJECT, 45 | \r 46 | \r 47 | subversion_test\r 48 | \r 49 | \r 50 | \r 51 | \r 52 | \r 53 | \r 54 | \r 55 | \r 56 | PROJECT 57 | "custom_field_id" => null_number, 58 | "is_private" => null_boolean, 59 | "tag_ids" => [], 60 | }, 61 | ], 62 | [ 63 | records.collect(&:title).sort, 64 | records.first.attributes.except("id"), 65 | ]) 66 | end 67 | 68 | def test_fetch_changesets_sub_path 69 | url = "#{self.class.subversion_repository_url}/subversion_test" 70 | repository = Repository::Subversion.create(:project => @project, 71 | :url => url) 72 | repository.fetch_changesets 73 | repository_info = RepositoryInfo.new(repository) 74 | sub_path_files = repository_info.files.collect do |file| 75 | "/subversion_test/#{file}" 76 | end 77 | records = Target. 78 | where(container_id: repository.id, 79 | container_type_id: Type.repository.id). 80 | order(source_id: :asc) 81 | first_change = Change.find_by!(path: "/subversion_test/.project") 82 | assert_equal([ 83 | sub_path_files.sort, 84 | { 85 | "project_id" => @project.id, 86 | "source_id" => first_change.id, 87 | "source_type_id" => Type.change.id, 88 | "last_modified_at" => parse_time("2007-09-10T16:54:52.203Z"), 89 | "registered_at" => parse_time("2007-09-10T16:54:52.203Z"), 90 | "container_id" => repository.id, 91 | "container_type_id" => Type.repository.id, 92 | "title" => "/subversion_test/.project", 93 | "content" => <<-PROJECT, 94 | \r 95 | \r 96 | subversion_test\r 97 | \r 98 | \r 99 | \r 100 | \r 101 | \r 102 | \r 103 | \r 104 | \r 105 | PROJECT 106 | "custom_field_id" => null_number, 107 | "is_private" => null_boolean, 108 | "tag_ids" => [], 109 | }, 110 | ], 111 | [ 112 | records.collect(&:title).sort, 113 | records.first.attributes.except("id"), 114 | ]) 115 | end 116 | end 117 | end 118 | -------------------------------------------------------------------------------- /app/models/full_text_search/target.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | class Target < ApplicationRecord 3 | self.table_name = :fts_targets 4 | 5 | case connection_db_config.adapter 6 | when "postgresql" 7 | include Pgroonga 8 | when "mysql2" 9 | include Mroonga 10 | attribute :tag_ids, 11 | MroongaIntegerArrayType.new(mroonga_vector_load_is_supported?) 12 | unless mroonga_vector_load_is_supported? 13 | around_save :tag_ids_around_save 14 | private def tag_ids_around_save 15 | if tag_ids_changed? 16 | raw_tag_ids = tag_ids.dup 17 | yield 18 | values = [ 19 | {"_key" => id, "tag_ids" => raw_tag_ids}, 20 | ] 21 | arguments = { 22 | "values" => values.to_json, 23 | } 24 | command = Groonga::Command::Load.new("load", arguments) 25 | self.class.select(command) 26 | else 27 | yield 28 | end 29 | end 30 | end 31 | end 32 | 33 | @highlight_keyword_extraction_is_broken = nil 34 | @use_slices = nil 35 | class << self 36 | def highlight_keyword_extraction_is_broken? 37 | if @highlight_keyword_extraction_is_broken.nil? 38 | @highlight_keyword_extraction_is_broken = 39 | (Gem::Version.new(groonga_version) < 40 | Gem::Version.new("9.0.5")) 41 | end 42 | @highlight_keyword_extraction_is_broken 43 | end 44 | 45 | def use_slices? 46 | if @use_slices.nil? 47 | @use_slices = (Gem::Version.new(groonga_version) >= 48 | Gem::Version.new("9.0.7")) 49 | end 50 | @use_slices 51 | end 52 | 53 | def truncate 54 | connection.truncate(table_name) 55 | end 56 | 57 | def pgroonga_index_name 58 | "fts_targets_index_pgroonga" 59 | end 60 | end 61 | 62 | scope :attachments, -> {where(source_type_id: Type.attachment.id)} 63 | scope :changes, -> {where(source_type_id: Type.change.id)} 64 | scope :changesets, -> {where(source_type_id: Type.changeset.id)} 65 | scope :custom_values, -> {where(source_type_id: Type.custom_value.id)} 66 | scope :documents, -> {where(source_type_id: Type.document.id)} 67 | scope :files, -> {where(source_type_id: Type.file.id)} 68 | scope :issues, -> {where(source_type_id: Type.issue.id)} 69 | scope :journals, -> {where(source_type_id: Type.journal.id)} 70 | scope :messages, -> {where(source_type_id: Type.message.id)} 71 | scope :news, -> {where(source_type_id: Type.news.id)} 72 | scope :projects, -> {where(source_type_id: Type.project.id)} 73 | scope :repositories, -> {where(source_type_id: Type.repository.id)} 74 | scope :versions, -> {where(source_type_id: Type.version.id)} 75 | scope :wiki_pages, -> {where(source_type_id: Type.wiki_page.id)} 76 | 77 | attr_accessor :_score 78 | attr_accessor :highlighted_title 79 | attr_accessor :content_snippets 80 | 81 | acts_as_event(type: :_type, 82 | datetime: :_datetime, 83 | title: :_title, 84 | description: :_description, 85 | author: :_author, 86 | url: :_url) 87 | 88 | def score 89 | _score 90 | end 91 | alias rank score 92 | 93 | def mapper 94 | @mapper ||= FullTextSearch.resolver.resolve(self) 95 | end 96 | 97 | def source_record 98 | @source_record ||= mapper.redmine_record 99 | end 100 | 101 | def project 102 | @project ||= Project.find(project_id) 103 | end 104 | 105 | def _type 106 | mapper.type 107 | end 108 | 109 | def _datetime 110 | mapper.datetime 111 | end 112 | 113 | def _title 114 | mapper.title 115 | end 116 | 117 | def _description 118 | mapper.description 119 | end 120 | 121 | def _author 122 | # Not in use /search 123 | nil 124 | end 125 | 126 | def _url 127 | mapper.url 128 | end 129 | 130 | def event_group 131 | # Not in use /search 132 | nil 133 | end 134 | 135 | def event_id 136 | mapper.id 137 | end 138 | 139 | def event_highlighted_title 140 | @event_highlighted_title ||= 141 | if highlighted_title.present? 142 | "#{h(mapper.title_prefix)}#{highlighted_title}#{h(mapper.title_suffix)}".html_safe 143 | else 144 | h(event_title).html_safe 145 | end 146 | end 147 | 148 | def event_content_snippets 149 | @event_content_snippets ||= 150 | if content_snippets.present? 151 | content_snippets 152 | else 153 | [h((event_description || "").truncate(255)).html_safe] 154 | end 155 | end 156 | 157 | def tags 158 | Tag.where(id: tag_ids || []) 159 | end 160 | 161 | private 162 | def h(string) 163 | CGI.escape_html(string) 164 | end 165 | end 166 | end 167 | -------------------------------------------------------------------------------- /lib/full_text_search/scm_adapter_all_file_entries.rb: -------------------------------------------------------------------------------- 1 | require "chupa-text/sax-parser" 2 | 3 | # For auto load 4 | Redmine::Scm::Adapters::GitAdapter 5 | Redmine::Scm::Adapters::SubversionAdapter 6 | 7 | # TODO: Submit a patch to Redmine 8 | 9 | module Redmine 10 | module Scm 11 | module Adapters 12 | class GitAdapter 13 | def all_file_entries(identifier=nil) 14 | entries = Entries.new 15 | identifier ||= "HEAD" 16 | cmd_args = [ 17 | "ls-tree", 18 | "-l", 19 | "-r", 20 | "--full-tree", 21 | "#{identifier}:", 22 | ] 23 | git_cmd(cmd_args) do |io| 24 | io.each_line do |line| 25 | e = line.chomp.to_s 26 | if e =~ /^\d+\s+(\w+)\s+([0-9a-f]{40})\s+([0-9-]+)\t(.+)$/ 27 | type = $1 28 | sha = $2 29 | size = $3 30 | full_path = $4.force_encoding(@path_encoding) 31 | next if type != "blob" 32 | full_path_utf8 = scm_iconv('UTF-8', @path_encoding, full_path) 33 | attributes = { 34 | :name => full_path_utf8, 35 | :path => full_path_utf8, 36 | :kind => "file", 37 | :size => size.to_i(10), 38 | :lastrev => lastrev(full_path, identifier), 39 | } 40 | entries << Entry.new(attributes) 41 | end 42 | end 43 | end 44 | entries.sort_by do |entry| 45 | entry.path 46 | end 47 | rescue ScmCommandAborted 48 | [] 49 | end 50 | end 51 | 52 | class SubversionAdapter 53 | class ListListener < ChupaText::SAXListener 54 | def initialize(path_prefix, entries) 55 | @path_prefix = path_prefix 56 | @entries = entries 57 | @tag_names = [] 58 | @entry_attributes = nil 59 | @revision_attributes = nil 60 | end 61 | 62 | def start_element(uri, local_name, qname, attributes) 63 | @tag_names.push(local_name) 64 | case local_name 65 | when "entry" 66 | @entry_attributes = { 67 | kind: attributes["kind"], 68 | } 69 | when "commit" 70 | @revision_attributes = { 71 | identifier: attributes["revision"], 72 | } 73 | end 74 | end 75 | 76 | def end_element(uri, local_name, qname) 77 | case local_name 78 | when "entry" 79 | if @entry_attributes[:kind] == "file" 80 | @entries << Entry.new(@entry_attributes) 81 | end 82 | @entry_attributes = nil 83 | when "commit" 84 | @entry_attributes[:lastrev] = Revision.new(@revision_attributes) 85 | @revision_attributes = nil 86 | end 87 | @tag_names.pop 88 | end 89 | 90 | def characters(text) 91 | case @tag_names.last 92 | when "name" 93 | path = "#{@path_prefix}#{text}" 94 | @entry_attributes[:name] = CGI.unescape(path) 95 | @entry_attributes[:path] = path 96 | when "size" 97 | @entry_attributes[:size] = Integer(text, 10) 98 | when "author" 99 | @revision_attributes[:author] = text 100 | when "date" 101 | @revision_attributes[:date] = Time.iso8601(text).localtime 102 | end 103 | end 104 | end 105 | 106 | def all_file_entries(identifier=nil) 107 | prefix = url.gsub(root_url, "") 108 | prefix = "/#{prefix}" unless prefix.start_with?("/") 109 | if identifier and identifier.to_i > 0 110 | identifier = identifier.to_i 111 | else 112 | identifier = "HEAD" 113 | end 114 | entries = Entries.new 115 | root = target("") 116 | cmd = "#{self.class.sq_bin} list --recursive --xml " 117 | cmd << "--revision #{identifier} " 118 | cmd << root 119 | cmd << credentials_string 120 | shellout(cmd) do |io| 121 | listener = ListListener.new(prefix, entries) 122 | begin 123 | parser = ChupaText::SAXParser.new(io, listener) 124 | parser.parse 125 | rescue ChupaText::SAXParser::ParseError => e 126 | logger.error("Error parsing svn output: #{e.message}") 127 | logger.error(e.backtrace.join("\n")) 128 | end 129 | end 130 | return [] if $? && $?.exitstatus != 0 131 | logger.debug("Found #{entries.size} entries in the repository for #{root}") if logger && logger.debug? 132 | entries.sort_by do |entry| 133 | entry.path 134 | end 135 | end 136 | end 137 | end 138 | end 139 | end 140 | 141 | # For auto load 142 | module FullTextSearch 143 | module ScmAdapterAllFileEntries 144 | end 145 | end 146 | -------------------------------------------------------------------------------- /lib/full_text_search/attachment_mapper.rb: -------------------------------------------------------------------------------- 1 | module FullTextSearch 2 | class AttachmentMapper < Mapper 3 | class << self 4 | def redmine_mapper_class 5 | RedmineAttachmentMapper 6 | end 7 | 8 | def fts_mapper_class 9 | FtsAttachmentMapper 10 | end 11 | end 12 | end 13 | resolver.register(Attachment, AttachmentMapper) 14 | 15 | class RedmineAttachmentMapper < RedmineMapper 16 | class << self 17 | def with_project(redmine_class) 18 | redmine_class 19 | .joins(<<-JOIN) 20 | LEFT OUTER JOIN documents 21 | ON container_type = 'Document' AND documents.id = container_id 22 | JOIN 23 | .joins(<<-JOIN) 24 | LEFT OUTER JOIN issues 25 | ON container_type = 'Issue' AND issues.id = container_id 26 | JOIN 27 | .joins(<<-JOIN) 28 | LEFT OUTER JOIN messages 29 | ON container_type = 'Message' AND messages.id = container_id 30 | JOIN 31 | .joins(<<-JOIN) 32 | LEFT OUTER JOIN boards 33 | ON container_type = 'Message' AND boards.id = messages.board_id 34 | JOIN 35 | .joins(<<-JOIN) 36 | LEFT OUTER JOIN wiki_pages 37 | ON container_type = 'WikiPage' AND wiki_pages.id = container_id 38 | JOIN 39 | .joins(<<-JOIN) 40 | LEFT OUTER JOIN wikis 41 | ON container_type = 'WikiPage' AND wikis.id = wiki_pages.wiki_id 42 | JOIN 43 | .joins(<<-JOIN) 44 | JOIN projects 45 | ON (container_type = 'Document' AND projects.id = documents.project_id) OR 46 | (container_type = 'Issue' AND projects.id = issues.project_id) OR 47 | (container_type = 'Message' AND projects.id = boards.project_id) OR 48 | (container_type = 'Project' AND projects.id = container_id) OR 49 | (container_type = 'WikiPage' AND projects.id = wikis.project_id) 50 | JOIN 51 | end 52 | end 53 | 54 | def upsert_fts_target(options={}) 55 | # container is not specified when initial upload 56 | return if @record.container_type.nil? 57 | return unless Type.available?(@record.container_type) 58 | 59 | fts_target = find_fts_target 60 | fts_target.source_id = @record.id 61 | fts_target.source_type_id = Type[@record.class].id 62 | fts_target.title = @record.filename 63 | fts_target.content = @record.description 64 | fts_target.last_modified_at = @record.created_on 65 | fts_target.registered_at = @record.created_on 66 | tag_ids = [] 67 | case @record.container_type 68 | when "Issue" 69 | issue = @record.container 70 | return if issue.nil? 71 | fts_target.project_id = issue.project_id 72 | fts_target.is_private = issue.is_private 73 | tag_ids << Tag.issue_status(issue.status_id).id 74 | when "Message" 75 | fts_target.project_id = @record.container.board.project_id 76 | when "Project" 77 | fts_target.project_id = @record.container_id 78 | when "WikiPage" 79 | wiki_page = @record.container 80 | fts_target.project_id = wiki_page.wiki.project_id 81 | else 82 | return unless @record.container.respond_to?(:project_id) 83 | fts_target.project_id = @record.container.project_id 84 | end 85 | fts_target.container_id = @record.container_id 86 | fts_target.container_type_id = Type[@record.container_type].id 87 | tag_ids.concat(extract_tag_ids_from_path(@record.filename)) 88 | fts_target.tag_ids = tag_ids 89 | prepare_text_extraction(fts_target) 90 | fts_target.save! 91 | extract_content(fts_target, options) 92 | end 93 | 94 | def extract_text 95 | return unless @record.readable? 96 | 97 | fts_target = find_fts_target 98 | return unless fts_target.persisted? 99 | 100 | disk_path = @record.diskfile 101 | path = @record.filename 102 | content_type = resolve_content_type(path, @record.content_type) 103 | metadata = [ 104 | ["path", path], 105 | ["content-type", content_type], 106 | ] 107 | content = run_text_extractor(fts_target, metadata) do |extractor| 108 | if @record.respond_to?(:raw_data) 109 | input = StringIO.new(@record.raw_data) 110 | else 111 | input = nil 112 | end 113 | extractor.extract(Pathname(disk_path), input, content_type) 114 | end 115 | set_extracted_content(fts_target, 116 | content, 117 | [@record.description.presence]) 118 | fts_target.save! 119 | end 120 | 121 | private 122 | def resolve_content_type(path, content_type) 123 | case content_type 124 | when "application/x-tar" 125 | case File.extname(path).downcase 126 | when ".gz" 127 | "application/gzip" 128 | when ".bz2" 129 | "application/x-bzip2" 130 | else 131 | content_type 132 | end 133 | else 134 | content_type 135 | end 136 | end 137 | end 138 | 139 | class FtsAttachmentMapper < FtsMapper 140 | def title 141 | @record.filename 142 | end 143 | 144 | def url 145 | { 146 | controller: "attachments", 147 | action: "show", 148 | id: @record.source_id, 149 | filename: @record.title, 150 | } 151 | end 152 | end 153 | end 154 | --------------------------------------------------------------------------------