├── .ackrc ├── .gitignore ├── .travis.yml ├── Makefile ├── README.md ├── cache.ts ├── control-panel.tid ├── demo ├── A_fox_in_the_garden.tid ├── Examples_Fuzzy_Searching.tid ├── Examples_Relevance.tid ├── Examples_Simple_Stemming.tid ├── Examples_Synonyms.tid ├── Examples_Wildcard_Searches.tid ├── Foxes_foxes_foxes.tid ├── Introduction.tid ├── Query_Examples.tid ├── The_quick_fox_jumped_over_the_lazy_dog.tid ├── Wiping_a_harddrive.tid ├── ___DefaultTiddlers.tid ├── ___Ribbon.tid ├── ___SiteSubtitle.tid ├── ___SiteTitle.tid ├── ___plugins_hoelzro_full-text-search_RelatedTerms.json.tid └── ___plugins_hoelzro_full-text-search_use-cache.tid ├── files ├── .gitignore ├── localforage.min.js ├── lunr-mutable.js ├── lunr.min.js └── tiddlywiki.files ├── fts-action-generate-index.ts ├── ftsearch.ts ├── ftsfeedback.ts ├── history.tid ├── hooks.ts ├── index-worker.ts ├── license.tid ├── plugin.info.in ├── query-expander.ts ├── readme.tid ├── search-results.tid ├── shared-index.ts ├── state.tid └── tests ├── .gitignore └── test-simple.js /.ackrc: -------------------------------------------------------------------------------- 1 | --type-add=ts:ext:ts 2 | --nojs 3 | --nohtml 4 | --ignore-dir=.build-wiki 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.js 2 | /.build-wiki 3 | /fts.html 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - 10 4 | install: npm install localforage typescript tiddlywiki@$TIDDLYWIKI @types/node 5 | script: make test 6 | 7 | matrix: 8 | include: 9 | - env: TIDDLYWIKI=5.1.17 10 | - env: TIDDLYWIKI=5.1.16 11 | - env: TIDDLYWIKI=5.1.15 12 | 13 | git: 14 | depth: 200 15 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | TSC=tsc 2 | TSCFLAGS=--pretty --module commonjs --alwaysStrict --noEmitOnError 3 | TSCLIBS=--lib esnext,dom 4 | 5 | JS_FILES=$(patsubst %.ts,%.js,$(filter-out $(wildcard *.d.ts),$(wildcard *.ts))) 6 | TID_FILES=$(shell ls *.tid | fgrep -v fts.json.tid) 7 | DEMO_FILES=$(wildcard demo/*.tid) 8 | TEST_FILES=$(wildcard tests/*.js) 9 | 10 | all: dist.html fts.json.tid 11 | 12 | test: run-tests check-plugin-contents 13 | 14 | run-tests: .test-wiki $(JS_FILES) $(TID_FILES) $(TEST_FILES) plugin.info 15 | mkdir -p $ 0) { $$failed = 1 } END { exit(1) if($$failed) }' 24 | 25 | check-plugin-contents: .build-wiki 26 | output=$$(mktemp) && tiddlywiki $< --rendertiddler '$$:/core/ui/PluginInfo/Default/contents' $$output text/plain '' currentTiddler '$$:/plugins/hoelzro/progress-bar' && perl -nle 'exit 1 if /\S/ && !m{^\$$:}' $$output 27 | output=$$(mktemp) && tiddlywiki $< --rendertiddler '$$:/core/ui/PluginInfo/Default/contents' $$output text/plain '' currentTiddler '$$:/plugins/hoelzro/full-text-search' && perl -nle 'exit 1 if /\S/ && !m{^\$$:}' $$output 28 | 29 | dist.html: .build-wiki $(JS_FILES) $(TID_FILES) $(DEMO_FILES) plugin.info 30 | mkdir -p $ .test-wiki/tiddlywiki.info.tmp 48 | git clone https://github.com/hoelzro/tw-modern-jasmine .test-wiki/plugins/jasmine3 49 | (cd .test-wiki/plugins/jasmine3; npm install) 50 | mv .test-wiki/tiddlywiki.info.tmp .test-wiki/tiddlywiki.info 51 | 52 | .build-wiki: 53 | tiddlywiki .build-wiki --init empty 54 | jq '(.plugins) |= . + ["tiddlywiki/github-fork-ribbon", "hoelzro/progress-bar", "hoelzro/full-text-search"]' .build-wiki/tiddlywiki.info > .build-wiki/tiddlywiki.info.tmp 55 | mv .build-wiki/tiddlywiki.info.tmp .build-wiki/tiddlywiki.info 56 | mkdir .build-wiki/plugins/ 57 | git clone https://github.com/hoelzro/tw-progress-bar .build-wiki/plugins/progress-bar 58 | rm .build-wiki/plugins/progress-bar/README.md 59 | 60 | plugin.info: plugin.info.in 61 | jq --arg version $(shell git describe) '.version |= $$version' $^ > $@ 62 | 63 | clean: 64 | rm -f $(JS_FILES) dist.html fts.json.tid plugin.info 65 | 66 | realclean: clean 67 | rm -rf .build-wiki/ .test-wiki/ 68 | 69 | %.js: %.ts 70 | $(TSC) $(TSCFLAGS) $(TSCLIBS) $^ 71 | 72 | index-worker.js: index-worker.ts 73 | $(TSC) $(TSCFLAGS) --lib esnext,webworker,webworker.importscripts $^ 74 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Purpose 2 | 3 | Provides an alternative search result list that orders results by search relevance and ignores differences in word forms (ex. *tag* vs *tags*). 4 | 5 | On my personal wiki, I have the problem that there are terms I use across a lot of tiddlers, and sometimes I'll use different forms (such as the aforementioned *tag* vs *tags*). I wanted a plugin to allow me to find the tiddler I'm looking for quickly and didn't require me to worry about how I declined a noun or inflected a verb - so I wrote this plugin, which provides an alternative search list powered by [lunr.js](https://lunrjs.com/). 6 | 7 | This plugin should be considered as **BETA** quality - I use it pretty much every day, but there's definitely room for improvement. Please [let me know](https://github.com/hoelzro/tw-full-text-search/issues) if there are any bugs! 8 | 9 | # Demo 10 | 11 | https://hoelz.ro/files/fts.html 12 | 13 | # Installation 14 | 15 | https://hoelz.ro/files/fts.html 16 | 17 | # Usage 18 | 19 | Each time you start a new TiddlyWiki session, you'll need to build the FTS index. You can do this from a tab in the `$:/ControlPanel`. Older versions of the index are retained in web storage, so it should be pretty quick after the first time! After you build the index, you can just search as you would normally. 20 | 21 | # Ideas for Future Enhancement 22 | 23 | * Display score for search results 24 | * Specify a filter for tiddlers to be included in the index. 25 | * Custom stemmers for non-English/mixed language wikis 26 | 27 | # Source Code 28 | 29 | If you want to help out, you can check out the source for this plugin (or its dependency, the progress bar plugin) on GitHub: 30 | 31 | https://github.com/hoelzro/tw-full-text-search/ 32 | 33 | https://github.com/hoelzro/tw-progress-bar 34 | 35 | Requires [$:/plugins/hoelzro/progress-bar](https://github.com/hoelzro/tw-progress-bar) to display progress when generating the index. 36 | -------------------------------------------------------------------------------- /cache.ts: -------------------------------------------------------------------------------- 1 | /*\ 2 | title: $:/plugins/hoelzro/full-text-search/cache.js 3 | type: application/javascript 4 | module-type: library 5 | 6 | \*/ 7 | 8 | declare var $tw; 9 | declare var window; 10 | declare var require; 11 | 12 | import * as LocalForageModule from 'localforage'; 13 | var localForage : typeof LocalForageModule = require('$:/plugins/hoelzro/full-text-search/localforage.min.js'); 14 | 15 | module FTSCache { 16 | const RELATED_TERMS_TIDDLER = '$:/plugins/hoelzro/full-text-search/RelatedTerms.json'; 17 | 18 | function hasFunctionalCache() { 19 | return localForage.driver() != null; 20 | } 21 | 22 | interface CacheMeta { 23 | age: number; 24 | ftsPluginVersion: string; 25 | relatedTermsModified: number; 26 | } 27 | 28 | function currentPluginVersion() { 29 | let pluginTiddler = $tw.wiki.getTiddler('$:/plugins/hoelzro/full-text-search'); 30 | return pluginTiddler.fields.version; 31 | } 32 | 33 | function relatedTermsModified() { 34 | let relatedTerms = $tw.wiki.getTiddler(RELATED_TERMS_TIDDLER); 35 | if(!relatedTerms || !relatedTerms.fields.modified) { 36 | return 0; 37 | } 38 | return relatedTerms.fields.modified.getTime(); 39 | } 40 | 41 | async function getCacheMetadata() { 42 | if(!hasFunctionalCache()) { 43 | return; 44 | } 45 | 46 | var metaKey = 'tw-fts-index.meta.' + $tw.wiki.getTiddler('$:/SiteTitle').fields.text; 47 | // XXX how does TS handle the case where the cache item doesn't have the right keys? 48 | var cacheMeta = await localForage.getItem(metaKey); 49 | if(cacheMeta === null) { 50 | return; 51 | } 52 | 53 | if(!('ftsPluginVersion' in cacheMeta) || cacheMeta.ftsPluginVersion != currentPluginVersion()) { 54 | return; 55 | } 56 | 57 | let cacheRelatedTermsModified = ('relatedTermsModified' in cacheMeta) ? cacheMeta.relatedTermsModified : 0; 58 | let relatedTerms = $tw.wiki.getTiddler(RELATED_TERMS_TIDDLER); 59 | let ourRelatedTermsModified = relatedTermsModified(); 60 | 61 | if(cacheRelatedTermsModified != ourRelatedTermsModified) { 62 | return; 63 | } 64 | 65 | return cacheMeta; 66 | } 67 | 68 | // XXX what about migrating between lunr versions? what about invalid data under the key? 69 | async function getCacheData() { 70 | if(!hasFunctionalCache()) { 71 | return; 72 | } 73 | 74 | let metaData = await getCacheMetadata(); 75 | 76 | if(metaData == null) { 77 | return null; 78 | } 79 | 80 | var dataKey = 'tw-fts-index.data.' + $tw.wiki.getTiddler('$:/SiteTitle').fields.text; 81 | var cacheData = await localForage.getItem(dataKey); 82 | 83 | if(cacheData === null) { 84 | return null; 85 | } 86 | 87 | return JSON.parse(cacheData as string); 88 | } 89 | 90 | export async function getAge() { 91 | if(!hasFunctionalCache()) { 92 | return 0; 93 | } 94 | 95 | var cacheMeta = await getCacheMetadata(); 96 | if(!cacheMeta) { 97 | return 0; 98 | } 99 | 100 | return cacheMeta.age; 101 | } 102 | 103 | export function load() { 104 | if(!hasFunctionalCache()) { 105 | return null; 106 | } 107 | 108 | var cacheData = getCacheData(); 109 | if(!cacheData) { 110 | return; 111 | } 112 | 113 | return cacheData; 114 | } 115 | 116 | export async function save(age, data) { 117 | if(!hasFunctionalCache()) { 118 | return; 119 | } 120 | var dataKey = 'tw-fts-index.data.' + $tw.wiki.getTiddler('$:/SiteTitle').fields.text; 121 | var metaKey = 'tw-fts-index.meta.' + $tw.wiki.getTiddler('$:/SiteTitle').fields.text; 122 | var dataPromise = localForage.setItem(dataKey, JSON.stringify(data)); 123 | var metaPromise = localForage.setItem(metaKey, { age: age, ftsPluginVersion: currentPluginVersion(), relatedTermsModified: relatedTermsModified() }); 124 | await Promise.all([ dataPromise, metaPromise ]); 125 | } 126 | 127 | export async function invalidate() { 128 | if(!hasFunctionalCache()) { 129 | return; 130 | } 131 | var dataKey = 'tw-fts-index.data.' + $tw.wiki.getTiddler('$:/SiteTitle').fields.text; 132 | var metaKey = 'tw-fts-index.meta.' + $tw.wiki.getTiddler('$:/SiteTitle').fields.text; 133 | var dataPromise = localForage.removeItem(dataKey); 134 | var metaPromise = localForage.removeItem(metaKey); 135 | await Promise.all([ dataPromise, metaPromise ]); 136 | } 137 | } 138 | 139 | export = FTSCache; 140 | -------------------------------------------------------------------------------- /control-panel.tid: -------------------------------------------------------------------------------- 1 | title: $:/plugins/hoelzro/full-text-search/control-panel 2 | type: text/vnd.tiddlywiki 3 | tags: $:/tags/ControlPanel 4 | caption: Full Text Configuration 5 | 6 | <$set name="state" value="$:/temp/FTS-state"> 7 | 8 | <$reveal type="match" state=<> text="uninitialized"> 9 | 10 | In order to use full text search, you'll need to generate an index. 11 | 12 | <$button> 13 | Click here to generate the index 14 | <$fts-action-generate-index /> 15 | 16 | 17 | 18 | 19 | <$reveal type="match" state=<> text="initializing"> 20 | Generating index... 21 | 22 | <$hoelzro-progressbar current={{$:/temp/FTS-state!!progressCurrent}} total={{$:/temp/FTS-state!!progressTotal}} /> 23 | 24 | 25 | <$reveal type="match" state=<> text="initialized"> 26 | 27 | Index generated. Happy searching! 28 | 29 | <$button> 30 | Click here to rebuild the index 31 | <$fts-action-generate-index rebuild="true" /> 32 | 33 | 34 | 35 | 36 | ! Auto Indexing 37 | 38 | <$list variable="fiveSixteenOrBetter" filter="5.1.16 [title] +[sort[]first[]] +[prefix[5.1.16]]" emptyMessage="Sorry, you need TiddlyWiki 5.1.16 or greater to enable auto-indexing."> 39 | 40 | <$set name="autoIndexTiddler" value="$:/plugins/hoelzro/full-text-search/auto-index"> 41 | 42 | Automatic indexing refers to a feature where your wiki is automatically indexed in the background after it's been loaded. 43 | 44 | <$list filter="[titleget[title]]" emptyMessage="""<$button><$action-createtiddler $basetitle=<> text="<$fts-action-generate-index />" tags="$:/tags/StartupAction/Browser" />Click here to enable automatic indexing"""> 45 | <$button> 46 | <$action-deletetiddler $tiddler=<> /> 47 | Click here to disable automatic indexing 48 | 49 | 50 | 51 | 52 | 53 | 54 | ! Wildcard/Fuzzy Searching (experimental) 55 | 56 | Wildcard searches allow you to use wildcards to do things like use `format*ing` to match both "formating" and "formatting". 57 | Please consult [[the lunr documentation|https://lunrjs.com/guides/searching.html#wildcards]] for more information. 58 | 59 | Fuzzy searches allow you to compensate for spelling mistakes by specifying an //edit distance// - for example, if you want 60 | to tolerate being off by one character, you'd searching for `formattign~1`. For performance, it's recommended to keep 61 | the distance below 3. You can read more about fuzzy searching in [[the lunr documentation|https://lunrjs.com/guides/searching.html#fuzzy-matches]]. 62 | 63 | Note that you don't need wildcards to have the search system treat "format", "formatting", and "formatted" as the same - 64 | the default setup is smart enough to handle this! 65 | 66 | Enabling wildcard and fuzzy searches takes more computing power, memory, and storage space, but does prove useful for various users. If 67 | you'd like to try wildcard searches, you can enable them here: 68 | 69 | <$checkbox tiddler="$:/plugins/hoelzro/full-text-search/EnableFuzzySearching" field="text" checked="yes" unchecked="" default=""> 70 | Enable wildcard/fuzzy searches? (this will require an index rebuild) 71 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /demo/A_fox_in_the_garden.tid: -------------------------------------------------------------------------------- 1 | created: 20181007160452834 2 | modified: 20181007160456586 3 | tags: 4 | title: A fox in the garden 5 | 6 | -------------------------------------------------------------------------------- /demo/Examples_Fuzzy_Searching.tid: -------------------------------------------------------------------------------- 1 | caption: Fuzzy Searches (experimental) 2 | created: 20181125020408306 3 | modified: 20181125021125541 4 | tags: [[Query Examples]] 5 | title: Examples/Fuzzy Searching 6 | 7 | Fuzzy searches give your queries a little wiggle room; they allow you to specify an //edit distance// between word in your query and the words it matches in your wiki. This is chiefly used to work around mispellings. You create a fuzzy search by adding a tilde and a numeric edit distance after the word you'd like to make fuzzy. 8 | 9 | <$reveal type="match" text="uninitialized" state="$:/temp/FTS-state"> 10 | 11 | In order to do anything, you'll need to build the index first: 12 | 13 | <$button> 14 | Click here to generate the index 15 | <$fts-action-generate-index /> 16 | 17 | 18 | 19 | <$reveal type="match" text="initialized" state="$:/temp/FTS-state"> 20 | <$reveal type="match" text="yes" state="$:/plugins/hoelzro/full-text-search/EnableFuzzySearching"> 21 | 22 | Here's the results looking for `formattign~1`: 23 | 24 | <> 25 | 26 | Notice how even though we misspelled "formatting", `Wiping a harddrive` still came up in our results! Fuzzy searches are powerful, but use them wisely - if you specify two large of an edit distance, it can bring your wiki to a halt. They are also experimental, so please use them with a grain of salt and [[report any bugs|https://github.com/hoelzro/tw-full-text-search/issues]] you find. 27 | 28 | 29 | 30 | <$reveal type="nomatch" text="yes" state="$:/plugins/hoelzro/full-text-search/EnableFuzzySearching"> 31 | 32 | Fuzzy searches aren't enabled; in order to try them out, you'll need to toggle this experimental setting, either here or in the control panel. Afterwards, you'll need to rebuild the index. 33 | 34 | <$checkbox tiddler="$:/plugins/hoelzro/full-text-search/EnableFuzzySearching" field="text" checked="yes" unchecked="" default=""> 35 | Enable fuzzy searches 36 | 37 | 38 | -------------------------------------------------------------------------------- /demo/Examples_Relevance.tid: -------------------------------------------------------------------------------- 1 | caption: Relevance 2 | created: 20181007155154524 3 | modified: 20181007160844128 4 | tags: [[Query Examples]] 5 | title: Examples/Relevance 6 | 7 | Now here are the results for `fox`: 8 | 9 | <> 10 | 11 | Note how here [[Foxes foxes foxes]] appears first in the result list, even though alphabetically it follows [[A fox in the garden]]. This is because the FTS plugin orders search results by relevance by default. The way relevance is calculated is beyond the scope of these examples, but it's roughly equivalent to the number of times a token occurs in a tiddler. 12 | -------------------------------------------------------------------------------- /demo/Examples_Simple_Stemming.tid: -------------------------------------------------------------------------------- 1 | caption: Simple Stemming 2 | created: 20181007154232198 3 | modified: 20181007154857758 4 | tags: [[Query Examples]] 5 | title: Examples/Simple Stemming 6 | 7 | Here's the results looking for `jumps`: 8 | 9 | <> 10 | 11 | Note how [[The quick fox jumped over the lazy dog]] is in the search results, even though "jumps" is itself not present - this is because the FTS plugin uses a process known as //stemming// to reduce both "jumped" and "jumps" to a root form. 12 | -------------------------------------------------------------------------------- /demo/Examples_Synonyms.tid: -------------------------------------------------------------------------------- 1 | caption: Synonyms (experimental) 2 | created: 20181007160916290 3 | modified: 20181007162846487 4 | tags: [[Query Examples]] 5 | title: Examples/Synonyms 6 | 7 | Sometimes two words have the same or similar meaning; for example, a //vixen// is a specifc word that means "female fox". Occasionally we want our search engines to treat certain sets of words as equivalent - the FTS plugin enables you to do that! 8 | 9 | Here's what the results look like for `vixen`: 10 | 11 | <> 12 | 13 | What's interesting here is that //none// of the tiddlers here have the word "vixen" in them - only fox! The FTS plugin allows you to define synonyms in the [[$:/plugins/hoelzro/full-text-search/RelatedTerms.json]] tiddler - the format is a data tiddler with a list of strings, each of which is a ~TiddlyWiki list. Here are the current contents that define "fox" and "vixen" as synonyms: 14 | 15 | {{ $:/plugins/hoelzro/full-text-search/RelatedTerms.json }} 16 | 17 | If you want to define a multi-word synonym, you'll use the `[[...]]` syntax: 18 | 19 | ``` 20 | ["FTS [[Full Text Search]]"] 21 | ``` 22 | 23 | Since this feature is newer and experimental, there's no fancy UI for editing this tiddler - yet! If you //do// edit it, you'll need to rebuild your FTS index from the control panel. 24 | -------------------------------------------------------------------------------- /demo/Examples_Wildcard_Searches.tid: -------------------------------------------------------------------------------- 1 | caption: Wildcard (experimental) 2 | created: 20181124234340089 3 | modified: 20181125021112605 4 | tags: [[Query Examples]] 5 | title: Examples/Wildcard Searches 6 | 7 | Wildcards are an experimental search feature that, when enabled in the settings, take a little more computing power and memory, but can provide a lot of power. The way to use wildcards is to insert an asterisk (`*`) character into your query - this character stands in for "0 or more of any character". For example, the query `*oo` would match any documents that contain a word that ends in "oo". The full text search plugin uses lunr.js for its implementation, so you can naturally read more about wildcards in the [[lunr documentation|https://lunrjs.com/guides/searching.html#wildcards]]. 8 | 9 | <$reveal type="match" text="uninitialized" state="$:/temp/FTS-state"> 10 | 11 | In order to do anything, you'll need to build the index first: 12 | 13 | <$button> 14 | Click here to generate the index 15 | <$fts-action-generate-index /> 16 | 17 | 18 | 19 | <$reveal type="match" text="initialized" state="$:/temp/FTS-state"> 20 | <$reveal type="match" text="yes" state="$:/plugins/hoelzro/full-text-search/EnableFuzzySearching"> 21 | 22 | Here's the results looking for `format*ing`: 23 | 24 | <> 25 | 26 | The tiddler `Wiping a harddrive` contains the word "formatting", which matches our wildcard query. If it said "formating" instead, it would also match, because the `*` in the query matches any number of characters - this means that even words like "formattering" would match that query! Since wildcards are experimental, please [[report any bugs|https://github.com/hoelzro/tw-full-text-search/issues]] you find with them! 27 | 28 | 29 | 30 | <$reveal type="nomatch" text="yes" state="$:/plugins/hoelzro/full-text-search/EnableFuzzySearching"> 31 | 32 | Wildcards aren't enabled; in order to try them out, you'll need to toggle this experimental setting, either here or in the control panel. Afterwards, you'll need to rebuild the index. 33 | 34 | <$checkbox tiddler="$:/plugins/hoelzro/full-text-search/EnableFuzzySearching" field="text" checked="yes" unchecked="" default=""> 35 | Enable wildcard searches 36 | 37 | 38 | -------------------------------------------------------------------------------- /demo/Foxes_foxes_foxes.tid: -------------------------------------------------------------------------------- 1 | created: 20181007160526376 2 | modified: 20181007160530317 3 | tags: 4 | title: Foxes foxes foxes 5 | 6 | -------------------------------------------------------------------------------- /demo/Introduction.tid: -------------------------------------------------------------------------------- 1 | created: 20181007164006666 2 | modified: 20181007171028534 3 | tags: 4 | title: Introduction 5 | 6 | ! What is This? 7 | 8 | This plugin adds full text search capabilities to ~TiddlyWiki, which provides features like language awareness (ex. treating jump, jumps, jumping, and jumped as different forms of the same word) and ordering results by search relevance, meaning the first result is going to be the one that the search engine figures as the closest match to what you're looking for. 9 | 10 | ! Usage 11 | 12 | For examples of the plugin in action, have a look at [[Query Examples]]. 13 | 14 | Since this plugin uses lunr.js under the hood to work its magic, you may find [[lunr's documentation on searching|https://lunrjs.com/guides/searching.html]] useful in crafting more advanced queries. 15 | 16 | ! Installation 17 | 18 | To install, drag and drop these plugins into your wiki: 19 | 20 | <$list filter="$:/plugins/hoelzro/progress-bar $:/plugins/hoelzro/full-text-search"> 21 | <$set name="plugin-type" value={{!!plugin-type}}> 22 | <$set name="default-popup-state" value="no"> 23 | <$set name="qualified-state" value=<>> 24 | {{||$:/core/ui/Components/plugin-info}} 25 | 26 | 27 | 28 | 29 | 30 | ! Using FTS machinery in your own ~TiddlyWiki creations 31 | 32 | This plugin provides a filter operator named `ftsearch` which you can use in conjunction with other filter operators; it works exactly like ~TiddlyWiki's `search` operator, only you can't specify which field to search on - it always searches on title, tags, and text. 33 | 34 | There's also a filter operator named `ftsfeedback`, which allows the `ftsearch` filter operator to gather feedback into a tiddler to show the user. This is currently used to let the user know if they're using features not supported by their configuration. Here's an example of how to use the two in concert: 35 | 36 | ``` 37 | <$set name="ftsFeedback" value=<> > 38 | <$list filter="[ftsearchftsfeedback]"> 39 | 40 | 41 | <> is a data tiddler containing feedback messages from the FTS machinery. 42 | 43 | ``` 44 | 45 | Because of how `ftsearch` passes information to `ftsfeedback`, you may notice that queries not supported by your configuration return a list with a single blank item - you can use `ftsfeedback` to handle that situation. 46 | 47 | ! Reporting Bugs & Source Code 48 | 49 | The source code for this plugin is available on [[GitHub|https://github.com/hoelzro/tw-full-text-search]], and ~GitHub is also where the issue tracker for this plugin lives. 50 | 51 | ! Credits 52 | 53 | This plugin is powered by [[lunr.js|https://lunrjs.com]]. 54 | -------------------------------------------------------------------------------- /demo/Query_Examples.tid: -------------------------------------------------------------------------------- 1 | created: 20181007153959728 2 | modified: 20181007162358907 3 | tags: 4 | title: Query Examples 5 | list: [[Examples/Simple Stemming]] Examples/Relevance Examples/Synonyms [[Examples/Wildcard Searches]] [[Examples/Fuzzy Searching]] 6 | 7 | \define reveal-example(title) 8 | <$set name="revealState" value=<>> 9 |

10 | <$reveal type="nomatch" state=<> text="yes"> 11 | <$button class="tc-btn-invisible tc-tiddlylink" set=<> setTo="yes"> 12 | {{$:/core/images/right-arrow}} 13 | 14 | 15 | <$reveal type="match" state=<> text="yes"> 16 | <$button class="tc-btn-invisible tc-tiddlylink" set=<> setTo="no"> 17 | {{$:/core/images/down-arrow}} 18 | 19 | 20 | {{$title$!!caption}} 21 |

22 | 23 | <$reveal type="match" state=<> text="yes"> 24 | <$transclude tiddler="$title$" mode="block" /> 25 | 26 | 27 | \end 28 | 29 | <$set name="state" value="$:/temp/FTS-state"> 30 | <$reveal type="nomatch" state=<> text="initialized"> 31 | Before we look at some examples, we need to initialize the search index: 32 | 33 | <$button> 34 | Click here to generate the index 35 | <$fts-action-generate-index /> 36 | 37 | 38 | 39 | <$reveal type="match" state=<> text="initialized"> 40 | Here are some examples of queries: 41 | 42 | <$list filter="[tag]"> 43 | <$macrocall $name="reveal-example" title=<> /> 44 | 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /demo/The_quick_fox_jumped_over_the_lazy_dog.tid: -------------------------------------------------------------------------------- 1 | created: 20181007154316922 2 | modified: 20181007154324052 3 | tags: 4 | title: The quick fox jumped over the lazy dog 5 | 6 | -------------------------------------------------------------------------------- /demo/Wiping_a_harddrive.tid: -------------------------------------------------------------------------------- 1 | created: 20181125002738701 2 | modified: 20181125002813922 3 | tags: 4 | title: Wiping a harddrive 5 | 6 | Wiping a hardrrive is sometimes known as //formatting// -------------------------------------------------------------------------------- /demo/___DefaultTiddlers.tid: -------------------------------------------------------------------------------- 1 | created: 20181007171004045 2 | modified: 20181007171005170 3 | title: $:/DefaultTiddlers 4 | 5 | Introduction 6 | -------------------------------------------------------------------------------- /demo/___Ribbon.tid: -------------------------------------------------------------------------------- 1 | created: 20181007163423946 2 | modified: 20181007163855880 3 | tags: $:/tags/PageTemplate 4 | title: $:/Ribbon 5 | 6 |
Fork me on ~GitHub
7 |
-------------------------------------------------------------------------------- /demo/___SiteSubtitle.tid: -------------------------------------------------------------------------------- 1 | created: 20181007170950170 2 | modified: 20181007171236803 3 | title: $:/SiteSubtitle 4 | 5 | Add full text search capabilities to ~TiddlyWiki -------------------------------------------------------------------------------- /demo/___SiteTitle.tid: -------------------------------------------------------------------------------- 1 | created: 20181007170943988 2 | modified: 20181007170946575 3 | title: $:/SiteTitle 4 | 5 | Full Text Search -------------------------------------------------------------------------------- /demo/___plugins_hoelzro_full-text-search_RelatedTerms.json.tid: -------------------------------------------------------------------------------- 1 | created: 20180106190218243 2 | modified: 20181007161250209 3 | title: $:/plugins/hoelzro/full-text-search/RelatedTerms.json 4 | type: application/json 5 | 6 | ["fox vixen"] -------------------------------------------------------------------------------- /demo/___plugins_hoelzro_full-text-search_use-cache.tid: -------------------------------------------------------------------------------- 1 | created: 20181008022742368 2 | modified: 20181008022756830 3 | tags: 4 | title: $:/plugins/hoelzro/full-text-search/use-cache 5 | 6 | no -------------------------------------------------------------------------------- /files/.gitignore: -------------------------------------------------------------------------------- 1 | !*.js 2 | -------------------------------------------------------------------------------- /files/localforage.min.js: -------------------------------------------------------------------------------- 1 | /*! 2 | localForage -- Offline Storage, Improved 3 | Version 1.5.0 4 | https://localforage.github.io/localForage 5 | (c) 2013-2017 Mozilla, Apache License 2.0 6 | */ 7 | !function(a){if("object"==typeof exports&&"undefined"!=typeof module)module.exports=a();else if("function"==typeof define&&define.amd)define([],a);else{var b;b="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof self?self:this,b.localforage=a()}}(function(){return function a(b,c,d){function e(g,h){if(!c[g]){if(!b[g]){var i="function"==typeof require&&require;if(!h&&i)return i(g,!0);if(f)return f(g,!0);var j=new Error("Cannot find module '"+g+"'");throw j.code="MODULE_NOT_FOUND",j}var k=c[g]={exports:{}};b[g][0].call(k.exports,function(a){var c=b[g][1][a];return e(c?c:a)},k,k.exports,a,b,c,d)}return c[g].exports}for(var f="function"==typeof require&&require,g=0;g=43)}}).catch(function(){return!1})}function n(a){return"boolean"==typeof ha?ja.resolve(ha):m(a).then(function(a){return ha=a})}function o(a){var b=ia[a.name],c={};c.promise=new ja(function(a){c.resolve=a}),b.deferredOperations.push(c),b.dbReady?b.dbReady=b.dbReady.then(function(){return c.promise}):b.dbReady=c.promise}function p(a){var b=ia[a.name],c=b.deferredOperations.pop();c&&c.resolve()}function q(a,b){return new ja(function(c,d){if(a.db){if(!b)return c(a.db);o(a),a.db.close()}var e=[a.name];b&&e.push(a.version);var f=ga.open.apply(ga,e);b&&(f.onupgradeneeded=function(b){var c=f.result;try{c.createObjectStore(a.storeName),b.oldVersion<=1&&c.createObjectStore(ka)}catch(c){if("ConstraintError"!==c.name)throw c;console.warn('The database "'+a.name+'" has been upgraded from version '+b.oldVersion+" to version "+b.newVersion+', but the storage "'+a.storeName+'" already exists.')}}),f.onerror=function(a){a.preventDefault(),d(f.error)},f.onsuccess=function(){c(f.result),p(a)}})}function r(a){return q(a,!1)}function s(a){return q(a,!0)}function t(a,b){if(!a.db)return!0;var c=!a.db.objectStoreNames.contains(a.storeName),d=a.versiona.db.version;if(d&&(a.version!==b&&console.warn('The database "'+a.name+"\" can't be downgraded from version "+a.db.version+" to version "+a.version+"."),a.version=a.db.version),e||c){if(c){var f=a.db.version+1;f>a.version&&(a.version=f)}return!0}return!1}function u(a){return new ja(function(b,c){var d=new FileReader;d.onerror=c,d.onloadend=function(c){var d=btoa(c.target.result||"");b({__local_forage_encoded_blob:!0,data:d,type:a.type})},d.readAsBinaryString(a)})}function v(a){var b=l(atob(a.data));return i([b],{type:a.type})}function w(a){return a&&a.__local_forage_encoded_blob}function x(a){var b=this,c=b._initReady().then(function(){var a=ia[b._dbInfo.name];if(a&&a.dbReady)return a.dbReady});return k(c,a,a),c}function y(a){function b(){return ja.resolve()}var c=this,d={db:null};if(a)for(var e in a)d[e]=a[e];ia||(ia={});var f=ia[d.name];f||(f={forages:[],db:null,dbReady:null,deferredOperations:[]},ia[d.name]=f),f.forages.push(c),c._initReady||(c._initReady=c.ready,c.ready=x);for(var g=[],h=0;h>4,k[i++]=(15&d)<<4|e>>2,k[i++]=(3&e)<<6|63&f;return j}function I(a){var b,c=new Uint8Array(a),d="";for(b=0;b>2],d+=na[(3&c[b])<<4|c[b+1]>>4],d+=na[(15&c[b+1])<<2|c[b+2]>>6],d+=na[63&c[b+2]];return c.length%3===2?d=d.substring(0,d.length-1)+"=":c.length%3===1&&(d=d.substring(0,d.length-2)+"=="),d}function J(a,b){var c="";if(a&&(c=Ea.call(a)),a&&("[object ArrayBuffer]"===c||a.buffer&&"[object ArrayBuffer]"===Ea.call(a.buffer))){var d,e=qa;a instanceof ArrayBuffer?(d=a,e+=sa):(d=a.buffer,"[object Int8Array]"===c?e+=ua:"[object Uint8Array]"===c?e+=va:"[object Uint8ClampedArray]"===c?e+=wa:"[object Int16Array]"===c?e+=xa:"[object Uint16Array]"===c?e+=za:"[object Int32Array]"===c?e+=ya:"[object Uint32Array]"===c?e+=Aa:"[object Float32Array]"===c?e+=Ba:"[object Float64Array]"===c?e+=Ca:b(new Error("Failed to get type for BinaryArray"))),b(e+I(d))}else if("[object Blob]"===c){var f=new FileReader;f.onload=function(){var c=oa+a.type+"~"+I(this.result);b(qa+ta+c)},f.readAsArrayBuffer(a)}else try{b(JSON.stringify(a))}catch(c){console.error("Couldn't convert value into a JSON string: ",a),b(null,c)}}function K(a){if(a.substring(0,ra)!==qa)return JSON.parse(a);var b,c=a.substring(Da),d=a.substring(ra,Da);if(d===ta&&pa.test(c)){var e=c.match(pa);b=e[1],c=c.substring(e[0].length)}var f=H(c);switch(d){case sa:return f;case ta:return i([f],{type:b});case ua:return new Int8Array(f);case va:return new Uint8Array(f);case wa:return new Uint8ClampedArray(f);case xa:return new Int16Array(f);case za:return new Uint16Array(f);case ya:return new Int32Array(f);case Aa:return new Uint32Array(f);case Ba:return new Float32Array(f);case Ca:return new Float64Array(f);default:throw new Error("Unkown type: "+d)}}function L(a){var b=this,c={db:null};if(a)for(var d in a)c[d]="string"!=typeof a[d]?a[d].toString():a[d];var e=new ja(function(a,d){try{c.db=openDatabase(c.name,String(c.version),c.description,c.size)}catch(a){return d(a)}c.db.transaction(function(e){e.executeSql("CREATE TABLE IF NOT EXISTS "+c.storeName+" (id INTEGER PRIMARY KEY, key unique, value)",[],function(){b._dbInfo=c,a()},function(a,b){d(b)})})});return c.serializer=Fa,e}function M(a,b){var c=this;"string"!=typeof a&&(console.warn(a+" used as a key, but it is not a string."),a=String(a));var d=new ja(function(b,d){c.ready().then(function(){var e=c._dbInfo;e.db.transaction(function(c){c.executeSql("SELECT * FROM "+e.storeName+" WHERE key = ? LIMIT 1",[a],function(a,c){var d=c.rows.length?c.rows.item(0).value:null;d&&(d=e.serializer.deserialize(d)),b(d)},function(a,b){d(b)})})}).catch(d)});return j(d,b),d}function N(a,b){var c=this,d=new ja(function(b,d){c.ready().then(function(){var e=c._dbInfo;e.db.transaction(function(c){c.executeSql("SELECT * FROM "+e.storeName,[],function(c,d){for(var f=d.rows,g=f.length,h=0;h0)return void f(O.apply(e,[a,h,c,d-1]));g(b)}})})}).catch(g)});return j(f,c),f}function P(a,b,c){return O.apply(this,[a,b,c,1])}function Q(a,b){var c=this;"string"!=typeof a&&(console.warn(a+" used as a key, but it is not a string."),a=String(a));var d=new ja(function(b,d){c.ready().then(function(){var e=c._dbInfo;e.db.transaction(function(c){c.executeSql("DELETE FROM "+e.storeName+" WHERE key = ?",[a],function(){b()},function(a,b){d(b)})})}).catch(d)});return j(d,b),d}function R(a){var b=this,c=new ja(function(a,c){b.ready().then(function(){var d=b._dbInfo;d.db.transaction(function(b){b.executeSql("DELETE FROM "+d.storeName,[],function(){a()},function(a,b){c(b)})})}).catch(c)});return j(c,a),c}function S(a){var b=this,c=new ja(function(a,c){b.ready().then(function(){var d=b._dbInfo;d.db.transaction(function(b){b.executeSql("SELECT COUNT(key) as c FROM "+d.storeName,[],function(b,c){var d=c.rows.item(0).c;a(d)},function(a,b){c(b)})})}).catch(c)});return j(c,a),c}function T(a,b){var c=this,d=new ja(function(b,d){c.ready().then(function(){var e=c._dbInfo;e.db.transaction(function(c){c.executeSql("SELECT key FROM "+e.storeName+" WHERE id = ? LIMIT 1",[a+1],function(a,c){var d=c.rows.length?c.rows.item(0).key:null;b(d)},function(a,b){d(b)})})}).catch(d)});return j(d,b),d}function U(a){var b=this,c=new ja(function(a,c){b.ready().then(function(){var d=b._dbInfo;d.db.transaction(function(b){b.executeSql("SELECT key FROM "+d.storeName,[],function(b,c){for(var d=[],e=0;e=0;c--){var d=localStorage.key(c);0===d.indexOf(a)&&localStorage.removeItem(d)}});return j(c,a),c}function X(a,b){var c=this;"string"!=typeof a&&(console.warn(a+" used as a key, but it is not a string."),a=String(a));var d=c.ready().then(function(){var b=c._dbInfo,d=localStorage.getItem(b.keyPrefix+a);return d&&(d=b.serializer.deserialize(d)),d});return j(d,b),d}function Y(a,b){var c=this,d=c.ready().then(function(){for(var b=c._dbInfo,d=b.keyPrefix,e=d.length,f=localStorage.length,g=1,h=0;h 6 | * 7 | * Permission is hereby granted, free of charge, to any person obtaining a copy 8 | * of this software and associated documentation files (the "Software"), to 9 | * deal in the Software without restriction, including without limitation the 10 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 11 | * sell copies of the Software, and to permit persons to whom the Software is 12 | * furnished to do so, subject to the following conditions: 13 | * 14 | * The above copyright notice and this permission notice shall be included in 15 | * all copies or substantial portions of the Software. 16 | * 17 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 22 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 23 | * IN THE SOFTWARE. 24 | */ 25 | 26 | var MutableBuilder = function () { 27 | lunr.Builder.call(this) 28 | } 29 | 30 | MutableBuilder.prototype = new lunr.Builder() 31 | 32 | MutableBuilder.prototype.build = function build () { 33 | this.calculateAverageFieldLengths() 34 | this.createFieldVectors() 35 | this.createTokenSet() 36 | 37 | return new MutableIndex({ 38 | invertedIndex: this.invertedIndex, 39 | fieldVectors: this.fieldVectors, 40 | tokenSet: this.tokenSet, 41 | fields: Object.keys(this._fields), 42 | pipeline: this.searchPipeline, 43 | builder: this 44 | }) 45 | } 46 | 47 | MutableBuilder.prototype.remove = function remove (doc) { 48 | var docRef = doc[this._ref] 49 | var fields = Object.keys(this._fields) 50 | 51 | var isDirty = false 52 | 53 | for (var i = 0; i < fields.length; i++) { 54 | var fieldName = fields[i], 55 | fieldRef = new lunr.FieldRef (docRef, fieldName) 56 | 57 | if (fieldRef in this.fieldTermFrequencies || fieldRef in this.fieldLengths) { 58 | isDirty = true 59 | } 60 | 61 | delete this.fieldTermFrequencies[fieldRef] 62 | delete this.fieldLengths[fieldRef] 63 | } 64 | 65 | if (!isDirty) { 66 | return 67 | } 68 | 69 | this.documentCount -= 1 70 | 71 | // XXX what if a term disappears from the index? 72 | for (var term in this.invertedIndex) { 73 | for (var fieldName in this.invertedIndex[term]) { // XXX what about "_index"? 74 | delete this.invertedIndex[term][fieldName][docRef] 75 | } 76 | } 77 | } 78 | 79 | MutableBuilder.prototype.toJSON = function toJSON () { 80 | var fieldRefs = [] 81 | var fieldTermFrequencies = [] 82 | var fieldLengths = [] 83 | 84 | for (var fieldRef in this.fieldTermFrequencies) { 85 | if (this.fieldTermFrequencies.hasOwnProperty(fieldRef)) { 86 | fieldRefs.push(fieldRef) 87 | fieldTermFrequencies.push(this.fieldTermFrequencies[fieldRef]) 88 | fieldLengths.push(this.fieldLengths[fieldRef]) 89 | } 90 | } 91 | 92 | // XXX omit tokenizer for now 93 | // some properties (invertedIndex, searchPipeline) are omitted 94 | // from here because they're on the index, and serializing them twice 95 | // would be redundant 96 | return { 97 | _ref: this._ref, 98 | _fields: this._fields, 99 | _documents: this._documents, 100 | fieldRefs: fieldRefs, 101 | fieldTermFrequencies: fieldTermFrequencies, 102 | fieldLengths: fieldLengths, 103 | pipeline: this.pipeline.toJSON(), 104 | documentCount: this.documentCount, 105 | _b: this._b, // XXX special (due to precision)? 106 | _k1: this._k1, // XXX special (due to precision)? 107 | termIndex: this.termIndex, 108 | metadataWhitelist: this.metadataWhitelist 109 | } 110 | } 111 | 112 | MutableBuilder.load = function load (serializedBuilder) { 113 | var builder = new MutableBuilder() 114 | 115 | for (var k in serializedBuilder) { 116 | if (serializedBuilder.hasOwnProperty(k)) { 117 | builder[k] = serializedBuilder[k] 118 | if(k == '_fields' || k == '_documents') { 119 | var noProtoObject = Object.create(null) 120 | for(var innerK in builder[k]) { 121 | noProtoObject[innerK] = builder[k][innerK] 122 | } 123 | builder[k] = noProtoObject 124 | } 125 | } 126 | } 127 | 128 | var fieldRefs = builder.fieldRefs 129 | var fieldTermFrequencies = builder.fieldTermFrequencies 130 | var fieldLengths = builder.fieldLengths 131 | delete builder.fieldRefs 132 | 133 | builder.fieldTermFrequencies = {} 134 | builder.fieldLengths = {} 135 | 136 | for (var i = 0; i < fieldRefs.length; i++) { 137 | var fieldRef = fieldRefs[i] 138 | builder.fieldTermFrequencies[fieldRef] = fieldTermFrequencies[i] 139 | builder.fieldLengths[fieldRef] = fieldLengths[i] 140 | } 141 | 142 | // builder.tokenizer is initialized to the default by the MutableBuilder 143 | // constructor 144 | builder.pipeline = lunr.Pipeline.load(builder.pipeline) 145 | 146 | return builder 147 | } 148 | /* 149 | * Copyright 2018 Rob Hoelz 150 | * 151 | * Permission is hereby granted, free of charge, to any person obtaining a copy 152 | * of this software and associated documentation files (the "Software"), to 153 | * deal in the Software without restriction, including without limitation the 154 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 155 | * sell copies of the Software, and to permit persons to whom the Software is 156 | * furnished to do so, subject to the following conditions: 157 | * 158 | * The above copyright notice and this permission notice shall be included in 159 | * all copies or substantial portions of the Software. 160 | * 161 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 162 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 163 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 164 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 165 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 166 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 167 | * IN THE SOFTWARE. 168 | */ 169 | 170 | var MutableIndex = function (attrs) { 171 | lunr.Index.call(this, attrs) 172 | this.builder = attrs.builder 173 | this._dirty = false 174 | } 175 | 176 | MutableIndex.prototype = new lunr.Index({}) 177 | 178 | MutableIndex.prototype.add = function add (doc) { 179 | this.builder.add(doc) 180 | this._dirty = true 181 | } 182 | 183 | MutableIndex.prototype.update = function update (doc) { 184 | this.remove(doc) 185 | this.add(doc) 186 | } 187 | 188 | MutableIndex.prototype.remove = function remove (doc) { 189 | this.builder.remove(doc) 190 | this._dirty = true 191 | } 192 | 193 | // XXX rebuilds the entire index =( 194 | // XXX refreshing this from newIndex is kinda wonky =( 195 | MutableIndex.prototype.checkDirty = function checkDirty () { 196 | if (this._dirty) { 197 | this._dirty = false 198 | var newIndex = this.builder.build() 199 | for (var k in newIndex) { 200 | if (newIndex.hasOwnProperty(k)) { 201 | this[k] = newIndex[k] 202 | } 203 | } 204 | } 205 | } 206 | 207 | MutableIndex.prototype.toJSON = function toJSON () { 208 | this.checkDirty() 209 | 210 | // XXX do you need to serialize things that we could calculate post-load via builder.build? 211 | var json = lunr.Index.prototype.toJSON.call(this) 212 | json.builder = this.builder.toJSON() 213 | return json 214 | } 215 | 216 | MutableIndex.load = function load (serializedIndex) { 217 | var index = lunr.Index.load(serializedIndex) 218 | var mutableIndex = new MutableIndex({}) 219 | 220 | for (var k in index) { 221 | if (index.hasOwnProperty(k)) { 222 | mutableIndex[k] = index[k] 223 | } 224 | } 225 | 226 | mutableIndex.builder = MutableBuilder.load(serializedIndex.builder) 227 | mutableIndex.builder.invertedIndex = mutableIndex.invertedIndex 228 | mutableIndex.builder.searchPipeline = mutableIndex.pipeline 229 | mutableIndex.dirty = false 230 | 231 | return mutableIndex 232 | } 233 | 234 | MutableIndex.prototype.query = function query (fn) { 235 | this.checkDirty() 236 | 237 | return lunr.Index.prototype.query.call(this, fn) 238 | } 239 | /* 240 | * Copyright 2018 Rob Hoelz 241 | * 242 | * Permission is hereby granted, free of charge, to any person obtaining a copy 243 | * of this software and associated documentation files (the "Software"), to 244 | * deal in the Software without restriction, including without limitation the 245 | * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 246 | * sell copies of the Software, and to permit persons to whom the Software is 247 | * furnished to do so, subject to the following conditions: 248 | * 249 | * The above copyright notice and this permission notice shall be included in 250 | * all copies or substantial portions of the Software. 251 | * 252 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 253 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 254 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 255 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 256 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 257 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 258 | * IN THE SOFTWARE. 259 | */ 260 | 261 | /** 262 | * A convenience function for configuring and constructing 263 | * a new mutable lunr Index. 264 | * 265 | * A lunr.MutableBuilder instance is created and the pipeline setup 266 | * with a trimmer, stop word filter and stemmer. 267 | * 268 | * This mutable builder object is yielded to the configuration function 269 | * that is passed as a parameter, allowing the list of fields 270 | * and other builder parameters to be customised. 271 | * 272 | * All documents _must_ be added within the passed config function, but 273 | * you can always update the index later. ;) 274 | * 275 | * @example 276 | * var idx = lunrMutable(function () { 277 | * this.field('title') 278 | * this.field('body') 279 | * this.ref('id') 280 | * 281 | * documents.forEach(function (doc) { 282 | * this.add(doc) 283 | * }, this) 284 | * }) 285 | * 286 | * index.add({ 287 | * "title": "new title", 288 | * "body": "new body", 289 | * "id": "2" 290 | * }) 291 | * 292 | * index.remove({ id: "1" }); 293 | * 294 | * index.update({ 295 | * "body": "change", 296 | * "id": "2" 297 | * }) 298 | */ 299 | 300 | var lunrMutable = function (config) { 301 | var builder = new MutableBuilder(); 302 | 303 | builder.pipeline.add( 304 | lunr.trimmer, 305 | lunr.stopWordFilter, 306 | lunr.stemmer 307 | ) 308 | 309 | builder.searchPipeline.add( 310 | lunr.stemmer 311 | ) 312 | 313 | config.call(builder, builder) 314 | return builder.build() 315 | } 316 | 317 | lunrMutable.version = "2.3.2" 318 | 319 | lunrMutable.Builder = MutableBuilder 320 | lunrMutable.Index = MutableIndex 321 | /** 322 | * export the module via AMD, CommonJS or as a browser global 323 | * Export code from https://github.com/umdjs/umd/blob/master/returnExports.js 324 | */ 325 | ;(function (root, factory) { 326 | if (typeof define === 'function' && define.amd) { 327 | // AMD. Register as an anonymous module. 328 | define(factory) 329 | } else if (typeof exports === 'object') { 330 | /** 331 | * Node. Does not work with strict CommonJS, but 332 | * only CommonJS-like enviroments that support module.exports, 333 | * like Node. 334 | */ 335 | module.exports = factory() 336 | } else { 337 | // Browser globals (root is window) 338 | root.lunr = factory() 339 | } 340 | }(this, function () { 341 | /** 342 | * Just return a value to define the module export. 343 | * This example returns an object, but the module 344 | * can return a function as the exported value. 345 | */ 346 | return lunrMutable 347 | })) 348 | })(); 349 | -------------------------------------------------------------------------------- /files/lunr.min.js: -------------------------------------------------------------------------------- 1 | /** 2 | * lunr - http://lunrjs.com - A bit like Solr, but much smaller and not as bright - 2.3.5 3 | * Copyright (C) 2018 Oliver Nightingale 4 | * @license MIT 5 | */ 6 | !function(){var t,l,c,e,r,h,d,f,p,y,m,g,x,v,w,Q,k,S,E,L,b,P,T,O,I,i,n,s,z=function(e){var t=new z.Builder;return t.pipeline.add(z.trimmer,z.stopWordFilter,z.stemmer),t.searchPipeline.add(z.stemmer),e.call(t,t),t.build()};z.version="2.3.5",z.utils={},z.utils.warn=(t=this,function(e){t.console&&console.warn&&console.warn(e)}),z.utils.asString=function(e){return null==e?"":e.toString()},z.utils.clone=function(e){if(null==e)return e;for(var t=Object.create(null),r=Object.keys(e),i=0;i=this.length)return z.QueryLexer.EOS;var e=this.str.charAt(this.pos);return this.pos+=1,e},z.QueryLexer.prototype.width=function(){return this.pos-this.start},z.QueryLexer.prototype.ignore=function(){this.start==this.pos&&(this.pos+=1),this.start=this.pos},z.QueryLexer.prototype.backup=function(){this.pos-=1},z.QueryLexer.prototype.acceptDigitRun=function(){for(var e,t;47<(t=(e=this.next()).charCodeAt(0))&&t<58;);e!=z.QueryLexer.EOS&&this.backup()},z.QueryLexer.prototype.more=function(){return this.pos= 0; i--) { 72 | var title = titles[i]; 73 | var tiddler = this.wiki.getTiddler(title); 74 | if(!('modified' in tiddler.fields)) { 75 | break; 76 | } 77 | let modified = $tw.utils.stringifyDate(tiddler.fields.modified); 78 | if(modified <= cacheAge) { 79 | break; 80 | } 81 | tiddlers.push(title); 82 | } 83 | for(let i = 0; i < titles.length; i++) { 84 | var title = titles[i]; 85 | var tiddler = this.wiki.getTiddler(title); 86 | if('modified' in tiddler.fields) { 87 | break; 88 | } 89 | tiddlers.push(title); 90 | } 91 | let relatedTerms = $tw.wiki.getTiddlerDataCached(RELATED_TERMS_TIDDLER, []); 92 | relatedTerms = relatedTerms.map($tw.utils.parseStringArray); 93 | 94 | let lunr = require('$:/plugins/hoelzro/full-text-search/lunr.min.js'); 95 | let expandQuery = generateQueryExpander(lunr, relatedTerms); 96 | 97 | sharedIndex.load(cacheData); 98 | isFresh = false; 99 | } else { 100 | tiddlers = this.wiki.filterTiddlers(filter); 101 | isFresh = true; 102 | } 103 | var age = this.wiki.filterTiddlers(filter + ' +[nsort[modified]last[]get[modified]]')[0]; 104 | age = age == null ? '0' : age; 105 | var stateTiddler = this.wiki.getTiddler(STATE_TIDDLER); 106 | if(tiddlers.length > 0) { 107 | var fields = { 108 | text: 'initializing', 109 | progressCurrent: 0, 110 | progressTotal: tiddlers.length 111 | }; 112 | this.wiki.addTiddler(new $tw.Tiddler(stateTiddler, fields, this.wiki.getModificationFields())); 113 | 114 | var self = this; 115 | var lastUpdate = 0; 116 | await sharedIndex.buildIndex(this.wiki, tiddlers, isFresh, async function(progressCurrent) { 117 | if((progressCurrent - lastUpdate) >= UPDATE_FREQUENCY) { 118 | var stateTiddler = self.wiki.getTiddler(STATE_TIDDLER); 119 | self.wiki.addTiddler(new $tw.Tiddler(stateTiddler, { progressCurrent: progressCurrent }, self.wiki.getModificationFields())); 120 | lastUpdate = progressCurrent; 121 | } 122 | if(progressCurrent == tiddlers.length) { 123 | try { 124 | if(!shouldSuppressCache) { 125 | await cache.save(age, sharedIndex.getIndex().toJSON()); 126 | } 127 | } catch(e) { 128 | // failure to save the cache isn't great, but it's tolerable, so ignore it 129 | } 130 | 131 | var stateTiddler = self.wiki.getTiddler(STATE_TIDDLER); 132 | self.wiki.addTiddler(new $tw.Tiddler(stateTiddler, { text: 'initialized', progressCurrent: progressCurrent }, self.wiki.getModificationFields())); 133 | } 134 | }); 135 | } else { 136 | this.wiki.addTiddler(new $tw.Tiddler(stateTiddler, { text: 'initialized', progressCurrent: 1, progressTotal: 1 }, this.wiki.getModificationFields())); 137 | } 138 | } 139 | 140 | invokeAction(triggeringWidget, event) { 141 | this.asyncInvokeAction().then(function() { 142 | }, function(err) { 143 | console.log(err); 144 | }); 145 | } 146 | } 147 | 148 | exports['fts-action-generate-index'] = FTSActionGenerateIndexWidget; 149 | } 150 | 151 | // vim:sts=4:sw=4 152 | -------------------------------------------------------------------------------- /ftsearch.ts: -------------------------------------------------------------------------------- 1 | /*\ 2 | title: $:/plugins/hoelzro/full-text-search/ftsearch.js 3 | type: application/javascript 4 | module-type: filteroperator 5 | 6 | \*/ 7 | 8 | declare var require; 9 | 10 | module FTSearch { 11 | const FUZZY_SEARCH_TIDDLER = '$:/plugins/hoelzro/full-text-search/EnableFuzzySearching'; 12 | var lunr = require('$:/plugins/hoelzro/full-text-search/lunr.min.js'); 13 | var getIndex = require('$:/plugins/hoelzro/full-text-search/shared-index.js').getIndex; 14 | 15 | export function ftsearch(source, operator, options) { 16 | let sourceLookup = Object.create(null); 17 | source(function(tiddler, title) { 18 | sourceLookup[title] = tiddler; 19 | }); 20 | 21 | var index = getIndex(); 22 | if(!index) { 23 | return []; 24 | } 25 | 26 | return function(callback) { 27 | let results; 28 | 29 | try { 30 | let fuzzySearchesEnabled = options.wiki.getTiddlerText(FUZZY_SEARCH_TIDDLER, '') == 'yes'; 31 | if(!fuzzySearchesEnabled) { 32 | let qp = new lunr.QueryParser(operator.operand, new lunr.Query(['title', 'tags', 'text'])); 33 | let query = qp.parse(); 34 | for(let clause of query.clauses) { 35 | if(!clause.usePipeline) { 36 | // we're using a wildcard, but the index isn't prepared for 37 | // fuzzy searches - so pass information on this down the pipeline 38 | return callback(null, null, "It looks like you're trying to perform a wildcard search; you'll need to enable wildcard/fuzzy searching in the FTS settings"); 39 | } 40 | if('editDistance' in clause) { 41 | // we're using a fuzzy search, but the index isn't prepared for 42 | // fuzzy searches - so pass information on this down the pipeline 43 | return callback(null, null, "It looks like you're trying to perform a fuzzy search; you'll need to enable wildcard/fuzzy searching in the FTS settings"); 44 | } 45 | } 46 | } 47 | 48 | results = index.search(operator.operand); 49 | } catch(e) { 50 | if(e instanceof lunr.QueryParseError) { 51 | results = []; 52 | } else { 53 | throw e; 54 | } 55 | } 56 | 57 | for(let match of results) { 58 | if(match.ref in sourceLookup) { 59 | callback(sourceLookup[match.ref], match.ref); 60 | } 61 | } 62 | }; 63 | }; 64 | } 65 | 66 | export = FTSearch; 67 | 68 | // vim:sts=4:sw=4 69 | -------------------------------------------------------------------------------- /ftsfeedback.ts: -------------------------------------------------------------------------------- 1 | /*\ 2 | title: $:/plugins/hoelzro/full-text-search/ftsfeedback.js 3 | type: application/javascript 4 | module-type: filteroperator 5 | 6 | \*/ 7 | 8 | declare var require; 9 | 10 | module FTSFeedback { 11 | export function ftsfeedback(source, operator, options) { 12 | return function(callback) { 13 | let targetTiddler = operator.operand; 14 | let listOfFeedback = []; 15 | 16 | source(function(tiddler, title, feedback) { 17 | if(tiddler == null && title == null) { 18 | listOfFeedback.push(feedback); 19 | } else { 20 | callback(tiddler, title); 21 | } 22 | }); 23 | 24 | options.wiki.setTiddlerData(targetTiddler, listOfFeedback); 25 | }; 26 | } 27 | } 28 | 29 | export = FTSFeedback; 30 | -------------------------------------------------------------------------------- /history.tid: -------------------------------------------------------------------------------- 1 | title: $:/plugins/hoelzro/full-text-search/history 2 | type: text/vnd.tiddlywiki 3 | 4 | ! Release History 5 | 6 | !! 1.1.0 (2018-11-24) 7 | 8 | !!! Bug Fixes 9 | 10 | * The plugin no longer indexes draft tiddlers 11 | * Refresh query results when embedding ftsearch directly ([[GH #24|https://github.com/hoelzro/tw-full-text-search/issues/24]]) 12 | * Fixed a bug where wikis with `__proto__` in them would fail to load from cache 13 | * Fix some query relevance issues 14 | * Fix a bug where indexing didn't work on non-Node installations on Chrome 15 | 16 | !!! User-facing changes 17 | 18 | * ~TiddlyWiki 5.1.15 is now required! 19 | * Added the ability to automatically index your wiki on startup (~TiddlyWiki 5.1.16 is required for this) 20 | * Added lots of examples 21 | * Improved indexing for wildcards/fuzzy searches, but these are now hidden behind a configuration flag due the overhead incurred 22 | * Alert the user if changing their FTS settings requires an index rebuild 23 | 24 | !!! Developer changes 25 | 26 | * Updated to lunr 2.3.5 27 | * Tests now use Jasmine 3 28 | 29 | !! 1.0.3 (2018-10-07) 30 | 31 | !!! User-facing changes 32 | 33 | * Fixed advanced search result listing (GH #8) 34 | 35 | !!! Developer changes 36 | 37 | * Allow wikis to disable the cache, primarily for the demo wiki that lives on hoelz.ro (GH #11) 38 | 39 | !! 1.0.2 (2018-08-02) 40 | 41 | !!! User-facing changes 42 | 43 | * Upgraded to lunr.js 2.3.1, which includes new features like additional query operators. 44 | 45 | !! 1.0.1 (2018-06-29) 46 | 47 | !!! Bug Fixes 48 | 49 | * Fixed a bug where incomplete lunr.js queries would throw red boxes in the user's face. Thanks to Diego Mesa for reporting! 50 | 51 | !! 1.0.0 (2017-11-29) 52 | 53 | !!! User-facing changes 54 | 55 | * Enabled index creation using web workers, resulting in a 10x speedup 56 | * Fixed a bug causing images and other non-text data to get indexed 57 | * Fixed a bug causing tags to be improperly indexed 58 | 59 | !!! Developer changes 60 | 61 | * Upgraded the FTS plugin to lunr.js 2.1.4, plus some modifications of my own to enable mutable indexes 62 | * Added tests 63 | * Added an experimental feature called "query expansion" - I don't know if it'll stay around or how well it works, so there's no UI around it at the moment 64 | 65 | !! 0.0.4 (2017-06-11) 66 | 67 | Fix bug where deleted content from a tiddler would still affect search results if a 68 | partial index is loaded from the web storage cache. 69 | 70 | !! 0.0.3 (2017-06-06) 71 | 72 | Fix bug with web storage index serialization/deserialization. 73 | 74 | !! 0.0.2 (2017-06-02) 75 | 76 | Added stashing away current index in web storage to avoid repeated index creation/cut down on index creation time. 77 | 78 | !! 0.0.1 (2017-02-27) 79 | 80 | Basic full-text search functionality. 81 | -------------------------------------------------------------------------------- /hooks.ts: -------------------------------------------------------------------------------- 1 | /*\ 2 | title: $:/plugins/hoelzro/full-text-search/hooks.js 3 | type: text/vnd.tiddlywiki 4 | module-type: startup 5 | 6 | \*/ 7 | 8 | declare var $tw; 9 | 10 | module SaveTiddlerHook { 11 | const RELATED_TERMS_TIDDLER = '$:/plugins/hoelzro/full-text-search/RelatedTerms.json'; 12 | const FUZZY_SEARCH_TIDDLER = '$:/plugins/hoelzro/full-text-search/EnableFuzzySearching'; 13 | const STATE_TIDDLER = '$:/temp/FTS-state'; 14 | 15 | export function startup() { 16 | var { updateTiddler, getIndex, clearIndex } = require('$:/plugins/hoelzro/full-text-search/shared-index.js'); 17 | let cache = require('$:/plugins/hoelzro/full-text-search/cache.js'); 18 | 19 | let logger = new $tw.utils.Logger('full-text-search'); 20 | 21 | $tw.wiki.addEventListener('change', function(changes) { 22 | let index = getIndex(); 23 | 24 | let isIndexDirty = false; 25 | 26 | for(var title in changes) { 27 | if(title == RELATED_TERMS_TIDDLER || title == FUZZY_SEARCH_TIDDLER) { 28 | clearIndex(); 29 | let stateTiddler = $tw.wiki.getTiddler(STATE_TIDDLER); 30 | if(stateTiddler && stateTiddler.fields.text != 'uninitialized') { 31 | logger.alert('A configuration change occurred that has invalidated your FTS index; please rebuild the index from the control panel in order to use full text search'); 32 | } 33 | $tw.wiki.addTiddler(new $tw.Tiddler( 34 | stateTiddler, 35 | { text: 'uninitialized' }, 36 | $tw.wiki.getModificationFields())); 37 | cache.invalidate(); 38 | } 39 | 40 | if(!index) { 41 | continue; 42 | } 43 | 44 | if($tw.wiki.isSystemTiddler(title)) { 45 | continue; 46 | } 47 | 48 | var change = changes[title]; 49 | if(change.modified) { 50 | var tiddler = $tw.wiki.getTiddler(title); 51 | if(tiddler !== undefined) { 52 | let type = tiddler.fields.type || 'text/vnd.tiddlywiki'; 53 | if(!type.startsWith('text/')) { 54 | continue; 55 | } 56 | if('draft.of' in tiddler.fields) { 57 | continue; 58 | } 59 | 60 | isIndexDirty = true; 61 | 62 | updateTiddler(index, tiddler); 63 | } 64 | } else { // change.deleted 65 | isIndexDirty = true; 66 | index.remove({ title: title }); 67 | } 68 | } 69 | 70 | // Since actual changes are happening to lunr data structures outside of 71 | // TiddlyWiki, we need to tell TiddlyWiki to rerender the page and any 72 | // tiddlers whose contents may have changed due to the change in the index 73 | if(isIndexDirty) { 74 | let stateTiddler = $tw.wiki.getTiddler(STATE_TIDDLER); 75 | $tw.wiki.addTiddler(stateTiddler); 76 | } 77 | }); 78 | 79 | } 80 | } 81 | 82 | export = SaveTiddlerHook; 83 | 84 | // vim:sts=4:sw=4 85 | -------------------------------------------------------------------------------- /index-worker.ts: -------------------------------------------------------------------------------- 1 | /*\ 2 | title: $:/plugins/hoelzro/full-text-search/index-worker.js 3 | type: application/javascript 4 | module-type: library 5 | 6 | \*/ 7 | 8 | (async function() { 9 | async function getNextMessage() { 10 | return new Promise(function(resolve, reject) { 11 | onmessage = function(msg) { 12 | onmessage = function() {}; 13 | 14 | resolve(msg.data); 15 | }; 16 | }); 17 | } 18 | 19 | async function requireFromPage(name, sandbox?) : Promise { 20 | postMessage({ 21 | type: 'require', 22 | name: name, 23 | }); 24 | 25 | return getNextMessage().then(function(msg) { 26 | let mod = { exports: {} }; 27 | self['module'] = mod; 28 | self['exports'] = mod.exports; 29 | if(sandbox != null) { 30 | for(let k in sandbox) { 31 | if(sandbox.hasOwnProperty(k)) { 32 | self[k] = sandbox[k]; 33 | } 34 | } 35 | } 36 | importScripts(msg); 37 | if(sandbox != null) { 38 | for(let k in sandbox) { 39 | if(sandbox.hasOwnProperty(k)) { 40 | delete self[k]; 41 | } 42 | } 43 | } 44 | delete self['module']; 45 | delete self['exports']; 46 | 47 | return mod.exports; 48 | }); 49 | } 50 | 51 | async function getRelatedTerms() { 52 | postMessage({ 53 | type: 'getRelatedTerms' 54 | }); 55 | 56 | return await getNextMessage(); 57 | } 58 | 59 | async function getFuzzySetting() { 60 | postMessage({ 61 | type: 'getFuzzySetting' 62 | }); 63 | 64 | return await getNextMessage(); 65 | } 66 | 67 | async function* readTiddlers() { 68 | postMessage({ type: 'sendTiddlers' }); 69 | 70 | let msg = await getNextMessage(); 71 | 72 | while(msg != null) { 73 | yield JSON.parse(msg); 74 | msg = await getNextMessage(); 75 | } 76 | } 77 | 78 | let lunr : any = await requireFromPage('$:/plugins/hoelzro/full-text-search/lunr.min.js'); 79 | let lunrMutable : any = await requireFromPage('$:/plugins/hoelzro/full-text-search/lunr-mutable.js', { 80 | require: function(modName) { 81 | if(modName != '$:/plugins/hoelzro/full-text-search/lunr.min.js') { 82 | throw new Error("Invalid module name for lunr-mutable!"); 83 | } 84 | return lunr; 85 | } 86 | }); 87 | let { generateQueryExpander } = await requireFromPage('$:/plugins/hoelzro/full-text-search/query-expander.js'); 88 | let relatedTerms = await getRelatedTerms(); 89 | let fuzzySetting = await getFuzzySetting(); 90 | 91 | let expandQuery = generateQueryExpander(lunr, relatedTerms); 92 | 93 | let builder = new lunrMutable.Builder(); 94 | 95 | let stemmer; 96 | 97 | if(fuzzySetting == 'yes') { 98 | stemmer = function(unstemmedToken) { 99 | let stemmedToken = lunr.stemmer(unstemmedToken.clone()); 100 | 101 | return [ unstemmedToken, stemmedToken ]; 102 | }; 103 | 104 | lunr.Pipeline.registerFunction(stemmer, 'stemmedAndUnstemmed'); 105 | } else { 106 | stemmer = lunr.stemmer; 107 | } 108 | 109 | builder.pipeline.add( 110 | lunr.trimmer, 111 | lunr.stopWordFilter, 112 | expandQuery, 113 | 114 | stemmer 115 | ); 116 | 117 | builder.searchPipeline.add( 118 | lunr.stemmer 119 | ); 120 | 121 | let count = 0; 122 | let previousUpdate = new Date(); 123 | 124 | // XXX configurable fields? 125 | builder.field('title'); 126 | builder.field('tags'); 127 | builder.field('text'); 128 | 129 | builder.ref('title'); 130 | 131 | for await (let tiddlerFields of readTiddlers()) { 132 | // XXX duplication sucks 133 | var fields : any = { 134 | title: tiddlerFields.title 135 | }; 136 | 137 | if('text' in tiddlerFields) { 138 | fields.text = tiddlerFields.text; 139 | } 140 | 141 | if('tags' in tiddlerFields) { 142 | fields.tags = tiddlerFields.tags.join(' '); 143 | } 144 | 145 | builder.add(fields); 146 | count++; 147 | let now = new Date(); 148 | if((now.getTime() - previousUpdate.getTime()) > 200) { 149 | previousUpdate = now; 150 | postMessage({ type: 'progress', count: count }); 151 | } 152 | } 153 | 154 | postMessage({ type: 'index', index: JSON.stringify(builder.build()) }); 155 | close(); 156 | })().catch(function(err) { 157 | postMessage({ type: 'error', error: err.toString() }); 158 | }); 159 | 160 | // vim:sts=4:sw=4 161 | -------------------------------------------------------------------------------- /license.tid: -------------------------------------------------------------------------------- 1 | title: $:/plugins/hoelzro/full-text-search/license 2 | type: text/vnd.tiddlywiki 3 | 4 | ``` 5 | Copyright 2017-2018 Rob Hoelz 6 | 7 | Permission is hereby granted, free of charge, to any person obtaining a copy of 8 | this software and associated documentation files (the "Software"), to deal in 9 | the Software without restriction, including without limitation the rights to 10 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 11 | of the Software, and to permit persons to whom the Software is furnished to do 12 | so, subject to the following conditions: 13 | 14 | The above copyright notice and this permission notice shall be included in all 15 | copies or substantial portions of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | SOFTWARE. 24 | ``` 25 | -------------------------------------------------------------------------------- /plugin.info.in: -------------------------------------------------------------------------------- 1 | { 2 | "title": "$:/plugins/hoelzro/full-text-search", 3 | "description": "Full text search", 4 | "author": "hoelzro", 5 | "version": "FILLED IN BY MAKE", 6 | "core-version": ">=5.1.15", 7 | "source": "https://github.com/hoelzro/tw-full-text-search", 8 | "plugin-type": "plugin", 9 | "list": "readme license history" 10 | } 11 | -------------------------------------------------------------------------------- /query-expander.ts: -------------------------------------------------------------------------------- 1 | /*\ 2 | title: $:/plugins/hoelzro/full-text-search/query-expander.js 3 | type: application/javascript 4 | module-type: library 5 | 6 | \*/ 7 | 8 | 9 | module QueryExpander { 10 | if(! ('asyncIterator' in Symbol)) { 11 | (Symbol as any).asyncIterator = (Symbol as any).for('Symbol.asyncIterator'); 12 | } 13 | 14 | function buildAliasTree(lunr, listOfAliases) { 15 | let topTree : any = {}; 16 | 17 | for(let aliases of listOfAliases) { 18 | for(let i = 0; i < aliases.length; i++) { 19 | // XXX do you want to run the full pipeline? what if we tweak the tokenizer? 20 | let iTokens = lunr.tokenizer(aliases[i]).map(token => token.toString()); 21 | 22 | for(let j = 0; j < aliases.length; j++) { 23 | if(i == j) { 24 | continue; 25 | } 26 | 27 | let jTokens = lunr.tokenizer(aliases[j]).map(token => token.toString()); 28 | 29 | let tree = topTree; 30 | 31 | for(let token of jTokens) { 32 | if(! (token in tree)) { 33 | tree[token] = {}; 34 | } 35 | tree = tree[token]; 36 | } 37 | if(! ('.expansion' in tree)) { 38 | tree['.expansion'] = []; 39 | } 40 | for(let token of iTokens) { 41 | tree['.expansion'].push(token); 42 | } 43 | } 44 | } 45 | } 46 | 47 | return topTree; 48 | } 49 | 50 | export function generateQueryExpander(lunr, relatedTerms) { 51 | let treeTop = buildAliasTree(lunr, relatedTerms); 52 | let currentTree = treeTop; 53 | 54 | let expandQuery = function expandQuery(token) { 55 | if(token.metadata.index == 0) { 56 | currentTree = treeTop; 57 | } 58 | 59 | let tokenStr = token.toString(); 60 | if(currentTree.hasOwnProperty(tokenStr)) { 61 | currentTree = currentTree[tokenStr]; 62 | if('.expansion' in currentTree) { 63 | let originalToken = token; 64 | let tokens = [ originalToken ]; 65 | 66 | for(let token of currentTree['.expansion']) { 67 | tokens.push(originalToken.clone(function(str, meta) { 68 | return token; 69 | })); 70 | } 71 | 72 | return tokens; 73 | } 74 | } else { 75 | currentTree = treeTop; 76 | } 77 | return token; 78 | }; 79 | 80 | lunr.Pipeline.registerFunction(expandQuery, 'expandQuery'); 81 | return expandQuery; 82 | } 83 | } 84 | 85 | export = QueryExpander; 86 | 87 | // vim:sts=4:sw=4 88 | -------------------------------------------------------------------------------- /readme.tid: -------------------------------------------------------------------------------- 1 | title: $:/plugins/hoelzro/full-text-search/readme 2 | type: text/vnd.tiddlywiki 3 | 4 | ! Purpose 5 | 6 | Provides an alternative search result list that orders results by search relevance and ignores differences in word forms (ex. //tag// vs //tags//). 7 | 8 | On my personal wiki, I have the problem that there are terms I use across a lot of tiddlers, and sometimes I'll use different forms (such as the aforementioned //tag// vs //tags//). I wanted a plugin to allow me to find the tiddler I'm looking for quickly and didn't require me to worry about how I declined a noun or inflected a verb - so I wrote this plugin, which provides an alternative search list powered by [[lunr.js|https://lunrjs.com/]]. 9 | 10 | I use it pretty much every day, but there's definitely room for improvement. Please [[let me know|https://github.com/hoelzro/tw-full-text-search/issues]] if there are any bugs! 11 | 12 | ! Demo 13 | 14 | Please check out the [[examples|https://hoelz.ro/files/fts.html#Query Examples]] on the demo wiki. 15 | 16 | ! Usage 17 | 18 | Each time you start a new ~TiddlyWiki session, you'll need to build the FTS index. You can do this from a tab in the [[$:/ControlPanel]]. Older versions of the index are retained in web storage, so it should be pretty quick after the first time! After you build the index, you can just search as you would normally. 19 | 20 | ! Source Code/Reporting Bugs 21 | 22 | If you want to help out, you can report bugs or check out the source for this plugin (or its dependency, the progress bar plugin) on ~GitHub: 23 | 24 | https://github.com/hoelzro/tw-full-text-search/ 25 | 26 | https://github.com/hoelzro/tw-progress-bar 27 | 28 | Requires [[$:/plugins/hoelzro/progress-bar]] to display progress when generating the index. 29 | -------------------------------------------------------------------------------- /search-results.tid: -------------------------------------------------------------------------------- 1 | title: $:/plugins/hoelzro/full-text-search/search-results 2 | type: text/vnd.tiddlywiki 3 | tags: $:/tags/SearchResults 4 | caption: Full Text Results 5 | 6 | \define searchResults() 7 | <$set name="resultCount" value="""<$count filter="[ftsearch{$(searchTiddler)$}ftsfeedback]"/>"""> 8 | {{$:/language/Search/Matches}} 9 | 10 | 11 | <$list filter="[ftsearch{$(searchTiddler)$}ftsfeedback]" template="$:/core/ui/ListItemTemplate"/> 12 | \end 13 | 14 | <$set name="state" value="$:/temp/FTS-state"> 15 | <$reveal type="match" state=<> text="initialized"> 16 | <$set name="ftsFeedback" value=<> > 17 | <> 18 | 19 | <$list variable="index" filter="[titleindexes[]]"> 20 | 21 | <$set name="message" tiddler=<> index=<> > 22 | <> 23 | 24 | 25 | 26 | 27 | 28 | 29 | <$reveal type="match" state=<> text="uninitialized"> 30 | Search index not initialized; please <$button> 31 | Click here to generate the index 32 | <$fts-action-generate-index /> 33 | 34 | 35 | 36 | <$reveal type="match" state=<> text="initializing"> 37 | Generating index... 38 | 39 | <$hoelzro-progressbar current={{$:/temp/FTS-state!!progressCurrent}} total={{$:/temp/FTS-state!!progressTotal}} /> 40 | 41 | 42 | -------------------------------------------------------------------------------- /shared-index.ts: -------------------------------------------------------------------------------- 1 | /*\ 2 | title: $:/plugins/hoelzro/full-text-search/shared-index.js 3 | type: application/javascript 4 | module-type: library 5 | 6 | \*/ 7 | 8 | declare var require; 9 | declare var $tw; 10 | declare var setTimeout; 11 | declare var setInterval; 12 | 13 | module SharedIndex { 14 | const RELATED_TERMS_TIDDLER = '$:/plugins/hoelzro/full-text-search/RelatedTerms.json'; 15 | const FUZZY_SEARCH_TIDDLER = '$:/plugins/hoelzro/full-text-search/EnableFuzzySearching'; 16 | let lunr = require('$:/plugins/hoelzro/full-text-search/lunr.min.js'); 17 | let lunrMutable = require('$:/plugins/hoelzro/full-text-search/lunr-mutable.js'); 18 | // XXX import? 19 | let { generateQueryExpander } = require('$:/plugins/hoelzro/full-text-search/query-expander.js'); 20 | 21 | lunr.utils.warn = function() {}; 22 | 23 | let index = null; 24 | 25 | async function tick() { 26 | return new Promise(resolve => { 27 | $tw.utils.nextTick(resolve); 28 | }); 29 | } 30 | 31 | async function buildIndexIncremental(wiki, tiddlers, rebuilding, progressCallback) { 32 | let builder = null; 33 | if(rebuilding || !index) { 34 | let relatedTerms = $tw.wiki.getTiddlerDataCached(RELATED_TERMS_TIDDLER, []); 35 | relatedTerms = relatedTerms.map($tw.utils.parseStringArray); 36 | 37 | let expandQuery = generateQueryExpander(lunr, relatedTerms); 38 | 39 | builder = new lunrMutable.Builder(); 40 | 41 | let stemmer; 42 | 43 | if(wiki.getTiddlerText(FUZZY_SEARCH_TIDDLER, '') == 'yes') { 44 | stemmer = function(unstemmedToken) { 45 | let stemmedToken = lunr.stemmer(unstemmedToken.clone()); 46 | 47 | return [ unstemmedToken, stemmedToken ]; 48 | }; 49 | lunr.Pipeline.registerFunction(stemmer, 'stemmedAndUnstemmed'); 50 | } else { 51 | stemmer = lunr.stemmer; 52 | } 53 | 54 | builder.pipeline.add( 55 | lunr.trimmer, 56 | lunr.stopWordFilter, 57 | expandQuery, 58 | stemmer 59 | ); 60 | 61 | builder.searchPipeline.add( 62 | lunr.stemmer 63 | ); 64 | 65 | // XXX configurable fields? 66 | builder.field('title'); 67 | builder.field('tags'); 68 | builder.field('text'); 69 | 70 | builder.ref('title'); 71 | } else { 72 | builder = index.builder; 73 | } 74 | 75 | let i = 0; 76 | for(let title of tiddlers) { 77 | let tiddler = wiki.getTiddler(title); 78 | i++; 79 | 80 | if(tiddler === undefined) { // avoid drafts that were open when we started 81 | continue; 82 | } 83 | var type = tiddler.fields.type || 'text/vnd.tiddlywiki'; 84 | if(!type.startsWith('text/')) { 85 | continue; 86 | } 87 | 88 | if('draft.of' in tiddler.fields) { 89 | continue; 90 | } 91 | 92 | updateTiddler(builder, tiddler); 93 | await progressCallback(i); 94 | await tick(); 95 | } 96 | index = builder.build(); 97 | await progressCallback(tiddlers.length); 98 | } 99 | 100 | async function buildIndexWorker(wiki, tiddlers, progressCallback) { 101 | var workerSource = wiki.getTiddlerText('$:/plugins/hoelzro/full-text-search/index-worker.js'); 102 | var worker = new Worker(URL.createObjectURL(new Blob([ workerSource ]))); 103 | 104 | // XXX this needs to happen - not great how "action at a distance"y this is 105 | let relatedTerms = $tw.wiki.getTiddlerDataCached(RELATED_TERMS_TIDDLER, []); 106 | relatedTerms = relatedTerms.map($tw.utils.parseStringArray); 107 | let expandQuery = generateQueryExpander(lunr, relatedTerms); 108 | 109 | // more action at a distance =/ 110 | let stemmer = function(unstemmedToken) { 111 | let stemmedToken = lunr.stemmer(unstemmedToken.clone()); 112 | 113 | return [ unstemmedToken, stemmedToken ]; 114 | }; 115 | lunr.Pipeline.registerFunction(stemmer, 'stemmedAndUnstemmed'); 116 | 117 | var workerFinished = new Promise(function(resolve, reject) { 118 | worker.onmessage = function(msg) { 119 | let payload = msg.data; 120 | 121 | if(payload.type == 'require') { 122 | let moduleName = payload.name; 123 | let moduleSource = wiki.getTiddlerText(moduleName); 124 | 125 | worker.postMessage(URL.createObjectURL(new Blob( [ moduleSource ]))); 126 | } else if(payload.type == 'index') { 127 | index = lunrMutable.Index.load(JSON.parse(payload.index)); 128 | resolve(); 129 | } else if(payload.type == 'sendTiddlers') { 130 | for(let title of tiddlers) { 131 | let tiddler = wiki.getTiddler(title); 132 | 133 | if(tiddler === undefined) { // avoid drafts that were open when we started 134 | continue; 135 | } 136 | var type = tiddler.fields.type || 'text/vnd.tiddlywiki'; 137 | if(!type.startsWith('text/')) { 138 | continue; 139 | } 140 | worker.postMessage(JSON.stringify(tiddler.fields)); 141 | } 142 | worker.postMessage(null); 143 | } else if(payload.type == 'progress') { 144 | progressCallback(payload.count); 145 | } else if(payload.type == 'getRelatedTerms') { 146 | worker.postMessage(relatedTerms); 147 | } else if(payload.type == 'getFuzzySetting') { 148 | let fuzzySetting = wiki.getTiddlerText(FUZZY_SEARCH_TIDDLER, ''); 149 | 150 | worker.postMessage(fuzzySetting); 151 | } else if(payload.type == 'error') { 152 | reject(payload.error); 153 | } 154 | }; 155 | }); 156 | 157 | await workerFinished; 158 | await progressCallback(tiddlers.length); 159 | } 160 | 161 | export async function buildIndex(wiki, tiddlers, isFresh, progressCallback) { 162 | if($tw.browser && isFresh) { 163 | try { 164 | return await buildIndexWorker(wiki, tiddlers, progressCallback); 165 | } catch(e) { 166 | console.log(e); 167 | console.log('falling back to incremental indexing...'); 168 | } 169 | } 170 | 171 | return await buildIndexIncremental(wiki, tiddlers, isFresh, progressCallback); 172 | } 173 | 174 | export function updateTiddler(builder, tiddler) { 175 | var fields : any = { 176 | title: tiddler.fields.title 177 | }; 178 | 179 | if('text' in tiddler.fields) { 180 | fields.text = tiddler.fields.text; 181 | } 182 | 183 | if('tags' in tiddler.fields) { 184 | fields.tags = tiddler.fields.tags.join(' '); 185 | } 186 | 187 | builder.remove(fields); 188 | builder.add(fields); 189 | } 190 | 191 | export function getIndex() { 192 | return index; 193 | }; 194 | 195 | export function clearIndex() { 196 | index = null; 197 | } 198 | 199 | export function load(data) { 200 | index = lunrMutable.Index.load(data); 201 | } 202 | } 203 | export = SharedIndex; 204 | 205 | // vim:sts=4:sw=4 206 | -------------------------------------------------------------------------------- /state.tid: -------------------------------------------------------------------------------- 1 | title: $:/temp/FTS-state 2 | type: text/vnd.tiddlywiki 3 | 4 | uninitialized -------------------------------------------------------------------------------- /tests/.gitignore: -------------------------------------------------------------------------------- 1 | !*.js 2 | -------------------------------------------------------------------------------- /tests/test-simple.js: -------------------------------------------------------------------------------- 1 | /*\ 2 | title: test-simple.js 3 | type: application/javascript 4 | tags: [[$:/tags/test-spec]] 5 | 6 | \*/ 7 | (function() { 8 | var localforage = require('$:/plugins/hoelzro/full-text-search/localforage.min.js'); 9 | var wiki = $tw.wiki; 10 | 11 | var nullDriver = { 12 | _driver: 'nullDriver', 13 | _initStorage: function(options) { 14 | return Promise.resolve(); 15 | }, 16 | clear: function(callback) { 17 | return callback ? callback() : Promise.resolve(); 18 | }, 19 | getItem: function(key, callback) { 20 | return callback ? callback() : Promise.resolve(null); 21 | }, 22 | iterate: function(iterator, callback) { 23 | return callback ? callback() : Promise.resolve(); 24 | }, 25 | key: function(n, callback) { 26 | return callback ? callback(n) : Promise.resolve(n); 27 | }, 28 | keys: function(callback) { 29 | return callback ? callback([]) : Promise.resolve([]); 30 | }, 31 | length: function(callback) { 32 | return callback ? callback(0) : Promise.resolve(0); 33 | }, 34 | removeItem: function(key, callback) { 35 | return callback ? callback() : Promise.resolve(); 36 | }, 37 | setItem: function(key, value, callback) { 38 | return callback ? callback(null) : Promise.resolve(null); 39 | }, 40 | dropInstance: function(options, callback) { 41 | return callback ? callback() : Promise.resolve(); 42 | } 43 | }; 44 | 45 | function waitForNextTick() { 46 | return new Promise(function(resolve, reject) { 47 | $tw.utils.nextTick(resolve); 48 | }); 49 | } 50 | 51 | async function buildIndex() { 52 | var FTSActionGenerateIndexWidget = require('$:/plugins/hoelzro/full-text-search/fts-action-generate-index.js')['fts-action-generate-index']; 53 | var widget = new FTSActionGenerateIndexWidget(null, { 54 | wiki: wiki 55 | }); 56 | await widget.asyncInvokeAction(); 57 | } 58 | 59 | async function prepare() { 60 | await buildIndex(); 61 | } 62 | 63 | var fauxStorage = Object.create(null); 64 | 65 | var inMemoryDriver = { 66 | _driver: 'inMemoryDriver', 67 | // XXX re-init between tests 68 | _initStorage: function(options) { 69 | return Promise.resolve(); 70 | }, 71 | clear: function(callback) { 72 | fauxStorage = Object.create(null); 73 | return callback ? callback() : Promise.resolve(); 74 | }, 75 | getItem: function(key, callback) { 76 | var value = fauxStorage[key]; 77 | if(value != undefined) { 78 | value = JSON.parse(value); 79 | } else { 80 | value = null; 81 | } 82 | return callback ? callback(value) : Promise.resolve(value); 83 | }, 84 | iterate: function(iterator, callback) { 85 | return callback ? callback() : Promise.resolve(); 86 | }, 87 | key: function(n, callback) { 88 | return callback ? callback(n) : Promise.resolve(n); 89 | }, 90 | keys: function(callback) { 91 | return callback ? callback(Object.keys(fauxStorage)) : Promise.resolve(Object.keys(fauxStorage)); 92 | }, 93 | length: function(callback) { 94 | return callback ? callback(Object.keys(fauxStorage).length) : Promise.resolve(Object.keys(fauxStorage).length); 95 | }, 96 | removeItem: function(key, callback) { 97 | delete fauxStorage[key]; 98 | return callback ? callback() : Promise.resolve(); 99 | }, 100 | setItem: function(key, value, callback) { 101 | var oldValue = fauxStorage[key]; 102 | if(oldValue != undefined) { 103 | oldValue = JSON.parse(oldValue); 104 | } else { 105 | oldValue = null; 106 | } 107 | fauxStorage[key] = JSON.stringify(value); 108 | return callback ? callback(oldValue) : Promise.resolve(oldValue); 109 | }, 110 | dropInstance: function(options, callback) { 111 | return callback ? callback() : Promise.resolve(); 112 | } 113 | }; 114 | 115 | async function setupInMemoryDriver() { 116 | // XXX how do I remove this driver after this test to make sure it doesn't interfere? 117 | await localforage.defineDriver(inMemoryDriver); 118 | await localforage.setDriver('inMemoryDriver'); 119 | } 120 | 121 | function clearIndex() { 122 | require('$:/plugins/hoelzro/full-text-search/shared-index.js').clearIndex(); 123 | return Promise.resolve(); 124 | } 125 | 126 | var nullDriverReady; 127 | 128 | var initialTitles = Object.create(null); 129 | for(var title of wiki.compileFilter('[!is[system]]')()) { 130 | initialTitles[title] = true; 131 | } 132 | 133 | async function addTiddler(fields) { 134 | wiki.addTiddler(new $tw.Tiddler( 135 | wiki.getCreationFields(), 136 | fields, 137 | wiki.getModificationFields() 138 | )); 139 | 140 | while(wiki.getSizeOfTiddlerEventQueue() > 0) { 141 | await waitForNextTick(); 142 | } 143 | } 144 | 145 | async function deleteTiddler(title) { 146 | wiki.deleteTiddler(title); 147 | 148 | while(wiki.getSizeOfTiddlerEventQueue() > 0) { 149 | await waitForNextTick(); 150 | } 151 | } 152 | 153 | beforeEach(async function() { 154 | // XXX clear localforage in memory cache? 155 | require('$:/plugins/hoelzro/full-text-search/shared-index.js').clearIndex(); 156 | wiki.addTiddler({ 157 | title: 'NoModified', 158 | text: 'No modification date' 159 | }); 160 | 161 | wiki.addTiddler(new $tw.Tiddler( 162 | wiki.getCreationFields(), 163 | { title: 'JustSomeText', text: 'This one has a modification date' }, 164 | wiki.getModificationFields())); 165 | 166 | wiki.addTiddler(new $tw.Tiddler( 167 | { 168 | title: 'Draft of New Tiddler', 169 | 'draft.of': 'New Tiddler', 170 | 'draft.title': 'New Tiddler', 171 | text: 'test tiddler', 172 | }, 173 | wiki.getCreationFields(), 174 | wiki.getModificationFields())); 175 | // XXX wait for wiki to settle 176 | 177 | await localforage.defineDriver(nullDriver); 178 | await localforage.setDriver('nullDriver'); 179 | nullDriverReady = true; 180 | 181 | let lunr = require('$:/plugins/hoelzro/full-text-search/lunr.min.js'); 182 | 183 | delete lunr.Pipeline.registeredFunctions.expandQuery; 184 | }); 185 | 186 | afterEach(async function() { 187 | var titles = wiki.compileFilter('[!is[system]]')(); 188 | var pending = []; 189 | for(var title of titles) { 190 | if(! (title in initialTitles)) { 191 | pending.push(deleteTiddler(title)); 192 | } 193 | } 194 | await Promise.all(pending); 195 | }); 196 | 197 | describe('Simple test', function() { 198 | it('should start with an uninitialized FTS state', function() { 199 | expect(wiki.getTiddlerText('$:/temp/FTS-state')).toBe('uninitialized'); 200 | }); 201 | 202 | it('should find matching documents without a modified field', async function() { 203 | await prepare(); 204 | 205 | expect(wiki.getTiddlerText('$:/temp/FTS-state')).toBe('initialized'); 206 | var results = wiki.compileFilter('[ftsearch[modification]]')(); 207 | expect(results).toContain('NoModified'); 208 | }); 209 | 210 | it("should pick up changes to tiddlers' contents", async function() { 211 | await prepare(); 212 | 213 | var tiddler = wiki.getTiddler('NoModified'); 214 | var newTiddler = new $tw.Tiddler( 215 | tiddler, 216 | {text: "New text without that word we're looking for"}, 217 | wiki.getModificationFields()); 218 | wiki.addTiddler(newTiddler); 219 | 220 | await waitForNextTick(); 221 | 222 | var results = wiki.compileFilter('[ftsearch[modification]]')(); 223 | expect(results).not.toContain('NoModified'); 224 | 225 | results = wiki.compileFilter('[ftsearch[looking]]')(); 226 | expect(results).toContain('NoModified'); 227 | }); 228 | 229 | it("should pick up on renames after initial index", async function() { 230 | await prepare(); 231 | 232 | $tw.wiki.renameTiddler('NoModified', 'BrandNewName'); 233 | 234 | await waitForNextTick(); 235 | 236 | var results = wiki.compileFilter('[ftsearch[modification]]')(); 237 | expect(results).toContain('BrandNewName'); 238 | }); 239 | 240 | it("should pick up on deletions after initial index", async function() { 241 | await prepare(); 242 | await deleteTiddler('NoModified'); 243 | 244 | var results = wiki.compileFilter('[ftsearch[modification]]')(); 245 | expect(results).not.toContain('NoModified'); 246 | }); 247 | 248 | it('should pick reason with "reason programming language"', async function() { 249 | await prepare(); 250 | 251 | var text = ` 252 | A kind of OCaml that compiles down to JavaScript 253 | 254 | https://facebook.github.io/reason/ 255 | 256 | https://jaredforsyth.com/2017/07/05/a-reason-react-tutorial/ 257 | `; 258 | $tw.wiki.addTiddler(new $tw.Tiddler( 259 | $tw.wiki.getCreationFields(), 260 | { title: 'Reason', tags: 'Someday/Maybe Play Coding [[Programming Languages]]', type: 'text/vnd.tiddlywiki', text: text }, 261 | $tw.wiki.getModificationFields() 262 | )); 263 | await waitForNextTick(); 264 | 265 | var results = wiki.compileFilter('[ftsearch[reason programming language]]')(); 266 | expect(results).toContain('Reason'); 267 | }); 268 | 269 | xit('should pick up "twitter" in a URL', async function() { 270 | await prepare(); 271 | var text = 'https://twitter.com/hoelzro/status/877901644125663232'; 272 | $tw.wiki.addTiddler(new $tw.Tiddler( 273 | $tw.wiki.getCreationFields(), 274 | { title: 'ContainsTweetLink', type: 'text/vnd.tiddlywiki', text: text }, 275 | $tw.wiki.getModificationFields() 276 | )); 277 | await waitForNextTick(); 278 | 279 | var results = wiki.compileFilter('[ftsearch[twitter]]')(); 280 | expect(results).toContain('ContainsTweetLink'); 281 | }); 282 | 283 | it('should not pick up a non-text tiddler on an update', async function() { 284 | await prepare(); 285 | 286 | var newTiddler = new $tw.Tiddler( 287 | wiki.getCreationFields(), 288 | {type: 'application/x-tiddler-data', text: 'foo bar', title: 'MyDataTiddler'}, 289 | wiki.getModificationFields()); 290 | wiki.addTiddler(newTiddler); 291 | 292 | await waitForNextTick(); 293 | 294 | var results = wiki.compileFilter('[ftsearch[foo]]')(); 295 | expect(results).not.toContain('MyDataTiddler'); 296 | }); 297 | 298 | it('should not pick up JavaScript code', async function() { 299 | await prepare(); 300 | 301 | expect(wiki.getTiddlerText('$:/temp/FTS-state')).toBe('initialized'); 302 | var results = wiki.compileFilter('[ftsearch[tag]]')(); 303 | expect(results).not.toContain('test-simple.js'); 304 | }); 305 | 306 | it('should not fail upon an incomplete query', async function() { 307 | await prepare(); 308 | 309 | expect(wiki.getTiddlerText('$:/temp/FTS-state')).toBe('initialized'); 310 | try { 311 | var results = wiki.compileFilter('[ftsearch[date~]]')(); 312 | expect(results.length).toBe(0); 313 | } catch(e) { 314 | console.log(e); 315 | expect(true).toBe(false); 316 | } 317 | }); 318 | 319 | it('should not index new draft tiddlers', async function() { 320 | await prepare(); 321 | 322 | var draftTiddler = new $tw.Tiddler( 323 | { 324 | title: 'Draft of New Tiddler 2', 325 | 'draft.of': 'New Tiddler 2', 326 | 'draft.title': 'New Tiddler 2', 327 | text: 'test tiddler', 328 | }, 329 | wiki.getCreationFields(), 330 | wiki.getModificationFields()); 331 | 332 | wiki.addTiddler(draftTiddler); 333 | 334 | await waitForNextTick(); 335 | 336 | var results = wiki.filterTiddlers('[ftsearch[tiddler]has[draft.of]]'); 337 | expect(results.length).toBe(0); 338 | }); 339 | 340 | it('should not index draft tiddlers from the start', async function() { 341 | await prepare(); 342 | 343 | var results = wiki.filterTiddlers('[ftsearch[tiddler]has[draft.of]]'); 344 | expect(results.length).toBe(0); 345 | }); 346 | 347 | it('should order results by query relevance', async function() { 348 | await prepare(); 349 | 350 | var results = wiki.filterTiddlers('[ftsearch[fox]]'); 351 | let foxesFoxesFoxesIndex = results.indexOf('Foxes foxes foxes'); 352 | let foxInGardenIndex = results.indexOf('A fox in the garden'); 353 | expect(foxesFoxesFoxesIndex).not.toBe(-1); 354 | expect(foxInGardenIndex).not.toBe(-1); 355 | expect(foxesFoxesFoxesIndex).toBeLessThan(foxInGardenIndex); 356 | }); 357 | }); 358 | 359 | describe('Cache tests', function() { 360 | function freshBuildIndex() { 361 | let lunr = require('$:/plugins/hoelzro/full-text-search/lunr.min.js'); 362 | delete lunr.Pipeline.registeredFunctions.expandQuery; 363 | return clearIndex().then(buildIndex); 364 | } 365 | 366 | it('should work with the cache', async function() { 367 | async function modifyTiddler() { 368 | var tiddler = $tw.wiki.getTiddler('JustSomeText'); 369 | 370 | var newTiddler = new $tw.Tiddler( 371 | tiddler, 372 | {text: "New text without that word we're looking for"}, 373 | wiki.getModificationFields()); 374 | wiki.addTiddler(newTiddler); 375 | 376 | await waitForNextTick(); 377 | } 378 | 379 | await setupInMemoryDriver(); 380 | await localforage.clear(); 381 | await buildIndex(); 382 | await modifyTiddler(); 383 | await freshBuildIndex(); 384 | 385 | var results = wiki.compileFilter('[ftsearch[modification]]')(); 386 | expect(results).not.toContain('JustSomeText'); 387 | }); 388 | 389 | it('should not pick up tiddlers deleted between a save and cache load', async function() { 390 | await setupInMemoryDriver(); 391 | await localforage.clear(); 392 | await buildIndex(); 393 | await clearIndex(); 394 | await deleteTiddler('JustSomeText'); 395 | await buildIndex(); 396 | 397 | var results = wiki.compileFilter('[ftsearch[modification]]')(); 398 | expect(results).not.toContain('JustSomeText'); 399 | }); 400 | 401 | it('should not pick up tiddlers deleted and re-added between a save and cache load', async function() { 402 | async function readdTiddler() { 403 | var tiddler = $tw.wiki.getTiddler('JustSomeText'); 404 | 405 | var newTiddler = new $tw.Tiddler( 406 | tiddler, 407 | {text: "New text without that word we're looking for"}, 408 | wiki.getModificationFields()); 409 | wiki.addTiddler(newTiddler); 410 | 411 | await waitForNextTick(); 412 | } 413 | 414 | await setupInMemoryDriver(); 415 | await localforage.clear(); 416 | await buildIndex(); 417 | await clearIndex(); 418 | await deleteTiddler('JustSomeText'); 419 | await readdTiddler(); 420 | await buildIndex(); 421 | 422 | var results = wiki.compileFilter('[ftsearch[modification]]')(); 423 | expect(results).not.toContain('JustSomeText'); 424 | }); 425 | }); 426 | 427 | describe('Query expansion tests', function() { 428 | async function clearRelatedTerms() { 429 | await deleteTiddler('$:/plugins/hoelzro/full-text-search/RelatedTerms.json'); 430 | } 431 | 432 | it('should expand change to modification', async function() { 433 | async function setupRelatedTerms() { 434 | wiki.addTiddler(new $tw.Tiddler( 435 | wiki.getCreationFields(), 436 | {title: '$:/plugins/hoelzro/full-text-search/RelatedTerms.json', text: '["modification change"]', type: 'application/json'}, 437 | wiki.getModificationFields(), 438 | )); 439 | 440 | await waitForNextTick(); 441 | } 442 | 443 | await setupRelatedTerms(); 444 | await buildIndex(); 445 | 446 | expect(wiki.getTiddlerText('$:/temp/FTS-state')).toBe('initialized'); 447 | var results = wiki.compileFilter('[ftsearch[change]]')(); 448 | expect(results).toContain('NoModified'); 449 | expect(results).toContain('JustSomeText'); 450 | }); 451 | 452 | it("shouldn't expand anything if the config tiddler has no data", async function() { 453 | await clearRelatedTerms(); 454 | await buildIndex(); 455 | 456 | expect(wiki.getTiddlerText('$:/temp/FTS-state')).toBe('initialized'); 457 | var results = wiki.compileFilter('[ftsearch[change]]')(); 458 | expect(results).not.toContain('NoModified'); 459 | expect(results).not.toContain('JustSomeText'); 460 | }); 461 | 462 | it("should invalidate the index if the config tiddler is changed", async function() { 463 | async function setupRelatedTerms() { 464 | wiki.addTiddler(new $tw.Tiddler( 465 | wiki.getCreationFields(), 466 | {title: '$:/plugins/hoelzro/full-text-search/RelatedTerms.json', text: '["modification change"]', type: 'application/json'}, 467 | wiki.getModificationFields(), 468 | )); 469 | 470 | await waitForNextTick(); 471 | } 472 | 473 | await setupRelatedTerms(); 474 | await buildIndex(); 475 | await clearRelatedTerms(); 476 | 477 | expect(wiki.getTiddlerText('$:/temp/FTS-state')).toBe('uninitialized'); 478 | }); 479 | 480 | it('should rebuild the whole index if the config tiddler is changed and loaded from cache', async function() { 481 | async function setupRelatedTerms() { 482 | wiki.addTiddler(new $tw.Tiddler( 483 | wiki.getCreationFields(), 484 | {title: '$:/plugins/hoelzro/full-text-search/RelatedTerms.json', text: '["modification change"]', type: 'application/json'}, 485 | wiki.getModificationFields(), 486 | )); 487 | 488 | await waitForNextTick(); 489 | } 490 | 491 | await setupInMemoryDriver(); 492 | await localforage.clear(); 493 | await setupRelatedTerms(); 494 | await buildIndex(); 495 | await clearRelatedTerms(); 496 | await clearIndex(); 497 | await buildIndex(); 498 | 499 | expect(wiki.getTiddlerText('$:/temp/FTS-state')).toBe('initialized'); 500 | var results = wiki.compileFilter('[ftsearch[change]]')(); 501 | expect(results).not.toContain('NoModified'); 502 | expect(results).not.toContain('JustSomeText'); 503 | }); 504 | 505 | it('should not break the indexer to use Related terms with a number as a member', async function() { 506 | async function setupRelatedTerms() { 507 | let relatedTermList = [ 508 | "foobar [[foo 2]]" 509 | ]; 510 | 511 | wiki.addTiddler(new $tw.Tiddler( 512 | wiki.getCreationFields(), 513 | {title: '$:/plugins/hoelzro/full-text-search/RelatedTerms.json', text: JSON.stringify(relatedTermList), type: 'application/json'}, 514 | wiki.getModificationFields(), 515 | )); 516 | 517 | wiki.addTiddler(new $tw.Tiddler( 518 | wiki.getCreationFields(), 519 | {title: 'Empty', text: 'foo', type: 'text/vnd.tiddlywiki', tags: ''}, 520 | wiki.getModificationFields(), 521 | )); 522 | 523 | await waitForNextTick(); 524 | } 525 | 526 | await setupRelatedTerms(); 527 | await buildIndex(); 528 | 529 | expect(true).toBe(true); 530 | }); 531 | }); 532 | 533 | describe('Wildcard tests', function() { 534 | async function enableFuzzySearch() { 535 | await addTiddler({ 536 | title: '$:/plugins/hoelzro/full-text-search/EnableFuzzySearching', 537 | text: 'yes', 538 | type: 'text/vnd.tiddlywiki', 539 | }); 540 | } 541 | 542 | async function disableFuzzySearch() { 543 | await deleteTiddler('$:/plugins/hoelzro/full-text-search/EnableFuzzySearching'); 544 | } 545 | 546 | it('should return "formatting" if the user searches for "format*ing"', async function () { 547 | await enableFuzzySearch(); 548 | await addTiddler({ 549 | title: 'Experiment with Formatting' 550 | }); 551 | await buildIndex(); 552 | 553 | expect(wiki.getTiddlerText('$:/temp/FTS-state')).toBe('initialized'); 554 | var results = wiki.filterTiddlers('[ftsearch[format*ing]]'); 555 | expect(results).toContain('Experiment with Formatting'); 556 | }); 557 | 558 | it('should not return "formatting" if a fuzzy search is used and fuzzy searching is disabled', async function() { 559 | await disableFuzzySearch(); 560 | await addTiddler({ 561 | title: 'Experiment with Formatting' 562 | }); 563 | await buildIndex(); 564 | 565 | expect(wiki.getTiddlerText('$:/temp/FTS-state')).toBe('initialized'); 566 | var results = wiki.filterTiddlers('[ftsearch[format*ing]]'); 567 | expect(results).not.toContain('Experiment with Formatting'); 568 | }); 569 | 570 | it('should invalidate the index if the user changes their fuzzy settings', async function() { 571 | await enableFuzzySearch(); 572 | await buildIndex(); 573 | 574 | expect(wiki.getTiddlerText('$:/temp/FTS-state')).toBe('initialized'); 575 | 576 | await disableFuzzySearch(); 577 | 578 | expect(wiki.getTiddlerText('$:/temp/FTS-state')).toBe('uninitialized'); 579 | }); 580 | 581 | it('should detect wildcard queries when fuzzy matching is off and notify the user', async function() { 582 | await disableFuzzySearch(); 583 | await addTiddler({ 584 | title: 'Experiment with Formatting' 585 | }); 586 | await buildIndex(); 587 | 588 | var results = wiki.filterTiddlers('[ftsearch[format*ing]ftsfeedback[$:/temp/fts-feedback]]'); 589 | expect(results).not.toContain('Experiment with Formatting'); 590 | 591 | let feedback = wiki.getTiddlerData('$:/temp/fts-feedback', []); 592 | expect(feedback).toContain("It looks like you're trying to perform a wildcard search; you'll need to enable wildcard/fuzzy searching in the FTS settings"); 593 | }); 594 | 595 | it('should detect fuzzy queries when fuzzy matching is off and notify the user', async function() { 596 | await disableFuzzySearch(); 597 | await addTiddler({ 598 | title: 'Experiment with Formatting' 599 | }); 600 | await buildIndex(); 601 | 602 | var results = wiki.filterTiddlers('[ftsearch[formattign~1]ftsfeedback[$:/temp/fts-feedback]]'); 603 | expect(results).not.toContain('Experiment with Formatting'); 604 | 605 | let feedback = wiki.getTiddlerData('$:/temp/fts-feedback', []); 606 | expect(feedback).toContain("It looks like you're trying to perform a fuzzy search; you'll need to enable wildcard/fuzzy searching in the FTS settings"); 607 | }); 608 | 609 | it('should not prevent results from being returned if ftsfeedback is used', async function() { 610 | await disableFuzzySearch(); 611 | await addTiddler({ 612 | title: 'Experiment with Formatting' 613 | }); 614 | await buildIndex(); 615 | 616 | var results = wiki.filterTiddlers('[ftsearch[formatting]ftsfeedback[$:/temp/fts-feedback]]'); 617 | expect(results).toContain('Experiment with Formatting'); 618 | 619 | await enableFuzzySearch(); 620 | await buildIndex(); 621 | 622 | results = wiki.filterTiddlers('[ftsearch[format*ing]ftsfeedback[$:/temp/fts-feedback]]'); 623 | expect(results).toContain('Experiment with Formatting'); 624 | }); 625 | 626 | it('should not populate the feedback tiddler if fuzzy searching is on', async function() { 627 | await enableFuzzySearch(); 628 | await addTiddler({ 629 | title: 'Experiment with Formatting' 630 | }); 631 | await buildIndex(); 632 | 633 | var results = wiki.filterTiddlers('[ftsearch[format*ing]ftsfeedback[$:/temp/fts-feedback]]'); 634 | expect(results).toContain('Experiment with Formatting'); 635 | var messages = wiki.getTiddlerData('$:/temp/fts-feedback', []); 636 | expect(messages.length).toBe(0); 637 | }); 638 | 639 | it('should always clear feedback upon search', async function() { 640 | await disableFuzzySearch(); 641 | await addTiddler({ 642 | title: 'Experiment with Formatting' 643 | }); 644 | await buildIndex(); 645 | 646 | var results = wiki.filterTiddlers('[ftsearch[format*ing]ftsfeedback[$:/temp/fts-feedback]]'); 647 | 648 | await enableFuzzySearch(); 649 | await buildIndex(); 650 | 651 | results = wiki.filterTiddlers('[ftsearch[format*ing]ftsfeedback[$:/temp/fts-feedback]]'); 652 | var messages = wiki.getTiddlerData('$:/temp/fts-feedback', []); 653 | expect(messages.length).toBe(0); 654 | }); 655 | }); 656 | })(); 657 | --------------------------------------------------------------------------------