├── LICENSE ├── NOTICE ├── README.md ├── errorcorrect.go ├── example ├── dictionary.dat ├── dictionaryexample.go └── example.go ├── ferret.go └── unicodeconvert.go /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Ferret Substring Search Engine 2 | Copyright 2012-2013 Mark Canning 3 | 4 | This software was developed at Tamber, Inc. 5 | (http://www.tamber.com/) by Mark Canning. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Ferret 2 | ====== 3 | ## An optimized substring search engine written in Go. 4 | Ferret makes use of a combination of an Inverted Index and a Suffix Array to allow log-time lookups with a relatively small memory footprint. 5 | Also incorporates error-correction (Levenshtein distance 1) and simple Unicode-to-ASCII conversion. 6 | Allows for arbitrary sorting functions 7 | Allows you to map arbitrary data to your results, and quickly update this data. 8 | 9 | ***Author:*** Mark Canning
10 | ***Developed at/for:*** Tamber - http://www.tamber.com/ 11 | 12 | Installing 13 | ---------- 14 | Install: `go get github.com/argusdusty/Ferret`
15 | Update: `go get -u github.com/argusdusty/Ferret`
16 | User: `import "github.com/argusdusty/Ferret"`
17 | 18 | Performance 19 | ----------- 20 | Uses linear memory (~10-18 bytes per character) 21 | Searches performed in log time with the number of characters in the dictionary. 22 | Sorted searches can be slow, taking ~linear time with the number of matches, rather than linear time with the results limit. 23 | Initialization takes linearithmic (ln(n)\*n) time (being a sorting algorithm) 24 | 25 | 26 | The code is meant to be as fast as possible for a substring dictionary search, and as such is best suited for medium-large dictionaries with ~1-100 million total characters. I've timed 10s initialization for 3.5 million characters on a modern CPU, and 10us search time (4000us with error-correction), so this system is capable of ~100,000 queries per second on a single processor - feel free to try the benchmarks in dictionaryexample.go. 27 | 28 | Sample usage 29 | ------------ 30 | 31 | ### Initializing the search engine: 32 | ```go 33 | // Allows for exact (case-sensitive) substring searches over a list of songs 34 | // mapping their respective artists, allowing sorting by the song popularity 35 | SearchEngine := ferret.New(Songs, Artists, SongPopularities, func(s string) []byte { return []byte(s) }) 36 | 37 | // Allows for lowercase-ASCII substring searches over a list of songs 38 | // mapping their respective artists, allowing sorting by the song popularity 39 | SearchEngine := ferret.New(Songs, Artists, SongPopularities, ferret.UnicodeToLowerASCII) 40 | 41 | // Allows for lowercase-ASCII substring searches over a list of artists, 42 | // allowing sorting by the artist popularity 43 | SearchEngine := ferret.New(Artists, Artists, ArtistPopularities, ferret.UnicodeToLowerASCII) 44 | ``` 45 | 46 | ### Inserting a new element into the search engine: 47 | ```go 48 | // Add a song to an existing SearchEngine, written by Artist, 49 | // and with popularity SongPopularity 50 | SearchEngine.Insert(Song, Artist, SongPopularity) 51 | ``` 52 | 53 | ### Performing simple unsorted substring search: 54 | ```go 55 | // For songs - returns a list of up to 25 artists of the matching songs, 56 | // and the song popularities 57 | SearchEngine.Query(SongQuery, 25) 58 | ``` 59 | 60 | ### Performing a sorted substring search: 61 | ```go 62 | // For songs - returns a list of up to 25 artists of the matching songs, 63 | // and the song popularities, sorted by the song popularities 64 | // assuming the song popularities are float64s 65 | SearchEngine.SortedQuery(SongQuery, 25, func(s string, v interface{}, l int, i int) float64 { return v.(float64) }) 66 | ``` 67 | 68 | ### More examples 69 | Check out example/example.go and example/dictionaryexample.go for more example usage. 70 | -------------------------------------------------------------------------------- /errorcorrect.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Mark Canning 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // 16 | // Author: Mark Canning 17 | // Developed at: Tamber, Inc. (http://www.tamber.com/). 18 | 19 | package ferret 20 | 21 | // AllASCII is all ASCII bytes (0-127) 22 | var AllASCII = []byte{ 23 | 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 17, 18, 24 | 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 25 | 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 26 | 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 27 | 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 28 | 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 29 | 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 30 | 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 31 | } 32 | 33 | // PrintableASCII is all printable ASCII bytes 34 | var PrintableASCII = []byte{ 35 | 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 36 | 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 37 | 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 38 | 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 39 | 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 40 | 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 41 | } 42 | 43 | // LowercaseASCII is all printable ASCII bytes excluding capitalized letters (A-Z) 44 | var LowercaseASCII = []byte{ 45 | 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 46 | 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 47 | 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 48 | 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 49 | 119, 120, 121, 122, 123, 124, 125, 126, 50 | } 51 | 52 | // LowercaseLetters is All lowercase ASCII bytes (a-z/97-122) 53 | var LowercaseLetters = []byte{ 54 | 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 55 | 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 56 | } 57 | 58 | // ErrorCorrect returns all byte-arrays which are Levenshtein distance of 1 away from Word 59 | // within an allowed array of byte characters. 60 | func ErrorCorrect(Word []byte, AllowedBytes []byte) [][]byte { 61 | results := make([][]byte, 0) 62 | N := len(Word) 63 | for i := 0; i < N; i++ { 64 | t := Word[i] 65 | // Remove Character 66 | temp := make([]byte, N) 67 | copy(temp, Word) 68 | temp = append(temp[:i], temp[i+1:]...) 69 | results = append(results, temp) 70 | if i != 0 { 71 | // Add Character 72 | for _, c := range AllowedBytes { 73 | temp := make([]byte, N) 74 | copy(temp, Word) 75 | temp = append(temp[:i], append([]byte{c}, temp[i:]...)...) 76 | results = append(results, temp) 77 | } 78 | // Transpose Character 79 | temp := make([]byte, N) 80 | copy(temp, Word) 81 | temp[i], temp[i-1] = temp[i-1], temp[i] 82 | results = append(results, temp) 83 | } 84 | // Insert Character 85 | for _, c := range AllowedBytes { 86 | if c == t { 87 | continue 88 | } 89 | temp := make([]byte, N) 90 | copy(temp, Word) 91 | temp[i] = c 92 | results = append(results, temp) 93 | } 94 | } 95 | return results 96 | } 97 | -------------------------------------------------------------------------------- /example/dictionaryexample.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Mark Canning 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // 16 | // Author: Mark Canning 17 | // Developed at: Tamber, Inc. (http://www.tamber.com/). 18 | 19 | package main 20 | 21 | import ( 22 | "bytes" 23 | "fmt" 24 | "github.com/argusdusty/Ferret" 25 | "io/ioutil" 26 | "strconv" 27 | "time" 28 | ) 29 | 30 | var Correction = func(b []byte) [][]byte { return ferret.ErrorCorrect(b, ferret.LowercaseLetters) } 31 | var LengthSorter = func(s string, v interface{}, l int, i int) float64 { return -float64(l + i) } 32 | var FreqSorter = func(s string, v interface{}, l int, i int) float64 { return float64(v.(uint64)) } 33 | var Converter = ferret.UnicodeToLowerASCII 34 | 35 | func main() { 36 | t := time.Now() 37 | Data, err := ioutil.ReadFile("dictionary.dat") 38 | if err != nil { 39 | panic(err) 40 | } 41 | Words := make([]string, 0) 42 | Values := make([]interface{}, 0) 43 | for _, Vals := range bytes.Split(Data, []byte("\n")) { 44 | Vals = bytes.TrimSpace(Vals) 45 | WordFreq := bytes.Split(Vals, []byte(" ")) 46 | if len(WordFreq) != 2 { 47 | continue 48 | } 49 | Freq, err := strconv.ParseUint(string(WordFreq[1]), 10, 64) 50 | if err != nil { 51 | continue 52 | } 53 | Words = append(Words, string(WordFreq[0])) 54 | Values = append(Values, Freq) 55 | } 56 | fmt.Println("Loaded dictionary in:", time.Now().Sub(t)) 57 | t = time.Now() 58 | 59 | SearchEngine := ferret.New(Words, Words, Values, Converter) 60 | fmt.Println("Created index in:", time.Now().Sub(t)) 61 | t = time.Now() 62 | fmt.Println(SearchEngine.Query("ar", 5)) 63 | fmt.Println("Performed search in:", time.Now().Sub(t)) 64 | t = time.Now() 65 | fmt.Println(SearchEngine.Query("test", 5)) 66 | fmt.Println("Performed search in:", time.Now().Sub(t)) 67 | t = time.Now() 68 | fmt.Println(SearchEngine.ErrorCorrectingQuery("tsst", 5, Correction)) 69 | fmt.Println("Performed error correcting search in:", time.Now().Sub(t)) 70 | t = time.Now() 71 | fmt.Println(SearchEngine.SortedErrorCorrectingQuery("tssst", 5, Correction, LengthSorter)) 72 | fmt.Println("Performed sorted error correcting search in:", time.Now().Sub(t)) 73 | t = time.Now() 74 | fmt.Println(SearchEngine.SortedErrorCorrectingQuery("tssst", 5, Correction, FreqSorter)) 75 | fmt.Println("Performed sorted error correcting search in:", time.Now().Sub(t)) 76 | t = time.Now() 77 | fmt.Println(SearchEngine.SortedQuery("a", 5, LengthSorter)) 78 | fmt.Println("Performed sorted search in:", time.Now().Sub(t)) 79 | t = time.Now() 80 | fmt.Println(SearchEngine.SortedQuery("a", 5, FreqSorter)) 81 | fmt.Println("Performed sorted search in:", time.Now().Sub(t)) 82 | t = time.Now() 83 | fmt.Println(SearchEngine.Query("a", 5)) 84 | fmt.Println("Performed search in:", time.Now().Sub(t)) 85 | t = time.Now() 86 | fmt.Println(SearchEngine.Query("the", 25)) 87 | fmt.Println("Performed search in:", time.Now().Sub(t)) 88 | t = time.Now() 89 | fmt.Println(SearchEngine.SortedQuery("the", 25, FreqSorter)) 90 | fmt.Println("Performed sorted search in:", time.Now().Sub(t)) 91 | t = time.Now() 92 | SearchEngine.Insert("asdfghjklqwertyuiopzxcvbnm", "asdfghjklqwertyuiopzxcvbnm", uint64(0)) 93 | fmt.Println("Performed insert in:", time.Now().Sub(t)) 94 | t = time.Now() 95 | fmt.Println(SearchEngine.Query("sdfghjklqwert", 5)) 96 | fmt.Println("Performed search in:", time.Now().Sub(t)) 97 | fmt.Println("Running benchmarks...") 98 | t = time.Now() 99 | n := 0 100 | for _, Query := range SearchEngine.Words { 101 | SearchEngine.Query(string(Query), 5) 102 | n++ 103 | } 104 | fmt.Println("Performed", n, "limit-5 searches in:", time.Now().Sub(t)) 105 | t = time.Now() 106 | n = 0 107 | for _, Query := range SearchEngine.Words { 108 | SearchEngine.Query(string(Query), 25) 109 | n++ 110 | } 111 | fmt.Println("Performed", n, "limit-25 searches in:", time.Now().Sub(t)) 112 | t = time.Now() 113 | n = 0 114 | for _, Query := range SearchEngine.Words { 115 | SearchEngine.SortedQuery(string(Query), 5, LengthSorter) 116 | n++ 117 | } 118 | fmt.Println("Performed", n, "limit-5 length sorted searches in:", time.Now().Sub(t)) 119 | t = time.Now() 120 | n = 0 121 | for _, Query := range SearchEngine.Words { 122 | SearchEngine.SortedQuery(string(Query), 25, LengthSorter) 123 | n++ 124 | } 125 | fmt.Println("Performed", n, "limit-25 length sorted searches in:", time.Now().Sub(t)) 126 | t = time.Now() 127 | n = 0 128 | for _, Query := range SearchEngine.Words { 129 | SearchEngine.SortedQuery(string(Query), 5, FreqSorter) 130 | n++ 131 | } 132 | fmt.Println("Performed", n, "limit-5 frequency sorted searches in:", time.Now().Sub(t)) 133 | t = time.Now() 134 | n = 0 135 | for _, Query := range SearchEngine.Words { 136 | SearchEngine.SortedQuery(string(Query), 25, FreqSorter) 137 | n++ 138 | } 139 | fmt.Println("Performed", n, "limit-25 frequency sorted searches in:", time.Now().Sub(t)) 140 | t = time.Now() 141 | n = 0 142 | for _, Query := range SearchEngine.Words[:2048] { 143 | SearchEngine.ErrorCorrectingQuery(string(Query)+"0", 5, Correction) 144 | n++ 145 | } 146 | fmt.Println("Performed", n, "limit-5 error correcting searches in:", time.Now().Sub(t)) 147 | t = time.Now() 148 | n = 0 149 | for _, Query := range SearchEngine.Words[:2048] { 150 | SearchEngine.ErrorCorrectingQuery(string(Query)+"0", 25, Correction) 151 | n++ 152 | } 153 | fmt.Println("Performed", n, "limit-25 error correcting searches in:", time.Now().Sub(t)) 154 | t = time.Now() 155 | n = 0 156 | for _, Query := range SearchEngine.Words[:2048] { 157 | SearchEngine.SortedErrorCorrectingQuery(string(Query)+"0", 5, Correction, LengthSorter) 158 | n++ 159 | } 160 | fmt.Println("Performed", n, "limit-5 length sorted error correcting searches in:", time.Now().Sub(t)) 161 | t = time.Now() 162 | n = 0 163 | for _, Query := range SearchEngine.Words[:2048] { 164 | SearchEngine.SortedErrorCorrectingQuery(string(Query)+"0", 25, Correction, LengthSorter) 165 | n++ 166 | } 167 | fmt.Println("Performed", n, "limit-25 length sorted error correcting searches in:", time.Now().Sub(t)) 168 | t = time.Now() 169 | n = 0 170 | for _, Query := range SearchEngine.Words[:2048] { 171 | SearchEngine.SortedErrorCorrectingQuery(string(Query)+"0", 5, Correction, FreqSorter) 172 | n++ 173 | } 174 | fmt.Println("Performed", n, "limit-5 frequency sorted error correcting searches in:", time.Now().Sub(t)) 175 | t = time.Now() 176 | n = 0 177 | for _, Query := range SearchEngine.Words[:2048] { 178 | SearchEngine.SortedErrorCorrectingQuery(string(Query)+"0", 25, Correction, FreqSorter) 179 | n++ 180 | } 181 | fmt.Println("Performed", n, "limit-25 frequency sorted error correcting searches in:", time.Now().Sub(t)) 182 | } 183 | -------------------------------------------------------------------------------- /example/example.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Mark Canning 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // 16 | // Author: Mark Canning 17 | // Developed at: Tamber, Inc. (http://www.tamber.com/). 18 | 19 | package main 20 | 21 | import ( 22 | "fmt" 23 | "github.com/argusdusty/Ferret" 24 | ) 25 | 26 | // Some words to search for and return 27 | var ExampleWords = []string{ 28 | "abdeblah", 29 | "foobar", 30 | "barfoo", 31 | "qwerty", 32 | "testing", 33 | "example", 34 | "dictionary", 35 | "dvorak", 36 | "ferret", 37 | } 38 | 39 | // Length of data 40 | var ExampleData = []interface{}{ 41 | []uint64{8}, 42 | []uint64{6}, 43 | []uint64{6}, 44 | []uint64{6}, 45 | []uint64{7}, 46 | []uint64{7}, 47 | []uint64{10}, 48 | []uint64{6}, 49 | []uint64{6}, 50 | } 51 | 52 | var ExampleCorrection = func(b []byte) [][]byte { return ferret.ErrorCorrect(b, ferret.LowercaseLetters) } 53 | var ExampleSorter = func(s string, v interface{}, l int, i int) float64 { return -float64(l + i) } 54 | var ExampleConverter = func(s string) []byte { return []byte(s) } 55 | 56 | func main() { 57 | ExampleSearchEngine := ferret.New(ExampleWords, ExampleWords, ExampleData, ExampleConverter) 58 | fmt.Println(ExampleSearchEngine.Query("ar", 5)) 59 | fmt.Println(ExampleSearchEngine.Query("test", 5)) 60 | fmt.Println(ExampleSearchEngine.ErrorCorrectingQuery("tsst", 5, ExampleCorrection)) 61 | fmt.Println(ExampleSearchEngine.SortedErrorCorrectingQuery("tsst", 5, ExampleCorrection, ExampleSorter)) 62 | fmt.Println(ExampleSearchEngine.SortedQuery("a", 5, ExampleSorter)) 63 | fmt.Println(ExampleSearchEngine.Query("a", 5)) 64 | ExampleSearchEngine.Insert("asdfghjklqwertyuiopzxcvbnm", "asdfghjklqwertyuiopzxcvbnm", []uint64{26}) 65 | fmt.Println(ExampleSearchEngine.Query("sdfghjklqwert", 5)) 66 | fmt.Println(ExampleSearchEngine.Query("ferret", 5)) 67 | ExampleSearchEngine.Insert("ferret", "ferret", []uint64{7}) 68 | fmt.Println(ExampleSearchEngine.Query("ferret", 5)) 69 | 70 | fmt.Println(ExampleSearchEngine.Query("e", -1)) 71 | fmt.Println(ExampleSearchEngine.SortedQuery("e", -1, ExampleSorter)) 72 | fmt.Println(ExampleSearchEngine.ErrorCorrectingQuery("e", -1, ExampleCorrection)) 73 | fmt.Println(ExampleSearchEngine.SortedErrorCorrectingQuery("e", -1, ExampleCorrection, ExampleSorter)) 74 | } 75 | -------------------------------------------------------------------------------- /ferret.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Mark Canning 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // 16 | // Author: Mark Canning 17 | // Developed at: Tamber, Inc. (http://www.tamber.com/). 18 | 19 | // Package ferret implements a fast in-memory substring search engine 20 | package ferret // import "github.com/argusdusty/Ferret" 21 | 22 | import "sort" 23 | 24 | // InvertedSuffix implements the data structure for substring searches 25 | type InvertedSuffix struct { 26 | WordIndex []int // WordIndex and SuffixIndex are sorted by Words[WordIndex[i]][SuffixIndex[i]:] 27 | SuffixIndex []int // WordIndex and SuffixIndex are sorted by Words[WordIndex[i]][SuffixIndex[i]:] 28 | Words [][]byte // Words is the list of words (in []byte form) to perform substring searches over 29 | Results []string // Results is the string value of the words. Used as a return value 30 | Values []interface{} // Values is some data mapped to the words. Can be used for sorting, or as a return value 31 | Converter func(string) []byte // Converter converts an inserted word/query to a byte array to search for/with 32 | } 33 | 34 | // A wrapper type used to sort the three arrays according to sort.sort 35 | type sortWrapper struct { 36 | WordIndex []int 37 | SuffixIndex []int 38 | Words [][]byte 39 | } 40 | 41 | func (SW *sortWrapper) Swap(i, j int) { 42 | SW.WordIndex[i], SW.WordIndex[j] = SW.WordIndex[j], SW.WordIndex[i] 43 | SW.SuffixIndex[i], SW.SuffixIndex[j] = SW.SuffixIndex[j], SW.SuffixIndex[i] 44 | } 45 | 46 | func (SW *sortWrapper) Len() int { 47 | return len(SW.WordIndex) 48 | } 49 | 50 | // Equivalent to: 51 | // bytes.Compare(S.Words[S.WordIndex[i]][S.SuffixIndex[i]:], S.Words[S.WordIndex[j]][S.SuffixIndex[j]:]) <= 0 52 | // but faster 53 | func (SW *sortWrapper) Less(i, j int) bool { 54 | x := SW.WordIndex[i] 55 | y := SW.WordIndex[j] 56 | a := SW.Words[x] 57 | b := SW.Words[y] 58 | pa := SW.SuffixIndex[i] 59 | pb := SW.SuffixIndex[j] 60 | na := len(a) 61 | nb := len(b) 62 | for pa < na && pb < nb { 63 | wa := a[pa] 64 | wb := b[pb] 65 | if wa < wb { 66 | return true 67 | } 68 | if wb < wa { 69 | return false 70 | } 71 | pa++ 72 | pb++ 73 | } 74 | return pa == na 75 | } 76 | 77 | // New creates an inverted suffix from a dictionary of byte arrays, mapping data, and a string->[]byte converter 78 | func New(Words, Results []string, Data []interface{}, Converter func(string) []byte) *InvertedSuffix { 79 | CharCount := 0 80 | NewWords := make([][]byte, len(Words)) 81 | for i, Word := range Words { 82 | NewWord := Converter(Word) 83 | NewWords[i] = NewWord 84 | CharCount += len(NewWord) 85 | } 86 | WordIndex := make([]int, 0, CharCount) 87 | SuffixIndex := make([]int, 0, CharCount) 88 | for i, NewWord := range NewWords { 89 | for j := 0; j < len(NewWord); j++ { 90 | WordIndex = append(WordIndex, i) 91 | SuffixIndex = append(SuffixIndex, j) 92 | } 93 | } 94 | sort.Sort(&sortWrapper{WordIndex, SuffixIndex, NewWords}) 95 | Suffixes := &InvertedSuffix{WordIndex, SuffixIndex, NewWords, Results, Data, Converter} 96 | return Suffixes 97 | } 98 | 99 | // Insert adds a word to the dictionary that IS was built on. 100 | // This is pretty slow, because of linear-time insertion into an array, 101 | // so stick to New when you can 102 | func (IS *InvertedSuffix) Insert(Word, Result string, Data interface{}) { 103 | Query := IS.Converter(Word) 104 | low, high := IS.Search(Query) 105 | for k := low; k < high; k++ { 106 | if IS.Results[IS.WordIndex[k]] == Word { 107 | IS.Values[IS.WordIndex[k]] = Data 108 | return 109 | } 110 | } 111 | i := len(IS.Words) 112 | IS.Words = append(IS.Words, Query) 113 | Length := len(Query) 114 | IS.Results = append(IS.Results, Result) 115 | IS.Values = append(IS.Values, Data) 116 | for j := 0; j < Length; j++ { 117 | k, _ := IS.Search(Query[j:]) 118 | IS.WordIndex = append(IS.WordIndex, 0) 119 | copy(IS.WordIndex[k+1:], IS.WordIndex[k:]) 120 | IS.WordIndex[k] = i 121 | IS.SuffixIndex = append(IS.SuffixIndex, 0) 122 | copy(IS.SuffixIndex[k+1:], IS.SuffixIndex[k:]) 123 | IS.SuffixIndex[k] = j 124 | } 125 | } 126 | 127 | // Search performs an exact substring search for the query in the word dictionary 128 | // Returns the boundaries (low/high) of sorted suffixes which have the query as a prefix 129 | // This is a low-level interface. I wouldn't recommend using this yourself 130 | func (IS *InvertedSuffix) Search(Query []byte) (int, int) { 131 | low := 0 132 | high := len(IS.WordIndex) 133 | n := len(Query) 134 | for a := 0; a < n; a++ { 135 | c := Query[a] 136 | oldlow := low 137 | oldhigh := high 138 | i := low 139 | j := high 140 | // Raise the lower-bound 141 | for i < j { 142 | h := (i + j) >> 1 143 | Index := IS.WordIndex[h] 144 | Word := IS.Words[Index] 145 | Length := len(Word) 146 | d := IS.SuffixIndex[h] + a 147 | if d >= Length { 148 | i = h + 1 149 | } else { 150 | e := Word[d] 151 | if e < c { 152 | i = h + 1 153 | } else { 154 | j = h 155 | if e > c { 156 | high = h 157 | } 158 | } 159 | } 160 | } 161 | low = i 162 | if low == high { 163 | break 164 | } 165 | j = high 166 | if low == oldlow && high == oldhigh { 167 | // nothing to do here: Word[IS.SuffixIndex[(i + j) >> 1] + a] == c 168 | continue 169 | } 170 | // Lower the upper-bound 171 | for i < j { 172 | h := (i + j) >> 1 173 | Index := IS.WordIndex[h] 174 | Word := IS.Words[Index] 175 | Length := len(Word) 176 | d := IS.SuffixIndex[h] + a 177 | if d >= Length { 178 | i = h + 1 179 | } else { 180 | e := Word[d] 181 | if e <= c { 182 | i = h + 1 183 | if e < c { 184 | low = i 185 | } 186 | } else { 187 | j = h 188 | } 189 | } 190 | } 191 | high = j 192 | if low == high { 193 | break 194 | } 195 | } 196 | return low, high 197 | } 198 | 199 | // Query returns the strings which contain the query, and their stored values unsorted 200 | // Input: 201 | // Word: The substring to search for. 202 | // ResultsLimit: Limit the results to some number of values. Set to -1 for no limit 203 | func (IS *InvertedSuffix) Query(Word string, ResultsLimit int) ([]string, []interface{}) { 204 | Query := IS.Converter(Word) 205 | if ResultsLimit == 0 { 206 | return []string{}, []interface{}{} 207 | } 208 | if ResultsLimit < 0 { 209 | ResultsLimit = 0 210 | } 211 | Results := make([]string, 0, ResultsLimit) 212 | Values := make([]interface{}, 0, ResultsLimit) 213 | low, high := IS.Search(Query) 214 | a := 0 215 | used := make(map[int]bool, 0) 216 | for k := low; k < high; k++ { 217 | x := IS.WordIndex[k] 218 | if _, ok := used[x]; ok { 219 | continue 220 | } 221 | used[x] = true 222 | Results = append(Results, IS.Results[x]) 223 | Values = append(Values, IS.Values[x]) 224 | a++ 225 | if a == ResultsLimit { 226 | return Results, Values 227 | } 228 | } 229 | return Results, Values 230 | } 231 | 232 | // SortedQuery returns the strings which contain the query sorted 233 | // The function sorter produces a value to sort by (largest first) 234 | // Input: 235 | // Word: The substring to search for. 236 | // ResultsLimit: Limit the results to some number of values. Set to -1 for no limit 237 | // Sorter: Takes (Result, Value, Length, Index (where Query begins in Result)) (string, []byte, int, int) 238 | // and produces a value (float64) to sort by (largest first). 239 | func (IS *InvertedSuffix) SortedQuery(Word string, ResultsLimit int, Sorter func(string, interface{}, int, int) float64) ([]string, []interface{}, []float64) { 240 | Query := IS.Converter(Word) 241 | if ResultsLimit == 0 { 242 | return []string{}, []interface{}{}, []float64{} 243 | } 244 | if ResultsLimit < 0 { 245 | ResultsLimit = 0 246 | } 247 | Results := make([]string, 0, ResultsLimit) 248 | Values := make([]interface{}, 0, ResultsLimit) 249 | Scores := make([]float64, 0, ResultsLimit) 250 | if ResultsLimit == 0 { 251 | ResultsLimit = -1 252 | } 253 | low, high := IS.Search(Query) 254 | a := 0 255 | used := make(map[int]float64, 0) 256 | for k := low; k < high; k++ { 257 | x := IS.WordIndex[k] 258 | w := IS.Results[x] 259 | v := IS.Values[x] 260 | s := Sorter(w, v, len(IS.Words[x]), IS.SuffixIndex[k]) 261 | if ps, ok := used[x]; ok && ps >= s { 262 | continue 263 | } 264 | used[x] = s 265 | i := 0 266 | j := a 267 | for i < j { 268 | h := (i + j) >> 1 269 | if Scores[h] > s { 270 | i = h + 1 271 | } else { 272 | j = h 273 | } 274 | } 275 | if a == ResultsLimit { 276 | if i < a { 277 | copy(Results[i+1:], Results[i:a-1]) 278 | Results[i] = w 279 | copy(Values[i+1:], Values[i:a-1]) 280 | Values[i] = v 281 | copy(Scores[i+1:], Scores[i:a-1]) 282 | Scores[i] = s 283 | } 284 | } else { 285 | a++ 286 | if i < a { 287 | Results = append(Results, "") 288 | copy(Results[i+1:], Results[i:]) 289 | Results[i] = w 290 | Values = append(Values, nil) 291 | copy(Values[i+1:], Values[i:a-1]) 292 | Values[i] = v 293 | Scores = append(Scores, 0.0) 294 | copy(Scores[i+1:], Scores[i:a-1]) 295 | Scores[i] = s 296 | } else { 297 | Results = append(Results, w) 298 | Values = append(Values, v) 299 | Scores = append(Scores, s) 300 | } 301 | } 302 | } 303 | return Results, Values, Scores 304 | } 305 | 306 | // ErrorCorrectingQuery returns the strings which contain the query 307 | // Unsorted, I think it's partially sorted alphabetically 308 | // Will search for all substrings defined by ErrorCorrection 309 | // if no results are found on the initial query 310 | // Input: 311 | // Query: The substring to search for. 312 | // ResultsLimit: Limit the results so you don't return your whole dictionary by accident. Set to -1 for no limit 313 | // ErrorCorrection: Returns a list of alternate queries 314 | func (IS *InvertedSuffix) ErrorCorrectingQuery(Word string, ResultsLimit int, ErrorCorrection func([]byte) [][]byte) ([]string, []interface{}) { 315 | Query := IS.Converter(Word) 316 | if ResultsLimit == 0 { 317 | return []string{}, []interface{}{} 318 | } 319 | if ResultsLimit < 0 { 320 | ResultsLimit = 0 321 | } 322 | Results := make([]string, 0, ResultsLimit) 323 | Values := make([]interface{}, 0, ResultsLimit) 324 | low, high := IS.Search(Query) 325 | a := 0 326 | used := make(map[int]bool, 0) 327 | for k := low; k < high; k++ { 328 | x := IS.WordIndex[k] 329 | if _, ok := used[x]; ok { 330 | continue 331 | } 332 | used[x] = true 333 | Results = append(Results, IS.Results[x]) 334 | Values = append(Values, IS.Values[x]) 335 | a++ 336 | if a == ResultsLimit { 337 | return Results, Values 338 | } 339 | } 340 | if a != ResultsLimit { 341 | for _, q := range ErrorCorrection(Query) { 342 | low, high := IS.Search(q) 343 | for k := low; k < high; k++ { 344 | x := IS.WordIndex[k] 345 | if _, ok := used[x]; ok { 346 | continue 347 | } 348 | used[x] = true 349 | Results = append(Results, IS.Results[x]) 350 | Values = append(Values, IS.Values[x]) 351 | a++ 352 | if a == ResultsLimit { 353 | return Results, Values 354 | } 355 | } 356 | } 357 | } 358 | return Results, Values 359 | } 360 | 361 | // SortedErrorCorrectingQuery returns the strings which contain the query 362 | // Sorted. The function sorter produces a value to sort by (largest first) 363 | // Will search for all substrings defined by ErrorCorrection 364 | // if no results are found on the initial query 365 | // Input: 366 | // Query: The substring to search for. 367 | // ResultsLimit: Limit the results so you don't return your whole dictionary by accident. Set to -1 for no limit 368 | // ErrorCorrection: Returns a list of alternate queries 369 | // Sorter: Takes (Result, Value, Length, Index (where Query begins in Result)) 370 | // (string, []byte, int, int), and produces a value (float64) to sort by (largest first). 371 | func (IS *InvertedSuffix) SortedErrorCorrectingQuery(Word string, ResultsLimit int, ErrorCorrection func([]byte) [][]byte, Sorter func(string, interface{}, int, int) float64) ([]string, []interface{}, []float64) { 372 | Query := IS.Converter(Word) 373 | if ResultsLimit == 0 { 374 | return []string{}, []interface{}{}, []float64{} 375 | } 376 | if ResultsLimit < 0 { 377 | ResultsLimit = 0 378 | } 379 | Results := make([]string, 0, ResultsLimit) 380 | Values := make([]interface{}, 0, ResultsLimit) 381 | Scores := make([]float64, 0, ResultsLimit) 382 | if ResultsLimit == 0 { 383 | ResultsLimit = -1 384 | } 385 | low, high := IS.Search(Query) 386 | a := 0 387 | used := make(map[int]float64, 0) 388 | for k := low; k < high; k++ { 389 | x := IS.WordIndex[k] 390 | w := IS.Results[x] 391 | v := IS.Values[x] 392 | s := Sorter(w, v, len(IS.Words[x]), IS.SuffixIndex[k]) 393 | if ps, ok := used[x]; ok && ps >= s { 394 | continue 395 | } 396 | used[x] = s 397 | i := 0 398 | j := a 399 | for i < j { 400 | h := (i + j) >> 1 401 | if Scores[h] > s { 402 | i = h + 1 403 | } else { 404 | j = h 405 | } 406 | } 407 | if a == ResultsLimit { 408 | if i < a { 409 | copy(Results[i+1:], Results[i:a-1]) 410 | Results[i] = w 411 | copy(Values[i+1:], Values[i:a-1]) 412 | Values[i] = v 413 | copy(Scores[i+1:], Scores[i:a-1]) 414 | Scores[i] = s 415 | } 416 | } else { 417 | a++ 418 | if i < a { 419 | Results = append(Results, "") 420 | copy(Results[i+1:], Results[i:]) 421 | Results[i] = w 422 | Values = append(Values, nil) 423 | copy(Values[i+1:], Values[i:a-1]) 424 | Values[i] = v 425 | Scores = append(Scores, 0.0) 426 | copy(Scores[i+1:], Scores[i:a-1]) 427 | Scores[i] = s 428 | } else { 429 | Results = append(Results, w) 430 | Values = append(Values, v) 431 | Scores = append(Scores, s) 432 | } 433 | } 434 | } 435 | if a == 0 { 436 | for _, q := range ErrorCorrection(Query) { 437 | low, high := IS.Search(q) 438 | for k := low; k < high; k++ { 439 | x := IS.WordIndex[k] 440 | w := IS.Results[x] 441 | v := IS.Values[x] 442 | s := Sorter(w, v, len(IS.Words[x]), IS.SuffixIndex[k]) 443 | if ps, ok := used[x]; ok && ps >= s { 444 | continue 445 | } 446 | used[x] = s 447 | i := 0 448 | j := a 449 | for i < j { 450 | h := (i + j) >> 1 451 | if Scores[h] > s { 452 | i = h + 1 453 | } else { 454 | j = h 455 | } 456 | } 457 | if a == ResultsLimit { 458 | if i < a { 459 | copy(Results[i+1:], Results[i:a-1]) 460 | Results[i] = w 461 | copy(Values[i+1:], Values[i:a-1]) 462 | Values[i] = v 463 | copy(Scores[i+1:], Scores[i:a-1]) 464 | Scores[i] = s 465 | } 466 | } else { 467 | a++ 468 | if i < a { 469 | Results = append(Results, "") 470 | copy(Results[i+1:], Results[i:]) 471 | Results[i] = w 472 | Values = append(Values, nil) 473 | copy(Values[i+1:], Values[i:a-1]) 474 | Values[i] = v 475 | Scores = append(Scores, 0.0) 476 | copy(Scores[i+1:], Scores[i:a-1]) 477 | Scores[i] = s 478 | } else { 479 | Results = append(Results, w) 480 | Values = append(Values, v) 481 | Scores = append(Scores, s) 482 | } 483 | } 484 | } 485 | } 486 | } 487 | return Results, Values, Scores 488 | } 489 | -------------------------------------------------------------------------------- /unicodeconvert.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Mark Canning 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | // 16 | // Author: Mark Canning 17 | // Developed at: Tamber, Inc. (http://www.tamber.com/). 18 | 19 | package ferret 20 | 21 | import ( 22 | "strings" 23 | "unicode" 24 | ) 25 | 26 | // UnicodeToASCII maps the unicode Latin-1 supplement to ASCII characters without accents 27 | var UnicodeToASCII = map[rune]rune{ 28 | 'À': 'A', 'Á': 'A', 'Â': 'A', 'Ã': 'A', 'Ä': 'A', 'Å': 'A', 29 | 'Ç': 'C', 30 | 'È': 'E', 'É': 'E', 'Ê': 'E', 'Ë': 'E', 31 | 'Ì': 'I', 'Í': 'I', 'Î': 'I', 'Ï': 'I', 32 | 'Ñ': 'N', 33 | 'Ò': 'O', 'Ó': 'O', 'Ô': 'O', 'Õ': 'O', 'Ö': 'O', 'Ø': 'O', 34 | 'Ù': 'U', 'Ú': 'U', 'Û': 'U', 'Ü': 'U', 35 | 'Ý': 'Y', 36 | 'à': 'a', 'á': 'a', 'â': 'a', 'ã': 'a', 'ä': 'a', 'å': 'a', 37 | 'ç': 'c', 38 | 'è': 'e', 'é': 'e', 'ê': 'e', 'ë': 'e', 39 | 'ì': 'i', 'í': 'i', 'î': 'i', 'ï': 'i', 40 | 'ð': 'o', 41 | 'ñ': 'n', 42 | 'ò': 'o', 'ó': 'o', 'ô': 'o', 'õ': 'o', 'ö': 'o', 'ø': 'o', 43 | 'ù': 'u', 'ú': 'u', 'û': 'u', 'ü': 'u', 44 | 'ý': 'y', 'ÿ': 'y', 45 | } 46 | 47 | // ToASCII converts a single unicode rune to the ASCII equivalent using the UnicodeToASCII table 48 | func ToASCII(r rune) rune { 49 | a, ok := UnicodeToASCII[r] 50 | if ok { 51 | return a 52 | } 53 | return r 54 | } 55 | 56 | // UnicodeToLowerASCII converts a unicode string to ASCII bytes using the UnicodeToASCII table 57 | func UnicodeToLowerASCII(s string) []byte { 58 | return []byte(strings.Map(func(r rune) rune { return unicode.ToLower(ToASCII(r)) }, s)) 59 | } 60 | --------------------------------------------------------------------------------