├── .github └── workflows │ └── maven.yml ├── .gitignore ├── .travis.yml ├── LICENSE.md ├── README.md ├── checkstyle.xml ├── nbactions.xml ├── pom.xml └── src ├── main └── java │ └── info │ └── debatty │ └── java │ └── stringsimilarity │ ├── CharacterInsDelInterface.java │ ├── CharacterSubstitutionInterface.java │ ├── Cosine.java │ ├── Damerau.java │ ├── Jaccard.java │ ├── JaroWinkler.java │ ├── Levenshtein.java │ ├── LongestCommonSubsequence.java │ ├── MetricLCS.java │ ├── NGram.java │ ├── NormalizedLevenshtein.java │ ├── OptimalStringAlignment.java │ ├── QGram.java │ ├── RatcliffObershelp.java │ ├── ShingleBased.java │ ├── SorensenDice.java │ ├── WeightedLevenshtein.java │ ├── examples │ ├── Examples.java │ ├── MetricLCS.java │ ├── PrecomputedCosine.java │ └── nischay21.java │ ├── experimental │ └── Sift4.java │ └── interfaces │ ├── MetricStringDistance.java │ ├── NormalizedStringDistance.java │ ├── NormalizedStringSimilarity.java │ ├── StringDistance.java │ └── StringSimilarity.java └── test ├── java └── info │ └── debatty │ └── java │ └── stringsimilarity │ ├── CosineTest.java │ ├── DamerauTest.java │ ├── JaccardTest.java │ ├── JaroWinklerTest.java │ ├── LevenshteinTest.java │ ├── LongestCommonSubsequenceTest.java │ ├── MetricLCSTest.java │ ├── NGramTest.java │ ├── NormalizedLevenshteinTest.java │ ├── OptimalStringAlignmentTest.java │ ├── QGramTest.java │ ├── RatcliffObershelpTest.java │ ├── SorensenDiceTest.java │ ├── WeightedLevenshteinTest.java │ ├── experimental │ └── Sift4Test.java │ └── testutil │ └── NullEmptyTests.java └── resources ├── 11328-1.txt └── 71816-2.txt /.github/workflows/maven.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a Java project with Maven 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven 3 | 4 | name: Java CI with Maven 5 | 6 | on: 7 | push: 8 | branches: [ master ] 9 | pull_request: 10 | branches: [ master ] 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - uses: actions/checkout@v2 19 | - name: Set up JDK 11 20 | uses: actions/setup-java@v2 21 | with: 22 | java-version: '11' 23 | distribution: 'adopt' 24 | - name: Build with Maven 25 | run: mvn -B package --file pom.xml 26 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /nbproject/private/ 2 | /build/ 3 | /dist/ 4 | /target/ 5 | .idea/ 6 | *.iml 7 | *~ 8 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | install: mvn install -DskipTests=true -Dmaven.javadoc.skip=true -Dgpg.skip=true 3 | 4 | after_success: 5 | - mvn clean cobertura:cobertura coveralls:report 6 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | # License 2 | 3 | This project is licensed under the terms of the **MIT license**. 4 | 5 | https://opensource.org/licenses/MIT 6 | 7 | > Copyright 2015 Thibault Debatty. 8 | > 9 | > Permission is hereby granted, free of charge, to any person obtaining 10 | > a copy of this software and associated documentation files (the 11 | > "Software"), to deal in the Software without restriction, including 12 | > without limitation the rights to use, copy, modify, merge, publish, 13 | > distribute, sublicense, and/or sell copies of the Software, and to 14 | > permit persons to whom the Software is furnished to do so, subject to 15 | > the following conditions: 16 | > 17 | > The above copyright notice and this permission notice shall be 18 | > included in all copies or substantial portions of the Software. 19 | > 20 | > THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 21 | > EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 22 | > MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 23 | > NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 24 | > LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 25 | > OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 26 | > WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # java-string-similarity 2 | 3 | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/info.debatty/java-string-similarity/badge.svg)](https://maven-badges.herokuapp.com/maven-central/info.debatty/java-string-similarity) 4 | [![Build Status](https://github.com/tdebatty/java-string-similarity/actions/workflows/maven.yml/badge.svg)](https://github.com/tdebatty/java-string-similarity/actions/workflows/maven.yml) 5 | [![Coverage Status](https://coveralls.io/repos/tdebatty/java-string-similarity/badge.svg?branch=master&service=github)](https://coveralls.io/github/tdebatty/java-string-similarity?branch=master) 6 | [![Javadocs](http://www.javadoc.io/badge/info.debatty/java-string-similarity.svg)](http://www.javadoc.io/doc/info.debatty/java-string-similarity) 7 | 8 | A library implementing different string similarity and distance measures. A dozen of algorithms (including Levenshtein edit distance and sibblings, Jaro-Winkler, Longest Common Subsequence, cosine similarity etc.) are currently implemented. Check the summary table below for the complete list... 9 | 10 | * [Download](#download) 11 | * [Overview](#overview) 12 | * [Normalized, metric, similarity and distance](#normalized-metric-similarity-and-distance) 13 | * [Shingles (n-gram) based similarity and distance](#shingles-n-gram-based-similarity-and-distance) 14 | * [Levenshtein](#levenshtein) 15 | * [Normalized Levenshtein](#normalized-levenshtein) 16 | * [Weighted Levenshtein](#weighted-levenshtein) 17 | * [Damerau-Levenshtein](#damerau-levenshtein) 18 | * [Optimal String Alignment](#optimal-string-alignment) 19 | * [Jaro-Winkler](#jaro-winkler) 20 | * [Longest Common Subsequence](#longest-common-subsequence) 21 | * [Metric Longest Common Subsequence](#metric-longest-common-subsequence) 22 | * [N-Gram](#n-gram) 23 | * [Shingle (n-gram) based algorithms](#shingle-n-gram-based-algorithms) 24 | * [Q-Gram](#shingle-n-gram-based-algorithms) 25 | * [Cosine similarity](#shingle-n-gram-based-algorithms) 26 | * [Jaccard index](#shingle-n-gram-based-algorithms) 27 | * [Sorensen-Dice coefficient](#shingle-n-gram-based-algorithms) 28 | * [Ratcliff-Obershelp](#ratcliff-obershelp) 29 | * [Experimental](#experimental) 30 | * [SIFT4](#sift4) 31 | * [Users](#users) 32 | 33 | 34 | ## Download 35 | 36 | Using maven: 37 | 38 | ``` 39 | 40 | info.debatty 41 | java-string-similarity 42 | RELEASE 43 | 44 | ``` 45 | 46 | Or check the [releases](https://github.com/tdebatty/java-string-similarity/releases). 47 | 48 | This library requires Java 8 or more recent. 49 | 50 | ## Overview 51 | 52 | The main characteristics of each implemented algorithm are presented below. The "cost" column gives an estimation of the computational cost to compute the similarity between two strings of length m and n respectively. 53 | 54 | | | | Normalized? | Metric? | Type | Cost | Typical usage | 55 | | -------- |------- |------------- |-------- | ------ | ---- | --- | 56 | | [Levenshtein](#levenshtein) |distance | No | Yes | | O(m*n) 1 | | 57 | | [Normalized Levenshtein](#normalized-levenshtein) |distance
similarity | Yes | No | | O(m*n) 1 | | 58 | | [Weighted Levenshtein](#weighted-levenshtein) |distance | No | No | | O(m*n) 1 | OCR | 59 | | [Damerau-Levenshtein](#damerau-levenshtein) 3 |distance | No | Yes | | O(m*n) 1 | | 60 | | [Optimal String Alignment](#optimal-string-alignment) 3 |distance | No | No | | O(m*n) 1 | | 61 | | [Jaro-Winkler](#jaro-winkler) |similarity
distance | Yes | No | | O(m*n) | typo correction | 62 | | [Longest Common Subsequence](#longest-common-subsequence) |distance | No | No | | O(m*n) 1,2 | diff utility, GIT reconciliation | 63 | | [Metric Longest Common Subsequence](#metric-longest-common-subsequence) |distance | Yes | Yes | | O(m*n) 1,2 | | 64 | | [N-Gram](#n-gram) |distance | Yes | No | | O(m*n) | | 65 | | [Q-Gram](#q-gram) |distance | No | No | Profile | O(m+n) | | 66 | | [Cosine similarity](#cosine-similarity) |similarity
distance | Yes | No | Profile | O(m+n) | | 67 | | [Jaccard index](#jaccard-index) |similarity
distance | Yes | Yes | Set | O(m+n) | | 68 | | [Sorensen-Dice coefficient](#sorensen-dice-coefficient) |similarity
distance | Yes | No | Set | O(m+n) | | 69 | | [Ratcliff-Obershelp](#ratcliff-obershelp) |similarity
distance | Yes | No | | ? | | 70 | 71 | [1] In this library, Levenshtein edit distance, LCS distance and their sibblings are computed using the **dynamic programming** method, which has a cost O(m.n). For Levenshtein distance, the algorithm is sometimes called **Wagner-Fischer algorithm** ("The string-to-string correction problem", 1974). The original algorithm uses a matrix of size m x n to store the Levenshtein distance between string prefixes. 72 | 73 | If the alphabet is finite, it is possible to use the **method of four russians** (Arlazarov et al. "On economic construction of the transitive closure of a directed graph", 1970) to speedup computation. This was published by Masek in 1980 ("A Faster Algorithm Computing String Edit Distances"). This method splits the matrix in blocks of size t x t. Each possible block is precomputed to produce a lookup table. This lookup table can then be used to compute the string similarity (or distance) in O(nm/t). Usually, t is choosen as log(m) if m > n. The resulting computation cost is thus O(mn/log(m)). This method has not been implemented (yet). 74 | 75 | [2] In "Length of Maximal Common Subsequences", K.S. Larsen proposed an algorithm that computes the length of LCS in time O(log(m).log(n)). But the algorithm has a memory requirement O(m.n²) and was thus not implemented here. 76 | 77 | [3] There are two variants of Damerau-Levenshtein string distance: Damerau-Levenshtein with adjacent transpositions (also sometimes called unrestricted Damerau–Levenshtein distance) and Optimal String Alignment (also sometimes called restricted edit distance). For Optimal String Alignment, no substring can be edited more than once. 78 | 79 | ## Normalized, metric, similarity and distance 80 | Although the topic might seem simple, a lot of different algorithms exist to measure text similarity or distance. Therefore the library defines some interfaces to categorize them. 81 | 82 | ### (Normalized) similarity and distance 83 | 84 | - StringSimilarity : Implementing algorithms define a similarity between strings (0 means strings are completely different). 85 | - NormalizedStringSimilarity : Implementing algorithms define a similarity between 0.0 and 1.0, like Jaro-Winkler for example. 86 | - StringDistance : Implementing algorithms define a distance between strings (0 means strings are identical), like Levenshtein for example. The maximum distance value depends on the algorithm. 87 | - NormalizedStringDistance : This interface extends StringDistance. For implementing classes, the computed distance value is between 0.0 and 1.0. NormalizedLevenshtein is an example of NormalizedStringDistance. 88 | 89 | Generally, algorithms that implement NormalizedStringSimilarity also implement NormalizedStringDistance, and similarity = 1 - distance. But there are a few exceptions, like N-Gram similarity and distance (Kondrak)... 90 | 91 | ### Metric distances 92 | The MetricStringDistance interface : A few of the distances are actually metric distances, which means that verify the triangle inequality d(x, y) <= d(x,z) + d(z,y). For example, Levenshtein is a metric distance, but NormalizedLevenshtein is not. 93 | 94 | A lot of nearest-neighbor search algorithms and indexing structures rely on the triangle inequality. You can check "Similarity Search, The Metric Space Approach" by Zezula et al. for a survey. These cannot be used with non metric similarity measures. 95 | 96 | [Read Javadoc for a detailed description](http://www.javadoc.io/doc/info.debatty/java-string-similarity) 97 | 98 | ## Shingles (n-gram) based similarity and distance 99 | A few algorithms work by converting strings into sets of n-grams (sequences of n characters, also sometimes called k-shingles). The similarity or distance between the strings is then the similarity or distance between the sets. 100 | 101 | Some of them, like jaccard, consider strings as sets of shingles, and don't consider the number of occurences of each shingle. Others, like cosine similarity, work using what is sometimes called the profile of the strings, which takes into account the number of occurences of each shingle. 102 | 103 | For these algorithms, another use case is possible when dealing with large datasets: 104 | 1. compute the set or profile representation of all the strings 105 | 2. compute the similarity between sets or profiles 106 | 107 | ## Levenshtein 108 | The Levenshtein distance between two words is the minimum number of single-character edits (insertions, deletions or substitutions) required to change one word into the other. 109 | 110 | It is a metric string distance. This implementation uses dynamic programming (Wagner–Fischer algorithm), with only 2 rows of data. The space requirement is thus O(m) and the algorithm runs in O(m.n). 111 | 112 | ```java 113 | import info.debatty.java.stringsimilarity.*; 114 | 115 | public class MyApp { 116 | 117 | public static void main (String[] args) { 118 | Levenshtein l = new Levenshtein(); 119 | 120 | System.out.println(l.distance("My string", "My $tring")); 121 | System.out.println(l.distance("My string", "My $tring")); 122 | System.out.println(l.distance("My string", "My $tring")); 123 | } 124 | } 125 | ``` 126 | 127 | ## Normalized Levenshtein 128 | This distance is computed as levenshtein distance divided by the length of the longest string. The resulting value is always in the interval [0.0 1.0] but it is not a metric anymore! 129 | 130 | The similarity is computed as 1 - normalized distance. 131 | 132 | ```java 133 | import info.debatty.java.stringsimilarity.*; 134 | 135 | public class MyApp { 136 | 137 | public static void main (String[] args) { 138 | NormalizedLevenshtein l = new NormalizedLevenshtein(); 139 | 140 | System.out.println(l.distance("My string", "My $tring")); 141 | System.out.println(l.distance("My string", "My $tring")); 142 | System.out.println(l.distance("My string", "My $tring")); 143 | } 144 | } 145 | ``` 146 | 147 | ## Weighted Levenshtein 148 | An implementation of Levenshtein that allows to define different weights for different character substitutions. 149 | 150 | This algorithm is usually used for optical character recognition (OCR) applications. For OCR, the cost of substituting P and R is lower then the cost of substituting P and M for example because because from and OCR point of view P is similar to R. 151 | 152 | It can also be used for keyboard typing auto-correction. Here the cost of substituting E and R is lower for example because these are located next to each other on an AZERTY or QWERTY keyboard. Hence the probability that the user mistyped the characters is higher. 153 | 154 | ```java 155 | import info.debatty.java.stringsimilarity.*; 156 | 157 | public class MyApp { 158 | 159 | public static void main(String[] args) { 160 | WeightedLevenshtein wl = new WeightedLevenshtein( 161 | new CharacterSubstitutionInterface() { 162 | public double cost(char c1, char c2) { 163 | 164 | // The cost for substituting 't' and 'r' is considered 165 | // smaller as these 2 are located next to each other 166 | // on a keyboard 167 | if (c1 == 't' && c2 == 'r') { 168 | return 0.5; 169 | } 170 | 171 | // For most cases, the cost of substituting 2 characters 172 | // is 1.0 173 | return 1.0; 174 | } 175 | }); 176 | 177 | System.out.println(wl.distance("String1", "Srring2")); 178 | } 179 | } 180 | ``` 181 | 182 | ## Damerau-Levenshtein 183 | Similar to Levenshtein, Damerau-Levenshtein distance with transposition (also sometimes calls unrestricted Damerau-Levenshtein distance) is the minimum number of operations needed to transform one string into the other, where an operation is defined as an insertion, deletion, or substitution of a single character, or a **transposition of two adjacent characters**. 184 | 185 | It does respect triangle inequality, and is thus a metric distance. 186 | 187 | This is not to be confused with the optimal string alignment distance, which is an extension where no substring can be edited more than once. 188 | 189 | ```java 190 | import info.debatty.java.stringsimilarity.*; 191 | 192 | public class MyApp { 193 | 194 | 195 | public static void main(String[] args) { 196 | Damerau d = new Damerau(); 197 | 198 | // 1 substitution 199 | System.out.println(d.distance("ABCDEF", "ABDCEF")); 200 | 201 | // 2 substitutions 202 | System.out.println(d.distance("ABCDEF", "BACDFE")); 203 | 204 | // 1 deletion 205 | System.out.println(d.distance("ABCDEF", "ABCDE")); 206 | System.out.println(d.distance("ABCDEF", "BCDEF")); 207 | System.out.println(d.distance("ABCDEF", "ABCGDEF")); 208 | 209 | // All different 210 | System.out.println(d.distance("ABCDEF", "POIU")); 211 | } 212 | } 213 | ``` 214 | 215 | Will produce: 216 | 217 | ``` 218 | 1.0 219 | 2.0 220 | 1.0 221 | 1.0 222 | 1.0 223 | 6.0 224 | ``` 225 | 226 | ## Optimal String Alignment 227 | The Optimal String Alignment variant of Damerau–Levenshtein (sometimes called the restricted edit distance) computes the number of edit operations needed to make the strings equal under the condition that **no substring is edited more than once**, whereas the true Damerau–Levenshtein presents no such restriction. 228 | The difference from the algorithm for Levenshtein distance is the addition of one recurrence for the transposition operations. 229 | 230 | Note that for the optimal string alignment distance, the triangle inequality does not hold and so it is not a true metric. 231 | 232 | ```java 233 | import info.debatty.java.stringsimilarity.*; 234 | 235 | public class MyApp { 236 | 237 | 238 | public static void main(String[] args) { 239 | OptimalStringAlignment osa = new OptimalStringAlignment(); 240 | 241 | System.out.println(osa.distance("CA", "ABC"));; 242 | } 243 | } 244 | ``` 245 | 246 | Will produce: 247 | 248 | ``` 249 | 3.0 250 | ``` 251 | 252 | ## Jaro-Winkler 253 | Jaro-Winkler is a string edit distance that was developed in the area of record linkage (duplicate detection) (Winkler, 1990). The Jaro–Winkler distance metric is designed and best suited for short strings such as person names, and to detect transposition typos. 254 | 255 | Jaro-Winkler computes the similarity between 2 strings, and the returned value lies in the interval [0.0, 1.0]. 256 | It is (roughly) a variation of Damerau-Levenshtein, where the transposition of 2 close characters is considered less important than the transposition of 2 characters that are far from each other. Jaro-Winkler penalizes additions or substitutions that cannot be expressed as transpositions. 257 | 258 | The distance is computed as 1 - Jaro-Winkler similarity. 259 | 260 | ```java 261 | import info.debatty.java.stringsimilarity.*; 262 | 263 | public class MyApp { 264 | 265 | 266 | public static void main(String[] args) { 267 | JaroWinkler jw = new JaroWinkler(); 268 | 269 | // substitution of s and t 270 | System.out.println(jw.similarity("My string", "My tsring")); 271 | 272 | // substitution of s and n 273 | System.out.println(jw.similarity("My string", "My ntrisg")); 274 | } 275 | } 276 | ``` 277 | 278 | will produce: 279 | 280 | ``` 281 | 0.9740740656852722 282 | 0.8962963223457336 283 | ``` 284 | 285 | ## Longest Common Subsequence 286 | 287 | The longest common subsequence (LCS) problem consists in finding the longest subsequence common to two (or more) sequences. It differs from problems of finding common substrings: unlike substrings, subsequences are not required to occupy consecutive positions within the original sequences. 288 | 289 | It is used by the diff utility, by Git for reconciling multiple changes, etc. 290 | 291 | The LCS distance between strings X (of length n) and Y (of length m) is n + m - 2 |LCS(X, Y)| 292 | min = 0 293 | max = n + m 294 | 295 | LCS distance is equivalent to Levenshtein distance when only insertion and deletion is allowed (no substitution), or when the cost of the substitution is the double of the cost of an insertion or deletion. 296 | 297 | This class implements the dynamic programming approach, which has a space requirement O(m.n), and computation cost O(m.n). 298 | 299 | In "Length of Maximal Common Subsequences", K.S. Larsen proposed an algorithm that computes the length of LCS in time O(log(m).log(n)). But the algorithm has a memory requirement O(m.n²) and was thus not implemented here. 300 | 301 | ```java 302 | import info.debatty.java.stringsimilarity.*; 303 | 304 | public class MyApp { 305 | public static void main(String[] args) { 306 | LongestCommonSubsequence lcs = new LongestCommonSubsequence(); 307 | 308 | // Will produce 4.0 309 | System.out.println(lcs.distance("AGCAT", "GAC")); 310 | 311 | // Will produce 1.0 312 | System.out.println(lcs.distance("AGCAT", "AGCT")); 313 | } 314 | } 315 | ``` 316 | 317 | ## Metric Longest Common Subsequence 318 | Distance metric based on Longest Common Subsequence, from the notes "An LCS-based string metric" by Daniel Bakkelund. 319 | http://heim.ifi.uio.no/~danielry/StringMetric.pdf 320 | 321 | The distance is computed as 1 - |LCS(s1, s2)| / max(|s1|, |s2|) 322 | ```java 323 | public class MyApp { 324 | 325 | public static void main(String[] args) { 326 | 327 | info.debatty.java.stringsimilarity.MetricLCS lcs = 328 | new info.debatty.java.stringsimilarity.MetricLCS(); 329 | 330 | String s1 = "ABCDEFG"; 331 | String s2 = "ABCDEFHJKL"; 332 | // LCS: ABCDEF => length = 6 333 | // longest = s2 => length = 10 334 | // => 1 - 6/10 = 0.4 335 | System.out.println(lcs.distance(s1, s2)); 336 | 337 | // LCS: ABDF => length = 4 338 | // longest = ABDEF => length = 5 339 | // => 1 - 4 / 5 = 0.2 340 | System.out.println(lcs.distance("ABDEF", "ABDIF")); 341 | } 342 | } 343 | ``` 344 | 345 | ## N-Gram 346 | 347 | Normalized N-Gram distance as defined by Kondrak, "N-Gram Similarity and Distance", String Processing and Information Retrieval, Lecture Notes in Computer Science Volume 3772, 2005, pp 115-126. 348 | 349 | http://webdocs.cs.ualberta.ca/~kondrak/papers/spire05.pdf 350 | 351 | The algorithm uses affixing with special character '\n' to increase the weight of first characters. The normalization is achieved by dividing the total similarity score the original length of the longest word. 352 | 353 | In the paper, Kondrak also defines a similarity measure, which is not implemented (yet). 354 | 355 | ```java 356 | import info.debatty.java.stringsimilarity.*; 357 | 358 | public class MyApp { 359 | 360 | public static void main(String[] args) { 361 | 362 | // produces 0.583333 363 | NGram twogram = new NGram(2); 364 | System.out.println(twogram.distance("ABCD", "ABTUIO")); 365 | 366 | // produces 0.97222 367 | String s1 = "Adobe CreativeSuite 5 Master Collection from cheap 4zp"; 368 | String s2 = "Adobe CreativeSuite 5 Master Collection from cheap d1x"; 369 | NGram ngram = new NGram(4); 370 | System.out.println(ngram.distance(s1, s2)); 371 | } 372 | } 373 | ``` 374 | 375 | ## Shingle (n-gram) based algorithms 376 | A few algorithms work by converting strings into sets of n-grams (sequences of n characters, also sometimes called k-shingles). The similarity or distance between the strings is then the similarity or distance between the sets. 377 | 378 | The cost for computing these similarities and distances is mainly domnitated by k-shingling (converting the strings into sequences of k characters). Therefore there are typically two use cases for these algorithms: 379 | 380 | Directly compute the distance between strings: 381 | 382 | ```java 383 | import info.debatty.java.stringsimilarity.*; 384 | 385 | public class MyApp { 386 | 387 | public static void main(String[] args) { 388 | QGram dig = new QGram(2); 389 | 390 | // AB BC CD CE 391 | // 1 1 1 0 392 | // 1 1 0 1 393 | // Total: 2 394 | 395 | System.out.println(dig.distance("ABCD", "ABCE")); 396 | } 397 | } 398 | ``` 399 | 400 | Or, for large datasets, pre-compute the profile of all strings. The similarity can then be computed between profiles: 401 | 402 | ```java 403 | import info.debatty.java.stringsimilarity.KShingling; 404 | import info.debatty.java.stringsimilarity.StringProfile; 405 | 406 | 407 | /** 408 | * Example of computing cosine similarity with pre-computed profiles. 409 | */ 410 | public class PrecomputedCosine { 411 | 412 | public static void main(String[] args) throws Exception { 413 | String s1 = "My first string"; 414 | String s2 = "My other string..."; 415 | 416 | // Let's work with sequences of 2 characters... 417 | Cosine cosine = new Cosine(2); 418 | 419 | // Pre-compute the profile of strings 420 | Map profile1 = cosine.getProfile(s1); 421 | Map profile2 = cosine.getProfile(s2); 422 | 423 | // Prints 0.516185 424 | System.out.println(cosine.similarity(profile1, profile2)); 425 | } 426 | } 427 | ``` 428 | 429 | Pay attention, this only works if the same KShingling object is used to parse all input strings ! 430 | 431 | 432 | ### Q-Gram 433 | Q-gram distance, as defined by Ukkonen in "Approximate string-matching with q-grams and maximal matches" 434 | http://www.sciencedirect.com/science/article/pii/0304397592901434 435 | 436 | The distance between two strings is defined as the L1 norm of the difference of their profiles (the number of occurences of each n-gram): SUM( |V1_i - V2_i| ). Q-gram distance is a lower bound on Levenshtein distance, but can be computed in O(m + n), where Levenshtein requires O(m.n) 437 | 438 | 439 | ### Cosine similarity 440 | The similarity between the two strings is the cosine of the angle between these two vectors representation, and is computed as V1 . V2 / (|V1| * |V2|) 441 | 442 | Distance is computed as 1 - cosine similarity. 443 | 444 | ### Jaccard index 445 | Like Q-Gram distance, the input strings are first converted into sets of n-grams (sequences of n characters, also called k-shingles), but this time the cardinality of each n-gram is not taken into account. Each input string is simply a set of n-grams. The Jaccard index is then computed as |V1 inter V2| / |V1 union V2|. 446 | 447 | Distance is computed as 1 - similarity. 448 | Jaccard index is a metric distance. 449 | 450 | ### Sorensen-Dice coefficient 451 | Similar to Jaccard index, but this time the similarity is computed as 2 * |V1 inter V2| / (|V1| + |V2|). 452 | 453 | Distance is computed as 1 - similarity. 454 | 455 | ## Ratcliff-Obershelp 456 | Ratcliff/Obershelp Pattern Recognition, also known as Gestalt Pattern Matching, is a string-matching algorithm for determining the similarity of two strings. It was developed in 1983 by John W. Ratcliff and John A. Obershelp and published in the Dr. Dobb's Journal in July 1988 457 | 458 | Ratcliff/Obershelp computes the similarity between 2 strings, and the returned value lies in the interval [0.0, 1.0]. 459 | 460 | The distance is computed as 1 - Ratcliff/Obershelp similarity. 461 | 462 | ```java 463 | import info.debatty.java.stringsimilarity.*; 464 | 465 | public class MyApp { 466 | 467 | 468 | public static void main(String[] args) { 469 | RatcliffObershelp ro = new RatcliffObershelp(); 470 | 471 | // substitution of s and t 472 | System.out.println(ro.similarity("My string", "My tsring")); 473 | 474 | // substitution of s and n 475 | System.out.println(ro.similarity("My string", "My ntrisg")); 476 | } 477 | } 478 | ``` 479 | 480 | will produce: 481 | 482 | ``` 483 | 0.8888888888888888 484 | 0.7777777777777778 485 | ``` 486 | 487 | ## Experimental 488 | 489 | ### SIFT4 490 | SIFT4 is a general purpose string distance algorithm inspired by JaroWinkler and Longest Common Subsequence. It was developed to produce a distance measure that matches as close as possible to the human perception of string distance. Hence it takes into account elements like character substitution, character distance, longest common subsequence etc. It was developed using experimental testing, and without theoretical background. 491 | 492 | ``` 493 | import info.debatty.java.stringsimilarity.experimental.Sift4; 494 | 495 | public class MyApp { 496 | 497 | public static void main(String[] args) { 498 | String s1 = "This is the first string"; 499 | String s2 = "And this is another string"; 500 | Sift4 sift4 = new Sift4(); 501 | sift4.setMaxOffset(5); 502 | double expResult = 11.0; 503 | double result = sift4.distance(s1, s2); 504 | assertEquals(expResult, result, 0.0); 505 | } 506 | } 507 | ``` 508 | 509 | ## Users 510 | * [StringSimilarity.NET](https://github.com/feature23/StringSimilarity.NET) a .NET port of java-string-similarity 511 | * [OrientDB string-metrics](https://github.com/orientechnologies/extra-functions/tree/master/string-metrics) wraps java-string-similarity to provide different string similarity and distance measures as SQL functions in [OrientDB](https://github.com/orientechnologies/orientdb) 512 | 513 | Use java-string-similarity in your project and want it to be mentioned here? Don't hesitate to drop me a line! 514 | 515 | ## Security & stability 516 | [![security status](https://www.meterian.io/badge/gh/tdebatty/java-string-similarity/security)](https://www.meterian.io/report/gh/tdebatty/java-string-similarity) 517 | [![stability status](https://www.meterian.io/badge/gh/tdebatty/java-string-similarity/stability)](https://www.meterian.io/report/gh/tdebatty/java-string-similarity) 518 | -------------------------------------------------------------------------------- /checkstyle.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 35 | 36 | 37 | 44 | 45 | 46 | 47 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | 225 | 226 | 227 | 228 | 229 | -------------------------------------------------------------------------------- /nbactions.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | run 5 | 6 | jar 7 | 8 | 9 | process-classes 10 | org.codehaus.mojo:exec-maven-plugin:1.2.1:exec 11 | 12 | 13 | -classpath %classpath info.debatty.java.stringsimilarity.Main 14 | java 15 | 16 | 17 | 18 | debug 19 | 20 | jar 21 | 22 | 23 | process-classes 24 | org.codehaus.mojo:exec-maven-plugin:1.2.1:exec 25 | 26 | 27 | -Xdebug -Xrunjdwp:transport=dt_socket,server=n,address=${jpda.address} -classpath %classpath info.debatty.java.stringsimilarity.Main 28 | java 29 | true 30 | 31 | 32 | 33 | profile 34 | 35 | jar 36 | 37 | 38 | process-classes 39 | org.codehaus.mojo:exec-maven-plugin:1.2.1:exec 40 | 41 | 42 | -classpath %classpath info.debatty.java.stringsimilarity.Main 43 | java 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 4.0.0 6 | info.debatty 7 | java-string-similarity 8 | 2.0.1-SNAPSHOT 9 | jar 10 | 11 | ${project.artifactId} 12 | https://github.com/tdebatty/java-string-similarity 13 | Implementation of various string similarity and distance algorithms: Levenshtein, Jaro-winkler, n-Gram, Q-Gram, Jaccard index, Longest Common Subsequence edit distance, cosine similarity... 14 | 15 | 16 | UTF-8 17 | 18 | 19 | 20 | 21 | MIT License 22 | http://www.opensource.org/licenses/mit-license.php 23 | 24 | 25 | 26 | 27 | 28 | Thibault Debatty 29 | thibault@debatty.info 30 | debatty.info 31 | http://debatty.info 32 | 33 | 34 | 35 | 36 | scm:git:git@github.com:tdebatty/java-string-similarity.git 37 | scm:git:git@github.com:tdebatty/java-string-similarity.git 38 | git@github.com:tdebatty/java-string-similarity.git 39 | v0.7 40 | 41 | 42 | 43 | 44 | ossrh 45 | https://oss.sonatype.org/content/repositories/snapshots 46 | 47 | 48 | ossrh 49 | https://oss.sonatype.org/service/local/staging/deploy/maven2/ 50 | 51 | 52 | 53 | 54 | 55 | 56 | org.sonatype.plugins 57 | nexus-staging-maven-plugin 58 | 1.6.8 59 | true 60 | 61 | ossrh 62 | https://oss.sonatype.org/ 63 | true 64 | 65 | 66 | 67 | 68 | org.apache.maven.plugins 69 | maven-source-plugin 70 | 3.0.1 71 | 72 | 73 | attach-sources 74 | 75 | jar-no-fork 76 | 77 | 78 | 79 | 80 | 81 | 82 | org.apache.maven.plugins 83 | maven-javadoc-plugin 84 | 2.10.4 85 | 86 | 8 87 | 88 | 89 | 90 | attach-javadocs 91 | 92 | jar 93 | 94 | 95 | 96 | 97 | 98 | 99 | org.apache.maven.plugins 100 | maven-gpg-plugin 101 | 1.6 102 | 103 | 104 | sign-artifacts 105 | verify 106 | 107 | sign 108 | 109 | 110 | 111 | 112 | 113 | 114 | org.apache.maven.plugins 115 | maven-compiler-plugin 116 | 3.6.1 117 | 118 | 8 119 | 8 120 | 121 | 122 | 123 | 124 | 125 | org.apache.maven.plugins 126 | maven-release-plugin 127 | 2.5.3 128 | 129 | v@{project.version} 130 | 131 | 132 | 133 | 134 | org.eluder.coveralls 135 | coveralls-maven-plugin 136 | 4.3.0 137 | 138 | 139 | 140 | 141 | 142 | 143 | org.codehaus.mojo 144 | cobertura-maven-plugin 145 | 2.7 146 | 147 | 148 | html 149 | xml 150 | 151 | 256m 152 | 153 | 154 | 155 | 156 | 157 | info/debatty/java/stringsimilarity/examples/**/*.class 158 | 159 | 160 | 161 | 162 | 163 | 164 | org.apache.maven.plugins 165 | maven-checkstyle-plugin 166 | 2.16 167 | 168 | 169 | validate 170 | verify 171 | 172 | checkstyle.xml 173 | target/checkstyle_cache 174 | UTF-8 175 | true 176 | false 177 | **/examples/** 178 | 179 | 180 | check 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | junit 191 | junit 192 | 4.12 193 | test 194 | 195 | 196 | net.jcip 197 | jcip-annotations 198 | 1.0 199 | 200 | 201 | 202 | 203 | 204 | -------------------------------------------------------------------------------- /src/main/java/info/debatty/java/stringsimilarity/CharacterInsDelInterface.java: -------------------------------------------------------------------------------- 1 | package info.debatty.java.stringsimilarity; 2 | 3 | 4 | /** 5 | * As an adjunct to CharacterSubstitutionInterface, this interface 6 | * allows you to specify the cost of deletion or insertion of a 7 | * character. 8 | */ 9 | public interface CharacterInsDelInterface { 10 | /** 11 | * @param c The character being deleted. 12 | * @return The cost to be allocated to deleting the given character, 13 | * in the range [0, 1]. 14 | */ 15 | double deletionCost(char c); 16 | 17 | /** 18 | * @param c The character being inserted. 19 | * @return The cost to be allocated to inserting the given character, 20 | * in the range [0, 1]. 21 | */ 22 | double insertionCost(char c); 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/info/debatty/java/stringsimilarity/CharacterSubstitutionInterface.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2015 Thibault Debatty. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | 25 | package info.debatty.java.stringsimilarity; 26 | 27 | /** 28 | * Used to indicate the cost of character substitution. 29 | * 30 | * Cost should always be in [0.0 .. 1.0] 31 | * For example, in an OCR application, cost('o', 'a') could be 0.4 32 | * In a checkspelling application, cost('u', 'i') could be 0.4 because these are 33 | * next to each other on the keyboard... 34 | * 35 | * @author Thibault Debatty 36 | */ 37 | public interface CharacterSubstitutionInterface { 38 | /** 39 | * Indicate the cost of substitution c1 and c2. 40 | * @param c1 The first character of the substitution. 41 | * @param c2 The second character of the substitution. 42 | * @return The cost in the range [0, 1]. 43 | */ 44 | double cost(char c1, char c2); 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/info/debatty/java/stringsimilarity/Cosine.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2015 Thibault Debatty. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | package info.debatty.java.stringsimilarity; 25 | 26 | import info.debatty.java.stringsimilarity.interfaces.NormalizedStringSimilarity; 27 | import info.debatty.java.stringsimilarity.interfaces.NormalizedStringDistance; 28 | import java.util.Map; 29 | 30 | import net.jcip.annotations.Immutable; 31 | 32 | /** 33 | * The similarity between the two strings is the cosine of the angle between 34 | * these two vectors representation. It is computed as V1 . V2 / (|V1| * |V2|) 35 | * The cosine distance is computed as 1 - cosine similarity. 36 | * 37 | * @author Thibault Debatty 38 | */ 39 | @Immutable 40 | public class Cosine extends ShingleBased implements 41 | NormalizedStringDistance, NormalizedStringSimilarity { 42 | 43 | /** 44 | * Implements Cosine Similarity between strings. The strings are first 45 | * transformed in vectors of occurrences of k-shingles (sequences of k 46 | * characters). In this n-dimensional space, the similarity between the two 47 | * strings is the cosine of their respective vectors. 48 | * 49 | * @param k 50 | */ 51 | public Cosine(final int k) { 52 | super(k); 53 | } 54 | 55 | /** 56 | * Implements Cosine Similarity between strings. The strings are first 57 | * transformed in vectors of occurrences of k-shingles (sequences of k 58 | * characters). In this n-dimensional space, the similarity between the two 59 | * strings is the cosine of their respective vectors. Default k is 3. 60 | */ 61 | public Cosine() { 62 | super(); 63 | } 64 | 65 | /** 66 | * Compute the cosine similarity between strings. 67 | * 68 | * @param s1 The first string to compare. 69 | * @param s2 The second string to compare. 70 | * @return The cosine similarity in the range [0, 1] 71 | * @throws NullPointerException if s1 or s2 is null. 72 | */ 73 | @Override 74 | public final double similarity(final String s1, final String s2) { 75 | if (s1 == null) { 76 | throw new NullPointerException("s1 must not be null"); 77 | } 78 | 79 | if (s2 == null) { 80 | throw new NullPointerException("s2 must not be null"); 81 | } 82 | 83 | if (s1.equals(s2)) { 84 | return 1; 85 | } 86 | 87 | if (s1.length() < getK() || s2.length() < getK()) { 88 | return 0; 89 | } 90 | 91 | Map profile1 = getProfile(s1); 92 | Map profile2 = getProfile(s2); 93 | 94 | return dotProduct(profile1, profile2) 95 | / (norm(profile1) * norm(profile2)); 96 | } 97 | 98 | /** 99 | * Compute the norm L2 : sqrt(Sum_i( v_i²)). 100 | * 101 | * @param profile 102 | * @return L2 norm 103 | */ 104 | private static double norm(final Map profile) { 105 | double agg = 0; 106 | 107 | for (Map.Entry entry : profile.entrySet()) { 108 | agg += 1.0 * entry.getValue() * entry.getValue(); 109 | } 110 | 111 | return Math.sqrt(agg); 112 | } 113 | 114 | private static double dotProduct( 115 | final Map profile1, 116 | final Map profile2) { 117 | 118 | // Loop over the smallest map 119 | Map small_profile = profile2; 120 | Map large_profile = profile1; 121 | if (profile1.size() < profile2.size()) { 122 | small_profile = profile1; 123 | large_profile = profile2; 124 | } 125 | 126 | double agg = 0; 127 | for (Map.Entry entry : small_profile.entrySet()) { 128 | Integer i = large_profile.get(entry.getKey()); 129 | if (i == null) { 130 | continue; 131 | } 132 | agg += 1.0 * entry.getValue() * i; 133 | } 134 | 135 | return agg; 136 | } 137 | 138 | /** 139 | * Return 1.0 - similarity. 140 | * 141 | * @param s1 The first string to compare. 142 | * @param s2 The second string to compare. 143 | * @return 1.0 - the cosine similarity in the range [0, 1] 144 | * @throws NullPointerException if s1 or s2 is null. 145 | */ 146 | @Override 147 | public final double distance(final String s1, final String s2) { 148 | return 1.0 - similarity(s1, s2); 149 | } 150 | 151 | /** 152 | * Compute similarity between precomputed profiles. 153 | * 154 | * @param profile1 155 | * @param profile2 156 | * @return 157 | */ 158 | public final double similarity( 159 | final Map profile1, 160 | final Map profile2) { 161 | 162 | return dotProduct(profile1, profile2) 163 | / (norm(profile1) * norm(profile2)); 164 | } 165 | 166 | } 167 | -------------------------------------------------------------------------------- /src/main/java/info/debatty/java/stringsimilarity/Damerau.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2015 Thibault Debatty. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | package info.debatty.java.stringsimilarity; 25 | 26 | import info.debatty.java.stringsimilarity.interfaces.MetricStringDistance; 27 | import java.util.HashMap; 28 | 29 | import net.jcip.annotations.Immutable; 30 | 31 | /** 32 | * Implementation of Damerau-Levenshtein distance with transposition (also 33 | * sometimes calls unrestricted Damerau-Levenshtein distance). 34 | * It is the minimum number of operations needed to transform one string into 35 | * the other, where an operation is defined as an insertion, deletion, or 36 | * substitution of a single character, or a transposition of two adjacent 37 | * characters. 38 | * It does respect triangle inequality, and is thus a metric distance. 39 | * 40 | * This is not to be confused with the optimal string alignment distance, which 41 | * is an extension where no substring can be edited more than once. 42 | * 43 | * @author Thibault Debatty 44 | */ 45 | @Immutable 46 | public class Damerau implements MetricStringDistance { 47 | 48 | /** 49 | * Compute the distance between strings: the minimum number of operations 50 | * needed to transform one string into the other (insertion, deletion, 51 | * substitution of a single character, or a transposition of two adjacent 52 | * characters). 53 | * @param s1 The first string to compare. 54 | * @param s2 The second string to compare. 55 | * @return The computed distance. 56 | * @throws NullPointerException if s1 or s2 is null. 57 | */ 58 | public final double distance(final String s1, final String s2) { 59 | 60 | if (s1 == null) { 61 | throw new NullPointerException("s1 must not be null"); 62 | } 63 | 64 | if (s2 == null) { 65 | throw new NullPointerException("s2 must not be null"); 66 | } 67 | 68 | if (s1.equals(s2)) { 69 | return 0; 70 | } 71 | 72 | // INFinite distance is the max possible distance 73 | int inf = s1.length() + s2.length(); 74 | 75 | // Create and initialize the character array indices 76 | HashMap da = new HashMap(); 77 | 78 | for (int d = 0; d < s1.length(); d++) { 79 | da.put(s1.charAt(d), 0); 80 | } 81 | 82 | for (int d = 0; d < s2.length(); d++) { 83 | da.put(s2.charAt(d), 0); 84 | } 85 | 86 | // Create the distance matrix H[0 .. s1.length+1][0 .. s2.length+1] 87 | int[][] h = new int[s1.length() + 2][s2.length() + 2]; 88 | 89 | // initialize the left and top edges of H 90 | for (int i = 0; i <= s1.length(); i++) { 91 | h[i + 1][0] = inf; 92 | h[i + 1][1] = i; 93 | } 94 | 95 | for (int j = 0; j <= s2.length(); j++) { 96 | h[0][j + 1] = inf; 97 | h[1][j + 1] = j; 98 | 99 | } 100 | 101 | // fill in the distance matrix H 102 | // look at each character in s1 103 | for (int i = 1; i <= s1.length(); i++) { 104 | int db = 0; 105 | 106 | // look at each character in b 107 | for (int j = 1; j <= s2.length(); j++) { 108 | int i1 = da.get(s2.charAt(j - 1)); 109 | int j1 = db; 110 | 111 | int cost = 1; 112 | if (s1.charAt(i - 1) == s2.charAt(j - 1)) { 113 | cost = 0; 114 | db = j; 115 | } 116 | 117 | h[i + 1][j + 1] = min( 118 | h[i][j] + cost, // substitution 119 | h[i + 1][j] + 1, // insertion 120 | h[i][j + 1] + 1, // deletion 121 | h[i1][j1] + (i - i1 - 1) + 1 + (j - j1 - 1)); 122 | } 123 | 124 | da.put(s1.charAt(i - 1), i); 125 | } 126 | 127 | return h[s1.length() + 1][s2.length() + 1]; 128 | } 129 | 130 | private static int min( 131 | final int a, final int b, final int c, final int d) { 132 | return Math.min(a, Math.min(b, Math.min(c, d))); 133 | } 134 | 135 | } 136 | -------------------------------------------------------------------------------- /src/main/java/info/debatty/java/stringsimilarity/Jaccard.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2015 Thibault Debatty. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | 25 | package info.debatty.java.stringsimilarity; 26 | 27 | import info.debatty.java.stringsimilarity.interfaces.MetricStringDistance; 28 | import info.debatty.java.stringsimilarity.interfaces.NormalizedStringSimilarity; 29 | import info.debatty.java.stringsimilarity.interfaces.NormalizedStringDistance; 30 | import java.util.HashSet; 31 | import java.util.Map; 32 | import java.util.Set; 33 | 34 | import net.jcip.annotations.Immutable; 35 | 36 | /** 37 | * Each input string is converted into a set of n-grams, the Jaccard index is 38 | * then computed as |V1 inter V2| / |V1 union V2|. 39 | * Like Q-Gram distance, the input strings are first converted into sets of 40 | * n-grams (sequences of n characters, also called k-shingles), but this time 41 | * the cardinality of each n-gram is not taken into account. 42 | * Distance is computed as 1 - cosine similarity. 43 | * Jaccard index is a metric distance. 44 | * @author Thibault Debatty 45 | */ 46 | @Immutable 47 | public class Jaccard extends ShingleBased implements 48 | MetricStringDistance, NormalizedStringDistance, 49 | NormalizedStringSimilarity { 50 | 51 | /** 52 | * The strings are first transformed into sets of k-shingles (sequences of k 53 | * characters), then Jaccard index is computed as |A inter B| / |A union B|. 54 | * The default value of k is 3. 55 | * 56 | * @param k 57 | */ 58 | public Jaccard(final int k) { 59 | super(k); 60 | } 61 | 62 | /** 63 | * The strings are first transformed into sets of k-shingles (sequences of k 64 | * characters), then Jaccard index is computed as |A inter B| / |A union B|. 65 | * The default value of k is 3. 66 | */ 67 | public Jaccard() { 68 | super(); 69 | } 70 | 71 | /** 72 | * Compute Jaccard index: |A inter B| / |A union B|. 73 | * @param s1 The first string to compare. 74 | * @param s2 The second string to compare. 75 | * @return The Jaccard index in the range [0, 1] 76 | * @throws NullPointerException if s1 or s2 is null. 77 | */ 78 | public final double similarity(final String s1, final String s2) { 79 | if (s1 == null) { 80 | throw new NullPointerException("s1 must not be null"); 81 | } 82 | 83 | if (s2 == null) { 84 | throw new NullPointerException("s2 must not be null"); 85 | } 86 | 87 | if (s1.equals(s2)) { 88 | return 1; 89 | } 90 | 91 | Map profile1 = getProfile(s1); 92 | Map profile2 = getProfile(s2); 93 | 94 | 95 | Set union = new HashSet(); 96 | union.addAll(profile1.keySet()); 97 | union.addAll(profile2.keySet()); 98 | 99 | int inter = profile1.keySet().size() + profile2.keySet().size() 100 | - union.size(); 101 | 102 | return 1.0 * inter / union.size(); 103 | } 104 | 105 | 106 | /** 107 | * Distance is computed as 1 - similarity. 108 | * @param s1 The first string to compare. 109 | * @param s2 The second string to compare. 110 | * @return 1 - the Jaccard similarity. 111 | * @throws NullPointerException if s1 or s2 is null. 112 | */ 113 | public final double distance(final String s1, final String s2) { 114 | return 1.0 - similarity(s1, s2); 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /src/main/java/info/debatty/java/stringsimilarity/JaroWinkler.java: -------------------------------------------------------------------------------- 1 | package info.debatty.java.stringsimilarity; 2 | 3 | import info.debatty.java.stringsimilarity.interfaces.NormalizedStringSimilarity; 4 | import info.debatty.java.stringsimilarity.interfaces.NormalizedStringDistance; 5 | import java.util.Arrays; 6 | 7 | import net.jcip.annotations.Immutable; 8 | 9 | /** 10 | * The Jaro–Winkler distance metric is designed and best suited for short 11 | * strings such as person names, and to detect typos; it is (roughly) a 12 | * variation of Damerau-Levenshtein, where the substitution of 2 close 13 | * characters is considered less important then the substitution of 2 characters 14 | * that a far from each other. 15 | * Jaro-Winkler was developed in the area of record linkage (duplicate 16 | * detection) (Winkler, 1990). It returns a value in the interval [0.0, 1.0]. 17 | * The distance is computed as 1 - Jaro-Winkler similarity. 18 | * @author Thibault Debatty 19 | */ 20 | @Immutable 21 | public class JaroWinkler 22 | implements NormalizedStringSimilarity, NormalizedStringDistance { 23 | 24 | private static final double DEFAULT_THRESHOLD = 0.7; 25 | private static final int THREE = 3; 26 | private static final double JW_COEF = 0.1; 27 | private final double threshold; 28 | 29 | /** 30 | * Instantiate with default threshold (0.7). 31 | * 32 | */ 33 | public JaroWinkler() { 34 | this.threshold = DEFAULT_THRESHOLD; 35 | } 36 | 37 | /** 38 | * Instantiate with given threshold to determine when Winkler bonus should 39 | * be used. 40 | * Set threshold to a negative value to get the Jaro distance. 41 | * @param threshold 42 | */ 43 | public JaroWinkler(final double threshold) { 44 | this.threshold = threshold; 45 | } 46 | 47 | /** 48 | * Returns the current value of the threshold used for adding the Winkler 49 | * bonus. The default value is 0.7. 50 | * 51 | * @return the current value of the threshold 52 | */ 53 | public final double getThreshold() { 54 | return threshold; 55 | } 56 | 57 | /** 58 | * Compute Jaro-Winkler similarity. 59 | * @param s1 The first string to compare. 60 | * @param s2 The second string to compare. 61 | * @return The Jaro-Winkler similarity in the range [0, 1] 62 | * @throws NullPointerException if s1 or s2 is null. 63 | */ 64 | public final double similarity(final String s1, final String s2) { 65 | if (s1 == null) { 66 | throw new NullPointerException("s1 must not be null"); 67 | } 68 | 69 | if (s2 == null) { 70 | throw new NullPointerException("s2 must not be null"); 71 | } 72 | 73 | if (s1.equals(s2)) { 74 | return 1; 75 | } 76 | 77 | int[] mtp = matches(s1, s2); 78 | float m = mtp[0]; 79 | if (m == 0) { 80 | return 0f; 81 | } 82 | double j = ((m / s1.length() + m / s2.length() + (m - mtp[1]) / m)) 83 | / THREE; 84 | double jw = j; 85 | 86 | if (j > getThreshold()) { 87 | jw = j + Math.min(JW_COEF, 1.0 / mtp[THREE]) * mtp[2] * (1 - j); 88 | } 89 | return jw; 90 | } 91 | 92 | 93 | /** 94 | * Return 1 - similarity. 95 | * @param s1 The first string to compare. 96 | * @param s2 The second string to compare. 97 | * @return 1 - similarity. 98 | * @throws NullPointerException if s1 or s2 is null. 99 | */ 100 | public final double distance(final String s1, final String s2) { 101 | return 1.0 - similarity(s1, s2); 102 | } 103 | 104 | private int[] matches(final String s1, final String s2) { 105 | String max, min; 106 | if (s1.length() > s2.length()) { 107 | max = s1; 108 | min = s2; 109 | } else { 110 | max = s2; 111 | min = s1; 112 | } 113 | int range = Math.max(max.length() / 2 - 1, 0); 114 | int[] match_indexes = new int[min.length()]; 115 | Arrays.fill(match_indexes, -1); 116 | boolean[] match_flags = new boolean[max.length()]; 117 | int matches = 0; 118 | for (int mi = 0; mi < min.length(); mi++) { 119 | char c1 = min.charAt(mi); 120 | for (int xi = Math.max(mi - range, 0), 121 | xn = Math.min(mi + range + 1, max.length()); 122 | xi < xn; 123 | xi++) { 124 | if (!match_flags[xi] && c1 == max.charAt(xi)) { 125 | match_indexes[mi] = xi; 126 | match_flags[xi] = true; 127 | matches++; 128 | break; 129 | } 130 | } 131 | } 132 | char[] ms1 = new char[matches]; 133 | char[] ms2 = new char[matches]; 134 | for (int i = 0, si = 0; i < min.length(); i++) { 135 | if (match_indexes[i] != -1) { 136 | ms1[si] = min.charAt(i); 137 | si++; 138 | } 139 | } 140 | for (int i = 0, si = 0; i < max.length(); i++) { 141 | if (match_flags[i]) { 142 | ms2[si] = max.charAt(i); 143 | si++; 144 | } 145 | } 146 | int transpositions = 0; 147 | for (int mi = 0; mi < ms1.length; mi++) { 148 | if (ms1[mi] != ms2[mi]) { 149 | transpositions++; 150 | } 151 | } 152 | int prefix = 0; 153 | for (int mi = 0; mi < min.length(); mi++) { 154 | if (s1.charAt(mi) == s2.charAt(mi)) { 155 | prefix++; 156 | } else { 157 | break; 158 | } 159 | } 160 | return new int[]{matches, transpositions / 2, prefix, max.length()}; 161 | } 162 | } 163 | -------------------------------------------------------------------------------- /src/main/java/info/debatty/java/stringsimilarity/Levenshtein.java: -------------------------------------------------------------------------------- 1 | package info.debatty.java.stringsimilarity; 2 | 3 | import info.debatty.java.stringsimilarity.interfaces.MetricStringDistance; 4 | import net.jcip.annotations.Immutable; 5 | 6 | /** 7 | * The Levenshtein distance between two words is the minimum number of 8 | * single-character edits (insertions, deletions or substitutions) required to 9 | * change one string into the other. 10 | * 11 | * @author Thibault Debatty 12 | */ 13 | @Immutable 14 | public class Levenshtein implements MetricStringDistance { 15 | 16 | /** 17 | * Equivalent to distance(s1, s2, Integer.MAX_VALUE). 18 | */ 19 | public final double distance(final String s1, final String s2) { 20 | return distance(s1, s2, Integer.MAX_VALUE); 21 | } 22 | 23 | /** 24 | * The Levenshtein distance, or edit distance, between two words is the 25 | * minimum number of single-character edits (insertions, deletions or 26 | * substitutions) required to change one word into the other. 27 | * 28 | * http://en.wikipedia.org/wiki/Levenshtein_distance 29 | * 30 | * It is always at least the difference of the sizes of the two strings. 31 | * It is at most the length of the longer string. 32 | * It is zero if and only if the strings are equal. 33 | * If the strings are the same size, the Hamming distance is an upper bound 34 | * on the Levenshtein distance. 35 | * The Levenshtein distance verifies the triangle inequality (the distance 36 | * between two strings is no greater than the sum Levenshtein distances from 37 | * a third string). 38 | * 39 | * Implementation uses dynamic programming (Wagner–Fischer algorithm), with 40 | * only 2 rows of data. The space requirement is thus O(m) and the algorithm 41 | * runs in O(mn). 42 | * 43 | * @param s1 The first string to compare. 44 | * @param s2 The second string to compare. 45 | * @param limit The maximum result to compute before stopping. This 46 | * means that the calculation can terminate early if you 47 | * only care about strings with a certain similarity. 48 | * Set this to Integer.MAX_VALUE if you want to run the 49 | * calculation to completion in every case. 50 | * @return The computed Levenshtein distance. 51 | * @throws NullPointerException if s1 or s2 is null. 52 | */ 53 | public final double distance(final String s1, final String s2, 54 | final int limit) { 55 | if (s1 == null) { 56 | throw new NullPointerException("s1 must not be null"); 57 | } 58 | 59 | if (s2 == null) { 60 | throw new NullPointerException("s2 must not be null"); 61 | } 62 | 63 | if (s1.equals(s2)) { 64 | return 0; 65 | } 66 | 67 | if (s1.length() == 0) { 68 | return s2.length(); 69 | } 70 | 71 | if (s2.length() == 0) { 72 | return s1.length(); 73 | } 74 | 75 | // create two work vectors of integer distances 76 | int[] v0 = new int[s2.length() + 1]; 77 | int[] v1 = new int[s2.length() + 1]; 78 | int[] vtemp; 79 | 80 | // initialize v0 (the previous row of distances) 81 | // this row is A[0][i]: edit distance for an empty s 82 | // the distance is just the number of characters to delete from t 83 | for (int i = 0; i < v0.length; i++) { 84 | v0[i] = i; 85 | } 86 | 87 | for (int i = 0; i < s1.length(); i++) { 88 | // calculate v1 (current row distances) from the previous row v0 89 | // first element of v1 is A[i+1][0] 90 | // edit distance is delete (i+1) chars from s to match empty t 91 | v1[0] = i + 1; 92 | 93 | int minv1 = v1[0]; 94 | 95 | // use formula to fill in the rest of the row 96 | for (int j = 0; j < s2.length(); j++) { 97 | int cost = 1; 98 | if (s1.charAt(i) == s2.charAt(j)) { 99 | cost = 0; 100 | } 101 | v1[j + 1] = Math.min( 102 | v1[j] + 1, // Cost of insertion 103 | Math.min( 104 | v0[j + 1] + 1, // Cost of remove 105 | v0[j] + cost)); // Cost of substitution 106 | 107 | minv1 = Math.min(minv1, v1[j + 1]); 108 | } 109 | 110 | if (minv1 >= limit) { 111 | return limit; 112 | } 113 | 114 | // copy v1 (current row) to v0 (previous row) for next iteration 115 | //System.arraycopy(v1, 0, v0, 0, v0.length); 116 | 117 | // Flip references to current and previous row 118 | vtemp = v0; 119 | v0 = v1; 120 | v1 = vtemp; 121 | 122 | } 123 | 124 | return v0[s2.length()]; 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /src/main/java/info/debatty/java/stringsimilarity/LongestCommonSubsequence.java: -------------------------------------------------------------------------------- 1 | package info.debatty.java.stringsimilarity; 2 | 3 | import info.debatty.java.stringsimilarity.interfaces.StringDistance; 4 | import net.jcip.annotations.Immutable; 5 | 6 | /** 7 | * The longest common subsequence (LCS) problem consists in finding the longest 8 | * subsequence common to two (or more) sequences. It differs from problems of 9 | * finding common substrings: unlike substrings, subsequences are not required 10 | * to occupy consecutive positions within the original sequences. 11 | * 12 | * It is used by the diff utility, by Git for reconciling multiple changes, etc. 13 | * 14 | * The LCS distance between Strings X (length n) and Y (length m) is n + m - 2 15 | * |LCS(X, Y)| min = 0 max = n + m 16 | * 17 | * LCS distance is equivalent to Levenshtein distance, when only insertion and 18 | * deletion is allowed (no substitution), or when the cost of the substitution 19 | * is the double of the cost of an insertion or deletion. 20 | * 21 | * ! This class currently implements the dynamic programming approach, which has 22 | * a space requirement O(m * n)! 23 | * 24 | * @author Thibault Debatty 25 | */ 26 | @Immutable 27 | public class LongestCommonSubsequence implements StringDistance { 28 | 29 | /** 30 | * Return the LCS distance between strings s1 and s2, computed as |s1| + 31 | * |s2| - 2 * |LCS(s1, s2)|. 32 | * 33 | * @param s1 The first string to compare. 34 | * @param s2 The second string to compare. 35 | * @return the LCS distance between strings s1 and s2, computed as |s1| + 36 | * |s2| - 2 * |LCS(s1, s2)| 37 | * @throws NullPointerException if s1 or s2 is null. 38 | */ 39 | public final double distance(final String s1, final String s2) { 40 | if (s1 == null) { 41 | throw new NullPointerException("s1 must not be null"); 42 | } 43 | 44 | if (s2 == null) { 45 | throw new NullPointerException("s2 must not be null"); 46 | } 47 | 48 | if (s1.equals(s2)) { 49 | return 0; 50 | } 51 | 52 | return s1.length() + s2.length() - 2 * length(s1, s2); 53 | } 54 | 55 | /** 56 | * Return the length of Longest Common Subsequence (LCS) between strings s1 57 | * and s2. 58 | * 59 | * @param s1 The first string to compare. 60 | * @param s2 The second string to compare. 61 | * @return the length of LCS(s1, s2) 62 | * @throws NullPointerException if s1 or s2 is null. 63 | */ 64 | public final int length(final String s1, final String s2) { 65 | if (s1 == null) { 66 | throw new NullPointerException("s1 must not be null"); 67 | } 68 | 69 | if (s2 == null) { 70 | throw new NullPointerException("s2 must not be null"); 71 | } 72 | 73 | /* function LCSLength(X[1..m], Y[1..n]) 74 | C = array(0..m, 0..n) 75 | 76 | for i := 0..m 77 | C[i,0] = 0 78 | 79 | for j := 0..n 80 | C[0,j] = 0 81 | 82 | for i := 1..m 83 | for j := 1..n 84 | if X[i] = Y[j] 85 | C[i,j] := C[i-1,j-1] + 1 86 | else 87 | C[i,j] := max(C[i,j-1], C[i-1,j]) 88 | return C[m,n] 89 | */ 90 | int s1_length = s1.length(); 91 | int s2_length = s2.length(); 92 | char[] x = s1.toCharArray(); 93 | char[] y = s2.toCharArray(); 94 | 95 | int[][] c = new int[s1_length + 1][s2_length + 1]; 96 | 97 | for (int i = 1; i <= s1_length; i++) { 98 | for (int j = 1; j <= s2_length; j++) { 99 | if (x[i - 1] == y[j - 1]) { 100 | c[i][j] = c[i - 1][j - 1] + 1; 101 | 102 | } else { 103 | c[i][j] = Math.max(c[i][j - 1], c[i - 1][j]); 104 | } 105 | } 106 | } 107 | 108 | return c[s1_length][s2_length]; 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /src/main/java/info/debatty/java/stringsimilarity/MetricLCS.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2015 Thibault Debatty. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | 25 | package info.debatty.java.stringsimilarity; 26 | 27 | import info.debatty.java.stringsimilarity.interfaces.MetricStringDistance; 28 | import info.debatty.java.stringsimilarity.interfaces.NormalizedStringDistance; 29 | import net.jcip.annotations.Immutable; 30 | 31 | /** 32 | * Distance metric based on Longest Common Subsequence, from the notes "An 33 | * LCS-based string metric" by Daniel Bakkelund. 34 | * 35 | * @author Thibault Debatty 36 | */ 37 | @Immutable 38 | public class MetricLCS 39 | implements MetricStringDistance, NormalizedStringDistance { 40 | 41 | private final LongestCommonSubsequence lcs = new LongestCommonSubsequence(); 42 | 43 | /** 44 | * Distance metric based on Longest Common Subsequence, computed as 45 | * 1 - |LCS(s1, s2)| / max(|s1|, |s2|). 46 | * 47 | * @param s1 The first string to compare. 48 | * @param s2 The second string to compare. 49 | * @return The computed distance metric value. 50 | * @throws NullPointerException if s1 or s2 is null. 51 | */ 52 | public final double distance(final String s1, final String s2) { 53 | if (s1 == null) { 54 | throw new NullPointerException("s1 must not be null"); 55 | } 56 | 57 | if (s2 == null) { 58 | throw new NullPointerException("s2 must not be null"); 59 | } 60 | 61 | if (s1.equals(s2)) { 62 | return 0; 63 | } 64 | 65 | int m_len = Math.max(s1.length(), s2.length()); 66 | if (m_len == 0) { 67 | return 0; 68 | } 69 | return 1.0 70 | - (1.0 * lcs.length(s1, s2)) 71 | / m_len; 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/main/java/info/debatty/java/stringsimilarity/NGram.java: -------------------------------------------------------------------------------- 1 | package info.debatty.java.stringsimilarity; 2 | 3 | import info.debatty.java.stringsimilarity.interfaces.NormalizedStringDistance; 4 | import net.jcip.annotations.Immutable; 5 | 6 | /** 7 | * N-Gram Similarity as defined by Kondrak, "N-Gram Similarity and Distance", 8 | * String Processing and Information Retrieval, Lecture Notes in Computer 9 | * Science Volume 3772, 2005, pp 115-126. 10 | * 11 | * The algorithm uses affixing with special character '\n' to increase the 12 | * weight of first characters. The normalization is achieved by dividing the 13 | * total similarity score the original length of the longest word. 14 | * 15 | * http://webdocs.cs.ualberta.ca/~kondrak/papers/spire05.pdf 16 | */ 17 | @Immutable 18 | public class NGram implements NormalizedStringDistance { 19 | 20 | private static final int DEFAULT_N = 2; 21 | private final int n; 22 | 23 | /** 24 | * Instantiate with given value for n-gram length. 25 | * @param n 26 | */ 27 | public NGram(final int n) { 28 | this.n = n; 29 | } 30 | 31 | /** 32 | * Instantiate with default value for n-gram length (2). 33 | */ 34 | public NGram() { 35 | this.n = DEFAULT_N; 36 | } 37 | 38 | /** 39 | * Compute n-gram distance. 40 | * @param s0 The first string to compare. 41 | * @param s1 The second string to compare. 42 | * @return The computed n-gram distance in the range [0, 1] 43 | * @throws NullPointerException if s0 or s1 is null. 44 | */ 45 | public final double distance(final String s0, final String s1) { 46 | if (s0 == null) { 47 | throw new NullPointerException("s0 must not be null"); 48 | } 49 | 50 | if (s1 == null) { 51 | throw new NullPointerException("s1 must not be null"); 52 | } 53 | 54 | if (s0.equals(s1)) { 55 | return 0; 56 | } 57 | 58 | final char special = '\n'; 59 | final int sl = s0.length(); 60 | final int tl = s1.length(); 61 | 62 | if (sl == 0 || tl == 0) { 63 | return 1; 64 | } 65 | 66 | int cost = 0; 67 | if (sl < n || tl < n) { 68 | for (int i = 0, ni = Math.min(sl, tl); i < ni; i++) { 69 | if (s0.charAt(i) == s1.charAt(i)) { 70 | cost++; 71 | } 72 | } 73 | return (float) cost / Math.max(sl, tl); 74 | } 75 | 76 | char[] sa = new char[sl + n - 1]; 77 | float[] p; //'previous' cost array, horizontally 78 | float[] d; // cost array, horizontally 79 | float[] d2; //placeholder to assist in swapping p and d 80 | 81 | //construct sa with prefix 82 | for (int i = 0; i < sa.length; i++) { 83 | if (i < n - 1) { 84 | sa[i] = special; //add prefix 85 | } else { 86 | sa[i] = s0.charAt(i - n + 1); 87 | } 88 | } 89 | p = new float[sl + 1]; 90 | d = new float[sl + 1]; 91 | 92 | // indexes into strings s and t 93 | int i; // iterates through source 94 | int j; // iterates through target 95 | 96 | char[] t_j = new char[n]; // jth n-gram of t 97 | 98 | for (i = 0; i <= sl; i++) { 99 | p[i] = i; 100 | } 101 | 102 | for (j = 1; j <= tl; j++) { 103 | //construct t_j n-gram 104 | if (j < n) { 105 | for (int ti = 0; ti < n - j; ti++) { 106 | t_j[ti] = special; //add prefix 107 | } 108 | for (int ti = n - j; ti < n; ti++) { 109 | t_j[ti] = s1.charAt(ti - (n - j)); 110 | } 111 | } else { 112 | t_j = s1.substring(j - n, j).toCharArray(); 113 | } 114 | d[0] = j; 115 | for (i = 1; i <= sl; i++) { 116 | cost = 0; 117 | int tn = n; 118 | //compare sa to t_j 119 | for (int ni = 0; ni < n; ni++) { 120 | if (sa[i - 1 + ni] != t_j[ni]) { 121 | cost++; 122 | } else if (sa[i - 1 + ni] == special) { 123 | //discount matches on prefix 124 | tn--; 125 | } 126 | } 127 | float ec = (float) cost / tn; 128 | // minimum of cell to the left+1, to the top+1, 129 | // diagonally left and up +cost 130 | d[i] = Math.min( 131 | Math.min(d[i - 1] + 1, p[i] + 1), p[i - 1] + ec); 132 | } 133 | // copy current distance counts to 'previous row' distance counts 134 | d2 = p; 135 | p = d; 136 | d = d2; 137 | } 138 | 139 | // our last action in the above loop was to switch d and p, so p now 140 | // actually has the most recent cost counts 141 | return p[sl] / Math.max(tl, sl); 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /src/main/java/info/debatty/java/stringsimilarity/NormalizedLevenshtein.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2015 Thibault Debatty. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | package info.debatty.java.stringsimilarity; 25 | 26 | import info.debatty.java.stringsimilarity.interfaces.NormalizedStringSimilarity; 27 | import info.debatty.java.stringsimilarity.interfaces.NormalizedStringDistance; 28 | import net.jcip.annotations.Immutable; 29 | 30 | /** 31 | * This distance is computed as levenshtein distance divided by the length of 32 | * the longest string. The resulting value is always in the interval [0.0 1.0] 33 | * but it is not a metric anymore! The similarity is computed as 1 - normalized 34 | * distance. 35 | * 36 | * @author Thibault Debatty 37 | */ 38 | @Immutable 39 | public class NormalizedLevenshtein implements 40 | NormalizedStringDistance, NormalizedStringSimilarity { 41 | 42 | private final Levenshtein l = new Levenshtein(); 43 | 44 | /** 45 | * Compute distance as Levenshtein(s1, s2) / max(|s1|, |s2|). 46 | * @param s1 The first string to compare. 47 | * @param s2 The second string to compare. 48 | * @return The computed distance in the range [0, 1] 49 | * @throws NullPointerException if s1 or s2 is null. 50 | */ 51 | public final double distance(final String s1, final String s2) { 52 | 53 | if (s1 == null) { 54 | throw new NullPointerException("s1 must not be null"); 55 | } 56 | 57 | if (s2 == null) { 58 | throw new NullPointerException("s2 must not be null"); 59 | } 60 | 61 | if (s1.equals(s2)) { 62 | return 0; 63 | } 64 | 65 | int m_len = Math.max(s1.length(), s2.length()); 66 | 67 | if (m_len == 0) { 68 | return 0; 69 | } 70 | 71 | return l.distance(s1, s2) / m_len; 72 | } 73 | 74 | /** 75 | * Return 1 - distance. 76 | * @param s1 The first string to compare. 77 | * @param s2 The second string to compare. 78 | * @return 1.0 - the computed distance 79 | * @throws NullPointerException if s1 or s2 is null. 80 | */ 81 | public final double similarity(final String s1, final String s2) { 82 | return 1.0 - distance(s1, s2); 83 | } 84 | 85 | } 86 | -------------------------------------------------------------------------------- /src/main/java/info/debatty/java/stringsimilarity/OptimalStringAlignment.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2016 Thibault Debatty. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | package info.debatty.java.stringsimilarity; 25 | 26 | import info.debatty.java.stringsimilarity.interfaces.StringDistance; 27 | import net.jcip.annotations.Immutable; 28 | 29 | /** 30 | * Implementation of the the Optimal String Alignment (sometimes called the 31 | * restricted edit distance) variant of the Damerau-Levenshtein distance. 32 | * 33 | * The difference between the two algorithms consists in that the Optimal String 34 | * Alignment algorithm computes the number of edit operations needed to make the 35 | * strings equal under the condition that no substring is edited more than once, 36 | * whereas Damerau-Levenshtein presents no such restriction. 37 | * 38 | * @author Michail Bogdanos 39 | */ 40 | @Immutable 41 | public final class OptimalStringAlignment implements StringDistance { 42 | 43 | /** 44 | * Compute the distance between strings: the minimum number of operations 45 | * needed to transform one string into the other (insertion, deletion, 46 | * substitution of a single character, or a transposition of two adjacent 47 | * characters) while no substring is edited more than once. 48 | * 49 | * @param s1 The first string to compare. 50 | * @param s2 The second string to compare. 51 | * @return the OSA distance 52 | * @throws NullPointerException if s1 or s2 is null. 53 | */ 54 | public double distance(final String s1, final String s2) { 55 | if (s1 == null) { 56 | throw new NullPointerException("s1 must not be null"); 57 | } 58 | 59 | if (s2 == null) { 60 | throw new NullPointerException("s2 must not be null"); 61 | } 62 | 63 | if (s1.equals(s2)) { 64 | return 0; 65 | } 66 | 67 | int n = s1.length(), m = s2.length(); 68 | 69 | if (n == 0) { 70 | return m; 71 | } 72 | 73 | if (m == 0) { 74 | return n; 75 | } 76 | 77 | // Create the distance matrix H[0 .. s1.length+1][0 .. s2.length+1] 78 | int[][] d = new int[n + 2][m + 2]; 79 | 80 | //initialize top row and leftmost column 81 | for (int i = 0; i <= n; i++) { 82 | d[i][0] = i; 83 | } 84 | for (int j = 0; j <= m; j++) { 85 | d[0][j] = j; 86 | } 87 | 88 | //fill the distance matrix 89 | int cost; 90 | 91 | for (int i = 1; i <= n; i++) { 92 | for (int j = 1; j <= m; j++) { 93 | 94 | //if s1[i - 1] = s2[j - 1] then cost = 0, else cost = 1 95 | cost = 1; 96 | if (s1.charAt(i - 1) == s2.charAt(j - 1)) { 97 | cost = 0; 98 | } 99 | 100 | d[i][j] = min( 101 | d[i - 1][j - 1] + cost, // substitution 102 | d[i][j - 1] + 1, // insertion 103 | d[i - 1][j] + 1 // deletion 104 | ); 105 | 106 | //transposition check 107 | if (i > 1 && j > 1 108 | && s1.charAt(i - 1) == s2.charAt(j - 2) 109 | && s1.charAt(i - 2) == s2.charAt(j - 1)) { 110 | d[i][j] = Math.min(d[i][j], d[i - 2][j - 2] + cost); 111 | } 112 | } 113 | } 114 | 115 | return d[n][m]; 116 | } 117 | 118 | private static int min( 119 | final int a, final int b, final int c) { 120 | return Math.min(a, Math.min(b, c)); 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /src/main/java/info/debatty/java/stringsimilarity/QGram.java: -------------------------------------------------------------------------------- 1 | package info.debatty.java.stringsimilarity; 2 | 3 | import info.debatty.java.stringsimilarity.interfaces.StringDistance; 4 | 5 | import java.util.HashSet; 6 | import java.util.Map; 7 | import java.util.Set; 8 | 9 | import net.jcip.annotations.Immutable; 10 | 11 | /** 12 | * Q-gram distance, as defined by Ukkonen in "Approximate string-matching with 13 | * q-grams and maximal matches". The distance between two strings is defined as 14 | * the L1 norm of the difference of their profiles (the number of occurences of 15 | * each n-gram): SUM( |V1_i - V2_i| ). Q-gram distance is a lower bound on 16 | * Levenshtein distance, but can be computed in O(m + n), where Levenshtein 17 | * requires O(m.n). 18 | * 19 | * @author Thibault Debatty 20 | */ 21 | @Immutable 22 | public class QGram extends ShingleBased implements StringDistance { 23 | 24 | /** 25 | * Q-gram similarity and distance. Defined by Ukkonen in "Approximate 26 | * string-matching with q-grams and maximal matches", 27 | * http://www.sciencedirect.com/science/article/pii/0304397592901434 The 28 | * distance between two strings is defined as the L1 norm of the difference 29 | * of their profiles (the number of occurences of each k-shingle). Q-gram 30 | * distance is a lower bound on Levenshtein distance, but can be computed in 31 | * O(|A| + |B|), where Levenshtein requires O(|A|.|B|) 32 | * 33 | * @param k 34 | */ 35 | public QGram(final int k) { 36 | super(k); 37 | } 38 | 39 | /** 40 | * Q-gram similarity and distance. Defined by Ukkonen in "Approximate 41 | * string-matching with q-grams and maximal matches", 42 | * http://www.sciencedirect.com/science/article/pii/0304397592901434 The 43 | * distance between two strings is defined as the L1 norm of the difference 44 | * of their profiles (the number of occurence of each k-shingle). Q-gram 45 | * distance is a lower bound on Levenshtein distance, but can be computed in 46 | * O(|A| + |B|), where Levenshtein requires O(|A|.|B|) 47 | * Default k is 3. 48 | */ 49 | public QGram() { 50 | super(); 51 | } 52 | 53 | /** 54 | * The distance between two strings is defined as the L1 norm of the 55 | * difference of their profiles (the number of occurence of each k-shingle). 56 | * 57 | * @param s1 The first string to compare. 58 | * @param s2 The second string to compare. 59 | * @return The computed Q-gram distance. 60 | * @throws NullPointerException if s1 or s2 is null. 61 | */ 62 | public final double distance(final String s1, final String s2) { 63 | if (s1 == null) { 64 | throw new NullPointerException("s1 must not be null"); 65 | } 66 | 67 | if (s2 == null) { 68 | throw new NullPointerException("s2 must not be null"); 69 | } 70 | 71 | if (s1.equals(s2)) { 72 | return 0; 73 | } 74 | 75 | Map profile1 = getProfile(s1); 76 | Map profile2 = getProfile(s2); 77 | 78 | return distance(profile1, profile2); 79 | } 80 | 81 | /** 82 | * Compute QGram distance using precomputed profiles. 83 | * 84 | * @param profile1 85 | * @param profile2 86 | * @return 87 | */ 88 | public final double distance( 89 | final Map profile1, 90 | final Map profile2) { 91 | 92 | Set union = new HashSet(); 93 | union.addAll(profile1.keySet()); 94 | union.addAll(profile2.keySet()); 95 | 96 | int agg = 0; 97 | for (String key : union) { 98 | int v1 = 0; 99 | int v2 = 0; 100 | Integer iv1 = profile1.get(key); 101 | if (iv1 != null) { 102 | v1 = iv1; 103 | } 104 | 105 | Integer iv2 = profile2.get(key); 106 | if (iv2 != null) { 107 | v2 = iv2; 108 | } 109 | agg += Math.abs(v1 - v2); 110 | } 111 | return agg; 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /src/main/java/info/debatty/java/stringsimilarity/RatcliffObershelp.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2015 Thibault Debatty. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | package info.debatty.java.stringsimilarity; 25 | 26 | import info.debatty.java.stringsimilarity.interfaces.NormalizedStringSimilarity; 27 | import info.debatty.java.stringsimilarity.interfaces.NormalizedStringDistance; 28 | import java.util.List; 29 | import java.util.ArrayList; 30 | 31 | import net.jcip.annotations.Immutable; 32 | 33 | /** 34 | * Ratcliff/Obershelp pattern recognition 35 | * The Ratcliff/Obershelp algorithm computes the similarity of two strings a 36 | * the doubled number of matching characters divided by the total number of 37 | * characters in the two strings. Matching characters are those in the longest 38 | * common subsequence plus, recursively, matching characters in the unmatched 39 | * region on either side of the longest common subsequence. 40 | * The Ratcliff/Obershelp distance is computed as 1 - Ratcliff/Obershelp 41 | * similarity. 42 | * 43 | * @author Ligi https://github.com/dxpux (as a patch for fuzzystring) 44 | * Ported to java from .net by denmase 45 | */ 46 | @Immutable 47 | public class RatcliffObershelp implements 48 | NormalizedStringSimilarity, NormalizedStringDistance { 49 | 50 | /** 51 | * Compute the Ratcliff-Obershelp similarity between strings. 52 | * 53 | * @param s1 The first string to compare. 54 | * @param s2 The second string to compare. 55 | * @return The RatcliffObershelp similarity in the range [0, 1] 56 | * @throws NullPointerException if s1 or s2 is null. 57 | */ 58 | @Override 59 | public final double similarity(final String s1, final String s2) { 60 | if (s1 == null) { 61 | throw new NullPointerException("s1 must not be null"); 62 | } 63 | 64 | if (s2 == null) { 65 | throw new NullPointerException("s2 must not be null"); 66 | } 67 | 68 | if (s1.equals(s2)) { 69 | return 1.0d; 70 | } 71 | 72 | List matches = getMatchList(s1, s2); 73 | int sum_of_matches = 0; 74 | 75 | for (String match : matches) { 76 | sum_of_matches += match.length(); 77 | } 78 | 79 | return 2.0d * sum_of_matches / (s1.length() + s2.length()); 80 | } 81 | 82 | /** 83 | * Return 1 - similarity. 84 | * 85 | * @param s1 The first string to compare. 86 | * @param s2 The second string to compare. 87 | * @return 1 - similarity 88 | * @throws NullPointerException if s1 or s2 is null. 89 | */ 90 | @Override 91 | public final double distance(final String s1, final String s2) { 92 | return 1.0d - similarity(s1, s2); 93 | } 94 | 95 | private static List getMatchList(final String s1, final String s2) { 96 | List list = new ArrayList(); 97 | String match = frontMaxMatch(s1, s2); 98 | 99 | if (match.length() > 0) { 100 | String frontsource = s1.substring(0, s1.indexOf(match)); 101 | String fronttarget = s2.substring(0, s2.indexOf(match)); 102 | List frontqueue = getMatchList(frontsource, fronttarget); 103 | 104 | String endsource = s1.substring(s1.indexOf(match) + match.length()); 105 | String endtarget = s2.substring(s2.indexOf(match) + match.length()); 106 | List endqueue = getMatchList(endsource, endtarget); 107 | 108 | list.add(match); 109 | list.addAll(frontqueue); 110 | list.addAll(endqueue); 111 | } 112 | 113 | return list; 114 | } 115 | 116 | private static String frontMaxMatch(final String s1, final String s2) { 117 | int longest = 0; 118 | String longestsubstring = ""; 119 | 120 | for (int i = 0; i < s1.length(); ++i) { 121 | for (int j = i + 1; j <= s1.length(); ++j) { 122 | String substring = s1.substring(i, j); 123 | if (s2.contains(substring) && substring.length() > longest) { 124 | longest = substring.length(); 125 | longestsubstring = substring; 126 | } 127 | } 128 | } 129 | 130 | return longestsubstring; 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /src/main/java/info/debatty/java/stringsimilarity/ShingleBased.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2015 Thibault Debatty. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | package info.debatty.java.stringsimilarity; 25 | 26 | import net.jcip.annotations.Immutable; 27 | 28 | import java.util.Collections; 29 | import java.util.HashMap; 30 | import java.util.Map; 31 | import java.util.regex.Pattern; 32 | 33 | /** 34 | * Abstract class for string similarities that rely on set operations (like 35 | * cosine similarity or jaccard index). 36 | * 37 | * k-shingling is the operation of transforming a string (or text document) into 38 | * a set of n-grams, which can be used to measure the similarity between two 39 | * strings or documents. 40 | * 41 | * Generally speaking, a k-gram is any sequence of k tokens. We use here the 42 | * definition from Leskovec, Rajaraman & Ullman (2014), "Mining of Massive 43 | * Datasets", Cambridge University Press: Multiple subsequent spaces are 44 | * replaced by a single space, and a k-gram is a sequence of k characters. 45 | * 46 | * Default value of k is 3. A good rule of thumb is to imagine that there are 47 | * only 20 characters and estimate the number of k-shingles as 20^k. For small 48 | * documents like e-mails, k = 5 is a recommended value. For large documents, 49 | * such as research articles, k = 9 is considered a safe choice. 50 | * 51 | * @author Thibault Debatty 52 | */ 53 | @Immutable 54 | public abstract class ShingleBased { 55 | 56 | private static final int DEFAULT_K = 3; 57 | 58 | private final int k; 59 | 60 | /** 61 | * Pattern for finding multiple following spaces. 62 | */ 63 | private static final Pattern SPACE_REG = Pattern.compile("\\s+"); 64 | 65 | /** 66 | * 67 | * @param k 68 | * @throws IllegalArgumentException if k is <= 0 69 | */ 70 | public ShingleBased(final int k) { 71 | if (k <= 0) { 72 | throw new IllegalArgumentException("k should be positive!"); 73 | } 74 | this.k = k; 75 | } 76 | 77 | /** 78 | * 79 | */ 80 | ShingleBased() { 81 | this(DEFAULT_K); 82 | } 83 | 84 | /** 85 | * Return k, the length of k-shingles (aka n-grams). 86 | * 87 | * @return The length of k-shingles. 88 | */ 89 | public final int getK() { 90 | return k; 91 | } 92 | 93 | /** 94 | * Compute and return the profile of s, as defined by Ukkonen "Approximate 95 | * string-matching with q-grams and maximal matches". 96 | * https://www.cs.helsinki.fi/u/ukkonen/TCS92.pdf The profile is the number 97 | * of occurrences of k-shingles, and is used to compute q-gram similarity, 98 | * Jaccard index, etc. Pay attention: the memory requirement of the profile 99 | * can be up to k * size of the string 100 | * 101 | * @param string 102 | * @return the profile of this string, as an unmodifiable Map 103 | */ 104 | public final Map getProfile(final String string) { 105 | HashMap shingles = new HashMap(); 106 | 107 | String string_no_space = SPACE_REG.matcher(string).replaceAll(" "); 108 | for (int i = 0; i < (string_no_space.length() - k + 1); i++) { 109 | String shingle = string_no_space.substring(i, i + k); 110 | Integer old = shingles.get(shingle); 111 | if (old != null) { 112 | shingles.put(shingle, old + 1); 113 | } else { 114 | shingles.put(shingle, 1); 115 | } 116 | } 117 | 118 | return Collections.unmodifiableMap(shingles); 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /src/main/java/info/debatty/java/stringsimilarity/SorensenDice.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2015 Thibault Debatty. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | package info.debatty.java.stringsimilarity; 25 | 26 | import info.debatty.java.stringsimilarity.interfaces.NormalizedStringSimilarity; 27 | import info.debatty.java.stringsimilarity.interfaces.NormalizedStringDistance; 28 | import java.util.HashSet; 29 | import java.util.Map; 30 | import java.util.Set; 31 | 32 | import net.jcip.annotations.Immutable; 33 | 34 | /** 35 | * Similar to Jaccard index, but this time the similarity is computed as 2 * |V1 36 | * inter V2| / (|V1| + |V2|). Distance is computed as 1 - cosine similarity. 37 | * 38 | * @author Thibault Debatty 39 | */ 40 | @Immutable 41 | public class SorensenDice extends ShingleBased implements 42 | NormalizedStringDistance, NormalizedStringSimilarity { 43 | 44 | /** 45 | * Sorensen-Dice coefficient, aka Sørensen index, Dice's coefficient or 46 | * Czekanowski's binary (non-quantitative) index. 47 | * 48 | * The strings are first converted to boolean sets of k-shingles (sequences 49 | * of k characters), then the similarity is computed as 2 * |A inter B| / 50 | * (|A| + |B|). Attention: Sorensen-Dice distance (and similarity) does not 51 | * satisfy triangle inequality. 52 | * 53 | * @param k 54 | */ 55 | public SorensenDice(final int k) { 56 | super(k); 57 | } 58 | 59 | /** 60 | * Sorensen-Dice coefficient, aka Sørensen index, Dice's coefficient or 61 | * Czekanowski's binary (non-quantitative) index. 62 | * 63 | * The strings are first converted to boolean sets of k-shingles (sequences 64 | * of k characters), then the similarity is computed as 2 * |A inter B| / 65 | * (|A| + |B|). Attention: Sorensen-Dice distance (and similarity) does not 66 | * satisfy triangle inequality. Default k is 3. 67 | */ 68 | public SorensenDice() { 69 | super(); 70 | } 71 | 72 | /** 73 | * Similarity is computed as 2 * |A inter B| / (|A| + |B|). 74 | * 75 | * @param s1 The first string to compare. 76 | * @param s2 The second string to compare. 77 | * @return The computed Sorensen-Dice similarity. 78 | * @throws NullPointerException if s1 or s2 is null. 79 | */ 80 | public final double similarity(final String s1, final String s2) { 81 | if (s1 == null) { 82 | throw new NullPointerException("s1 must not be null"); 83 | } 84 | 85 | if (s2 == null) { 86 | throw new NullPointerException("s2 must not be null"); 87 | } 88 | 89 | if (s1.equals(s2)) { 90 | return 1; 91 | } 92 | 93 | Map profile1 = getProfile(s1); 94 | Map profile2 = getProfile(s2); 95 | 96 | Set union = new HashSet(); 97 | union.addAll(profile1.keySet()); 98 | union.addAll(profile2.keySet()); 99 | 100 | int inter = 0; 101 | 102 | for (String key : union) { 103 | if (profile1.containsKey(key) && profile2.containsKey(key)) { 104 | inter++; 105 | } 106 | } 107 | 108 | return 2.0 * inter / (profile1.size() + profile2.size()); 109 | } 110 | 111 | /** 112 | * Returns 1 - similarity. 113 | * 114 | * @param s1 The first string to compare. 115 | * @param s2 The second string to compare. 116 | * @return 1.0 - the computed similarity 117 | * @throws NullPointerException if s1 or s2 is null. 118 | */ 119 | public final double distance(final String s1, final String s2) { 120 | return 1 - similarity(s1, s2); 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /src/main/java/info/debatty/java/stringsimilarity/WeightedLevenshtein.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2015 Thibault Debatty. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | package info.debatty.java.stringsimilarity; 25 | 26 | import info.debatty.java.stringsimilarity.interfaces.StringDistance; 27 | import net.jcip.annotations.Immutable; 28 | 29 | /** 30 | * Implementation of Levenshtein that allows to define different weights for 31 | * different character substitutions. 32 | * 33 | * @author Thibault Debatty 34 | */ 35 | @Immutable 36 | public class WeightedLevenshtein implements StringDistance { 37 | 38 | private final CharacterSubstitutionInterface charsub; 39 | private final CharacterInsDelInterface charchange; 40 | 41 | /** 42 | * Instantiate with provided character substitution. 43 | * @param charsub The strategy to determine character substitution weights. 44 | */ 45 | public WeightedLevenshtein(final CharacterSubstitutionInterface charsub) { 46 | this(charsub, null); 47 | } 48 | 49 | /** 50 | * Instantiate with provided character substitution, insertion, and 51 | * deletion weights. 52 | * @param charsub The strategy to determine character substitution weights. 53 | * @param charchange The strategy to determine character insertion / 54 | * deletion weights. 55 | */ 56 | public WeightedLevenshtein(final CharacterSubstitutionInterface charsub, 57 | final CharacterInsDelInterface charchange) { 58 | this.charsub = charsub; 59 | this.charchange = charchange; 60 | } 61 | 62 | /** 63 | * Equivalent to distance(s1, s2, Double.MAX_VALUE). 64 | */ 65 | public final double distance(final String s1, final String s2) { 66 | return distance(s1, s2, Double.MAX_VALUE); 67 | } 68 | 69 | /** 70 | * Compute Levenshtein distance using provided weights for substitution. 71 | * @param s1 The first string to compare. 72 | * @param s2 The second string to compare. 73 | * @param limit The maximum result to compute before stopping. This 74 | * means that the calculation can terminate early if you 75 | * only care about strings with a certain similarity. 76 | * Set this to Double.MAX_VALUE if you want to run the 77 | * calculation to completion in every case. 78 | * @return The computed weighted Levenshtein distance. 79 | * @throws NullPointerException if s1 or s2 is null. 80 | */ 81 | public final double distance(final String s1, final String s2, 82 | final double limit) { 83 | if (s1 == null) { 84 | throw new NullPointerException("s1 must not be null"); 85 | } 86 | 87 | if (s2 == null) { 88 | throw new NullPointerException("s2 must not be null"); 89 | } 90 | 91 | if (s1.equals(s2)) { 92 | return 0; 93 | } 94 | 95 | if (s1.length() == 0) { 96 | return s2.length(); 97 | } 98 | 99 | if (s2.length() == 0) { 100 | return s1.length(); 101 | } 102 | 103 | // create two work vectors of floating point (i.e. weighted) distances 104 | double[] v0 = new double[s2.length() + 1]; 105 | double[] v1 = new double[s2.length() + 1]; 106 | double[] vtemp; 107 | 108 | // initialize v0 (the previous row of distances) 109 | // this row is A[0][i]: edit distance for an empty s1 110 | // the distance is the cost of inserting each character of s2 111 | v0[0] = 0; 112 | for (int i = 1; i < v0.length; i++) { 113 | v0[i] = v0[i - 1] + insertionCost(s2.charAt(i - 1)); 114 | } 115 | 116 | for (int i = 0; i < s1.length(); i++) { 117 | char s1i = s1.charAt(i); 118 | double deletion_cost = deletionCost(s1i); 119 | 120 | // calculate v1 (current row distances) from the previous row v0 121 | // first element of v1 is A[i+1][0] 122 | // Edit distance is the cost of deleting characters from s1 123 | // to match empty t. 124 | v1[0] = v0[0] + deletion_cost; 125 | 126 | double minv1 = v1[0]; 127 | 128 | // use formula to fill in the rest of the row 129 | for (int j = 0; j < s2.length(); j++) { 130 | char s2j = s2.charAt(j); 131 | double cost = 0; 132 | if (s1i != s2j) { 133 | cost = charsub.cost(s1i, s2j); 134 | } 135 | double insertion_cost = insertionCost(s2j); 136 | v1[j + 1] = Math.min( 137 | v1[j] + insertion_cost, // Cost of insertion 138 | Math.min( 139 | v0[j + 1] + deletion_cost, // Cost of deletion 140 | v0[j] + cost)); // Cost of substitution 141 | 142 | minv1 = Math.min(minv1, v1[j + 1]); 143 | } 144 | 145 | if (minv1 >= limit) { 146 | return limit; 147 | } 148 | 149 | // copy v1 (current row) to v0 (previous row) for next iteration 150 | //System.arraycopy(v1, 0, v0, 0, v0.length); 151 | // Flip references to current and previous row 152 | vtemp = v0; 153 | v0 = v1; 154 | v1 = vtemp; 155 | 156 | } 157 | 158 | return v0[s2.length()]; 159 | } 160 | 161 | 162 | private double insertionCost(final char c) { 163 | if (charchange == null) { 164 | return 1.0; 165 | } else { 166 | return charchange.insertionCost(c); 167 | } 168 | } 169 | 170 | private double deletionCost(final char c) { 171 | if (charchange == null) { 172 | return 1.0; 173 | } else { 174 | return charchange.deletionCost(c); 175 | } 176 | } 177 | } 178 | -------------------------------------------------------------------------------- /src/main/java/info/debatty/java/stringsimilarity/examples/Examples.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2015 Thibault Debatty. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | package info.debatty.java.stringsimilarity.examples; 25 | 26 | import info.debatty.java.stringsimilarity.CharacterSubstitutionInterface; 27 | import info.debatty.java.stringsimilarity.Cosine; 28 | import info.debatty.java.stringsimilarity.Damerau; 29 | import info.debatty.java.stringsimilarity.OptimalStringAlignment; 30 | import info.debatty.java.stringsimilarity.Jaccard; 31 | import info.debatty.java.stringsimilarity.JaroWinkler; 32 | import info.debatty.java.stringsimilarity.Levenshtein; 33 | import info.debatty.java.stringsimilarity.LongestCommonSubsequence; 34 | import info.debatty.java.stringsimilarity.NGram; 35 | import info.debatty.java.stringsimilarity.NormalizedLevenshtein; 36 | import info.debatty.java.stringsimilarity.QGram; 37 | import info.debatty.java.stringsimilarity.SorensenDice; 38 | import info.debatty.java.stringsimilarity.WeightedLevenshtein; 39 | 40 | /** 41 | * 42 | * @author Thibault Debatty 43 | */ 44 | public class Examples { 45 | 46 | /** 47 | * @param args the command line arguments 48 | */ 49 | public static void main(String[] args) { 50 | // Levenshtein 51 | // =========== 52 | System.out.println("\nLevenshtein"); 53 | Levenshtein levenshtein = new Levenshtein(); 54 | System.out.println(levenshtein.distance("My string", "My $tring")); 55 | System.out.println(levenshtein.distance("My string", "M string2")); 56 | System.out.println(levenshtein.distance("My string", "My $tring")); 57 | 58 | // Jaccard index 59 | // ============= 60 | System.out.println("\nJaccard"); 61 | Jaccard j2 = new Jaccard(2); 62 | // AB BC CD DE DF 63 | // 1 1 1 1 0 64 | // 1 1 1 0 1 65 | // => 3 / 5 = 0.6 66 | System.out.println(j2.similarity("ABCDE", "ABCDF")); 67 | 68 | // Jaro-Winkler 69 | // ============ 70 | System.out.println("\nJaro-Winkler"); 71 | JaroWinkler jw = new JaroWinkler(); 72 | 73 | // substitution of s and t : 0.9740740656852722 74 | System.out.println(jw.similarity("My string", "My tsring")); 75 | 76 | // substitution of s and n : 0.8962963223457336 77 | System.out.println(jw.similarity("My string", "My ntrisg")); 78 | 79 | // Cosine 80 | // ====== 81 | System.out.println("\nCosine"); 82 | Cosine cos = new Cosine(3); 83 | 84 | // ABC BCE 85 | // 1 0 86 | // 1 1 87 | // angle = 45° 88 | // => similarity = .71 89 | System.out.println(cos.similarity("ABC", "ABCE")); 90 | 91 | cos = new Cosine(2); 92 | // AB BA 93 | // 2 1 94 | // 1 1 95 | // similarity = .95 96 | System.out.println(cos.similarity("ABAB", "BAB")); 97 | 98 | // Damerau 99 | // ======= 100 | System.out.println("\nDamerau"); 101 | Damerau damerau = new Damerau(); 102 | 103 | // 1 substitution 104 | System.out.println(damerau.distance("ABCDEF", "ABDCEF")); 105 | 106 | // 2 substitutions 107 | System.out.println(damerau.distance("ABCDEF", "BACDFE")); 108 | 109 | // 1 deletion 110 | System.out.println(damerau.distance("ABCDEF", "ABCDE")); 111 | System.out.println(damerau.distance("ABCDEF", "BCDEF")); 112 | 113 | System.out.println(damerau.distance("ABCDEF", "ABCGDEF")); 114 | 115 | // All different 116 | System.out.println(damerau.distance("ABCDEF", "POIU")); 117 | 118 | 119 | // Optimal String Alignment 120 | // ======= 121 | System.out.println("\nOptimal String Alignment"); 122 | OptimalStringAlignment osa = new OptimalStringAlignment(); 123 | 124 | //Will produce 3.0 125 | System.out.println(osa.distance("CA", "ABC")); 126 | 127 | 128 | // Longest Common Subsequence 129 | // ========================== 130 | System.out.println("\nLongest Common Subsequence"); 131 | LongestCommonSubsequence lcs = new LongestCommonSubsequence(); 132 | 133 | // Will produce 4.0 134 | System.out.println(lcs.distance("AGCAT", "GAC")); 135 | 136 | // Will produce 1.0 137 | System.out.println(lcs.distance("AGCAT", "AGCT")); 138 | 139 | // NGram 140 | // ===== 141 | // produces 0.416666 142 | System.out.println("\nNGram"); 143 | NGram twogram = new NGram(2); 144 | System.out.println(twogram.distance("ABCD", "ABTUIO")); 145 | 146 | // produces 0.97222 147 | String s1 = "Adobe CreativeSuite 5 Master Collection from cheap 4zp"; 148 | String s2 = "Adobe CreativeSuite 5 Master Collection from cheap d1x"; 149 | NGram ngram = new NGram(4); 150 | System.out.println(ngram.distance(s1, s2)); 151 | 152 | // Normalized Levenshtein 153 | // ====================== 154 | System.out.println("\nNormalized Levenshtein"); 155 | NormalizedLevenshtein l = new NormalizedLevenshtein(); 156 | 157 | System.out.println(l.distance("My string", "My $tring")); 158 | System.out.println(l.distance("My string", "M string2")); 159 | System.out.println(l.distance("My string", "abcd")); 160 | 161 | // QGram 162 | // ===== 163 | System.out.println("\nQGram"); 164 | QGram dig = new QGram(2); 165 | 166 | // AB BC CD CE 167 | // 1 1 1 0 168 | // 1 1 0 1 169 | // Total: 2 170 | System.out.println(dig.distance("ABCD", "ABCE")); 171 | 172 | System.out.println(dig.distance("", "QSDFGHJKLM")); 173 | 174 | System.out.println(dig.distance( 175 | "Best Deal Ever! Viagra50/100mg - $1.85 071", 176 | "Best Deal Ever! Viagra50/100mg - $1.85 7z3")); 177 | 178 | // Sorensen-Dice 179 | // ============= 180 | System.out.println("\nSorensen-Dice"); 181 | SorensenDice sd = new SorensenDice(2); 182 | 183 | // AB BC CD DE DF FG 184 | // 1 1 1 1 0 0 185 | // 1 1 1 0 1 1 186 | // => 2 x 3 / (4 + 5) = 6/9 = 0.6666 187 | System.out.println(sd.similarity("ABCDE", "ABCDFG")); 188 | 189 | // Weighted Levenshtein 190 | // ==================== 191 | System.out.println("\nWeighted Levenshtein"); 192 | WeightedLevenshtein wl = new WeightedLevenshtein( 193 | new CharacterSubstitutionInterface() { 194 | public double cost(char c1, char c2) { 195 | 196 | // The cost for substituting 't' and 'r' is considered 197 | // smaller as these 2 are located next to each other 198 | // on a keyboard 199 | if (c1 == 't' && c2 == 'r') { 200 | return 0.5; 201 | } 202 | 203 | // For most cases, the cost of substituting 2 characters 204 | // is 1.0 205 | return 1.0; 206 | } 207 | }); 208 | 209 | System.out.println(wl.distance("String1", "Srring2")); 210 | 211 | // K-Shingling 212 | System.out.println("\nK-Shingling"); 213 | s1 = "my string, \n my song"; 214 | s2 = "another string, from a song"; 215 | Cosine cosine = new Cosine(4); 216 | System.out.println(cosine.getProfile(s1)); 217 | System.out.println(cosine.getProfile(s2)); 218 | 219 | cosine = new Cosine(2); 220 | System.out.println(cosine.getProfile("ABCAB")); 221 | 222 | } 223 | 224 | } 225 | -------------------------------------------------------------------------------- /src/main/java/info/debatty/java/stringsimilarity/examples/MetricLCS.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2015 Thibault Debatty. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | 25 | package info.debatty.java.stringsimilarity.examples; 26 | 27 | /** 28 | * 29 | * @author Thibault Debatty 30 | */ 31 | public class MetricLCS { 32 | 33 | public static void main(String[] args) { 34 | 35 | info.debatty.java.stringsimilarity.MetricLCS lcs = 36 | new info.debatty.java.stringsimilarity.MetricLCS(); 37 | 38 | String s1 = "ABCDEFG"; 39 | String s2 = "ABCDEFHJKL"; 40 | // LCS: ABCDEF => length = 6 41 | // longest = s2 => length = 10 42 | // => 1 - 6/10 = 0.4 43 | System.out.println(lcs.distance(s1, s2)); 44 | 45 | // LCS: ABDF => length = 4 46 | // longest = ABDEF => length = 5 47 | // => 1 - 4 / 5 = 0.2 48 | System.out.println(lcs.distance("ABDEF", "ABDIF")); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/info/debatty/java/stringsimilarity/examples/PrecomputedCosine.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2015 Thibault Debatty. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | package info.debatty.java.stringsimilarity.examples; 25 | 26 | import info.debatty.java.stringsimilarity.Cosine; 27 | import java.util.Map; 28 | 29 | /** 30 | * Example of computing cosine similarity with pre-computed profiles. 31 | * 32 | * @author Thibault Debatty 33 | */ 34 | public class PrecomputedCosine { 35 | 36 | /** 37 | * @param args the command line arguments 38 | */ 39 | public static void main(String[] args) { 40 | String s1 = "My first string"; 41 | String s2 = "My other string..."; 42 | 43 | // Let's work with sequences of 2 characters... 44 | Cosine cosine = new Cosine(2); 45 | 46 | // Pre-compute the profile of strings 47 | Map profile1 = cosine.getProfile(s1); 48 | Map profile2 = cosine.getProfile(s2); 49 | 50 | // Prints 0.516185 51 | System.out.println(cosine.similarity(profile1, profile2)); 52 | 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/info/debatty/java/stringsimilarity/examples/nischay21.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2017 Thibault Debatty. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | 25 | package info.debatty.java.stringsimilarity.examples; 26 | 27 | import info.debatty.java.stringsimilarity.Cosine; 28 | import info.debatty.java.stringsimilarity.Damerau; 29 | import info.debatty.java.stringsimilarity.Jaccard; 30 | import info.debatty.java.stringsimilarity.JaroWinkler; 31 | import info.debatty.java.stringsimilarity.Levenshtein; 32 | import info.debatty.java.stringsimilarity.NGram; 33 | import info.debatty.java.stringsimilarity.SorensenDice; 34 | import info.debatty.java.stringsimilarity.interfaces.StringDistance; 35 | import java.util.LinkedList; 36 | 37 | /** 38 | * 39 | * @author Thibault Debatty 40 | */ 41 | public class nischay21 { 42 | 43 | /** 44 | * @param args the command line arguments 45 | */ 46 | public static void main(String[] args) { 47 | 48 | String s1 = "MINI GRINDER KIT"; 49 | String s2 = "Weiler 13001 Mini Grinder Accessory Kit, For Use With Small Right Angle Grinders"; 50 | String s3 = "Milwaukee Video Borescope, Rotating Inspection Scope, Series: M-SPECTOR 360, 2.7 in 640 x 480 pixels High-Resolution LCD, Plastic, Black/Red"; 51 | 52 | LinkedList algos = new LinkedList(); 53 | algos.add(new JaroWinkler()); 54 | algos.add(new Levenshtein()); 55 | algos.add(new NGram()); 56 | algos.add(new Damerau()); 57 | algos.add(new Jaccard()); 58 | algos.add(new SorensenDice()); 59 | algos.add(new Cosine()); 60 | 61 | 62 | System.out.println("S1 vs S2"); 63 | for (StringDistance algo : algos) { 64 | System.out.print(algo.getClass().getSimpleName() + " : "); 65 | System.out.println(algo.distance(s1, s2)); 66 | } 67 | System.out.println(); 68 | 69 | System.out.println("S1 vs S3"); 70 | for (StringDistance algo : algos) { 71 | System.out.print(algo.getClass().getSimpleName() + " : "); 72 | System.out.println(algo.distance(s1, s3)); 73 | } 74 | System.out.println(); 75 | 76 | System.out.println("With .toLower()"); 77 | System.out.println("S1 vs S2"); 78 | for (StringDistance algo : algos) { 79 | System.out.print(algo.getClass().getSimpleName() + " : "); 80 | System.out.println(algo.distance(s1.toLowerCase(), s2.toLowerCase())); 81 | } 82 | System.out.println(); 83 | 84 | System.out.println("S1 vs S3"); 85 | for (StringDistance algo : algos) { 86 | System.out.print(algo.getClass().getSimpleName() + " : "); 87 | System.out.println(algo.distance(s1.toLowerCase(), s3.toLowerCase())); 88 | } 89 | System.out.println(); 90 | 91 | } 92 | 93 | } 94 | -------------------------------------------------------------------------------- /src/main/java/info/debatty/java/stringsimilarity/experimental/Sift4.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2016 Thibault Debatty. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | package info.debatty.java.stringsimilarity.experimental; 25 | 26 | import info.debatty.java.stringsimilarity.interfaces.StringDistance; 27 | import java.util.LinkedList; 28 | 29 | /** 30 | * Sift4 - a general purpose string distance algorithm inspired by JaroWinkler 31 | * and Longest Common Subsequence. 32 | * Original JavaScript algorithm by siderite, java port by Nathan Fischer 2016. 33 | * https://siderite.dev/blog/super-fast-and-accurate-string-distance.html 34 | * https://blackdoor.github.io/blog/sift4-java/ 35 | * 36 | * @author Thibault Debatty 37 | */ 38 | public class Sift4 implements StringDistance { 39 | 40 | private static final int DEFAULT_MAX_OFFSET = 10; 41 | 42 | private int max_offset = DEFAULT_MAX_OFFSET; 43 | 44 | /** 45 | * Set the maximum distance to search for character transposition. 46 | * Compute cost of algorithm is O(n . max_offset) 47 | * @param max_offset 48 | */ 49 | public final void setMaxOffset(final int max_offset) { 50 | this.max_offset = max_offset; 51 | } 52 | 53 | /** 54 | * Sift4 - a general purpose string distance algorithm inspired by 55 | * JaroWinkler and Longest Common Subsequence. 56 | * Original JavaScript algorithm by siderite, java port by Nathan Fischer 57 | * 2016. 58 | * https://siderite.dev/blog/super-fast-and-accurate-string-distance.html 59 | * https://blackdoor.github.io/blog/sift4-java/ 60 | * 61 | * @param s1 62 | * @param s2 63 | * @return 64 | */ 65 | public final double distance(final String s1, final String s2) { 66 | 67 | /** 68 | * Used to store relation between same character in different positions 69 | * c1 and c2 in the input strings. 70 | */ 71 | class Offset { 72 | 73 | private final int c1; 74 | private final int c2; 75 | private boolean trans; 76 | 77 | Offset(final int c1, final int c2, final boolean trans) { 78 | this.c1 = c1; 79 | this.c2 = c2; 80 | this.trans = trans; 81 | } 82 | } 83 | 84 | if (s1 == null || s1.isEmpty()) { 85 | if (s2 == null) { 86 | return 0; 87 | } 88 | 89 | return s2.length(); 90 | } 91 | 92 | if (s2 == null || s2.isEmpty()) { 93 | return s1.length(); 94 | } 95 | 96 | int l1 = s1.length(); 97 | int l2 = s2.length(); 98 | 99 | int c1 = 0; //cursor for string 1 100 | int c2 = 0; //cursor for string 2 101 | int lcss = 0; //largest common subsequence 102 | int local_cs = 0; //local common substring 103 | int trans = 0; //number of transpositions ('ab' vs 'ba') 104 | 105 | // offset pair array, for computing the transpositions 106 | LinkedList offset_arr = new LinkedList(); 107 | 108 | while ((c1 < l1) && (c2 < l2)) { 109 | if (s1.charAt(c1) == s2.charAt(c2)) { 110 | local_cs++; 111 | boolean is_trans = false; 112 | // see if current match is a transposition 113 | int i = 0; 114 | while (i < offset_arr.size()) { 115 | Offset ofs = offset_arr.get(i); 116 | if (c1 <= ofs.c1 || c2 <= ofs.c2) { 117 | // when two matches cross, the one considered a 118 | // transposition is the one with the largest difference 119 | // in offsets 120 | is_trans = 121 | Math.abs(c2 - c1) >= Math.abs(ofs.c2 - ofs.c1); 122 | if (is_trans) { 123 | 124 | trans++; 125 | } else { 126 | if (!ofs.trans) { 127 | ofs.trans = true; 128 | trans++; 129 | } 130 | } 131 | 132 | break; 133 | } else { 134 | if (c1 > ofs.c2 && c2 > ofs.c1) { 135 | offset_arr.remove(i); 136 | } else { 137 | i++; 138 | } 139 | } 140 | } 141 | offset_arr.add(new Offset(c1, c2, is_trans)); 142 | 143 | } else { 144 | 145 | // s1.charAt(c1) != s2.charAt(c2) 146 | lcss += local_cs; 147 | local_cs = 0; 148 | if (c1 != c2) { 149 | //using min allows the computation of transpositions 150 | c1 = Math.min(c1, c2); 151 | c2 = c1; 152 | } 153 | 154 | // if matching characters are found, remove 1 from both cursors 155 | // (they get incremented at the end of the loop) 156 | // so that we can have only one code block handling matches 157 | for ( 158 | int i = 0; 159 | i < max_offset && (c1 + i < l1 || c2 + i < l2); 160 | i++) { 161 | 162 | if ((c1 + i < l1) && (s1.charAt(c1 + i) == s2.charAt(c2))) { 163 | c1 += i - 1; 164 | c2--; 165 | break; 166 | } 167 | 168 | if ((c2 + i < l2) && (s1.charAt(c1) == s2.charAt(c2 + i))) { 169 | c1--; 170 | c2 += i - 1; 171 | break; 172 | } 173 | } 174 | } 175 | c1++; 176 | c2++; 177 | // this covers the case where the last match is on the last token 178 | // in list, so that it can compute transpositions correctly 179 | if ((c1 >= l1) || (c2 >= l2)) { 180 | lcss += local_cs; 181 | local_cs = 0; 182 | c1 = Math.min(c1, c2); 183 | c2 = c1; 184 | } 185 | } 186 | lcss += local_cs; 187 | // add the cost of transpositions to the final result 188 | return Math.round(Math.max(l1, l2) - lcss + trans); 189 | } 190 | } 191 | -------------------------------------------------------------------------------- /src/main/java/info/debatty/java/stringsimilarity/interfaces/MetricStringDistance.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2015 Thibault Debatty. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | package info.debatty.java.stringsimilarity.interfaces; 25 | 26 | /** 27 | * String distances that implement this interface are metrics. 28 | * This means: 29 | * - d(x, y) ≥ 0 (non-negativity, or separation axiom) 30 | * - d(x, y) = 0 if and only if x = y (identity, or coincidence axiom) 31 | * - d(x, y) = d(y, x) (symmetry) 32 | * - d(x, z) ≤ d(x, y) + d(y, z) (triangle inequality). 33 | * 34 | * @author Thibault Debatty 35 | */ 36 | public interface MetricStringDistance extends StringDistance { 37 | 38 | /** 39 | * Compute and return the metric distance. 40 | * @param s1 41 | * @param s2 42 | * @return 43 | */ 44 | double distance(String s1, String s2); 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/info/debatty/java/stringsimilarity/interfaces/NormalizedStringDistance.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2015 Thibault Debatty. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | package info.debatty.java.stringsimilarity.interfaces; 25 | 26 | /** 27 | * Normalized string similarities return a similarity between 0.0 and 1.0. 28 | * 29 | * @author Thibault Debatty 30 | */ 31 | public interface NormalizedStringDistance extends StringDistance { 32 | 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/info/debatty/java/stringsimilarity/interfaces/NormalizedStringSimilarity.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2015 Thibault Debatty. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | package info.debatty.java.stringsimilarity.interfaces; 25 | 26 | /** 27 | * 28 | * @author Thibault Debatty 29 | */ 30 | public interface NormalizedStringSimilarity extends StringSimilarity { 31 | 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/info/debatty/java/stringsimilarity/interfaces/StringDistance.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2015 Thibault Debatty. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | package info.debatty.java.stringsimilarity.interfaces; 25 | 26 | import java.io.Serializable; 27 | 28 | /** 29 | * 30 | * @author Thibault Debatty 31 | */ 32 | public interface StringDistance extends Serializable { 33 | 34 | /** 35 | * Compute and return a measure of distance. 36 | * Must be >= 0. 37 | * @param s1 38 | * @param s2 39 | * @return 40 | */ 41 | double distance(String s1, String s2); 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/info/debatty/java/stringsimilarity/interfaces/StringSimilarity.java: -------------------------------------------------------------------------------- 1 | package info.debatty.java.stringsimilarity.interfaces; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * 7 | * @author Thibault Debatty 8 | */ 9 | public interface StringSimilarity extends Serializable { 10 | /** 11 | * Compute and return a measure of similarity between 2 strings. 12 | * @param s1 13 | * @param s2 14 | * @return similarity (0 means both strings are completely different) 15 | */ 16 | double similarity(String s1, String s2); 17 | } 18 | -------------------------------------------------------------------------------- /src/test/java/info/debatty/java/stringsimilarity/CosineTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2015 Thibault Debatty. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | 25 | package info.debatty.java.stringsimilarity; 26 | 27 | import java.io.BufferedReader; 28 | import java.io.IOException; 29 | import java.io.InputStream; 30 | import java.io.InputStreamReader; 31 | 32 | import info.debatty.java.stringsimilarity.testutil.NullEmptyTests; 33 | import org.junit.Test; 34 | import static org.junit.Assert.*; 35 | 36 | /** 37 | * 38 | * @author Thibault Debatty 39 | */ 40 | public class CosineTest { 41 | 42 | /** 43 | * Test of similarity method, of class Cosine. 44 | */ 45 | @Test 46 | public final void testSimilarity() { 47 | System.out.println("similarity"); 48 | Cosine instance = new Cosine(); 49 | double result = instance.similarity("ABC", "ABCE"); 50 | assertEquals(0.71, result, 0.01); 51 | 52 | NullEmptyTests.testSimilarity(instance); 53 | } 54 | 55 | /** 56 | * If one of the strings is smaller than k, the similarity should be 0. 57 | */ 58 | @Test 59 | public final void testSmallString() { 60 | System.out.println("test small string"); 61 | Cosine instance = new Cosine(3); 62 | double result = instance.similarity("AB", "ABCE"); 63 | assertEquals(0.0, result, 0.00001); 64 | } 65 | 66 | @Test 67 | public final void testLargeString() throws IOException { 68 | 69 | System.out.println("Test with large strings"); 70 | Cosine cos = new Cosine(); 71 | 72 | // read from 2 text files 73 | String string1 = readResourceFile("71816-2.txt"); 74 | String string2 = readResourceFile("11328-1.txt"); 75 | double similarity = cos.similarity(string1, string2); 76 | 77 | assertEquals(0.8115, similarity, 0.001); 78 | } 79 | 80 | @Test 81 | public final void testDistance() { 82 | Cosine instance = new Cosine(); 83 | 84 | double result = instance.distance("ABC", "ABCE"); 85 | assertEquals(0.29, result, 0.01); 86 | 87 | NullEmptyTests.testDistance(instance); 88 | } 89 | 90 | @Test 91 | public final void testDistanceSmallString() { 92 | System.out.println("test small string"); 93 | Cosine instance = new Cosine(3); 94 | double result = instance.distance("AB", "ABCE"); 95 | assertEquals(1, result, 0.00001); 96 | } 97 | 98 | @Test 99 | public final void testDistanceLargeString() throws IOException { 100 | 101 | System.out.println("Test with large strings"); 102 | Cosine cos = new Cosine(); 103 | 104 | // read from 2 text files 105 | String string1 = readResourceFile("71816-2.txt"); 106 | String string2 = readResourceFile("11328-1.txt"); 107 | double similarity = cos.distance(string1, string2); 108 | 109 | assertEquals(0.1885, similarity, 0.001); 110 | } 111 | 112 | private static String readResourceFile(String file) throws IOException { 113 | 114 | InputStream stream = Thread.currentThread() 115 | .getContextClassLoader() 116 | .getResourceAsStream(file); 117 | 118 | BufferedReader reader = new BufferedReader(new InputStreamReader(stream)); 119 | StringBuilder string_builder = new StringBuilder(); 120 | String ls = System.getProperty("line.separator"); 121 | String line = null; 122 | 123 | while (( line = reader.readLine() ) != null ) { 124 | string_builder.append(line); 125 | string_builder.append(ls); 126 | } 127 | 128 | string_builder.deleteCharAt(string_builder.length() - 1); 129 | return string_builder.toString(); 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /src/test/java/info/debatty/java/stringsimilarity/DamerauTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2015 Thibault Debatty. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | 25 | package info.debatty.java.stringsimilarity; 26 | 27 | import info.debatty.java.stringsimilarity.testutil.NullEmptyTests; 28 | import org.junit.Test; 29 | import static org.junit.Assert.assertEquals; 30 | 31 | /** 32 | * 33 | * @author Thibault Debatty 34 | */ 35 | public class DamerauTest { 36 | 37 | /** 38 | * Test of distance method, of class Damerau. 39 | */ 40 | @Test 41 | public final void testDistance() { 42 | System.out.println("distance"); 43 | Damerau instance = new Damerau(); 44 | assertEquals(1.0, instance.distance("ABCDEF", "ABDCEF"), 0.0); 45 | assertEquals(2.0, instance.distance("ABCDEF", "BACDFE"), 0.0); 46 | assertEquals(1.0, instance.distance("ABCDEF", "ABCDE"), 0.0); 47 | NullEmptyTests.testDistance(instance); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/test/java/info/debatty/java/stringsimilarity/JaccardTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2015 Thibault Debatty. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | 25 | package info.debatty.java.stringsimilarity; 26 | 27 | import info.debatty.java.stringsimilarity.testutil.NullEmptyTests; 28 | import org.junit.Test; 29 | 30 | import static org.junit.Assert.assertEquals; 31 | 32 | /** 33 | * 34 | * @author Thibault Debatty 35 | */ 36 | public class JaccardTest { 37 | 38 | /** 39 | * Test of similarity method, of class Jaccard. 40 | */ 41 | @Test 42 | public void testSimilarity() { 43 | System.out.println("similarity"); 44 | Jaccard instance = new Jaccard(2); 45 | 46 | // AB BC CD DE DF 47 | // 1 1 1 1 0 48 | // 1 1 1 0 1 49 | // => 3 / 5 = 0.6 50 | double result = instance.similarity("ABCDE", "ABCDF"); 51 | assertEquals(0.6, result, 0.0); 52 | 53 | NullEmptyTests.testSimilarity(instance); 54 | } 55 | 56 | /** 57 | * Test of distance method, of class Jaccard. 58 | */ 59 | @Test 60 | public void testDistance() { 61 | System.out.println("distance"); 62 | String s1 = ""; 63 | String s2 = ""; 64 | Jaccard instance = new Jaccard(2); 65 | double expResult = 0.4; 66 | double result = instance.distance("ABCDE", "ABCDF"); 67 | assertEquals(expResult, result, 0.0); 68 | 69 | NullEmptyTests.testDistance(instance); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/test/java/info/debatty/java/stringsimilarity/JaroWinklerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2015 Thibault Debatty. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | 25 | package info.debatty.java.stringsimilarity; 26 | 27 | import info.debatty.java.stringsimilarity.testutil.NullEmptyTests; 28 | import org.junit.Test; 29 | import static org.junit.Assert.*; 30 | 31 | /** 32 | * 33 | * @author Thibault Debatty 34 | */ 35 | public class JaroWinklerTest { 36 | 37 | 38 | /** 39 | * Test of similarity method, of class JaroWinkler. 40 | */ 41 | @Test 42 | public final void testSimilarity() { 43 | System.out.println("similarity"); 44 | JaroWinkler instance = new JaroWinkler(); 45 | assertEquals( 46 | 0.974074, 47 | instance.similarity("My string", "My tsring"), 48 | 0.000001); 49 | 50 | assertEquals( 51 | 0.896296, 52 | instance.similarity("My string", "My ntrisg"), 53 | 0.000001); 54 | 55 | NullEmptyTests.testSimilarity(instance); 56 | } 57 | 58 | @Test 59 | public final void testDistance() { 60 | JaroWinkler instance = new JaroWinkler(); 61 | NullEmptyTests.testDistance(instance); 62 | 63 | // TODO: regular (non-null/empty) distance tests 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/test/java/info/debatty/java/stringsimilarity/LevenshteinTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2015 Thibault Debatty. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | 25 | package info.debatty.java.stringsimilarity; 26 | 27 | import info.debatty.java.stringsimilarity.testutil.NullEmptyTests; 28 | import org.junit.Test; 29 | import static org.junit.Assert.assertEquals; 30 | 31 | /** 32 | * 33 | * @author Thibault Debatty 34 | */ 35 | public class LevenshteinTest { 36 | 37 | /** 38 | * Test of distance method, of class Levenshtein. 39 | */ 40 | @Test 41 | public final void testDistance() { 42 | System.out.println("distance"); 43 | Levenshtein instance = new Levenshtein(); 44 | assertEquals(1.0, instance.distance("My string", "My tring"), 0.0); 45 | assertEquals(2.0, instance.distance("My string", "M string2"), 0.0); 46 | assertEquals(1.0, instance.distance("My string", "My $tring"), 0.0); 47 | 48 | // With limits. 49 | assertEquals(2.0, instance.distance("My string", "M string2", 4), 0.0); 50 | assertEquals(2.0, instance.distance("My string", "M string2", 2), 0.0); 51 | assertEquals(1.0, instance.distance("My string", "M string2", 1), 0.0); 52 | 53 | NullEmptyTests.testDistance(instance); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/test/java/info/debatty/java/stringsimilarity/LongestCommonSubsequenceTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2015 Thibault Debatty. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | package info.debatty.java.stringsimilarity; 25 | 26 | import info.debatty.java.stringsimilarity.testutil.NullEmptyTests; 27 | import org.junit.Test; 28 | 29 | import static org.junit.Assert.assertEquals; 30 | 31 | /** 32 | * 33 | * @author Thibault Debatty 34 | */ 35 | public class LongestCommonSubsequenceTest { 36 | 37 | /** 38 | * Test of distance method, of class LongestCommonSubsequence. 39 | */ 40 | @Test 41 | public void testDistance() { 42 | System.out.println("distance"); 43 | LongestCommonSubsequence instance = new LongestCommonSubsequence(); 44 | // LCS = GA or GC => distance = 4 (remove 3 letters and add 1) 45 | assertEquals(4, instance.distance("AGCAT", "GAC"), 0.0); 46 | 47 | assertEquals(1, instance.distance("AGCAT", "AGCT"), 0.0); 48 | 49 | NullEmptyTests.testDistance(instance); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/test/java/info/debatty/java/stringsimilarity/MetricLCSTest.java: -------------------------------------------------------------------------------- 1 | package info.debatty.java.stringsimilarity; 2 | 3 | import info.debatty.java.stringsimilarity.testutil.NullEmptyTests; 4 | import org.junit.Test; 5 | 6 | import static org.junit.Assert.*; 7 | 8 | public class MetricLCSTest { 9 | @Test 10 | public final void testDistance() { 11 | MetricLCS instance = new MetricLCS(); 12 | NullEmptyTests.testDistance(instance); 13 | 14 | // TODO: regular (non-null/empty) distance tests 15 | } 16 | } -------------------------------------------------------------------------------- /src/test/java/info/debatty/java/stringsimilarity/NGramTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2016 Thibault Debatty. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | 25 | package info.debatty.java.stringsimilarity; 26 | 27 | import info.debatty.java.stringsimilarity.testutil.NullEmptyTests; 28 | import org.junit.Assert; 29 | import static org.junit.Assert.assertEquals; 30 | import org.junit.Test; 31 | 32 | /** 33 | * 34 | * @author Thibault Debatty 35 | */ 36 | public class NGramTest { 37 | 38 | /** 39 | * Test of distance method, of class NGram. 40 | */ 41 | @Test 42 | public void testDistance() { 43 | System.out.println("distance"); 44 | String s0 = "ABABABAB"; 45 | String s1 = "ABCABCABCABC"; 46 | String s2 = "POIULKJH"; 47 | NGram ngram = new NGram(); 48 | System.out.println(ngram.distance(s0, s1)); 49 | System.out.println(ngram.distance(s0, s2)); 50 | Assert.assertTrue(ngram.distance(s0, s1) < ngram.distance(s0, s2)); 51 | 52 | assertEquals(0.0, ngram.distance("SIJK", "SIJK"), 0.0); 53 | assertEquals(0.0, ngram.distance("S", "S"), 0.0); 54 | 55 | NullEmptyTests.testDistance(ngram); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/test/java/info/debatty/java/stringsimilarity/NormalizedLevenshteinTest.java: -------------------------------------------------------------------------------- 1 | package info.debatty.java.stringsimilarity; 2 | 3 | import info.debatty.java.stringsimilarity.testutil.NullEmptyTests; 4 | import org.junit.Test; 5 | 6 | import static org.junit.Assert.*; 7 | 8 | public class NormalizedLevenshteinTest { 9 | @Test 10 | public final void testDistance() { 11 | NormalizedLevenshtein instance = new NormalizedLevenshtein(); 12 | NullEmptyTests.testDistance(instance); 13 | 14 | // TODO: regular (non-null/empty) distance tests 15 | } 16 | 17 | @Test 18 | public final void testSimilarity() { 19 | NormalizedLevenshtein instance = new NormalizedLevenshtein(); 20 | NullEmptyTests.testSimilarity(instance); 21 | 22 | // TODO: regular (non-null/empty) similarity tests 23 | } 24 | } -------------------------------------------------------------------------------- /src/test/java/info/debatty/java/stringsimilarity/OptimalStringAlignmentTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2016 Thibault Debatty. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | package info.debatty.java.stringsimilarity; 25 | 26 | import static org.junit.Assert.assertEquals; 27 | 28 | import info.debatty.java.stringsimilarity.testutil.NullEmptyTests; 29 | import org.junit.Test; 30 | 31 | /** 32 | * 33 | * @author Michail Bogdanos 34 | */ 35 | public class OptimalStringAlignmentTest { 36 | 37 | /** 38 | * Test of distance method, of class OptimalStringAlignment. 39 | */ 40 | @Test 41 | public final void testDistance() { 42 | System.out.println("distance"); 43 | OptimalStringAlignment instance = new OptimalStringAlignment(); 44 | 45 | //equality 46 | assertEquals(0.0, instance.distance("ABDCEF", "ABDCEF"), 0.0); 47 | 48 | //single operation 49 | assertEquals(1.0, instance.distance("ABDCFE", "ABDCEF"), 0.0); 50 | assertEquals(1.0, instance.distance("BBDCEF", "ABDCEF"), 0.0); 51 | assertEquals(1.0, instance.distance("BDCEF", "ABDCEF"), 0.0); 52 | assertEquals(1.0, instance.distance("ABDCEF", "ADCEF"), 0.0); 53 | 54 | //other 55 | assertEquals(3.0, instance.distance("CA", "ABC"), 0.0); 56 | assertEquals(2.0, instance.distance("BAC", "CAB"), 0.0); 57 | assertEquals(4.0, instance.distance("abcde", "awxyz"), 0.0); 58 | assertEquals(5.0, instance.distance("abcde", "vwxyz"), 0.0); 59 | 60 | NullEmptyTests.testDistance(instance); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/test/java/info/debatty/java/stringsimilarity/QGramTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2015 Thibault Debatty. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | 25 | package info.debatty.java.stringsimilarity; 26 | 27 | import info.debatty.java.stringsimilarity.testutil.NullEmptyTests; 28 | import org.junit.Test; 29 | 30 | import static org.junit.Assert.assertEquals; 31 | 32 | /** 33 | * 34 | * @author Thibault Debatty 35 | */ 36 | public class QGramTest { 37 | 38 | /** 39 | * Test of distance method, of class QGram. 40 | */ 41 | @Test 42 | public final void testDistance() { 43 | System.out.println("distance"); 44 | QGram instance = new QGram(2); 45 | // AB BC CD CE 46 | // 1 1 1 0 47 | // 1 1 0 1 48 | // Total: 2 49 | double result = instance.distance("ABCD", "ABCE"); 50 | assertEquals(2.0, result, 0.0); 51 | 52 | assertEquals( 53 | 0.0, 54 | instance.distance("S", "S"), 55 | 0.0); 56 | 57 | assertEquals(0.0, 58 | instance.distance("012345", "012345"), 59 | 0.0); 60 | 61 | // NOTE: not using null/empty tests in NullEmptyTests because QGram is different 62 | assertEquals(0.0, instance.distance("", ""), 0.1); 63 | assertEquals(2.0, instance.distance("", "foo"), 0.1); 64 | assertEquals(2.0, instance.distance("foo", ""), 0.1); 65 | 66 | NullEmptyTests.assertNullPointerExceptions(instance); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/test/java/info/debatty/java/stringsimilarity/RatcliffObershelpTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2015 Thibault Debatty. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | 25 | package info.debatty.java.stringsimilarity; 26 | 27 | import info.debatty.java.stringsimilarity.testutil.NullEmptyTests; 28 | import org.junit.Test; 29 | import static org.junit.Assert.*; 30 | 31 | /** 32 | * 33 | * @author Agung Nugroho 34 | */ 35 | public class RatcliffObershelpTest { 36 | 37 | 38 | /** 39 | * Test of similarity method, of class RatcliffObershelp. 40 | */ 41 | @Test 42 | public final void testSimilarity() { 43 | System.out.println("similarity"); 44 | RatcliffObershelp instance = new RatcliffObershelp(); 45 | 46 | // test data from other algorithms 47 | // "My string" vs "My tsring" 48 | // Substrings: 49 | // "ring" ==> 4, "My s" ==> 3, "s" ==> 1 50 | // Ratcliff-Obershelp = 2*(sum of substrings)/(length of s1 + length of s2) 51 | // = 2*(4 + 3 + 1) / (9 + 9) 52 | // = 16/18 53 | // = 0.888888 54 | assertEquals( 55 | 0.888888, 56 | instance.similarity("My string", "My tsring"), 57 | 0.000001); 58 | 59 | // test data from other algorithms 60 | // "My string" vs "My tsring" 61 | // Substrings: 62 | // "My " ==> 3, "tri" ==> 3, "g" ==> 1 63 | // Ratcliff-Obershelp = 2*(sum of substrings)/(length of s1 + length of s2) 64 | // = 2*(3 + 3 + 1) / (9 + 9) 65 | // = 14/18 66 | // = 0.777778 67 | assertEquals( 68 | 0.777778, 69 | instance.similarity("My string", "My ntrisg"), 70 | 0.000001); 71 | 72 | // test data from essay by Ilya Ilyankou 73 | // "Comparison of Jaro-Winkler and Ratcliff/Obershelp algorithms 74 | // in spell check" 75 | // https://ilyankou.files.wordpress.com/2015/06/ib-extended-essay.pdf 76 | // p13, expected result is 0.857 77 | assertEquals( 78 | 0.857, 79 | instance.similarity("MATEMATICA", "MATHEMATICS"), 80 | 0.001); 81 | 82 | // test data from stringmetric 83 | // https://github.com/rockymadden/stringmetric 84 | // expected output is 0.7368421052631579 85 | assertEquals( 86 | 0.736842, 87 | instance.similarity("aleksander", "alexandre"), 88 | 0.000001); 89 | 90 | // test data from stringmetric 91 | // https://github.com/rockymadden/stringmetric 92 | // expected output is 0.6666666666666666 93 | assertEquals( 94 | 0.666666, 95 | instance.similarity("pennsylvania", "pencilvaneya"), 96 | 0.000001); 97 | 98 | // test data from wikipedia 99 | // https://en.wikipedia.org/wiki/Gestalt_Pattern_Matching 100 | // expected output is 14/18 = 0.7777777777777778‬ 101 | assertEquals( 102 | 0.777778, 103 | instance.similarity("WIKIMEDIA", "WIKIMANIA"), 104 | 0.000001); 105 | 106 | // test data from wikipedia 107 | // https://en.wikipedia.org/wiki/Gestalt_Pattern_Matching 108 | // expected output is 24/40 = 0.65 109 | assertEquals( 110 | 0.6, 111 | instance.similarity("GESTALT PATTERN MATCHING", "GESTALT PRACTICE"), 112 | 0.000001); 113 | 114 | NullEmptyTests.testSimilarity(instance); 115 | } 116 | 117 | @Test 118 | public final void testDistance() { 119 | RatcliffObershelp instance = new RatcliffObershelp(); 120 | NullEmptyTests.testDistance(instance); 121 | 122 | // TODO: regular (non-null/empty) distance tests 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /src/test/java/info/debatty/java/stringsimilarity/SorensenDiceTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2015 Thibault Debatty. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | 25 | package info.debatty.java.stringsimilarity; 26 | 27 | import info.debatty.java.stringsimilarity.testutil.NullEmptyTests; 28 | import org.junit.After; 29 | import org.junit.AfterClass; 30 | import org.junit.Before; 31 | import org.junit.BeforeClass; 32 | import org.junit.Test; 33 | import static org.junit.Assert.*; 34 | 35 | /** 36 | * 37 | * @author Thibault Debatty 38 | */ 39 | public class SorensenDiceTest { 40 | 41 | /** 42 | * Test of similarity method, of class SorensenDice. 43 | */ 44 | @Test 45 | public void testSimilarity() { 46 | System.out.println("similarity"); 47 | SorensenDice instance = new SorensenDice(2); 48 | // AB BC CD DE DF FG 49 | // 1 1 1 1 0 0 50 | // 1 1 1 0 1 1 51 | // => 2 x 3 / (4 + 5) = 6/9 = 0.6666 52 | double result = instance.similarity("ABCDE", "ABCDFG"); 53 | assertEquals(0.6666, result, 0.0001); 54 | 55 | NullEmptyTests.testSimilarity(instance); 56 | } 57 | 58 | @Test 59 | public final void testDistance() { 60 | SorensenDice instance = new SorensenDice(); 61 | NullEmptyTests.testDistance(instance); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/test/java/info/debatty/java/stringsimilarity/WeightedLevenshteinTest.java: -------------------------------------------------------------------------------- 1 | package info.debatty.java.stringsimilarity; 2 | 3 | import info.debatty.java.stringsimilarity.testutil.NullEmptyTests; 4 | import org.junit.Test; 5 | 6 | import static org.junit.Assert.*; 7 | 8 | public class WeightedLevenshteinTest { 9 | @Test 10 | public void testDistance() { 11 | WeightedLevenshtein instance = new WeightedLevenshtein(new CharacterSubstitutionInterface() { 12 | public double cost(char c1, char c2) { 13 | // The cost for substituting 't' and 'r' is considered 14 | // smaller as these 2 are located next to each other 15 | // on a keyboard 16 | if (c1 == 't' && c2 == 'r') { 17 | return 0.5; 18 | } 19 | 20 | // For most cases, the cost of substituting 2 characters 21 | // is 1.0 22 | return 1.0; 23 | } 24 | }); 25 | 26 | assertEquals(0.0, instance.distance("String1", "String1"), 0.1); 27 | assertEquals(0.5, instance.distance("String1", "Srring1"), 0.1); 28 | assertEquals(1.5, instance.distance("String1", "Srring2"), 0.1); 29 | 30 | // One insert or delete. 31 | assertEquals(1.0, instance.distance("Strng", "String"), 0.1); 32 | assertEquals(1.0, instance.distance("String", "Strng"), 0.1); 33 | 34 | // With limits. 35 | assertEquals(0.0, instance.distance("String1", "String1", Double.MAX_VALUE), 0.1); 36 | assertEquals(0.0, instance.distance("String1", "String1", 2.0), 0.1); 37 | assertEquals(1.5, instance.distance("String1", "Srring2", Double.MAX_VALUE), 0.1); 38 | assertEquals(1.5, instance.distance("String1", "Srring2", 2.0), 0.1); 39 | assertEquals(1.5, instance.distance("String1", "Srring2", 1.5), 0.1); 40 | assertEquals(1.0, instance.distance("String1", "Srring2", 1.0), 0.1); 41 | assertEquals(4.0, instance.distance("String1", "Potato", 4.0), 0.1); 42 | 43 | NullEmptyTests.testDistance(instance); 44 | } 45 | 46 | @Test 47 | public void testDistanceCharacterInsDelInterface() { 48 | WeightedLevenshtein instance = new WeightedLevenshtein( 49 | new CharacterSubstitutionInterface() { 50 | public double cost(char c1, char c2) { 51 | if (c1 == 't' && c2 == 'r') { 52 | return 0.5; 53 | } 54 | return 1.0; 55 | } 56 | }, 57 | new CharacterInsDelInterface() { 58 | public double deletionCost(char c) { 59 | if (c == 'i') { 60 | return 0.8; 61 | } 62 | return 1.0; 63 | } 64 | 65 | public double insertionCost(char c) { 66 | if (c == 'i') { 67 | return 0.5; 68 | } 69 | return 1.0; 70 | } 71 | }); 72 | 73 | // Same as testDistance above. 74 | assertEquals(0.0, instance.distance("String1", "String1"), 0.1); 75 | assertEquals(0.5, instance.distance("String1", "Srring1"), 0.1); 76 | assertEquals(1.5, instance.distance("String1", "Srring2"), 0.1); 77 | 78 | // Cost of insert of 'i' is less than normal, so these scores are 79 | // different than testDistance above. Note that the cost of delete 80 | // has been set differently than the cost of insert, so the distance 81 | // call is not symmetric in its arguments if an 'i' has changed. 82 | assertEquals(0.5, instance.distance("Strng", "String"), 0.1); 83 | assertEquals(0.8, instance.distance("String", "Strng"), 0.1); 84 | assertEquals(1.0, instance.distance("Strig", "String"), 0.1); 85 | assertEquals(1.0, instance.distance("String", "Strig"), 0.1); 86 | 87 | // Same as above with limits. 88 | assertEquals(0.0, instance.distance("String1", "String1", Double.MAX_VALUE), 0.1); 89 | assertEquals(0.0, instance.distance("String1", "String1", 2.0), 0.1); 90 | assertEquals(0.5, instance.distance("String1", "Srring1", Double.MAX_VALUE), 0.1); 91 | assertEquals(0.5, instance.distance("String1", "Srring1", 2.0), 0.1); 92 | assertEquals(1.5, instance.distance("String1", "Srring2", 2.0), 0.1); 93 | assertEquals(1.5, instance.distance("String1", "Srring2", 1.5), 0.1); 94 | assertEquals(1.0, instance.distance("String1", "Srring2", 1.0), 0.1); 95 | assertEquals(4.0, instance.distance("String1", "Potato", 4.0), 0.1); 96 | 97 | NullEmptyTests.testDistance(instance); 98 | } 99 | } -------------------------------------------------------------------------------- /src/test/java/info/debatty/java/stringsimilarity/experimental/Sift4Test.java: -------------------------------------------------------------------------------- 1 | /* 2 | * The MIT License 3 | * 4 | * Copyright 2016 Thibault Debatty. 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | 25 | package info.debatty.java.stringsimilarity.experimental; 26 | 27 | import static org.junit.Assert.assertEquals; 28 | import org.junit.Test; 29 | 30 | /** 31 | * 32 | * @author Thibault Debatty 33 | */ 34 | public class Sift4Test { 35 | 36 | /** 37 | * Test of distance method, of class Sift4. 38 | */ 39 | @Test 40 | public void testDistance() { 41 | System.out.println("SIFT4 distance"); 42 | String s1 = "This is the first string"; 43 | String s2 = "And this is another string"; 44 | Sift4 sift4 = new Sift4(); 45 | sift4.setMaxOffset(5); 46 | double expResult = 11.0; 47 | double result = sift4.distance(s1, s2); 48 | assertEquals(expResult, result, 0.0); 49 | 50 | sift4.setMaxOffset(10); 51 | assertEquals( 52 | 12.0, 53 | sift4.distance( 54 | "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", 55 | "Amet Lorm ispum dolor sit amet, consetetur adixxxpiscing elit."), 56 | 0.0); 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /src/test/java/info/debatty/java/stringsimilarity/testutil/NullEmptyTests.java: -------------------------------------------------------------------------------- 1 | package info.debatty.java.stringsimilarity.testutil; 2 | 3 | import info.debatty.java.stringsimilarity.interfaces.NormalizedStringDistance; 4 | import info.debatty.java.stringsimilarity.interfaces.NormalizedStringSimilarity; 5 | import info.debatty.java.stringsimilarity.interfaces.StringDistance; 6 | import org.junit.Assert; 7 | 8 | import static org.junit.Assert.assertEquals; 9 | import static org.junit.Assert.assertTrue; 10 | import static org.junit.Assert.fail; 11 | 12 | public final class NullEmptyTests { 13 | 14 | public static void testDistance(NormalizedStringDistance instance) { 15 | assertEquals(0.0, instance.distance("", ""), 0.1); 16 | assertEquals(1.0, instance.distance("", "foo"), 0.1); 17 | assertEquals(1.0, instance.distance("foo", ""), 0.1); 18 | 19 | assertNullPointerExceptions(instance); 20 | } 21 | 22 | public static void testDistance(StringDistance instance) { 23 | assertEquals(0.0, instance.distance("", ""), 0.1); 24 | assertEquals(3.0, instance.distance("", "foo"), 0.1); 25 | assertEquals(3.0, instance.distance("foo", ""), 0.1); 26 | 27 | assertNullPointerExceptions(instance); 28 | } 29 | 30 | public static void testSimilarity(NormalizedStringSimilarity instance) { 31 | assertEquals(1.0, instance.similarity("", ""), 0.1); 32 | assertEquals(0.0, instance.similarity("", "foo"), 0.1); 33 | assertEquals(0.0, instance.similarity("foo", ""), 0.1); 34 | 35 | try { 36 | instance.similarity(null, null); 37 | fail("A NullPointerException was not thrown."); 38 | } catch (NullPointerException ignored) { 39 | } 40 | 41 | try { 42 | instance.similarity(null, ""); 43 | fail("A NullPointerException was not thrown."); 44 | } catch (NullPointerException ignored) { 45 | } 46 | 47 | try { 48 | instance.similarity("", null); 49 | fail("A NullPointerException was not thrown."); 50 | } catch (NullPointerException ignored) { 51 | } 52 | } 53 | 54 | public static void assertNullPointerExceptions(StringDistance instance) { 55 | try { 56 | instance.distance(null, null); 57 | fail("A NullPointerException was not thrown."); 58 | } catch (NullPointerException ignored) { 59 | } 60 | 61 | try { 62 | instance.distance(null, ""); 63 | fail("A NullPointerException was not thrown."); 64 | } catch (NullPointerException ignored) { 65 | } 66 | 67 | try { 68 | instance.distance("", null); 69 | fail("A NullPointerException was not thrown."); 70 | } catch (NullPointerException ignored) { 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/test/resources/71816-2.txt: -------------------------------------------------------------------------------- 1 | diff --git a/org.eclipse.egit.ui.test/src/org/eclipse/egit/ui/test/team/actions/CommitActionStagingViewTest.java b/org.eclipse.egit.ui.test/src/org/eclipse/egit/ui/test/team/actions/CommitActionStagingViewTest.java 2 | new file mode 100644 3 | index 0000000..ad9d3e7 4 | --- /dev/null 5 | +++ b/org.eclipse.egit.ui.test/src/org/eclipse/egit/ui/test/team/actions/CommitActionStagingViewTest.java 6 | @@ -0,0 +1,110 @@ 7 | +/******************************************************************************* 8 | + * Copyright (c) 2016 Thomas Wolf 9 | + * All rights reserved. This program and the accompanying materials 10 | + * are made available under the terms of the Eclipse Public License v1.0 11 | + * which accompanies this distribution, and is available at 12 | + * http://www.eclipse.org/legal/epl-v10.html 13 | + *******************************************************************************/ 14 | +package org.eclipse.egit.ui.test.team.actions; 15 | + 16 | +import static org.junit.Assert.assertEquals; 17 | +import static org.junit.Assert.assertNotNull; 18 | +import static org.junit.Assert.assertTrue; 19 | + 20 | +import java.io.File; 21 | + 22 | +import org.eclipse.core.resources.IProject; 23 | +import org.eclipse.core.resources.ResourcesPlugin; 24 | +import org.eclipse.egit.ui.Activator; 25 | +import org.eclipse.egit.ui.UIPreferences; 26 | +import org.eclipse.egit.ui.common.LocalRepositoryTestCase; 27 | +import org.eclipse.egit.ui.internal.staging.StagingView; 28 | +import org.eclipse.egit.ui.test.ContextMenuHelper; 29 | +import org.eclipse.egit.ui.test.TestUtil; 30 | +import org.eclipse.jgit.lib.Repository; 31 | +import org.eclipse.swtbot.swt.finder.junit.SWTBotJunit4ClassRunner; 32 | +import org.eclipse.swtbot.swt.finder.widgets.SWTBotTree; 33 | +import org.eclipse.ui.PartInitException; 34 | +import org.eclipse.ui.PlatformUI; 35 | +import org.junit.After; 36 | +import org.junit.Before; 37 | +import org.junit.Test; 38 | +import org.junit.runner.RunWith; 39 | + 40 | +/** 41 | + * Tests for the Team->Commit action 42 | + */ 43 | +@RunWith(SWTBotJunit4ClassRunner.class) 44 | +public class CommitActionStagingViewTest extends LocalRepositoryTestCase { 45 | + private File repositoryFile; 46 | + 47 | + private boolean initialLinkWithSelection; 48 | + 49 | + private boolean initialUseStagingView; 50 | + 51 | + @Before 52 | + public void setup() throws Exception { 53 | + initialUseStagingView = Activator.getDefault().getPreferenceStore() 54 | + .getBoolean(UIPreferences.ALWAYS_USE_STAGING_VIEW); 55 | + initialLinkWithSelection = Activator.getDefault().getPreferenceStore() 56 | + .getBoolean(UIPreferences.STAGING_VIEW_SYNC_SELECTION); 57 | + Activator.getDefault().getPreferenceStore() 58 | + .setValue(UIPreferences.ALWAYS_USE_STAGING_VIEW, true); 59 | + Activator.getDefault().getPreferenceStore() 60 | + .setDefault(UIPreferences.STAGING_VIEW_SYNC_SELECTION, false); 61 | + Activator.getDefault().getPreferenceStore() 62 | + .setValue(UIPreferences.STAGING_VIEW_SYNC_SELECTION, false); 63 | + repositoryFile = createProjectAndCommitToRepository(); 64 | + Repository repo = lookupRepository(repositoryFile); 65 | + TestUtil.configureTestCommitterAsUser(repo); 66 | + // TODO delete the second project for the time being (.gitignore is 67 | + // currently not hiding the .project file from commit) 68 | + IProject project = ResourcesPlugin.getWorkspace().getRoot().getProject(PROJ2); 69 | + File dotProject = new File(project.getLocation().toOSString(), ".project"); 70 | + project.delete(false, false, null); 71 | + assertTrue(dotProject.delete()); 72 | + TestUtil.hideView(StagingView.VIEW_ID); 73 | + } 74 | + 75 | + @After 76 | + public void tearDown() { 77 | + Activator.getDefault().getPreferenceStore().setValue( 78 | + UIPreferences.ALWAYS_USE_STAGING_VIEW, initialUseStagingView); 79 | + Activator.getDefault().getPreferenceStore() 80 | + .setDefault(UIPreferences.STAGING_VIEW_SYNC_SELECTION, true); 81 | + Activator.getDefault().getPreferenceStore().setValue( 82 | + UIPreferences.STAGING_VIEW_SYNC_SELECTION, 83 | + initialLinkWithSelection); 84 | + } 85 | + 86 | + @Test 87 | + public void testOpenStagingViewNoLinkWithSelection() throws Exception { 88 | + setTestFileContent("I have changed this"); 89 | + SWTBotTree projectExplorerTree = TestUtil.getExplorerTree(); 90 | + util.getProjectItems(projectExplorerTree, PROJ1)[0].select(); 91 | + String menuString = util.getPluginLocalizedValue("CommitAction_label"); 92 | + ContextMenuHelper.clickContextMenu(projectExplorerTree, "Team", 93 | + menuString); 94 | + TestUtil.waitUntilViewWithGivenIdShows(StagingView.VIEW_ID); 95 | + final Repository[] repo = { null }; 96 | + PlatformUI.getWorkbench().getDisplay().syncExec(new Runnable() { 97 | + 98 | + @Override 99 | + public void run() { 100 | + StagingView view; 101 | + try { 102 | + view = (StagingView) PlatformUI.getWorkbench() 103 | + .getActiveWorkbenchWindow().getActivePage() 104 | + .showView(StagingView.VIEW_ID); 105 | + repo[0] = view.getCurrentRepository(); 106 | + } catch (PartInitException e) { 107 | + // Ignore, repo[0] remains null 108 | + } 109 | + } 110 | + }); 111 | + Repository repository = lookupRepository(repositoryFile); 112 | + assertNotNull("No repository found", repository); 113 | + assertEquals("Repository mismatch", repository, repo[0]); 114 | + } 115 | + 116 | +} 117 | --------------------------------------------------------------------------------