├── src
    └── rollinghash
    │   ├── CharacterHash.java
    │   ├── RabinKarpHash.java
    │   ├── CyclicHash.java
    │   └── Unit.java
└── README.md


/src/rollinghash/CharacterHash.java:
--------------------------------------------------------------------------------
 1 | package rollinghash;
 2 | 
 3 | import java.util.Random;
 4 | 
 5 | 
 6 | // not to be used directly, see CyclicHash and RabinKarpHash instead
 7 | public class CharacterHash {
 8 | 	public int hashvalues[] = new int[1<<16];
 9 | 	
10 | 	public CharacterHash() {
11 | 		Random r = new Random();
12 | 		for(int k = 0; k<hashvalues.length;++k)
13 | 			hashvalues[k] = r.nextInt();
14 | 	}
15 | 	
16 | 	public static CharacterHash getInstance() {
17 | 		return charhash;
18 | 	}
19 | 	
20 | 	static CharacterHash charhash = new CharacterHash();
21 | 
22 | }
23 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | ## Rolling hash functions in Java
 2 | 
 3 | License: Apache 2.0
 4 | 
 5 | ## What is this?
 6 | 
 7 | This is a set of Java classes implementing various recursive n-gram hashing techniques, also called rolling hashing (http://en.wikipedia.org/wiki/Rolling_hash), including:
 8 | 
 9 | * Randomized Karp-Rabin (sometimes called Rabin-Karp)
10 | * Hashing by Cyclic Polynomials (also known as Buzhash)
11 | 
12 | 
13 | ## Code sample
14 | 
15 | ```java
16 | String s = "some string";
17 | int n = 3; //hash all sequences of 3 characters
18 | CyclicHash ch = new CyclicHash(n);
19 | int k = 0;
20 | for(; k<n;++k) {
21 |   ch.eat(s.charAt(k));
22 | }
23 | int rollinghash = ch.eat(s.charAt(k)); // the first or last 32-(n-1) bits are strongly universal
24 | // do something with the hash value
25 | for(;k<s.length();++k) {
26 |   rollinghash = ch.update(s.charAt(k-n), s.charAt(k));
27 |   // do something with the hash value
28 | }
29 | ```
30 | 
31 | ## Reference 
32 | 
33 | Daniel Lemire, Owen Kaser: Recursive n-gram hashing is pairwise independent, at best, Computer Speech & Language, Volume 24, Issue 4, October 2010, Pages 698-710 http://arxiv.org/abs/0705.4676
34 | 


--------------------------------------------------------------------------------
/src/rollinghash/RabinKarpHash.java:
--------------------------------------------------------------------------------
 1 | package rollinghash;
 2 | 
 3 | public class RabinKarpHash {
 4 | 	
 5 | 	// myn is the length in characters of the blocks you want to hash
 6 | 	public RabinKarpHash(int myn) {
 7 | 		n = myn;
 8 | 		BtoN = 1;
 9 | 	    for (int i=0; i < n ; ++i) {
10 | 		      BtoN *= B;
11 | 	    }
12 | 	}
13 | 	
14 | 	// add new character  (useful to initiate the hasher)
15 | 	// return 32 bits (not even universal)
16 | 	public int eat(char c) {
17 | 		hashvalue = B*hashvalue + hasher.hashvalues[c];
18 | 		return hashvalue;
19 | 	}
20 | 	
21 | 	// remove old character and add new one
22 | 	// return 32 bits (not even universal)
23 | 	public int update(char outchar, char inchar) {
24 | 		hashvalue = B*hashvalue +hasher.hashvalues[inchar] - BtoN * hasher.hashvalues[outchar];
25 | 		return hashvalue;
26 | 	}
27 | 
28 | 	// this is purely for testing purposes
29 | 	public static int nonRollingHash(CharSequence s) {
30 | 		int value = 0;
31 | 		for(int i = 0; i<s.length();++i) {
32 | 			char c = s.charAt(i);
33 | 			int z = hasher.hashvalues[c];
34 | 			value = B*value + z;
35 | 		}
36 | 		return value;
37 | 	}
38 | 	public int hashvalue;
39 | 	int n;
40 | 	int BtoN;
41 | 	static CharacterHash hasher = CharacterHash.getInstance();
42 | 	final static int B = 31;
43 | 
44 | }
45 | 


--------------------------------------------------------------------------------
/src/rollinghash/CyclicHash.java:
--------------------------------------------------------------------------------
 1 | package rollinghash;
 2 | 
 3 | public class CyclicHash {
 4 | 	
 5 | 	// myn is the length in characters of the blocks you want to hash
 6 | 	public CyclicHash(int myn) {
 7 | 		n = myn;
 8 | 		if(n> wordsize) {
 9 | 			throw new IllegalArgumentException();
10 | 		}
11 | 
12 | 	}
13 | 	
14 | 
15 |     private int fastleftshiftn(int x)  {
16 |        return  (x << n ) | (x >>> (wordsize-n)) ;     
17 |     }
18 | 
19 |     private static int fastleftshift1(int x)  {
20 |     	return  (x << 1 ) | (x >>> (wordsize-1)) ;
21 |     }
22 | 	
23 | 	// add new character (useful to initiate the hasher)
24 | 	// to get a strongly universal hash value, you have to ignore the last or first (n-1) bits.
25 |     public int eat(char c) {
26 | 		hashvalue = fastleftshift1(hashvalue);
27 | 		hashvalue ^= hasher.hashvalues[c];
28 | 		return hashvalue;
29 | 	}
30 | 	
31 | 	// remove old character and add new one
32 | 	// to get a strongly universal hash value, you have to ignore the last or first (n-1) bits.
33 | 	public int update(char outchar, char inchar) {
34 | 		int z = fastleftshiftn(hasher.hashvalues[outchar]);
35 | 		hashvalue =  fastleftshift1(hashvalue) ^ z ^ hasher.hashvalues[inchar];
36 | 		return hashvalue;
37 | 	}
38 | 	
39 | 	// this is purely for testing purposes
40 | 	public static  int nonRollingHash(CharSequence s) {
41 | 		int value = 0;
42 | 		for(int i = 0; i<s.length();++i) {
43 | 			char c = s.charAt(i);
44 | 			int z = hasher.hashvalues[c];
45 | 			value = fastleftshift1(value) ^z;
46 | 		}
47 | 		return value;
48 | 	}
49 | 	
50 | 	public final static int wordsize = 32;
51 | 	public int hashvalue;
52 | 	int n;
53 | 	int myr;
54 | 	static CharacterHash hasher = CharacterHash.getInstance();
55 | 
56 | }


--------------------------------------------------------------------------------
/src/rollinghash/Unit.java:
--------------------------------------------------------------------------------
 1 | package rollinghash;
 2 | 
 3 | import java.util.Random;
 4 | 
 5 | public class Unit {
 6 | 	public static boolean testCyclic(String s, int n) {
 7 | 		System.out.println("Testing cyclic n = "+n);
 8 | 		CyclicHash ch = new CyclicHash(n);
 9 | 		int k = 0;
10 | 		if(n>=s.length()) return false;
11 | 		for(; k<n-1;++k) {
12 | 			ch.eat(s.charAt(k));
13 | 		}
14 | 		int rollinghash = ch.eat(s.charAt(k));
15 | 		++k;
16 | 		int longhash = CyclicHash.nonRollingHash(s.subSequence(k-n,k));
17 | 		
18 | 		if(rollinghash != longhash) {
19 | 			System.out.println("test failed! "+rollinghash+ " "+longhash);
20 | 			return false;
21 | 		} 
22 | 		for(;k<s.length();++k) {
23 | 			rollinghash = ch.update(s.charAt(k-n), s.charAt(k));
24 | 			longhash = CyclicHash.nonRollingHash(s.subSequence(k-n+1,k+1));
25 | 			if(rollinghash != longhash) {
26 | 				System.out.println("test failed! "+k+" "+rollinghash+ " "+longhash);
27 | 				return false;
28 | 			}
29 | 		}
30 | 		System.out.println("Ok!");
31 | 		return true;
32 | 	}
33 | 	public static boolean testRabinKarp(String s, int n) {
34 | 		System.out.println("Testing RabinKarpHash n = "+n);
35 | 		RabinKarpHash ch = new RabinKarpHash(n);
36 | 		int k = 0;
37 | 		if(n>=s.length()) return false;
38 | 		for(; k<n-1;++k) {
39 | 			ch.eat(s.charAt(k));
40 | 		}
41 | 		int rollinghash = ch.eat(s.charAt(k));
42 | 		++k;
43 | 		int longhash = RabinKarpHash.nonRollingHash(s.subSequence(k-n,k));
44 | 		
45 | 		if(rollinghash != longhash) {
46 | 			System.out.println("test failed! "+rollinghash+ " "+longhash);
47 | 			return false;
48 | 		} 
49 | 		for(;k<s.length();++k) {
50 | 			rollinghash = ch.update(s.charAt(k-n), s.charAt(k));
51 | 			longhash = RabinKarpHash.nonRollingHash(s.subSequence(k-n+1,k+1));
52 | 			if(rollinghash != longhash) {
53 | 				System.out.println("test failed! "+k+" "+rollinghash+ " "+longhash);
54 | 				return false;
55 | 			}
56 | 		}
57 | 		System.out.println("Ok!");
58 | 		return true;
59 | 	}
60 | 	
61 | 	public static void main(String[] s) {
62 | 		boolean ok = true;
63 | 		StringBuffer sb = new StringBuffer();
64 | 		Random r = new Random();
65 | 		for(int k = 0; k<1000000; ++k)
66 | 			sb.append((char)r.nextInt());
67 | 		String tests = sb.toString();
68 | 		for(int n = 1;n<10;++n)
69 |  		  if(!testCyclic(tests,n)) ok=false;
70 | 		for(int n = 1;n<10;++n)
71 | 		  if(!testRabinKarp(tests,n)) ok = false;
72 | 		if(!ok) System.out.println("The code did not pass the unit tests.");
73 | 		else System.out.println("The code is probably ok.");
74 | 	}
75 | 
76 | }
77 | 


--------------------------------------------------------------------------------