├── README.md
├── coveringLSH.py
└── testCoveringLSH.py


/README.md:
--------------------------------------------------------------------------------
1 | # coveringLSH
2 | Simple implementation of CoveringLSH.
3 | 
4 | The theory behind the implementation is described in http://arxiv.org/abs/1507.03225
5 | 


--------------------------------------------------------------------------------
/coveringLSH.py:
--------------------------------------------------------------------------------
 1 | # Reference implementation of CoveringLSH
 2 | # (c) Rasmus Pagh 2016
 3 | # Version 1.0
 4 | #
 5 | # Description: Encodes a collection of binary vectors as integers, creates a data structure to perform nearest neighbor queries under Hamming distance up to a maximum radius r. The procedure buildCovering must be called before buildDataStructure. For details see "Locality-sensitive Hashing without False Negatives" by Rasmus Pagh, Proceedings of SODA 2016.
 6 | #
 7 | # License: This implementation is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation; either version 2.1 of the License, or (at your option) any later version. The implementation is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more details.
 8 | 
 9 | from random import randint
10 | from math import floor, log
11 | from sets import Set 
12 | 
13 | A, infinity = {}, float("inf")
14 | def popcnt(x): return bin(x).count('1')
15 | def dist(x,y): return popcnt(x ^ y)
16 | 
17 | def buildCovering(d,r):
18 |     for v in xrange(1,2**(r+1)): A[v] = 0
19 |     for i in xrange(d):
20 |         m = randint(1, 2**(r+1)-1)
21 |         for v in xrange(1,2**(r+1)):
22 |             A[v] = A[v] + (1<<i) * (popcnt(m & v) % 2)
23 | 
24 | def buildDataStructure(S,r):
25 |     D = {}
26 |     for x in S:
27 |         for v in xrange(1,2**(r+1)):
28 |             if not (x & A[v]) in D: D[x & A[v]] = Set()
29 |             D[x & A[v]].add(x)
30 |     return D
31 | 
32 | def nearestNeighbor(D,r,y):
33 |     best, nn = infinity, None
34 |     for v in xrange(1,2**(r+1)):
35 |         if (y & A[v]) in D:
36 |             for x in D[y & A[v]]:
37 |                 if dist(x,y) < best:
38 |                     best, nn = dist(x,y), x
39 |         if best <= floor(log(v+1,2)): return nn
40 |     return None
41 | 


--------------------------------------------------------------------------------
/testCoveringLSH.py:
--------------------------------------------------------------------------------
 1 | # Test program for coveringLSH.py
 2 | # (c) Rasmus Pagh 2016
 3 | #
 4 | # License: This implementation is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation; either version 2.1 of the License, or (at your option) any later version. The implementation is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more details.
 5 | 
 6 | import code, random, time
 7 | from coveringLSH import dist, buildCovering, buildDataStructure, nearestNeighbor
 8 | 
 9 | n = 10000
10 | r = 5
11 | d = 25
12 | repetitions = 10000
13 | 
14 | print "Building data structure with",n,"vectors in",d,"dimensions and maximum radius",r,"..."
15 | buildCovering(d,r)
16 | S = [random.getrandbits(d) for i in xrange(n)] 
17 | D = buildDataStructure(S,r)
18 | 
19 | def bruteForceNearestNeighbor(S,y):
20 |     best, nn = float("inf"), None
21 |     for x in S:
22 |         if dist(x,y) < best:
23 |             best = dist(x,y)
24 |             nn = x
25 |     return nn
26 | 
27 | print "Testing correctness..."
28 | for y in xrange(100):
29 |     y = random.getrandbits(d)
30 |     x1 = nearestNeighbor(D,r,y)
31 |     x2 = bruteForceNearestNeighbor(S,y)
32 |     if dist(y,x2) <= r:
33 |         if x1==None: print (bin(y),None,bin(x2))
34 |         elif dist(y,x1)!=dist(y,x2):
35 |             print (bin(y),bin(x1),bin(x2))
36 | 
37 | print "Testing speed..."
38 | start = time.clock()
39 | for y in xrange(repetitions):
40 |     y = random.getrandbits(d)
41 |     x1 = bruteForceNearestNeighbor(S,y)
42 | print "Brute force:",int(repetitions/(time.clock()-start)),"queries per second"
43 | 
44 | start = time.clock()
45 | for y in xrange(repetitions):
46 |     y = random.getrandbits(d)
47 |     x1 = nearestNeighbor(D,r,y)
48 | print "CoveringLSH:",int(repetitions/(time.clock()-start)),"queries per second"
49 | 
50 | code.interact(local=locals())
51 | 


--------------------------------------------------------------------------------