├── .gitignore
├── LICENSE
├── README.md
├── SSE.pdf
├── SSE.pptx
├── client
    ├── sse_client.py
    └── time.sh
├── jmap
    └── jmap.py
└── server
    └── sse_server.py


/.gitignore:
--------------------------------------------------------------------------------
 1 | # Object files
 2 | *.o
 3 | *.ko
 4 | *.obj
 5 | *.elf
 6 | 
 7 | # Precompiled Headers
 8 | *.gch
 9 | *.pch
10 | 
11 | # Libraries
12 | *.lib
13 | *.a
14 | *.la
15 | *.lo
16 | 
17 | # Shared objects (inc. Windows DLLs)
18 | *.dll
19 | *.so
20 | *.so.*
21 | *.dylib
22 | 
23 | # Executables
24 | *.exe
25 | *.out
26 | *.app
27 | *.i*86
28 | *.x86_64
29 | *.hex
30 | 
31 | # Debug files
32 | *.dSYM/
33 | 
34 | # swp files
35 | *.swp
36 | *~
37 | 
38 | # compiled python files
39 | *.pyc
40 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | The MIT License (MIT)
 2 | 
 3 | Copyright (c) 2016 Ian Van Houdt
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 
23 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # SSE Implementation
 2 | 
 3 | # Dependencies
 4 | 
 5 | python-dev
 6 | 
 7 | libffi-dev
 8 | 	
 9 | ## Python packages
10 | 	
11 | bcrypt
12 | 
13 | Stemming
14 | 
15 | numpy (optional)
16 | 
17 | pyyaml
18 | 
19 | nltk
20 | 
21 | Flask
22 | 
23 | 	-install each with 'pip install <package>'
24 | 
25 | # How To
26 | To use this SSE implementation, you must first have the server running:
27 | 
28 | 	python sse_server.py
29 | 
30 | Then invoke the client with one of the requisite options:
31 | 
32 | 	python sse_client.py <OPTION>
33 | 
34 | It is also required that the user has access to some set of text documents. I recommend using the Enron corpus, which provides a huge number and variety of email documents.
35 | 
36 | # Options
37 |     -s, --search "<term(s)>"
38 |         Search for term or terms in quotations
39 | 
40 |     -S, --search-header "<header>" "<term(s)>"
41 |         Like --search, but only search for those terms in the listed header.
42 | 
43 |     -u, --update "<file>"
44 |         Updates a single file, included appending local index, appending encrypted remote index, encrypting "file", and sending it to server.
45 | 
46 |     -e, --encrypt "<infile>" "<outfile>"
47 |         Encrypts "infile", and writes out to "outfile". "Infile" is not modified.
48 | 
49 |     -d, --decrypt "<infile>" "<outfile>"
50 |         Decrypts "infile" and writes clear text to "outfile". "Infile" is not modified.
51 | 
52 |     -i, --inspect_index
53 |         Prints out local unencrypted index. 
54 |         BUG: require an argument, although unused.
55 | 


--------------------------------------------------------------------------------
/SSE.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/IanVanHoudt/Searchable-Symmetric-Encryption/11cbe7fdb969006f0b7d23c583e287f581d678ba/SSE.pdf


--------------------------------------------------------------------------------
/SSE.pptx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/IanVanHoudt/Searchable-Symmetric-Encryption/11cbe7fdb969006f0b7d23c583e287f581d678ba/SSE.pptx


--------------------------------------------------------------------------------
/client/sse_client.py:
--------------------------------------------------------------------------------
  1 | '''
  2 |  The MIT License (MIT)
  3 | 
  4 |  Copyright (c) 2016 Ian Van Houdt
  5 | 
  6 |  Permission is hereby granted, free of charge, to any person obtaining a copy
  7 |  of this software and associated documentation files (the "Software"), to deal
  8 |  in the Software without restriction, including without limitation the rights
  9 |  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 10 |  copies of the Software, and to permit persons to whom the Software is
 11 |  furnished to do so, subject to the following conditions:
 12 | 
 13 |  The above copyright notice and this permission notice shall be included in all
 14 |  copies or substantial portions of the Software.
 15 | 
 16 |  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 17 |  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 18 |  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 19 |  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 20 |  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 21 |  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 22 |  SOFTWARE.
 23 | '''
 24 | 
 25 | 
 26 | ############
 27 | #
 28 | #  sse_client.py
 29 | #
 30 | #  Serves as SSE implementation for mail client. The routines 
 31 | #  for SSE are invoked by the client module via the API.
 32 | #
 33 | ############
 34 | 
 35 | import socket
 36 | import os
 37 | import sys
 38 | sys.path.append(os.path.realpath('../jmap'))
 39 | from Crypto.Hash import HMAC
 40 | from Crypto.Hash import SHA256
 41 | from Crypto.Cipher import AES
 42 | from Crypto import Random
 43 | import bcrypt
 44 | import binascii
 45 | from argparse import ArgumentParser
 46 | import string
 47 | import anydbm
 48 | import json
 49 | from flask import Flask
 50 | import requests
 51 | import jmap
 52 | from nltk.stem.porter import PorterStemmer
 53 | import email
 54 | import re
 55 | 
 56 | DEBUG = 1
 57 | SEARCH = "search"
 58 | UPDATE = "update"
 59 | ADD_MAIL = "addmail"
 60 | 
 61 | # Default url is localhost, and the port 5000 is set by Flask on the server
 62 | DEFAULT_URL = "http://localhost:5000/"
 63 | 
 64 | NO_RESULTS = "Found no results for query"
 65 | 
 66 | # Some 'Enums' for different options in SEARCH and UPDATE
 67 | ENC_BODY = 0
 68 | ENC_HEADERS = 1
 69 | SRCH_BODY = 0
 70 | SRCH_HEADERS = 1
 71 | 
 72 | DELIMETER = "++?"
 73 | 
 74 | # TODO: Maybe strip out some of the excluded punctuation. Could be useful
 75 | # to keep some punct in the strings. We're mostly looking to strip the
 76 | # final punct (ie: '.' ',' '!' etc)
 77 | EXCLUDE = string.punctuation
 78 | 
 79 | app = Flask(__name__)
 80 | 
 81 | ########
 82 | #
 83 | # SSE_Client
 84 | #
 85 | ########
 86 | class SSE_Client():
 87 | 
 88 |     def __init__(self):
 89 | 
 90 |         # TODO: placeholder for password. Will eventually take
 91 |         # as an arg of some sort
 92 |         self.password = b"password"
 93 | 
 94 |         # TODO: need to sort out use of salt. Previously, salt was
 95 |         # randomly generated in initKeys, but the resulting pass-
 96 |         # words k & kPrime were different on each execution, and 
 97 |         # decryption was impossible. Hardcoding salt makes dectyption
 98 |         # possible but may be a bad short cut
 99 |         self.iv = None
100 |         self.salt = "$2b$12$ddTuco8zWXF2.kTqtOZa9O"
101 | 
102 |         # Two keys, generated/Initialized by KDF
103 |         (self.k, self.kPrime) = self.initKeys()
104 | 
105 |         # Two K's: generated/initialized by PRF
106 |         self.k1 = None
107 |         self.k2 = None
108 | 
109 |         # client's cipher (AES w/ CBC)
110 |         self.cipher = self.initCipher()
111 | 
112 |         # Stemming tool (cuts words to their roots/stems)
113 |         self.stemmer = PorterStemmer()
114 | 
115 |     def initKeys(self):
116 |         # initialize keys k & kPrime
117 |         # k used for PRF; kPrime used for Enc/Dec
118 |         # return (k, kPrime)
119 | 
120 |         #hashed = bcrypt.hashpw(self.password, bcrypt.gensalt())
121 |         hashed = bcrypt.hashpw(self.password, self.salt)
122 | 
123 |         if(DEBUG > 1):
124 |             print("len of k = %d" % len(hashed))
125 |             print("k = %s" % hashed)
126 | 
127 |         # Currently k and kPrime are equal
128 |         # TODO: Sort out requirements of k and kPrime
129 |         # Research uses both, but not sure the difference
130 |         return (hashed, hashed)
131 | 
132 | 
133 |     def initCipher(self):
134 |         # initialize Cipher, using kPrime
135 |         # return new Cipher object
136 | 
137 |         # TODO: fix key. Currently just a hack: AES keys must be
138 |         # 16, 24 or 32 bytes long, but kPrime is 60
139 |         key = self.kPrime[:16]
140 | 
141 |         # generates 16 byte random iv
142 |         self.iv = Random.new().read(AES.block_size)
143 |         cipher = AES.new(key, AES.MODE_CBC, self.iv)
144 | 
145 |         return cipher
146 | 
147 | 
148 |     def encryptMail(self, infile, outfile):
149 | 
150 |         # read in infile (opened file descriptor)
151 |         buf = infile.read()
152 |         if buf == '': 
153 |             print("[Enc] mail to encrypt is empty!\nExiting\n")
154 |             exit(1)
155 | 
156 |         if (DEBUG > 1): print("[Enc] mail to encrypt: %s\n" % (buf))
157 | 
158 |         # pad to mod 16
159 |         while len(buf)%16 != 0:
160 |             buf = buf + "\x08"
161 | 
162 |         # write encrypted data to new file
163 |         outfile.write(self.iv + self.cipher.encrypt(buf))
164 | 
165 |     def decryptMail(self, buf, outfile=None):
166 |         # Just pass in input file buf and fd in which to write out
167 | 
168 |         if buf == '': 
169 |             print("[Dec] mail to decrypt is empty!\nExiting\n")
170 |             exit(1)
171 | 
172 |         # self.kPrime[:16] is the  first 16 bytes of kPrime, ie: enc key
173 |         # buf[:16] is the iv of encrypted msg
174 |         cipher = AES.new(self.kPrime[:16], AES.MODE_CBC, buf[:16])
175 | 
176 |         # decrypt all but first 16 bytes (iv)
177 |         # if outfile is supplied, write to file
178 |         if (outfile):
179 |             outfile.write(cipher.decrypt(buf[16:]))
180 |         # else print to terminal
181 |         else:
182 |             tmp = cipher.decrypt(buf[16:])
183 |             print tmp
184 | 
185 | 
186 |     def encryptMailID(self, k2, document, word=None, index_IDs=None):
187 | 
188 |         # Encrypt doc id (document) with key passed in (k2)
189 | 
190 |         # set up new cipher using k2 and random iv
191 |         iv = Random.new().read(AES.block_size)
192 |         cipher = AES.new(k2[:16], AES.MODE_CBC, iv)
193 | 
194 |         # pad to mod 16
195 |         while len(document)%16 != 0:
196 |             document = document + "\x08"
197 | 
198 |         # If word and index_IDs are supplied, then we're updated the 
199 |         # existing list of ids for a corresponding word/term.
200 |         # This is used so that we can encrypt a list of mailIDs, rather
201 |         # than just a single one, speeding up SEARCH routine later.
202 |         if word and index_IDs:
203 |             IDs = index_IDs[word]
204 |             IDs = IDs + DELIMETER + document 
205 | 
206 |             while len(IDs)%16 != 0:
207 |                 IDs = IDs + "\x08"
208 | 
209 |             encId = iv + cipher.encrypt(IDs)
210 | 
211 |         # Else, encrypt single document (likely meaning it's the first time
212 |         # a particular word has been found in the mail, so the first ID to
213 |         # get added to the index for that word
214 |         else:
215 |             encId = iv + cipher.encrypt(document)
216 | 
217 |         if (DEBUG > 1):
218 |             print("New ID for '%s' = %s" % 
219 |                  (document, (binascii.hexlify(encId))))
220 | 
221 |         return binascii.hexlify(encId)
222 | 
223 | 
224 |     def update(self, infilename, outfilename):
225 | 
226 |         # First update index and send it
227 |         data = self.update_index(infilename)
228 |         message = jmap.pack(UPDATE, data, "1")
229 |         r = self.send(UPDATE, message)
230 |         data = r.json()
231 |         results = data['results']
232 |         print "Results of UPDATE: " + results 
233 |         
234 |         # Then encrypt msg
235 |         infile = open(infilename, "r")     
236 |         outfilename_full = "enc_mail/" + outfilename   
237 |         outfile = open(outfilename_full, "w+")
238 |         self.encryptMail(infile, outfile)
239 |         infile.close()
240 |         
241 |         outfile.seek(0)
242 |         data = binascii.hexlify(outfile.read())
243 |         message = jmap.pack(ADD_MAIL, data, "1", outfilename)
244 | 
245 |         # Then send message
246 |         r = self.send(ADD_MAIL, message, outfilename)        
247 |         data = r.json()
248 |         results = data['results']
249 |         print "Results of UPDATE/ADD FILE: " + results
250 | 
251 |         outfile.close()
252 | 
253 | 
254 |     def update_index(self, document):
255 | 
256 |         # Open file, read it's data, and close it
257 |         infile = open(document, "r")
258 |         msg = email.message_from_file(infile)
259 |         infile.close()
260 | 
261 |         # Parse body of email and return list of words
262 |         word_list = self.parseDocument(msg)
263 | 
264 |         # Parse headers of email, then extend list of words
265 |         (word_list_extended, header_list) = self.parseHeader(msg)
266 |         word_list.extend(word_list_extended)
267 | 
268 | 
269 |         if (DEBUG > 1): print "[Update] Words from doc: " + word_list
270 | 
271 |         # Encrypt the index to send to the server (body terms)
272 |         index = self.encryptIndex(document.split("/")[1], word_list,
273 |                                   ENC_BODY)
274 | 
275 |         # Encrypt index for each header found in message
276 |         for i in header_list:
277 |             index.extend(self.encryptIndex(document.split("/")[1], i,
278 |                                            ENC_HEADERS))
279 | 
280 |         if (DEBUG > 1):
281 |             print "\n[Client] Printing list elements to add to index"
282 |             for x in index:
283 |                 print "%s\n%s\n\n" % (x[0], x[1])
284 | 
285 |         return index
286 | 
287 | 
288 |     def parseDocument(self, infile):
289 | 
290 |         # Iterate through email's body, line-by-line, word-by-word,
291 |         # strip unwanted characters, skip duplicates, and add to list
292 |         word_list = None
293 |         for line in email.Iterators.body_line_iterator(infile):
294 |             for word in line.split():
295 |                 try:
296 |                     if any(s in EXCLUDE for s in word):
297 |                         word = self.removePunctuation(word)
298 |                         word = self.stemmer.stem(word)
299 |                     word = word.lower()
300 |                     word = word.encode('ascii', 'ignore')
301 |                     if  word not in word_list and '\x08' not in word:
302 |                         word_list.append(word)
303 |                 # except catches case of first word in doc, and an
304 |                 # empty list cannot be iterated over
305 |                 except:
306 |                     if any(s in EXCLUDE for s in word):
307 |                         word = self.removePunctuation(word)
308 |                     word = self.stemmer.stem(word)
309 |                     word = word.lower()
310 |                     word = word.encode('ascii', 'ignore')
311 |                     word_list = [word]
312 | 
313 |         return word_list
314 | 
315 | 
316 |     def parseHeader(self, msg):
317 | 
318 |         # TODO: need to strip some punctuation, but not all. Email addrs
319 |         # may want to be split on '.' and dates may need some stripping.
320 |         # Currently just adds entirety of header to index
321 |         header_list_standard = []
322 | 
323 |         # Set up list for each field to be searched. Sloppy, but will get
324 |         # the job done
325 |         header_list = []
326 |         header_list_from = ["from"]
327 |         header_list_sent = ["sent"]
328 |         header_list_date = ["date"]
329 |         header_list_to = ["to"]
330 |         header_list_subject = ["subject"]
331 |         header_list_cc = ["cc"]
332 | 
333 |         regex = r"[\w']+"
334 | 
335 |         # Go through each of the specified headers and add the words to the
336 |         # corresponding list, then append each of those lists to the main
337 |         # header list that gets returned (a list of lists, each sub-list 
338 |         # containing data for a particular header, the first term of those
339 |         # lists being the header name).
340 | 
341 |         for i in ["from", "sent", "date", "to", "subject", "cc"]:
342 |             if msg[i] != None:
343 |                 header_list_standard.append(msg[i])
344 | 
345 |                 if i == "from":
346 |                     header_list_from.append(msg[i])
347 |                     tmp = re.findall(regex, msg[i])
348 |                     for j in tmp:
349 |                         if j != "com":
350 |                             header_list_from.append(j)
351 |                             header_list_standard.append(j)
352 |                     header_list.append(header_list_from)
353 | 
354 |                 elif i == "sent":
355 |                     header_list_sent.append(msg[i])
356 |                     tmp = re.findall(regex, msg[i])
357 |                     for j in tmp:
358 |                         header_list_sent.append(j)
359 |                         header_list_standard.append(j)
360 |                     header_list.append(header_list_sent)
361 | 
362 |                 elif i == "date":
363 |                     header_list_date.append(msg[i])
364 |                     tmp = re.findall(regex, msg[i])
365 |                     for j in tmp:
366 |                         header_list_date.append(j)
367 |                         header_list_standard.append(j)
368 |                     header_list.append(header_list_date)
369 | 
370 |                 elif i == "to":
371 |                     header_list_to.append(msg[i])
372 |                     tmp = re.findall(regex, msg[i])
373 |                     for j in tmp:
374 |                         if j != "com":
375 |                             header_list_to.append(j)
376 |                             header_list_standard.append(j)
377 |                     header_list.append(header_list_to)
378 | 
379 |                 elif i == "subject":
380 |                     header_list_subject.append(msg[i])
381 |                     tmp = re.findall(regex, msg[i])
382 |                     for j in tmp:
383 |                         header_list_subject.append(j)
384 |                         header_list_standard.append(j)
385 |                     header_list.append(header_list_subject)
386 | 
387 |                 elif i == "cc":
388 |                     header_list_cc.append(msg[i])
389 |                     tmp = re.findall(regex, msg[i])
390 |                     for j in tmp:
391 |                         if j != "com": 
392 |                             header_list_cc.append(j)
393 |                             header_list_standard.append(j)
394 |                     header_list.append(header_list_cc)
395 | 
396 |         if (DEBUG > 1):
397 |             print header_list_standard
398 |             print header_list
399 |         return (header_list_standard, header_list)
400 | 
401 | 
402 |     def removePunctuation(self, string):
403 |         return ''.join(ch for ch in string if ch not in EXCLUDE)
404 | 
405 | 
406 |     def encryptIndex(self, document, word_list, TYPE):
407 | 
408 |         # This is where the meat of the SSE update routine is implemented
409 | 
410 |         if (DEBUG > 1): 
411 |             print "Encrypting index of words in '%s'" % document
412 | 
413 |         L = []
414 |         index = anydbm.open("index", "c")
415 |         index_IDs = anydbm.open("index_IDs", "c")
416 |        
417 |         # For each word, look through local index to see if it's there. If
418 |         # not, set c = 0, and apply the PRF. Otherwise c == number of 
419 |         # occurences of that word/term/number 
420 | 
421 |         for w in word_list:
422 | 
423 |             # Initialize K1 and K2
424 |             k1 = self.PRF(self.k, ("1" + w))
425 |             k2 = self.PRF(self.k, ("2" + w))
426 | 
427 |             # If TYPE == ENC_HEADERS, we're parsing a header list. [0] 
428 |             # will be type of header, which we'll use to disguise the 
429 |             # words of the header (instead of c).  Also, pass over if on 
430 |             # first iteration of list, because that will just be the type
431 |             # of header (see parseHeader()).
432 |             if TYPE == ENC_HEADERS:
433 |                 header = word_list[0]
434 |                 if w == header:
435 |                     continue
436 | 
437 |             if (DEBUG > 1): print("k1 = %s\nk2 = %s\n" % (k1, k2))
438 | 
439 |             # counter "c" (set as 0 if not in index), otherwise set
440 |             # as number found in index (refers to how many documents
441 |             # that word appears in
442 |             c = 0
443 |             found = 0
444 |             try:
445 |                 c = int(index[w])
446 |                 found = 1
447 |                 if (DEBUG > 1): 
448 |                     print("Found '%s' in db. C = %d" % (w, c))
449 |             except:
450 |                 c = 0
451 |  
452 |             # Set l as the PRF of k1 (1 || w) and c (num of occur) if 
453 |             # parsing the body (ENC_BODY).
454 |             # If parsing header list, then PRF k1 and header term.
455 |             if TYPE == ENC_BODY:
456 |                 l = self.PRF(k1, str(c))
457 |                 lprime = self.PRF(k1, str(c-1))
458 |             else:
459 |                 l = self.PRF(k1, header)
460 |                 lprime = None
461 | 
462 |             # Update encryptMailID() opens index_IDs and appends
463 |             # new document to list with DELIMETER and encrypts all.
464 |             # Set d as encrypted mail id [list]
465 |             if not found:
466 |                 d = self.encryptMailID(k2, document)
467 |             else:
468 |                 d = self.encryptMailID(k2, document, w, index_IDs)
469 | 
470 |             if (DEBUG > 1):
471 |                 print "w = " + w + "\tc = " + str(c)
472 |                 print("l = %s\nd = %s\n" % (l, d))
473 | 
474 |             # Increment c (1 indexed, not 0), then add unecrypted
475 |             # values to local index, and append encrypted/hashed
476 |             # values to L, the list that will extend the remote index
477 |             c = c + 1
478 |             if TYPE == ENC_BODY:
479 |                 index[w] = str(c)
480 |                 if found:
481 |                     IDs = index_IDs[w]
482 |                     if document not in IDs.split(DELIMETER):
483 |                         index_IDs[w] = IDs + DELIMETER + document
484 |                 else:
485 |                     index_IDs[w] = document
486 |             else:
487 |                 index[w] = str(c) + ":" + header
488 | 
489 |             L.append((l, d, lprime))
490 | 
491 |         index.close()
492 |         index_IDs.close()
493 | 
494 |         return L
495 | 
496 | 
497 |     def search(self, query, header=None, TYPE=SRCH_BODY):
498 | 
499 |         index = anydbm.open("index", "r")
500 |         query = query.split()
501 | 
502 |         # Generate list of querys (may be just 1)
503 |         L = []
504 |         for i in query:
505 |             if (DEBUG > 1): print repr(i)
506 |             i = i.lower()
507 | 
508 |             # For each term of query, first try to see if it's already in
509 |             # index. If it is, send c along with k1 and k2. This will 
510 |             # massively speed up search on server (1.5 minutes to < 1 sec)
511 |             try:
512 |                 c = index[i]
513 |             except:
514 |                 c = None
515 | 
516 |             # Use k, term ('i') and '1' or '2' as inputs to a pseudo-random
517 |             # function to generate k1 and k2. K1 will be used to find the 
518 |             # correct encrypted entry for the term on the server, and k2
519 |             # will be used to decrypt the mail ID(s)
520 |             k1 = self.PRF(self.k, ("1" + i))
521 |             k2 = self.PRF(self.k, ("2" + i))
522 | 
523 |             # If no 'c' (term not in local index so likely not on server),
524 |             # just send k1 and k2. Will take a long time to return false
525 |             # TODO, should the client just kill any search for a term not
526 |             # in local index?  Can we rely on the local index always being
527 |             # up to date?
528 |             if not c:
529 |                 L.append((k1, k2))
530 |             # Otherwise send along 'c'-1. 
531 |             else:
532 |                 c = str(int(c)-1)
533 |                 L.append((k1, k2, c))
534 | 
535 |             if (DEBUG > 1): 
536 |                 print "k1 = " + k1
537 |                 print "k2 = " + k2
538 | 
539 |         # FIXME: Don't like the additions made for searching headers
540 |         if TYPE == SRCH_HEADERS:
541 |             Lprime = [header]
542 |             Lprime.extend(L)
543 |             message = jmap.pack(SEARCH, Lprime, "1") 
544 | 
545 |         else:
546 |             message = jmap.pack(SEARCH, L, "1")
547 | 
548 |         # Send data and unpack results.
549 |         r = self.send(SEARCH, message) 
550 |         ret_data = r.json()
551 |         results = ret_data['results']
552 |         print "Results of SEARCH:"
553 | 
554 |         if results == NO_RESULTS:
555 |             print results
556 |             return -1
557 | 
558 |         # Print out messages that are returned from server
559 |         # TODO: Should recieve and print out mail, or just recieve mailIDs
560 |         # and resend requests for those messages?
561 | 
562 |         # FIXME: hack to decide if server is returning encrypted msgs (1)
563 |         # or just the decrypted IDs (0)
564 |         FILES = 1
565 |         for i in results:
566 |             if (FILES):
567 |                 self.decryptMail(binascii.unhexlify(i), )
568 |             else: 
569 |                 print i
570 | 
571 | 
572 |     def PRF(self, k, data):
573 |         hmac = HMAC.new(k, data, SHA256)
574 |         return hmac.hexdigest()
575 | 
576 | 
577 |     def send(self, routine, data, filename = None, in_url = DEFAULT_URL):
578 | 
579 |         url = in_url
580 | 
581 |         # Currently, each server url is just <IP>/<ROUTINE>, so just append
582 |         # routine to url, and set up headers with jmap package.
583 | 
584 |         if routine == SEARCH:
585 |             url = url + SEARCH
586 |             headers = jmap.jmap_header()
587 |         elif routine == UPDATE:
588 |             url = url + UPDATE
589 |             headers = jmap.jmap_header()
590 |         elif routine == ADD_MAIL:
591 |             url = url + ADD_MAIL
592 |             # For sending mail, need to do a little extra with the headers
593 |             headers = {'Content-Type': 'application/json',
594 |                        'Content-Disposition': 
595 |                        'attachment;filename=' + filename}
596 |         else:
597 |             print "[Client] Error: bad routine for send()"
598 |             exit(1)
599 | 
600 |         if (DEBUG > 1): 
601 |             print url
602 |             print values
603 | 
604 |         # Send to server using requests's post method, and return results
605 |         # to calling method
606 |         return requests.post(url, data, headers = headers)
607 | 
608 | 
609 |     def testSearch(self, index):
610 |         '''
611 |         Method for testing locally if the encryption in the update
612 |         routine is actually accurate. 
613 |         -create a static search term (ie: "the")
614 |         -generate hashes with self.k (ie generate k1 and k2)
615 |         -implement the backend get() and dec() methods to see if they
616 |          return the correct data
617 |         -try with search query that isn't in index
618 |         '''
619 | 
620 |         # 'Client' activities
621 |         query = "This"
622 |         k1 = self.PRF(self.k, ("1" + query))
623 |         k2 = self.PRF(self.k, ("2" + query))
624 | 
625 |         if (DEBUG > 1): 
626 |             print("[testSearch]\nk1:%s\nk2:%s" % (k1, k2))
627 | 
628 |         # 'Server' activities
629 |         c = 0
630 |         found = 0
631 |         while c < len(index):
632 |             if (DEBUG): print "c = " + str(c)
633 |             result = self.testGet(index, k1, c)
634 |             if result: break
635 |             c = c + 1
636 | 
637 |         if not result:
638 |             print "NOT FOUND in INDEX"
639 | 
640 |         else:
641 |             print "FOUND RESULT"
642 | 
643 | 
644 |     def testGet(self, index, k, c):
645 | 
646 |         cc = 0
647 |         while cc < len(index):
648 |             F = self.PRF(k, str(c))
649 |             if (DEBUG > 1):
650 |                 print "[Get] F: " + F
651 |                 print "[Get] Idx: " + index[cc][0] + "\n"
652 |             if F == index[cc][0]:
653 |                 return F
654 |             cc = cc + 1
655 | 
656 | 
657 | 
658 | def main():
659 | 
660 |     # Set-up a command-line argument parser
661 | 
662 |     # TODO: Fix argument parser. It works for what it is, but I don't 
663 |     # have a good enough grasp of the argparser package to fine tune it 
664 |     # ie: some options shouldnt require an argument but do (ie: '-i' 
665 |     # should be a standalone option, but currently requires a following,
666 |     # unused argument
667 | 
668 |     parser = ArgumentParser()
669 |     parser.add_argument('-s', '--search', metavar='search', dest='search',
670 |                         nargs='*')
671 |     parser.add_argument('-S', '--search-header', metavar='search_header', 
672 |                         dest='search_header', nargs=2)
673 |     parser.add_argument('-u', '--update', metavar='update', dest='update',
674 |                         nargs=1)
675 |     parser.add_argument('-e', '--encrypt', metavar='encrypt_file', 
676 |                         dest='encrypt_file', nargs=2)
677 |     parser.add_argument('-d', '--decrypt', metavar='decrypt_file',
678 |                         dest='decrypt_file', nargs=2)
679 |     parser.add_argument('-i', '--inspect_index', dest='inspect_index')
680 |     parser.add_argument('-t', '--test_http', dest='test_http')
681 |     args = parser.parse_args()
682 |  
683 |     sse = SSE_Client()
684 | 
685 |     if args.encrypt_file:
686 |         if (DEBUG): 
687 |             print("Encrypting %s\nOutput %s\n"
688 |             % (args.encrypt_file[0], args.encrypt_file[1]))
689 | 
690 |         infile = open(args.encrypt_file[0], "r")        
691 |         outfile = open(args.encrypt_file[1], "w+")
692 | 
693 |         sse.encryptMail(infile, outfile)
694 | 
695 |         infile.close()
696 |         outfile.close()
697 | 
698 |     elif args.decrypt_file:
699 |         if (DEBUG): 
700 |             print("Decrypting %s\nOutput %s" 
701 |             % (args.decrypt_file[0], args.decrypt_file[1]))
702 | 
703 |         infile = open(args.decrypt_file[0], "r")
704 |         buf = infile.read()
705 |         outfile = open(args.decrypt_file[1], "w+")
706 | 
707 |         sse.decryptMail(buf, outfile)
708 | 
709 |         infile.close()
710 |         outfile.close()
711 | 
712 |     elif args.update:
713 |         if (DEBUG):
714 |             print("Updating index with document %s" % args.update[0])
715 | 
716 |         infilename = args.update[0]
717 |         outfilename = args.update[0].split("/")[1]
718 |         sse.update(infilename, outfilename)
719 | 
720 |     elif args.search:
721 |         if (DEBUG):
722 |            print("Searching remote index for word(s): '%s'" 
723 |                   % args.search[0])
724 | 
725 |         sse.search(args.search[0])
726 | 
727 |     elif args.search_header:
728 |         if (DEBUG):
729 |            print("Searching remote index for word(s):'%s' in header: '%s'" 
730 |                   % (args.search_header[1], args.search_header[0]))
731 | 
732 |         query = args.search_header[1]
733 |         header = args.search_header[0].lower()
734 |         sse.search(query, header, SRCH_HEADERS)
735 | 
736 |     elif args.inspect_index:
737 |         if (DEBUG): print("Inspecting the index")
738 |         index = anydbm.open("index", "r")
739 |         for k, v in index.iteritems():
740 |             print "k:%s\tv:%s" % (k, v)
741 | 
742 |         index.close()
743 | 
744 |         index = anydbm.open("index_IDs", "r")
745 |         for k, v in index.iteritems():
746 |             print "k:%s\tv:%s" % (k, v)
747 | 
748 |         index.close()
749 | 
750 |     elif args.test_http:
751 |         url = "http://localhost:5000/search"
752 |         k1 = "c18d3a0d0a6278ee206447b13cbb46f182c7bb5d038398887a9506e673a1c016"
753 |         k2 = "ccb215ad2018660ad49668bca3c7f4222dc737f2346bf9853d06917d77771655"
754 |         k = []
755 |         k.append(k1)
756 |         k.append(k2)
757 |         #values = { 'k1' : k1, 'k2' : k2 }
758 |         values = { 'query' : k }
759 |         data = urllib.urlencode(values)
760 |         req = urllib2.Request(url, data)  
761 |         response = urllib2.urlopen(req)
762 |         data = response.read()
763 |         print data
764 | 
765 |     else:
766 |         print "Must specify a legitimate option"
767 |         exit(1)
768 | 
769 | 
770 | if __name__ == "__main__":
771 |     main()
772 | 


--------------------------------------------------------------------------------
/client/time.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | UPDATE='python sse_client.py -u'
 4 | SEARCH='python sse_client.py -s'
 5 | 
 6 | # 22 files to UPDATE
 7 | FILES=('1.' '12.' '15.' '18.' '20.' '23.' '26.' '29.' '31.' '34.' '37.' '4.' '42.' '45.' '62.' '65.' '68.' '70.' '73.' '78.' '83.' '86.')
 8 | 
 9 | # 10 'known' terms
10 | TERMS=( 'tuesday' 'such' 'pm' 'message' 'if' 'but' 'very' 'problem' 'see' 'the')
11 | 
12 | # 5 'unknown' terms
13 | UNKNOWN=('beluga' 'whippersnapper' 'cufflink' 'grigsby' 'described')
14 | 
15 | if [ "$1" == "" ]; then
16 |     echo -e "Must supply a command-line option!\n"
17 |     echo -e "\t'update' (clears and updates indices with new messages)"
18 |     echo -e "\t'search' (searches documents for a number of terms)"
19 |     echo -e "\t'searchfull' (includes terms that will not be found)"
20 |     echo -e "\t'all' (update and search)"
21 |     exit 1
22 | fi
23 | 
24 | if [ "$1" == "update" ] || [ "$1" == "all" ];then
25 |     rm index*
26 |     rm ../server/index
27 |     for i in "${FILES[@]}"; do
28 |         echo "$UPDATE inbox/$i"
29 |         $UPDATE inbox/$i
30 |     done
31 | fi
32 | if [ "$1" == "search" ] || [ "$1" == "all" ]; then
33 |     #$SEARCH 'tuesday such pm message if but very problem see the'
34 |     for i in "${TERMS[@]}"; do
35 |         echo "$SEARCH $i"
36 |         $SEARCH $i
37 |     done
38 | fi
39 | if [ "$1" == "searchfull" ]; then
40 |     for i in "${UNKNOWN[@]}"; do
41 |         echo "$SEARCH $i";
42 |         $SEARCH $i
43 |     done
44 | fi
45 | 


--------------------------------------------------------------------------------
/jmap/jmap.py:
--------------------------------------------------------------------------------
  1 | '''
  2 |  The MIT License (MIT)
  3 | 
  4 |  Copyright (c) 2016 Ian Van Houdt
  5 | 
  6 |  Permission is hereby granted, free of charge, to any person obtaining a copy
  7 |  of this software and associated documentation files (the "Software"), to deal
  8 |  in the Software without restriction, including without limitation the rights
  9 |  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 10 |  copies of the Software, and to permit persons to whom the Software is
 11 |  furnished to do so, subject to the following conditions:
 12 | 
 13 |  The above copyright notice and this permission notice shall be included in all
 14 |  copies or substantial portions of the Software.
 15 | 
 16 |  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 17 |  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 18 |  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 19 |  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 20 |  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 21 |  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 22 |  SOFTWARE.
 23 | '''
 24 | 
 25 | ############
 26 | #
 27 | #  jmap.py
 28 | #
 29 | #  Serves as the jmap library for the sse client/server.
 30 | #  Both client and server will use this lib to create 
 31 | #  and interpret JMAP messages
 32 | #
 33 | ############
 34 | 
 35 | import json
 36 | 
 37 | FILE = "[JMAP] "
 38 | SEARCH = "search"
 39 | UPDATE = "update"
 40 | ADD_FILE = "addmail"
 41 | SEARCH_METHOD = "getEncryptedMessages"
 42 | UPDATE_METHOD = "updateEncryptedIndex"
 43 | ADD_FILE_METHOD = "putEncryptedMessage"
 44 | 
 45 | JMAP_HEADER = {'Content-Type': 'application/json'} 
 46 | 
 47 | # Notes on JMAP spec from jmap.io/spec
 48 | '''
 49 | BASIC QUERY STRUCTURE:
 50 | [
 51 |   ["method1", {"arg1": "arg1data", "arg2": "arg2data"}, "#1"],
 52 |   ["method2", {"arg1": "arg1data"}, "#2"],
 53 |   ["method3", {}, "#3"]
 54 | ]
 55 | 
 56 | BASIC RESPONSE STRUCTURE:
 57 | [
 58 |   ["responseFromMethod1", {"arg1": 3, "arg2": "foo"}, "#1"],
 59 |   ["responseFromMethod2", {"isBlah": true}, "#2"],
 60 |   ["anotherResponseFromMethod2", {"data": 10, "yetmoredata": "Hello"}, "#2"],
 61 |   ["aResponseFromMethod3", {}, "#3"]
 62 | ] 
 63 | 
 64 | EXAMPLE REQ:
 65 | ["getMessages", {
 66 |   "ids": [ "f123u456", "f123u457" ],
 67 |   "properties": [ "threadId", "mailboxIds", "from", "subject", "date" ]
 68 | }, "#1"]
 69 | 
 70 | EXAMPLE RESP:
 71 | ["messages", {
 72 |   "state": "41234123231",
 73 |   "list": [
 74 |     {
 75 |       messageId: "f123u457",
 76 |       threadId: "ef1314a",
 77 |       mailboxIds: [ "f123" ],
 78 |       from: [{name: "Joe Bloggs", email: "joe@bloggs.com"}],
 79 |       subject: "Dinner on Thursday?",
 80 |       date: "2013-10-13T14:12:00Z"
 81 |     }
 82 |   ],
 83 |   notFound: [ "f123u456" ]
 84 | }, "#1"]
 85 | 
 86 | ========
 87 | 
 88 | JMAP CALLS FOR SSE:
 89 | 
 90 | ["method", {args}, "id"]
 91 | 
 92 | ["getEncryptedMessages", 
 93 |   {
 94 |   "query": [ "(k1 n, k2 n)", "(k1 n+1, k2 n+1)", ... ]
 95 |   },
 96 |   "#1" ]
 97 | 
 98 | ["encryptedMessages",
 99 |   {
100 |   "results": [ "data for msg n", "data for msg n+1", ... ]
101 |   }
102 |   "#1" ]
103 | 
104 |   -NOTE: for returning enc messages, possbility exists for returning each
105 |          message's data in a separate message. Just reuse id num
106 | 
107 | '''
108 | 
109 | def jmap_header():
110 |     return JMAP_HEADER
111 | 
112 | def pack_search(data, id_num):
113 |     return json.dumps([SEARCH_METHOD, {"query": data}, id_num])
114 | 
115 | def pack_update(data, id_num):
116 |     return json.dumps([UPDATE_METHOD, {"index": data}, id_num])
117 | 
118 | def pack_add_file(data, id_num, filename):
119 |     return json.dumps([ADD_FILE_METHOD, {"file": data, "filename": filename}, id_num])
120 | 
121 | def pack(METHOD, data, id_num, filename=None):
122 |     FUNC = "jmap.pack"
123 |     message = None
124 | 
125 |     if not METHOD:
126 |         print FILE + "Must provide a method to " + FUNC
127 |         return -1
128 | 
129 |     if METHOD == SEARCH:
130 |         message = pack_search(data, id_num)
131 | 
132 |     elif METHOD == UPDATE:
133 |         message = pack_update(data, id_num)
134 | 
135 |     elif METHOD == ADD_FILE:
136 |         message = pack_add_file(data, id_num, filename)
137 | 
138 |     else:
139 |         print FILE + "Unknown METHOD in " + FUNC
140 |         return -1
141 | 
142 |     return message
143 | 
144 | def unpack_search(data):
145 | 
146 |     if data[0] != SEARCH_METHOD:
147 |         return -1
148 | 
149 |     method = data[0]
150 |     id_num = data[2]
151 | 
152 |     # Limit scope to args (data[1])
153 |     data = data[1]
154 |     query = data['query']
155 | 
156 |     return (method, query, id_num)
157 | 
158 | 
159 | def unpack_update(data):
160 |     
161 |     if data[0] != UPDATE_METHOD:
162 |         return -1
163 | 
164 |     method = data[0]
165 |     id_num = data[2]
166 | 
167 |     # Limit scope to args (data[1])
168 |     data = data[1]
169 |     new_index = data['index']
170 | 
171 |     return (method, new_index, id_num)
172 | 
173 | def unpack_add_file(data):
174 | 
175 |     if data[0] != ADD_FILE_METHOD:
176 |         return -1
177 | 
178 |     method = data[0]
179 |     id_num = data[2]
180 | 
181 |     # Limit scope to args (data[1])
182 |     data = data[1]
183 | 
184 |     file = data['file']
185 |     filename = data['filename']
186 | 
187 |     return (method, file, filename, id_num)
188 | 
189 | def unpack(METHOD, data):
190 |     FUNC = "jmap.unpack"
191 | 
192 |     if not METHOD:
193 |         print FILE + "Must provide a method to " + FUNC
194 |         return -1
195 |   
196 |     if METHOD == SEARCH:
197 |         return unpack_search(data)
198 |     elif METHOD == UPDATE:
199 |         return unpack_update(data)
200 |     elif METHOD == ADD_FILE: 
201 |         return unpack_add_file(data)
202 |     else:
203 |         print FILE + "Unknown METHOD in " + FUNC
204 |         return -1
205 | 
206 | 
207 | 


--------------------------------------------------------------------------------
/server/sse_server.py:
--------------------------------------------------------------------------------
  1 | '''
  2 |  The MIT License (MIT)
  3 | 
  4 |  Copyright (c) 2016 Ian Van Houdt
  5 | 
  6 |  Permission is hereby granted, free of charge, to any person obtaining a copy
  7 |  of this software and associated documentation files (the "Software"), to deal
  8 |  in the Software without restriction, including without limitation the rights
  9 |  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 10 |  copies of the Software, and to permit persons to whom the Software is
 11 |  furnished to do so, subject to the following conditions:
 12 | 
 13 |  The above copyright notice and this permission notice shall be included in all
 14 |  copies or substantial portions of the Software.
 15 | 
 16 |  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 17 |  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 18 |  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 19 |  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 20 |  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 21 |  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 22 |  SOFTWARE.
 23 | '''
 24 | 
 25 | ############
 26 | #
 27 | #  sse_server.py
 28 | #
 29 | #  Serves as SSE implementation for mail server. The routines 
 30 | #  for SSE are invoked by the server module via the API.
 31 | #
 32 | ############
 33 | 
 34 | import socket
 35 | import os
 36 | import sys
 37 | sys.path.append(os.path.realpath('../jmap'))
 38 | from Crypto.Hash import HMAC
 39 | from Crypto.Hash import SHA256
 40 | from Crypto.Cipher import AES
 41 | import unicodedata
 42 | import binascii
 43 | import anydbm
 44 | import string
 45 | import json
 46 | from flask import Flask
 47 | from flask import request
 48 | from flask import render_template
 49 | from flask import jsonify
 50 | from werkzeug import secure_filename
 51 | import jmap
 52 | 
 53 | app = Flask(__name__)
 54 | app.config['UPLOAD_FOLDER'] = 'mail'
 55 | app.config['ALLOWED_EXTENSIONS'] = set(['txt', 'pdf', 'png', 'jpg', 
 56 |                                         'jpeg', 'gif'])
 57 | DEBUG = 1
 58 | 
 59 | # CMD list
 60 | UPDATE = "update"
 61 | SEARCH = "search"
 62 | ADD_FILE = "addmail"
 63 | SEARCH_METHOD = "getEncryptedMessages"
 64 | UPDATE_METHOD = "updateEncryptedIndex"
 65 | ADD_FILE_METHOD = "putEncryptedMessage"
 66 | 
 67 | SRCH_BODY = 0
 68 | SRCH_HEADERS = 1
 69 | 
 70 | HEADERS = ["from", "sent", "date", "to", "subject", "cc"]
 71 | 
 72 | DELIMETER = "++?"
 73 | 
 74 | ########
 75 | #
 76 | # SSE_Server
 77 | #
 78 | ########
 79 | 
 80 | @app.route('/addmail', methods=['POST'])
 81 | def add_mail():
 82 | 
 83 |     # Return error if request is not properly formatted
 84 |     if not request.json:
 85 |         return jsonify(results='Error: not json')
 86 | 
 87 |     # Unpack 'arguments'
 88 |     (method, file, filename, id_num) = jmap.unpack(ADD_FILE, request.get_json())
 89 | 
 90 |     if method != ADD_FILE_METHOD:
 91 |         return jsonify(results='Error: Wrong Method for url')
 92 | 
 93 |     # return file to binary
 94 |     file = binascii.unhexlify(file)
 95 | 
 96 |     # open file and write to it locally
 97 |     path = os.path.join(app.config['UPLOAD_FOLDER'], filename) 
 98 |     f = open(path, "w+")
 99 |     f.write(file)
100 |     f.close()
101 | 
102 |     return jsonify(results="GOOD ADD FILE")
103 | 
104 | 
105 | # TODO: Use this to request mail? Currently just sending them back after
106 | # SEARCH routine
107 | @app.route('/getmail', methods=['GET'])
108 | def get_mail():
109 |     pass
110 | 
111 | 
112 | @app.route('/update', methods=['POST'])
113 | def update():
114 | 
115 |     # Return error if request is not properly formatted
116 |     if not request.json:
117 |         return jsonify(results='Error: not json')
118 | 
119 |     # Unpack 'arguments'
120 |     (method, new_index, id_num) = jmap.unpack(UPDATE, request.get_json())
121 | 
122 |     if method != UPDATE_METHOD:
123 |         return jsonify(results='Error: Wrong Method for url')
124 | 
125 |     # Open local ecypted index and get length
126 |     index = anydbm.open("index", "c")
127 |     index_len = get_index_len(index)
128 | 
129 |     # Iterate through update list, replacing existing entries in local
130 |     # index if collisions
131 |     for i in new_index:
132 |         # i0 is the key (ie the hashed term), 
133 |         # i1 is the value (encrypted list of mailIDs where that word is
134 |         # present.
135 |         i0 = i[0].encode('ascii', 'ignore')
136 |         i1 = i[1].encode('ascii', 'ignore')
137 |         match = i0
138 | 
139 |         # if i2 exists, use that to match (hash of term with c - 1).
140 |         # Otherwise match with i0, representing word hahsed with header
141 |         try:
142 |             if i[2]:
143 |                 i2 = i[2].encode('ascii', 'ignore')
144 |                 match = i2
145 |         except:
146 |             pass
147 |         # Go through local index and compare, if match, then delete that 
148 |         # entry and add new one.
149 |         exists = 0
150 |         for k, v in index.iteritems():
151 |             if match == k: # and i1 == v:
152 |                 exists = 1
153 |                 del index[k]
154 |                 break
155 | 
156 |         index[i0] = i1
157 | 
158 |     if (DEBUG > 1): 
159 |         print "\nUpdate Complete! Index contents:" 
160 |         for k, v in index.iteritems():
161 |             print "k:%s\nv:%s\n\n" % (k, v)
162 | 
163 |     index.close()
164 |     return jsonify(results="GOOD UPDATE")
165 | 
166 | 
167 | @app.route('/search', methods=['POST'])
168 | def search():
169 | 
170 |     TYPE = SRCH_BODY
171 | 
172 |     if not request.json:
173 |         return jsonify(results='Error: not json')
174 | 
175 |     (method, query, id_num) = jmap.unpack(SEARCH, request.get_json())
176 | 
177 |     if method != SEARCH_METHOD:
178 |         return jsonify(results='Error: Wrong Method for url')
179 | 
180 |     # FIXME: This is a crap way to check for headers, and subsequently
181 |     # searching for them. Need to fix this, as well as following FIXME's
182 |     # in this method.
183 |     if query[0] in HEADERS:
184 |         header = query[0]
185 |         TYPE = SRCH_HEADERS
186 | 
187 |     index = anydbm.open("index", "r")
188 |     count = get_index_len(index)
189 | 
190 |     # query is a list of search terms, so each 'i' is a word/query
191 |     # each word/query is a tuple containing k1, a hash of the search term,
192 |     # and k2 for decrypting the document name(s).  Use k1 to match the key 
193 |     # and use k2 to decrypt each value (mail ID or name) that is associated
194 |     # with that key.
195 |     M = []
196 |     for i in query:
197 |         # Drop unicode
198 |         k1 = i[0].encode('ascii', 'ignore')
199 |         k2 = i[1].encode('ascii', 'ignore')
200 |         c = 0
201 | 
202 |         # If i2, then we have already recieved the correct 'c' with which
203 |         # to find 'key' term.
204 |         try:
205 |             if i[2]:
206 |                 c = i[2].encode('ascii', 'ignore')
207 |         except:
208 |             pass
209 |         if (DEBUG > 1): print "k1: %s\nk2: %s\n" % (k1, k2)
210 | 
211 |         # D [] is a list of mail IDs found for a term.
212 |         # Its leftover 'legacy' code. Used to be you had to iterate through
213 |         # entire encrypted index for repeated use of a term in different
214 |         # documents (values). Now, 'c' is included, and values are lists
215 |         # of documents, so each word has only one key in the index. 
216 |         # However, not fully tested, so I'm loathe to kick out the original
217 |         # idea of appending d's to a list D[] for now.  
218 |         # Plus, it should eventually change to have a limit of mail IDs, so
219 |         # a single term will show up multiple times, each key pointing to 
220 |         # some number of document IDs.
221 |         D = []
222 | 
223 |         # Find doc id list at that key in the index
224 |         d = new_get(index, k1, c)
225 | 
226 |         if not d:
227 |             print "get() returned None!"
228 |         else:
229 |             D.append(d)
230 | 
231 |         if not D: continue
232 | 
233 |         # 'd' represents an encrypted id number for a message (in the 
234 |         # simple case, just the message's name).
235 | 
236 |         # Go through list of d's in which the search query was found and
237 |         # dec() each and add to list of id's (M).
238 |         # Send those messages are found to the client
239 | 
240 |         for d in D:
241 |             # Decrypt d, getting list of docs that word is in
242 |             m = dec(k2, d)
243 |             m = filter(lambda x: x in string.printable, m)
244 |             for msg in m.split(DELIMETER):
245 |                 print "-----\t" + msg
246 |                 if msg not in M:
247 |                     M.append(msg) 
248 | 
249 |     if not M:
250 |         buf = "Found no results for query"
251 |         print "[Server] " +  buf
252 |         return jsonify(results=buf)
253 | 
254 |     # FIXME: sort out if we send back just IDs, or IDs and messages
255 |     '''
256 |     # quick hack to return early with just IDs, rather than the
257 |     # msgs themselves
258 |     no_files = 0
259 |     if no_files:
260 |         return jsonify(results=M)
261 |     '''
262 | 
263 |     if (DEBUG > 1): 
264 |         print "[Server] Found %d results for query" % len(M)
265 |         for m in M:
266 |             print "\t - %s" % repr(m)
267 |         print "\n"
268 | 
269 |     # TODO: Separate method for sending back files?  
270 |     # Should it be whole files or just msg ids?
271 |     # Currently sends msgs back in their entirety
272 | 
273 |     # TODO: Need to send back id_num and check at client side
274 | 
275 |     # For each doc in M[], send file back to Client
276 |     # buf is list of msgs so client can receive them all together 
277 |     # and parse
278 |     buf = []
279 |     for m in M:
280 |         path = os.path.join(app.config['UPLOAD_FOLDER'], m)
281 |         fd = open(path, "rb")
282 |         buf.append(binascii.hexlify(fd.read()))
283 |         fd.close()
284 | 
285 |     return jsonify(results=buf)
286 | 
287 | 
288 | # Depricated in favor of new_get(), used with new index structure
289 | def get(index_n, k1, count):
290 | 
291 |     cc = 0
292 |     while cc < count:
293 |         F = PRF(k1, str(cc))
294 |         if (DEBUG > 1): 
295 |             print "index key = " + index_n[0]
296 |             print "PRF of k1 and %d = %s\n" % (cc, F)
297 |         if F == index_n[0]:
298 |             print "C: " + str(cc)
299 |             return index_n[1]
300 |         cc = cc + 1
301 |     return 0
302 | 
303 | # Use k1 (hashed search term) and c (num of files it's in) to get key, and 
304 | # then value of index entry.
305 | def new_get(index, k1, c):
306 |     F = PRF(k1, c)
307 |     d = index[F]
308 |     return d
309 |     try:
310 |         F = PRF(k1, str(c))
311 |         d = index[F]
312 |     except:
313 |         d = None
314 | 
315 |     return d
316 | 
317 | # FIXME: But maybe not. This may be the only somewhat sane addition for the
318 | # header search.  Could be joined with get(), but is distinct enough that
319 | # it may be best to keep them separate.
320 | def get_header(index_n, k1, header):
321 | 
322 |     F = PRF(k1, header)
323 |     if (DEBUG > 1): 
324 |         print "index key = " + index_n[0]
325 |         print "PRF of k1 and %s = %s\n" % (header, F)
326 |     if F == index_n[0]:
327 |         return index_n[1]
328 | 
329 |     return 0
330 | 
331 | # Decrypt doc ID using k2
332 | def dec(k2, d):
333 | 
334 |     d_bin = binascii.unhexlify(d) 
335 |     iv = d_bin[:16]
336 |     cipher = AES.new(k2[:16], AES.MODE_CBC, iv)
337 |     doc = cipher.decrypt(d_bin[16:])
338 | 
339 |     if (DEBUG > 1): print "[Server] Retrieved Doc = %s" % (doc)
340 | 
341 |     return doc
342 | 
343 | 
344 | def PRF(k, data):
345 |     hmac = HMAC.new(k, data, SHA256)
346 |     return hmac.hexdigest()
347 | 
348 | def get_index_len(index):
349 | 
350 |     # TODO: crappy hack for now. Need to get size of index,
351 |     # but I'm not sure what the best method is. So for now, 
352 |     # just iterate through and grab the count.
353 |     count = 0
354 |     for k, v in index.iteritems():
355 |         count = count + 1
356 |         if (DEBUG > 1):
357 |             print "K: " + k
358 |             print "V: " + v
359 |             print "\n"
360 | 
361 |     return count
362 | 
363 | 
364 | if __name__ == '__main__':
365 |     app.run(debug=True)
366 | 
367 | 


--------------------------------------------------------------------------------