├── keyword-setup ├── openssl-library-rc4.txt ├── openssl-library-mdc2.txt ├── openssl-library-sha.txt ├── openssl-library.txt ├── openssl-library-hmac.txt ├── openssl-library-ripemd.txt ├── openssl-library-lhash.txt ├── openssl-library-md5.txt ├── openssl-library-blowfish.txt ├── openssl-library-rand.txt ├── openssl-library-dh.txt ├── openssl-library-threads.txt ├── openssl-library-err.txt ├── openssl-library-rsa.txt ├── openssl-library-dsa.txt ├── openssl-library-des.txt ├── openssl-library-x509.txt ├── openssl-library-pem.txt ├── openssl-library-evp.txt ├── openssl-library-bn.txt ├── openssl-library-bio.txt ├── nss-certificate-library.txt └── openssl-library-ssl.txt ├── gdiff └── __init__.py ├── .gitignore ├── templates ├── jan.png ├── logo.png ├── hacker.png ├── tile-grey.png ├── contributor.png ├── project-contributor.png ├── seals.txt ├── commit.html └── searchresults.html ├── config.sample.py ├── debian-init-script ├── gentoo-init-script ├── browsedb.py ├── regenkeywords.py ├── common.py ├── svnpuller.py ├── cronjob.py ├── gitpuller.py ├── repo.py ├── svncommit.py ├── database.py ├── gitcommit.py ├── README.md ├── webserver.py ├── synonymmapping.py ├── databaseQueries.py ├── commit.py ├── tags.txt ├── keywordsfilter.py └── setup.py /keyword-setup/openssl-library-rc4.txt: -------------------------------------------------------------------------------- 1 | RC4 2 | RC4_set_key 3 | -------------------------------------------------------------------------------- /gdiff/__init__.py: -------------------------------------------------------------------------------- 1 | from .diff_match_patch import diff_match_patch, patch_obj 2 | 3 | -------------------------------------------------------------------------------- /keyword-setup/openssl-library-mdc2.txt: -------------------------------------------------------------------------------- 1 | MDC2 2 | MDC2_Final 3 | MDC2_Init 4 | MDC2_Update 5 | -------------------------------------------------------------------------------- /keyword-setup/openssl-library-sha.txt: -------------------------------------------------------------------------------- 1 | SHA1 2 | SHA1_Final 3 | SHA1_Init 4 | SHA1_Update 5 | -------------------------------------------------------------------------------- /keyword-setup/openssl-library.txt: -------------------------------------------------------------------------------- 1 | OPENSSL_VERSION_NUMBER 2 | SSLeay 3 | SSLeay_version 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | git-repos 2 | *.pyc 3 | config.py 4 | parser.out 5 | parsetab.py 6 | debugging.txt 7 | -------------------------------------------------------------------------------- /templates/jan.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cryptodotis/code-peer-review/HEAD/templates/jan.png -------------------------------------------------------------------------------- /keyword-setup/openssl-library-hmac.txt: -------------------------------------------------------------------------------- 1 | HMAC_Final 2 | HMAC_Init 3 | HMAC_Update 4 | HMAC_cleanup 5 | -------------------------------------------------------------------------------- /templates/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cryptodotis/code-peer-review/HEAD/templates/logo.png -------------------------------------------------------------------------------- /templates/hacker.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cryptodotis/code-peer-review/HEAD/templates/hacker.png -------------------------------------------------------------------------------- /templates/tile-grey.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cryptodotis/code-peer-review/HEAD/templates/tile-grey.png -------------------------------------------------------------------------------- /keyword-setup/openssl-library-ripemd.txt: -------------------------------------------------------------------------------- 1 | RIPEMD160 2 | RIPEMD160_Final 3 | RIPEMD160_Init 4 | RIPEMD160_Update 5 | -------------------------------------------------------------------------------- /templates/contributor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cryptodotis/code-peer-review/HEAD/templates/contributor.png -------------------------------------------------------------------------------- /templates/project-contributor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cryptodotis/code-peer-review/HEAD/templates/project-contributor.png -------------------------------------------------------------------------------- /keyword-setup/openssl-library-lhash.txt: -------------------------------------------------------------------------------- 1 | lh_delete 2 | lh_doall 3 | lh_doall_arg 4 | lh_error 5 | lh_free 6 | lh_insert 7 | lh_new 8 | lh_retrieve 9 | -------------------------------------------------------------------------------- /keyword-setup/openssl-library-md5.txt: -------------------------------------------------------------------------------- 1 | MD2_Final 2 | MD2_Init 3 | MD2_Update 4 | MD4_Final 5 | MD4_Init 6 | MD4_Update 7 | MD5_Final 8 | MD5_Init 9 | MD5_Update 10 | -------------------------------------------------------------------------------- /keyword-setup/openssl-library-blowfish.txt: -------------------------------------------------------------------------------- 1 | BF_cbc_encrypt 2 | BF_cfb64_encrypt 3 | BF_decrypt 4 | BF_ecb_encrypt 5 | BF_encrypt 6 | BF_ofb64_encrypt 7 | BF_options 8 | BF_set_key 9 | -------------------------------------------------------------------------------- /templates/seals.txt: -------------------------------------------------------------------------------- 1 | http://www.says-it.com/seal/circle.php 2 | Contributor: Symbols, Glider Pg 101-125 3 | Hacker: SciFi, Superman Pg 126-150 4 | Project Contributor: Misc , Ninja, Pg 126-150 -------------------------------------------------------------------------------- /keyword-setup/openssl-library-rand.txt: -------------------------------------------------------------------------------- 1 | RAND_METHOD 2 | RAND_SSLeay 3 | RAND_add 4 | RAND_bytes 5 | RAND_cleanup 6 | RAND_egd 7 | RAND_event 8 | RAND_file_name 9 | RAND_get_rand_method 10 | RAND_load_file 11 | RAND_pseudo_bytes 12 | RAND_screen 13 | RAND_seed 14 | RAND_set_rand_engine 15 | RAND_set_rand_method 16 | RAND_status 17 | RAND_write_file 18 | -------------------------------------------------------------------------------- /config.sample.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | class Config: 4 | username="" 5 | password="" 6 | host="localhost" 7 | database="" 8 | 9 | tornadoport=8888# port you want Tornado to listen on 10 | rooturl="http://example.com" #no trailing slash 11 | fsdir="/var/www/htdocs/containingDirOfPyFiles/" #trailing slash 12 | logfile="/var/log/tornado/tornado_log" 13 | -------------------------------------------------------------------------------- /keyword-setup/openssl-library-dh.txt: -------------------------------------------------------------------------------- 1 | DH_METHOD 2 | DH_OpenSSL 3 | DH_check 4 | DH_compute_key 5 | DH_free 6 | DH_generate_key 7 | DH_generate_parameters 8 | DH_get_default_method 9 | DH_get_ex_data 10 | DH_get_ex_new_index 11 | DH_new 12 | DH_new_method 13 | DH_set_default_method 14 | DH_set_ex_data 15 | DH_set_method 16 | DH_size 17 | DHparams_print 18 | DHparams_print_fp 19 | d2i_DHparams 20 | i2d_DHparams 21 | -------------------------------------------------------------------------------- /debian-init-script: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | case "$1" in 4 | start) 5 | echo "Starting tornado" 6 | start-stop-daemon --start --user www-data --group www-data --quiet --background --make-pidfile --pidfile /var/run/cpr.pid --exec /usr/bin/python /var/www/htdocs/cpr/webserver.py 7 | ;; 8 | stop) 9 | echo "Stopping tornado" 10 | start-stop-daemon --stop --pidfile /var/run/cpr.pid 11 | ;; 12 | esac 13 | 14 | -------------------------------------------------------------------------------- /keyword-setup/openssl-library-threads.txt: -------------------------------------------------------------------------------- 1 | CRYPTO_THREADID_cmp 2 | CRYPTO_THREADID_cpy 3 | CRYPTO_THREADID_current 4 | CRYPTO_THREADID_get_callback 5 | CRYPTO_THREADID_hash 6 | CRYPTO_THREADID_set_callback 7 | CRYPTO_destroy_dynlockid 8 | CRYPTO_get_new_dynlockid 9 | CRYPTO_lock 10 | CRYPTO_num_locks 11 | CRYPTO_set_dynlock_create_callback 12 | CRYPTO_set_dynlock_destroy_callback 13 | CRYPTO_set_dynlock_lock_callback 14 | CRYPTO_set_locking_callback 15 | -------------------------------------------------------------------------------- /gentoo-init-script: -------------------------------------------------------------------------------- 1 | #!/sbin/runscript 2 | 3 | depend() { 4 | need net 5 | } 6 | 7 | start() { 8 | ebegin "Starting tornado" 9 | start-stop-daemon --start -i --user apache:apache --quiet --background --make-pidfile - 10 | -pidfile /var/run/tornado.pid --exec /usr/bin/python /web/crypto.is/webserver.py 11 | eend $? 12 | } 13 | 14 | stop() { 15 | ebegin "Stopping tornado" 16 | start-stop-daemon --stop --pidfile /var/run/tornado.pid 17 | eend $? 18 | } 19 | 20 | -------------------------------------------------------------------------------- /keyword-setup/openssl-library-err.txt: -------------------------------------------------------------------------------- 1 | ERR_GET_FUNC 2 | ERR_GET_LIB 3 | ERR_GET_REASON 4 | ERR_PACK 5 | ERR_add_error_data 6 | ERR_clear_error 7 | ERR_error_string 8 | ERR_free_strings 9 | ERR_func_error_string 10 | ERR_get_error 11 | ERR_get_error_line 12 | ERR_get_error_line_data 13 | ERR_get_next_error_library 14 | ERR_lib_error_string 15 | ERR_load_crypto_strings 16 | ERR_load_strings 17 | ERR_peek_error 18 | ERR_peek_error_line 19 | ERR_peek_error_line_data 20 | ERR_print_errors 21 | ERR_print_errors_fp 22 | ERR_put_error 23 | ERR_reason_error_string 24 | ERR_remove_state 25 | -------------------------------------------------------------------------------- /keyword-setup/openssl-library-rsa.txt: -------------------------------------------------------------------------------- 1 | RSARSA_new 2 | RSA_METHOD 3 | RSA_PKCS1_SSLeay 4 | RSA_blinding_off 5 | RSA_blinding_on 6 | RSA_check_key 7 | RSA_flags 8 | RSA_free 9 | RSA_generate_key 10 | RSA_get_default_method 11 | RSA_get_ex_data 12 | RSA_get_ex_new_index 13 | RSA_get_method 14 | RSA_new_method 15 | RSA_null_method 16 | RSA_print 17 | RSA_print_fp 18 | RSA_private_decrypt 19 | RSA_private_encrypt 20 | RSA_public_decrypt 21 | RSA_public_encrypt 22 | RSA_set_default_method 23 | RSA_set_ex_data 24 | RSA_set_method 25 | RSA_sign 26 | RSA_sign_ASN1_OCTET_STRING 27 | RSA_size 28 | RSA_verify 29 | RSA_verify_ASN1_OCTET_STRING 30 | -------------------------------------------------------------------------------- /browsedb.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import MySQLdb, argparse, datetime, unicodedata 4 | 5 | from common import * 6 | from config import Config 7 | from database import DB 8 | from databaseQueries import DBQ 9 | from repo import Repo 10 | from commit import Commit 11 | from keywordsfilter import * 12 | 13 | def browse(keywords): 14 | commits = DBQ.findByKeywordsAndFulltext(keywords) 15 | 16 | for c in commits: 17 | print "=========================================" 18 | c.pprint() 19 | 20 | if __name__ == "__main__": 21 | if sys.argv: 22 | browse(" ".join(sys.argv[1:])) 23 | else: 24 | browse('') 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /keyword-setup/openssl-library-dsa.txt: -------------------------------------------------------------------------------- 1 | DSA_METHOD 2 | DSA_OpenSSL 3 | DSA_SIG 4 | DSA_SIG DSA_SIG_new 5 | DSA_SIG DSA_do_sign 6 | DSA_SIG d2i_DSA_SIG 7 | DSA_SIG_free 8 | DSA_do_verify 9 | DSA_dup_DH 10 | DSA_free 11 | DSA_generate_key 12 | DSA_generate_parameters 13 | DSA_get_default_method 14 | DSA_get_ex_data 15 | DSA_get_ex_new_index 16 | DSA_new 17 | DSA_new_method 18 | DSA_print 19 | DSA_print_fp 20 | DSA_set_default_method 21 | DSA_set_ex_data 22 | DSA_set_method 23 | DSA_sign 24 | DSA_sign_setup 25 | DSA_size 26 | DSA_verify 27 | DSAparams_print 28 | DSAparams_print_fp 29 | d2i_DSAPrivateKey 30 | d2i_DSAPublicKey 31 | d2i_DSAparams 32 | i2d_DSAPrivateKey 33 | i2d_DSAPublicKey 34 | i2d_DSA_SIG 35 | i2d_DSAparams 36 | -------------------------------------------------------------------------------- /keyword-setup/openssl-library-des.txt: -------------------------------------------------------------------------------- 1 | DES_cbc_cksum 2 | DES_cfb64_encrypt 3 | DES_cfb_encrypt 4 | DES_crypt 5 | DES_ecb2_encrypt 6 | DES_ecb3_encrypt 7 | DES_ecb_encrypt 8 | DES_ede2_cbc_encrypt 9 | DES_ede2_cfb64_encrypt 10 | DES_ede2_ofb64_encrypt 11 | DES_ede3_cbc_encrypt 12 | DES_ede3_cbcm_encrypt 13 | DES_ede3_cfb64_encrypt 14 | DES_ede3_ofb64_encrypt 15 | DES_enc_read 16 | DES_enc_write 17 | DES_fcrypt 18 | DES_is_weak_key 19 | DES_key_sched 20 | DES_ncbc_encrypt 21 | DES_ofb64_encrypt 22 | DES_ofb_encrypt 23 | DES_pcbc_encrypt 24 | DES_quad_cksum 25 | DES_random_key 26 | DES_set_key 27 | DES_set_key_checked 28 | DES_set_key_unchecked 29 | DES_set_odd_parity 30 | DES_string_to_2keys 31 | DES_string_to_key 32 | DES_xcbc_encrypt 33 | -------------------------------------------------------------------------------- /keyword-setup/openssl-library-x509.txt: -------------------------------------------------------------------------------- 1 | X509_NAME_ENTRY_create_by_NID 2 | X509_NAME_ENTRY_create_by_OBJ 3 | X509_NAME_ENTRY_create_by_txt 4 | X509_NAME_ENTRY_get_data 5 | X509_NAME_ENTRY_get_object 6 | X509_NAME_ENTRY_set_data 7 | X509_NAME_ENTRY_set_object 8 | X509_NAME_add_entry 9 | X509_NAME_add_entry_by_NID 10 | X509_NAME_add_entry_by_OBJ 11 | X509_NAME_add_entry_by_txt 12 | X509_NAME_delete_entry 13 | X509_NAME_oneline 14 | X509_NAME_print 15 | X509_NAME_print_ex 16 | X509_NAME_print_ex_fp 17 | d2i_509_CRL_fp 18 | d2i_X509 19 | d2i_X509_ALGOR 20 | d2i_X509_CRL 21 | d2i_X509_CRL_bio 22 | d2i_X509_NAME 23 | d2i_X509_REQ 24 | d2i_X509_REQ_bio 25 | d2i_X509_REQ_fp 26 | d2i_X509_SIG 27 | d2i_X509_bio 28 | d2i_X509_fp 29 | i2d_X509 30 | i2d_X509_ALGOR 31 | i2d_X509_CRL 32 | i2d_X509_CRL_bio 33 | i2d_X509_CRL_fp 34 | i2d_X509_NAME 35 | i2d_X509_REQ 36 | i2d_X509_REQ_bio 37 | i2d_X509_REQ_fp 38 | i2d_X509_SIG 39 | i2d_X509_bio 40 | i2d_X509_fp 41 | -------------------------------------------------------------------------------- /regenkeywords.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import MySQLdb, argparse, datetime, unicodedata 4 | 5 | from common import * 6 | from config import Config 7 | from database import DB 8 | from databaseQueries import DBQ 9 | from repo import Repo 10 | from commit import Commit 11 | from keywordsfilter import * 12 | 13 | def regen(keywords): 14 | commits= DBQ.findByKeywords(keywords) 15 | 16 | for c in commits: 17 | sys.stdout.write("(" + str(c.repo.id) + ":" + c.uniqueid + ":") 18 | sys.stdout.flush() 19 | 20 | diffs = c.getChangedTexts(c.getChangedTextMetadata()) 21 | sys.stdout.write("1") 22 | sys.stdout.flush() 23 | 24 | c.dbkeywords = c.getSynonyms(diffs) 25 | sys.stdout.write("2") 26 | sys.stdout.flush() 27 | 28 | c.save() 29 | sys.stdout.write("3)") 30 | sys.stdout.flush() 31 | print "" 32 | 33 | if __name__ == "__main__": 34 | if sys.argv: 35 | regen(" ".join(sys.argv[1:])) 36 | else: 37 | regen('') 38 | 39 | 40 | -------------------------------------------------------------------------------- /common.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import time, os, urlparse, re, datetime 4 | 5 | def fla(a, b): 6 | if a and not b: 7 | return a 8 | elif b and not a: 9 | return b 10 | else: 11 | a.extend(b) 12 | return a 13 | 14 | def unixToGitDateFormat(t): 15 | return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(t)) 16 | 17 | def unixToDatetime(t): 18 | return datetime.datetime.utcfromtimestamp(t) 19 | 20 | def urlToFolder(url): 21 | scheme, netloc, path, parameters, query, fragment = urlparse.urlparse(url) 22 | folder = netloc + path 23 | folder = folder.replace('/', '-') 24 | return folder 25 | 26 | def fixDates(start, end): 27 | try: 28 | end = int(end) 29 | start = int(start) 30 | except ValueError: 31 | print "Invalid Start or End Date" 32 | exit 33 | 34 | if start < 0 and not end: 35 | end = time.time() 36 | start = end + int(start) 37 | elif end < start: 38 | tmp = end 39 | end = start 40 | start = tmp 41 | return start, end 42 | 43 | class MicroMock(object): 44 | def __init__(self, **kwargs): 45 | self.__dict__.update(kwargs) 46 | -------------------------------------------------------------------------------- /svnpuller.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import argparse, sys, time, os 3 | import pysvn 4 | 5 | from common import * 6 | from svncommit import SVNCommit 7 | from repo import Repo 8 | 9 | def getCommits(repo, startdate, enddate): 10 | end_rev = pysvn.Revision(pysvn.opt_revision_kind.date, enddate) 11 | start_rev = pysvn.Revision(pysvn.opt_revision_kind.date, startdate) 12 | 13 | client = pysvn.Client() 14 | 15 | commits = [] 16 | msgs = client.log(repo.url, revision_start=start_rev, revision_end=end_rev, discover_changed_paths=True) 17 | msgs.reverse() 18 | for m in msgs: 19 | date = m.data['revprops']['svn:date'] 20 | message = m.data['message'] 21 | paths = [p.path for p in m.data['changed_paths']] 22 | 23 | c = SVNCommit() 24 | alldiffs = c.getChangedTexts((m.data['revision'].number, repo)) 25 | c.loadFromSource(repo, message, date, paths, m.data['revision'].number, alldiffs) 26 | commits.append(c) 27 | return commits 28 | 29 | if __name__ == "__main__": 30 | parser = argparse.ArgumentParser(description='Given a repo url, a startdate and enddate, process the commits between.') 31 | parser.add_argument('repo') 32 | parser.add_argument('startdate') 33 | parser.add_argument('enddate') 34 | args = parser.parse_args() 35 | 36 | args.startdate, args.enddate = fixDates(args.startdate, args.enddate) 37 | 38 | r = Repo() 39 | r.loadFromValues(-1, "", Repo.Type.SVN, args.repo, '', '', '') 40 | commits = getCommits(r, args.startdate, args.enddate) 41 | for c in commits: 42 | print "=========================================" 43 | c.pprint() 44 | c.save() 45 | -------------------------------------------------------------------------------- /cronjob.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import MySQLdb, argparse, time 4 | 5 | from common import * 6 | from config import Config 7 | from database import DB 8 | from repo import Repo 9 | from commit import Commit 10 | import svnpuller, gitpuller 11 | 12 | if __name__ == "__main__": 13 | parser = argparse.ArgumentParser(description='Given a startdate and enddate, process the commits between for all repos in the database.') 14 | parser.add_argument('startdate') 15 | parser.add_argument('enddate') 16 | args = parser.parse_args() 17 | 18 | args.startdate, args.enddate = fixDates(args.startdate, args.enddate) 19 | 20 | print "Searching between", time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(args.startdate)), "-", time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(args.enddate)) 21 | 22 | conn = DB.getConn() 23 | c = conn.cursor() 24 | 25 | c.execute("SELECT * FROM " + DB.repo._table) 26 | rows = c.fetchall() 27 | 28 | for i in rows: 29 | r = Repo() 30 | r.loadFromDatabase(i) 31 | r.pprint() 32 | 33 | module = -1 34 | if r.type == Repo.Type.SVN: 35 | module = svnpuller 36 | elif r.type == Repo.Type.GIT: 37 | module = gitpuller 38 | 39 | if module != -1: 40 | try: 41 | commits = module.getCommits(r, args.startdate, args.enddate) 42 | except: 43 | print "Error pulling commits for", r.url 44 | commits = [] 45 | 46 | for c in commits: 47 | print ".", 48 | c.save() 49 | if commits: print "" 50 | else: 51 | print "Do not have a module for", r.url 52 | -------------------------------------------------------------------------------- /keyword-setup/openssl-library-pem.txt: -------------------------------------------------------------------------------- 1 | PEM_read_DHparams 2 | PEM_read_DSAPrivateKey 3 | PEM_read_DSA_PUBKEY 4 | PEM_read_DSAparams 5 | PEM_read_NETSCAPE_CERT_SEQUENCE 6 | PEM_read_PKCS7 7 | PEM_read_PUBKEY 8 | PEM_read_PrivateKey 9 | PEM_read_RSAPrivateKey 10 | PEM_read_RSAPublicKey 11 | PEM_read_RSA_PUBKEY 12 | PEM_read_X509 13 | PEM_read_X509_AUX 14 | PEM_read_X509_CRL 15 | PEM_read_X509_REQ 16 | PEM_read_bio_DHparams 17 | PEM_read_bio_DSAPrivateKey 18 | PEM_read_bio_DSA_PUBKEY 19 | PEM_read_bio_DSAparams 20 | PEM_read_bio_NETSCAPE_CERT_SEQUENCE 21 | PEM_read_bio_PKCS7 22 | PEM_read_bio_PUBKEY 23 | PEM_read_bio_PrivateKey 24 | PEM_read_bio_RSAPrivateKey 25 | PEM_read_bio_RSAPublicKey 26 | PEM_read_bio_RSA_PUBKEY 27 | PEM_read_bio_X509 28 | PEM_read_bio_X509_AUX 29 | PEM_read_bio_X509_CRL 30 | PEM_read_bio_X509_REQ 31 | PEM_write_DHparams 32 | PEM_write_DSAPrivateKey 33 | PEM_write_DSA_PUBKEY 34 | PEM_write_DSAparams 35 | PEM_write_NETSCAPE_CERT_SEQUENCE 36 | PEM_write_PKCS7 37 | PEM_write_PKCS8PrivateKey 38 | PEM_write_PKCS8PrivateKey_nid 39 | PEM_write_PUBKEY 40 | PEM_write_PrivateKey 41 | PEM_write_RSAPrivateKey 42 | PEM_write_RSAPublicKey 43 | PEM_write_RSA_PUBKEY 44 | PEM_write_X509 45 | PEM_write_X509_AUX 46 | PEM_write_X509_CRL 47 | PEM_write_X509_REQ 48 | PEM_write_X509_REQ_NEW 49 | PEM_write_bio_DHparams 50 | PEM_write_bio_DSAPrivateKey 51 | PEM_write_bio_DSA_PUBKEY 52 | PEM_write_bio_DSAparams 53 | PEM_write_bio_NETSCAPE_CERT_SEQUENCE 54 | PEM_write_bio_PKCS7 55 | PEM_write_bio_PKCS8PrivateKey 56 | PEM_write_bio_PKCS8PrivateKey_nid 57 | PEM_write_bio_PUBKEY 58 | PEM_write_bio_PrivateKey 59 | PEM_write_bio_RSAPrivateKey 60 | PEM_write_bio_RSAPublicKey 61 | PEM_write_bio_RSA_PUBKEY 62 | PEM_write_bio_X509 63 | PEM_write_bio_X509_AUX 64 | PEM_write_bio_X509_CRL 65 | PEM_write_bio_X509_REQ 66 | PEM_write_bio_X509_REQ_NEW 67 | -------------------------------------------------------------------------------- /keyword-setup/openssl-library-evp.txt: -------------------------------------------------------------------------------- 1 | EVP_CIPHER_CTX_block_size 2 | EVP_CIPHER_CTX_cipher 3 | EVP_CIPHER_CTX_cleanup 4 | EVP_CIPHER_CTX_ctrl 5 | EVP_CIPHER_CTX_flags 6 | EVP_CIPHER_CTX_get_app_data 7 | EVP_CIPHER_CTX_init 8 | EVP_CIPHER_CTX_iv_length 9 | EVP_CIPHER_CTX_key_length 10 | EVP_CIPHER_CTX_mode 11 | EVP_CIPHER_CTX_nid 12 | EVP_CIPHER_CTX_set_app_data 13 | EVP_CIPHER_CTX_set_key_length 14 | EVP_CIPHER_CTX_set_padding 15 | EVP_CIPHER_CTX_type 16 | EVP_CIPHER_asn1_to_param 17 | EVP_CIPHER_block_size 18 | EVP_CIPHER_flags 19 | EVP_CIPHER_iv_length 20 | EVP_CIPHER_key_length 21 | EVP_CIPHER_mode 22 | EVP_CIPHER_nid 23 | EVP_CIPHER_param_to_asn1 24 | EVP_CIPHER_type 25 | EVP_CipherFinal 26 | EVP_CipherFinal_ex 27 | EVP_CipherInit 28 | EVP_CipherInit_ex 29 | EVP_CipherUpdate 30 | EVP_DecryptFinal 31 | EVP_DecryptFinal_ex 32 | EVP_DecryptInit 33 | EVP_DecryptInit_ex 34 | EVP_DecryptUpdate 35 | EVP_DigestFinal_ex 36 | EVP_DigestInit_ex 37 | EVP_DigestUpdate 38 | EVP_EncryptFinal 39 | EVP_EncryptFinal_ex 40 | EVP_EncryptInit 41 | EVP_EncryptInit_ex 42 | EVP_EncryptUpdate 43 | EVP_MAX_MD_SIZE 44 | EVP_MD_CTX_block_size 45 | EVP_MD_CTX_cleanup 46 | EVP_MD_CTX_copy 47 | EVP_MD_CTX_copy_ex 48 | EVP_MD_CTX_create 49 | EVP_MD_CTX_destroy 50 | EVP_MD_CTX_init 51 | EVP_MD_CTX_md 52 | EVP_MD_CTX_size 53 | EVP_MD_CTX_type 54 | EVP_MD_block_size 55 | EVP_MD_pkey_type 56 | EVP_MD_size 57 | EVP_MD_type 58 | EVP_OpenFinal 59 | EVP_OpenInit 60 | EVP_OpenUpdate 61 | EVP_SealFinal 62 | EVP_SealInit 63 | EVP_SealUpdate 64 | EVP_SignFinal 65 | EVP_SignInit 66 | EVP_SignUpdate 67 | EVP_VerifyFinal 68 | EVP_VerifyInit 69 | EVP_VerifyUpdate 70 | EVP_dss 71 | EVP_dss1 72 | EVP_get_cipherbyname 73 | EVP_get_cipherbynid 74 | EVP_get_cipherbyobj 75 | EVP_get_digestbyname 76 | EVP_get_digestbynid 77 | EVP_get_digestbyobj 78 | EVP_md2 79 | EVP_md5 80 | EVP_md_null 81 | EVP_mdc2 82 | EVP_ripemd160 83 | EVP_sha 84 | EVP_sha1 85 | -------------------------------------------------------------------------------- /keyword-setup/openssl-library-bn.txt: -------------------------------------------------------------------------------- 1 | BN_BLINDING 2 | BN_BLINDING_convert 3 | BN_BLINDING_convert_ex 4 | BN_BLINDING_create_param 5 | BN_BLINDING_free 6 | BN_BLINDING_get_flags 7 | BN_BLINDING_get_thread_id 8 | BN_BLINDING_invert 9 | BN_BLINDING_invert_ex 10 | BN_BLINDING_new 11 | BN_BLINDING_set_flags 12 | BN_BLINDING_set_thread_id 13 | BN_BLINDING_update 14 | BN_CTX 15 | BN_CTX_free 16 | BN_CTX_init 17 | BN_CTX_new 18 | BN_MONT_CTX 19 | BN_MONT_CTX_copy 20 | BN_MONT_CTX_free 21 | BN_MONT_CTX_init 22 | BN_MONT_CTX_new 23 | BN_MONT_CTX_set 24 | BN_RECP_CTX 25 | BN_RECP_CTX_free 26 | BN_RECP_CTX_init 27 | BN_RECP_CTX_new 28 | BN_RECP_CTX_set 29 | BN_ULONG 30 | BN_add 31 | BN_add_word 32 | BN_bin2bn 33 | BN_bn2bin 34 | BN_bn2dec 35 | BN_bn2hex 36 | BN_bn2mpi 37 | BN_clear 38 | BN_clear_bit 39 | BN_clear_free 40 | BN_cmp 41 | BN_copy 42 | BN_dec2bn 43 | BN_div 44 | BN_div_word 45 | BN_dup 46 | BN_exp 47 | BN_free 48 | BN_from_montgomery 49 | BN_gcd 50 | BN_generate_prime 51 | BN_get_word 52 | BN_hex2bn 53 | BN_init 54 | BN_is_bit_set 55 | BN_is_negative 56 | BN_is_odd 57 | BN_is_one 58 | BN_is_prime 59 | BN_is_word 60 | BN_is_zero 61 | BN_lshift 62 | BN_lshift1 63 | BN_mask_bits 64 | BN_mod 65 | BN_mod_add 66 | BN_mod_exp 67 | BN_mod_inverse 68 | BN_mod_mul 69 | BN_mod_mul_montgomery 70 | BN_mod_mul_reciprocal 71 | BN_mod_sqr 72 | BN_mod_sub 73 | BN_mod_word 74 | BN_mpi2bn 75 | BN_mul 76 | BN_mul_word 77 | BN_new 78 | BN_nnmod 79 | BN_num_bits 80 | BN_num_bits_word 81 | BN_num_bytes 82 | BN_one 83 | BN_print 84 | BN_print_fp 85 | BN_pseudo_rand 86 | BN_pseudo_rand_range 87 | BN_rand 88 | BN_rand_range 89 | BN_rshift 90 | BN_rshift1 91 | BN_set_bit 92 | BN_set_negative 93 | BN_set_word 94 | BN_sqr 95 | BN_sub 96 | BN_sub_word 97 | BN_swap 98 | BN_to_montgomery 99 | BN_ucmp 100 | BN_value_one 101 | BN_zero 102 | -------------------------------------------------------------------------------- /gitpuller.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import argparse, sys, time, os 3 | import git as pygit 4 | import gdiff 5 | 6 | from common import * 7 | from config import * 8 | from gitcommit import GitCommit 9 | #from commit import Commit 10 | from repo import Repo 11 | 12 | def getCommits(repo, startdate, enddate): 13 | localfolder = urlToFolder(repo.url) 14 | differ = gdiff.diff_match_patch() 15 | 16 | repoloc = Config.fsdir + 'git-repos/' + localfolder + '/' 17 | if os.path.exists(repoloc): 18 | c = pygit.Repo(repoloc) 19 | else: 20 | os.makedirs(repoloc) 21 | c = pygit.Repo.init(repoloc) 22 | c.create_remote('origin', repo.url) 23 | 24 | c.remotes.origin.fetch() 25 | c.remotes.origin.pull('master') 26 | 27 | commits = [] 28 | msgs = c.iter_commits(since=unixToGitDateFormat(startdate)) 29 | for m in msgs: 30 | if m.committed_date > enddate: continue 31 | 32 | # TODO BUG: We are unable to get a git diff of the first commit. 33 | # http://osdir.com/ml/version-control.git/2005-05/msg01309.html 34 | # http://git.661346.n2.nabble.com/git-diff-tree-against-the-root-commit-td5685272.html 35 | if not m.parents: continue 36 | 37 | c = GitCommit() 38 | alldiffs = c.getChangedTexts(m) 39 | c.loadFromSource(repo, m.message, m.committed_date, m.stats.files.keys(), m.__str__(), alldiffs) 40 | commits.append(c) 41 | return commits 42 | 43 | if __name__ == "__main__": 44 | parser = argparse.ArgumentParser(description='Given a repo url, a startdate and enddate, process the commits between.') 45 | parser.add_argument('repo') 46 | parser.add_argument('startdate') 47 | parser.add_argument('enddate') 48 | args = parser.parse_args() 49 | 50 | args.startdate, args.enddate = fixDates(args.startdate, args.enddate) 51 | 52 | r = Repo() 53 | r.loadFromValues(-1, "", Repo.Type.GIT, args.repo, '', '', '') 54 | commits = getCommits(r, args.startdate, args.enddate) 55 | for c in commits: 56 | print "=========================================" 57 | c.pprint() 58 | c.save() 59 | -------------------------------------------------------------------------------- /repo.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import time, MySQLdb 4 | 5 | from database import DB 6 | 7 | class Repo: 8 | id = 0 9 | type = 0 10 | url = '' 11 | 12 | initialized = False 13 | def __init__(self): 14 | pass 15 | 16 | def loadFromDatabase(self, row): 17 | self.initialized = True 18 | 19 | self.id = row[DB.repo.id] 20 | self.name = row[DB.repo.name] 21 | self.type = row[DB.repo.repotypeid] 22 | self.url = row[DB.repo.url] 23 | self.viewlink = row[DB.repo.viewlink] 24 | self.tagname = row[DB.repo.tagname] 25 | self.tagmaturity = row[DB.repo.tagmaturity] 26 | 27 | def loadFromValues(self, i, n, t, u, vl, tagn, tagm): 28 | self.initialized = True 29 | 30 | self.id = i 31 | self.name = n 32 | self.type = t 33 | self.url = u 34 | self.viewlink = vl 35 | self.tagname = tagn 36 | self.tagmaturity = tagm 37 | 38 | def getpprint(self): 39 | if not self.initialized: 40 | raise Exception("called getBasePath on unitialized Commit object") 41 | 42 | s = "(" + str(self.id) + ", " + self.name + ", " 43 | 44 | for i in dir(Repo.Type): 45 | if self.type == getattr(Repo.Type, i): 46 | s += i 47 | 48 | s+= ", " + self.url + ", project-" + self.tagname + ", maturity-" + self.tagmaturity + ")" 49 | return s 50 | def pprint(self): 51 | print self.getpprint() 52 | 53 | 54 | class Type: 55 | SVN = 1 56 | GIT = 2 57 | CVS = 3 58 | RSS = 4 59 | EMAIL = 5 60 | BAZAAR = 6 61 | MERCURIAL = 7 62 | TAR = 8 63 | DARCS = 9 64 | 65 | #Maturity Values 66 | # pervasive - project is used by hundreds or thousands of people who regurally rely on it for protection 67 | # stable - project is in use and stable on public servers, but not pervasive 68 | # beta - project can be used on public servers but is marked as beta by the authors 69 | # development - project is in development 70 | -------------------------------------------------------------------------------- /svncommit.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import pysvn 4 | import gdiff 5 | 6 | from commit import * 7 | 8 | class SVNCommit(Commit): 9 | def getDiffsArray(self): 10 | if self.diffIsReallyBig: return [] 11 | 12 | alldiffs = [] 13 | differ = gdiff.diff_match_patch() 14 | client = pysvn.Client() 15 | 16 | for f in self.files: 17 | loc = self.repo.url + f 18 | loc = loc.replace("trunk//trunk", "trunk/") 19 | 20 | #Try/Catches are easier than seeing if the diff is an addition/deletion 21 | try: 22 | left = client.cat(url_or_path=loc, revision=pysvn.Revision(pysvn.opt_revision_kind.number, int(self.uniqueid)-1)) 23 | except: 24 | left = '' 25 | try: 26 | right = client.cat(url_or_path=loc, revision=pysvn.Revision(pysvn.opt_revision_kind.number, int(self.uniqueid))) 27 | except: 28 | right = '' 29 | 30 | alldiffs.append(differ.diff_main(left, right)) 31 | 32 | return alldiffs 33 | def getChangedTextMetadata(self): 34 | return (self.uniqueid, self.repo) 35 | def getChangedTexts(self, metadata): 36 | if self.changedTexts != None: 37 | return self.changedTexts 38 | elif self.changedTexts_data != None: 39 | return self._loadChangedTextFromBackingVar() 40 | elif metadata == None: 41 | raise Exception("NULL passed to getChangedTexts when local changedTexts was not set") 42 | 43 | uniqueid, repo = metadata 44 | if int(uniqueid) == 1: 45 | diff = '' 46 | else: 47 | client = pysvn.Client() 48 | diff = client.diff(tmp_path='./', url_or_path=repo.url, 49 | revision1=pysvn.Revision(pysvn.opt_revision_kind.number, int(uniqueid)-1), 50 | revision2=pysvn.Revision(pysvn.opt_revision_kind.number, int(uniqueid))) 51 | diff = svn_diff_header.sub('', diff) 52 | diff = svn_diff_newline.sub('', diff) 53 | diff = svn_diff_property.sub('', diff) 54 | diff = svn_diff_deletions.sub('', diff) 55 | diff = diff.lower() 56 | self.changedTexts = [diff] 57 | return self.changedTexts 58 | -------------------------------------------------------------------------------- /database.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import MySQLdb 4 | 5 | from config import Config 6 | 7 | class DB: 8 | @staticmethod 9 | def getConn(): 10 | conn = MySQLdb.connect (host = Config.host, 11 | user = Config.username, 12 | passwd = Config.password, 13 | db = Config.database, 14 | charset='utf8') 15 | return conn 16 | @staticmethod 17 | def execute(c, sql, *args): 18 | try: 19 | if not args: 20 | c.execute(sql) 21 | else: 22 | c.execute(sql, args[0]) 23 | except MySQLdb.Error as e: 24 | print e 25 | print sql 26 | if args: 27 | print args[0] 28 | raise e 29 | #@staticmethod 30 | #def execute(c, sql, args): 31 | # try: 32 | # c.execute(sql, args) 33 | # except _mysql_exceptions.OperationError as e: 34 | # print e 35 | # print sql, args 36 | # raise e 37 | 38 | #define table names and column indices 39 | class repotype: 40 | _table = "repotype_tbl" 41 | id = 0 42 | type = 1 43 | _numColumns = 2 44 | class repo: 45 | _table = "repo_tbl" 46 | id = 0 47 | name = 1 48 | repotypeid = 2 49 | url = 3 50 | viewlink = 4 51 | tagname = 5 52 | tagmaturity = 6 53 | _numColumns = 7 54 | class keyword: 55 | _table = "keyword_tbl" 56 | keyword = 0 57 | parent = 1 58 | type = 2 59 | _numColumns = 3 60 | class commit: 61 | _table = "commit_tbl" 62 | id = 0 63 | repoid = 1 64 | date = 2 65 | message = 3 66 | uniqueid = 4 67 | _numColumns = 5 68 | class commitfile: 69 | _table = "commitfile_tbl" 70 | commitid = 0 71 | file = 1 72 | _numColumns = 2 73 | class commitkeyword: 74 | _table = "commitkeyword_tbl" 75 | commitid = 0 76 | keyword = 1 77 | _numColumns = 2 78 | class commitdiffs: 79 | _table = "commitdiffs_tbl" 80 | commitid = 0 81 | data = 1 82 | _numColumns = 2 83 | class commitwordmap: 84 | _table = "commitwordmap_tbl" 85 | commitid = 0 86 | word = 1 87 | _numColumns = 2 88 | class searchqueries: 89 | _table = "searchqueries_tbl" 90 | timestamp = 0 91 | ip = 1 92 | terms = 2 93 | _numColumns = 3 94 | -------------------------------------------------------------------------------- /keyword-setup/openssl-library-bio.txt: -------------------------------------------------------------------------------- 1 | BIO_append_filename 2 | BIO_callback_ctrl 3 | BIO_ctrl 4 | BIO_ctrl_get_read_request 5 | BIO_ctrl_get_write_guarantee 6 | BIO_ctrl_pending 7 | BIO_ctrl_reset_read_request 8 | BIO_ctrl_wpending 9 | BIO_debug_callback 10 | BIO_destroy_bio_pair 11 | BIO_do_accept 12 | BIO_do_connect 13 | BIO_eof 14 | BIO_f_base64 15 | BIO_f_cipher 16 | BIO_f_md 17 | BIO_f_null 18 | BIO_f_ssl 19 | BIO_find_type 20 | BIO_flush 21 | BIO_free 22 | BIO_free_all 23 | BIO_get_accept_port 24 | BIO_get_bind_mode 25 | BIO_get_buffer_num_lines 26 | BIO_get_callback 27 | BIO_get_callback_arg 28 | BIO_get_cipher_ctx 29 | BIO_get_cipher_status 30 | BIO_get_close 31 | BIO_get_conn_hostname 32 | BIO_get_conn_int_port 33 | BIO_get_conn_ip 34 | BIO_get_conn_port 35 | BIO_get_fd 36 | BIO_get_fp 37 | BIO_get_info_callback 38 | BIO_get_md 39 | BIO_get_md_ctx 40 | BIO_get_mem_data 41 | BIO_get_mem_ptr 42 | BIO_get_num_renegotiates 43 | BIO_get_read_request 44 | BIO_get_retry_BIO 45 | BIO_get_retry_reason 46 | BIO_get_ssl 47 | BIO_get_write_buf_size 48 | BIO_get_write_guarantee 49 | BIO_gets 50 | BIO_int_ctrl 51 | BIO_make_bio_pair 52 | BIO_new 53 | BIO_new_bio_pair 54 | BIO_new_buffer_ssl_connect 55 | BIO_new_fd 56 | BIO_new_file 57 | BIO_new_fp 58 | BIO_new_mem_buf 59 | BIO_new_socket 60 | BIO_new_ssl 61 | BIO_new_ssl_connect 62 | BIO_next 63 | BIO_pending 64 | BIO_pop 65 | BIO_ptr_ctrl 66 | BIO_push 67 | BIO_puts 68 | BIO_read 69 | BIO_read_filename 70 | BIO_reset 71 | BIO_retry_type 72 | BIO_rw_filename 73 | BIO_s_accept 74 | BIO_s_bio 75 | BIO_s_connect 76 | BIO_s_fd 77 | BIO_s_file 78 | BIO_s_mem 79 | BIO_s_null 80 | BIO_s_socket 81 | BIO_seek 82 | BIO_set 83 | BIO_set_accept_bios 84 | BIO_set_accept_port 85 | BIO_set_bind_mode 86 | BIO_set_buffer_read_data 87 | BIO_set_buffer_size 88 | BIO_set_callback 89 | BIO_set_callback_arg 90 | BIO_set_cipher 91 | BIO_set_close 92 | BIO_set_conn_hostname 93 | BIO_set_conn_int_port 94 | BIO_set_conn_ip 95 | BIO_set_conn_port 96 | BIO_set_fd 97 | BIO_set_fp 98 | BIO_set_info_callback 99 | BIO_set_md 100 | BIO_set_mem_buf 101 | BIO_set_mem_eof_return 102 | BIO_set_nbio 103 | BIO_set_nbio_accept 104 | BIO_set_read_buffer_size 105 | BIO_set_ssl 106 | BIO_set_ssl_mode 107 | BIO_set_ssl_renegotiate_bytes 108 | BIO_set_ssl_renegotiate_timeout 109 | BIO_set_write_buf_size 110 | BIO_set_write_buffer_size 111 | BIO_should_io_special 112 | BIO_should_read 113 | BIO_should_retry 114 | BIO_should_write 115 | BIO_shutdown_wr 116 | BIO_ssl_copy_session_id 117 | BIO_ssl_shutdown 118 | BIO_tell 119 | BIO_vfree 120 | BIO_wpending 121 | BIO_write 122 | BIO_write_filename 123 | -------------------------------------------------------------------------------- /gitcommit.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import git as pygit 4 | import gdiff 5 | 6 | from config import * 7 | from commit import * 8 | 9 | class GitCommit(Commit): 10 | def getDiffsArray(self): 11 | if self.diffIsReallyBig: return [] 12 | 13 | alldiffs = [] 14 | differ = gdiff.diff_match_patch() 15 | 16 | commit = self.getChangedTextMetadata() 17 | for d in commit.diff(commit.__str__()+'^').iter_change_type('M'): #Changed 18 | left = d.a_blob.data_stream.read() 19 | right = d.b_blob.data_stream.read() 20 | alldiffs.append(differ.diff_main(left, right)) 21 | 22 | for d in commit.diff(commit.__str__()+'^').iter_change_type('A'): #Added 23 | addition = d.b_blob.data_stream.read() 24 | alldiffs.append(differ.diff_main('', addition)) 25 | 26 | return alldiffs 27 | def getChangedTextMetadata(self): 28 | localfolder = urlToFolder(self.repo.url) 29 | repoloc = Config.fsdir + 'git-repos/' + localfolder + '/' 30 | repo = pygit.Repo(repoloc) 31 | 32 | commit = repo.commit(self.uniqueid) 33 | return commit 34 | def getChangedTexts(self, commitobj): 35 | if self.changedTexts != None: 36 | return self.changedTexts 37 | elif self.changedTexts_data != None: 38 | return self._loadChangedTextFromBackingVar() 39 | elif commitobj == None: 40 | raise Exception("NULL passed to getChangedTexts when local changedTexts was not set") 41 | 42 | alldiffs = [] 43 | differ = gdiff.diff_match_patch() 44 | 45 | for d in commitobj.diff(commitobj.__str__()+'^').iter_change_type('M'): #Changed 46 | left = d.a_blob.data_stream.read() 47 | right = d.b_blob.data_stream.read() 48 | diffs = differ.diff_main(left, right) 49 | if diffs: differ.diff_cleanupSemantic(diffs) 50 | 51 | for d in diffs: 52 | if d[0] != 0 and d[1].strip(): 53 | alldiffs.append(d[1].lower()) 54 | 55 | for d in commitobj.diff(commitobj.__str__()+'^').iter_change_type('A'): #Added 56 | addition = d.b_blob.data_stream.read() 57 | alldiffs.append(addition.lower()) 58 | #for d in commitobj.diff(commitobj.__str__()+'^').iter_change_type('D'): #Deleted 59 | # pass 60 | #for d in commitobj.diff(commitobj.__str__()+'^').iter_change_type('R'): #Renamed 61 | # pass 62 | self.changedTexts = alldiffs 63 | return self.changedTexts 64 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | #About 2 | 3 | You're a developer. And you've spent the last 2 years working with Java sockets in an uninteresting trading app. But you also happen to support anonymity - but have no idea how to get involved. Or you're a security researcher who's spent the last two months understanding the padding oracle backwards and forwards. Wouldn't it be nice to see a personalized RSS feed of cryptography, anonymity, and privacy projects containing the keywords "java.net.Socket" or "CBC Mode"? Then you could skim commits, and if something interesting came up, you may be able to lend your expertise. That's exactly what the Code Peer Review is for. 4 | 5 | # How You Can Help Test the Code Peer Review 6 | 7 | From easiest to most time-intensive 8 | 9 | ## Help us add projects 10 | 11 | 1. Without looking at [setup.py](https://github.com/cryptodotis/code-peer-review/blob/master/setup.py) think of all the crypto and privacy open source projects you can, and write them down. 12 | 2. Check [setup.py](https://github.com/cryptodotis/code-peer-review/blob/master/setup.py) to see if you wrote down any we don't have yet 13 | 3. Either fork and edit setup.py to include them, and submit a pull request, or just [create an issue](https://github.com/cryptodotis/code-peer-review/issues/new) and link to the project. 14 | 15 | ## Help us add tags 16 | 17 | 1. Think of all the things you or someone else would be interested in code reviewing - crypto, threading, etc 18 | 2. Check [tags.txt](https://github.com/cryptodotis/code-peer-review/blob/master/tags.txt) and see if they're there. 19 | 3. Either fork and submit a pull request, or just [create an issue](https://github.com/cryptodotis/code-peer-review/issues/new) and tell us your tags 20 | 21 | ## Help us add library tags 22 | 23 | 1. Find the API documentation for a crypto library - like OpenSSL 24 | 2. Make textfiles that group the API function calls together (either into subgroups under the library or just all as the single library) 25 | 3. Follow the format of [OpenSSL textfiles](https://github.com/cryptodotis/code-peer-review/tree/master/keyword-setup) 26 | 4. Fork and submit a pull request 27 | 28 | ## Run your own instance, and inspect the commits: 29 | 30 | 1. Either set up your own instance of the Feed, or ask in #cryptodotis for the beta url 31 | 2. Put the raw RSS Feed in your favorite reader 32 | 3. Review the commits to see if the code peer review missed a keyword, or see if you think of a new keyword should be added 33 | 4. If you think of a UI feature that needs to be added, or something that's really missing, [create an issue](https://github.com/cryptodotis/code-peer-review/issues/new) 34 | 35 | ## Actively test the parser 36 | 37 | 1. Set up your own instance of the Feed 38 | 2. Change setup.py to contain your own test repositories 39 | 3. ./setup.py --testpopulate 40 | 4. Perform commits that should match tags, or perform commits on branches, or do strange commits (moves, renames, etc) 41 | 5. ./cronjob -30 0 (for commits made 30 minutes ago) 42 | 6. Make sure the Code Peer Review processes your test commits correctly. 43 | 44 | 45 | 46 | # Requirements 47 | 48 | - Python 49 | - MySQL 50 | 51 | - pysvn 52 | - GitPython. I'm using 0.3.2 RC1, I don't think 0.1 will work 53 | - [This patch applied](https://github.com/tomrittervg/GitPython/commit/0386f7ba2bedd8ae64040eb5f4d1246f0c4a2a1d) (Or figure out how to do diffing without it) 54 | - PLY (Python Lex-Yacc library), I'm using 3.4 55 | - PyRSS2Gen - I'm using 1.0.0 56 | - Jinja - I'm using 2.6 57 | - [Python-Graph](http://code.google.com/p/python-graph/) - I'm using 1.8.0 58 | - google-diff-match-patch - I'm using 20111010 59 | - Tornado Webserver - I'm using 2.1.1 60 | 61 | # Installation 62 | 63 | 1. Move config.sample.py to config.py and fill in variables 64 | 2. Run ./setup.py --populate or --testpopulate 65 | 3. Set up a cronjob to run nightly with ./cronjob.py -86400 0 66 | 4. python rss.py to run the rss feed/webserver temporarily, or use something similar to the included [gentoo-init-script](https://github.com/cryptodotis/code-peer-review/blob/master/gentoo-init-script) to run permanently 67 | 5. Browse to localhost:8888/rss/ for the RSS Feed. 68 | -------------------------------------------------------------------------------- /webserver.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import tornado.ioloop 4 | import tornado.web 5 | import tornado.options 6 | import logging 7 | 8 | import MySQLdb, argparse, datetime, unicodedata, time 9 | from PyRSS2Gen import RSS2 10 | from jinja2 import Environment, FileSystemLoader 11 | 12 | from common import * 13 | from config import Config 14 | from database import DB 15 | from databaseQueries import DBQ 16 | from repo import Repo 17 | from commit import Commit 18 | from keywordsfilter import * 19 | 20 | def getFeed(): 21 | feed = RSS2( 22 | title = "Crypto.is Code Audit Feed", 23 | description = "Just a thing, right?", 24 | link = "https://crypto.is", 25 | lastBuildDate = datetime.datetime.utcnow() 26 | ) 27 | return feed 28 | 29 | class RSSHandler(tornado.web.RequestHandler): 30 | def get(self, keywords): 31 | fiveDaysAgo = time.time() - (5 * 24 * 60 * 60) 32 | commits = DBQ.findByKeywordsAndFulltext(keywords, fiveDaysAgo) 33 | feed = getFeed() 34 | 35 | for c in commits: 36 | feed.items.append(c.toRSSItem()) 37 | 38 | self.set_header('Content-Type', 'application/rss+xml') 39 | 40 | xml = feed.to_xml() 41 | self.write(xml) 42 | return 43 | 44 | class KeywordsHandler(tornado.web.RequestHandler): 45 | def get(self, keywords): 46 | commits = DBQ.findByKeywords(keywords) 47 | feed = getFeed() 48 | 49 | for c in commits: 50 | feed.items.append(c.toRSSItem()) 51 | 52 | self.set_header('Content-Type', 'application/rss+xml') 53 | 54 | xml = feed.to_xml() 55 | self.write(xml) 56 | return 57 | 58 | class LandingHandler(tornado.web.RequestHandler): 59 | def get(self): 60 | commits=[] 61 | template = env.get_template('search.html') 62 | html = template.render(commits=commits) 63 | self.write(html) 64 | return 65 | 66 | class HallOfFameHandler(tornado.web.RequestHandler): 67 | def get(self): 68 | template = env.get_template('halloffame.html') 69 | html = template.render() 70 | self.write(html) 71 | return 72 | 73 | class CommitHandler(tornado.web.RequestHandler): 74 | def get(self, project, uniqueid): 75 | commit = DBQ.findByIDs(project, uniqueid) 76 | if len(commit) > 1: 77 | raise "More than one commit returned?" 78 | if not commit: 79 | self.write("Could not find commit") 80 | return 81 | commit = commit[0] 82 | 83 | template = env.get_template('commit.html') 84 | html = template.render(commit=commit) 85 | 86 | self.write(html) 87 | return 88 | 89 | class SearchHandler(tornado.web.RequestHandler): 90 | def get(self, keywords): 91 | commits = [] 92 | if not keywords: 93 | template = env.get_template('search.html') 94 | else: 95 | DBQ.logTerms(self.request.remote_ip, keywords) 96 | commits = DBQ.findByKeywordsAndFulltext(keywords) 97 | template = env.get_template('searchresults.html') 98 | 99 | html = template.render(commits=commits, rsssearchparams=keywords) 100 | self.write(html) 101 | return 102 | def post(self, keywords): 103 | keywords = self.request.arguments['terms'][0] 104 | DBQ.logTerms(self.request.remote_ip, keywords) 105 | commits = DBQ.findByKeywordsAndFulltext(keywords) 106 | template = env.get_template('searchresults.html') 107 | 108 | html = template.render(commits=commits, rsssearchparams=keywords) 109 | self.write(html) 110 | return 111 | 112 | env = Environment(autoescape=True, 113 | loader=FileSystemLoader(Config.fsdir + 'templates')) 114 | 115 | application = tornado.web.Application([ 116 | (r"/", LandingHandler), 117 | (r"/search/?(.*)", SearchHandler), 118 | (r"/commit/(.*)/(.*)", CommitHandler), 119 | (r"/halloffame", HallOfFameHandler), 120 | (r"/keywords/(.*)", KeywordsHandler),#Obsolete, For testing only 121 | (r"/rss/(.*)", RSSHandler), 122 | ]) 123 | tornado.options.parse_command_line() 124 | 125 | if __name__ == "__main__": 126 | tlog = logging.FileHandler(Config.logfile) 127 | logging.getLogger().addHandler(tlog) 128 | logging.debug('Starting up...') 129 | 130 | import socket 131 | if socket.has_ipv6: 132 | application.listen(Config.tornadoport) 133 | else: 134 | application.listen(Config.tornadoport, address="0.0.0.0") 135 | tornado.ioloop.IOLoop.instance().start() 136 | 137 | 138 | 139 | -------------------------------------------------------------------------------- /keyword-setup/nss-certificate-library.txt: -------------------------------------------------------------------------------- 1 | CERT_AddCertToListTail 2 | CERT_AddExtension 3 | CERT_AddOCSPAcceptableResponses 4 | CERT_AddOKDomainName 5 | CERT_AddRDN 6 | CERT_AsciiToName 7 | CERT_CacheCRL 8 | CERT_ClearOCSPCache 9 | CERT_CertChainFromCert 10 | CERT_CertListFromCert 11 | CERT_CertTimesValid 12 | CERT_ChangeCertTrust 13 | CERT_CheckCertValidTimes 14 | CERT_CheckNameSpace 15 | CERT_CheckCertUsage 16 | CERT_CompareName 17 | CERT_CompareValidityTimes 18 | CERT_CompleteCRLDecodeEntries 19 | CERT_ConvertAndDecodeCertificate 20 | CERT_CopyName 21 | CERT_CopyRDN 22 | CERT_CreateAVA 23 | CERT_CreateCertificate 24 | CERT_CreateCertificateRequest 25 | CERT_CreateName 26 | CERT_CreateOCSPCertID 27 | CERT_CreateOCSPRequest 28 | CERT_CreateRDN 29 | CERT_CreateSubjectCertList 30 | CERT_CreateValidity 31 | CERT_CRLCacheRefreshIssuer 32 | CERT_DecodeAltNameExtension 33 | CERT_DecodeAuthInfoAccessExtension 34 | CERT_DecodeAuthKeyID 35 | CERT_DecodeAVAValue 36 | CERT_DecodeBasicConstraintValue 37 | CERT_DecodeCertFromPackage 38 | CERT_DecodeCertificatePoliciesExtension 39 | CERT_DecodeCertPackage 40 | CERT_DecodeCRLDistributionPoints 41 | CERT_DecodeDERCrl 42 | CERT_DecodeDERCrlWithFlags 43 | CERT_DecodeGeneralName 44 | CERT_DecodeNameConstraintsExtension 45 | CERT_DecodeOCSPResponse 46 | CERT_DecodeOidSequence 47 | CERT_DecodePrivKeyUsagePeriodExtension 48 | CERT_DecodeTrustString 49 | CERT_DecodeUserNotice 50 | CERT_DerNameToAscii 51 | CERT_DestroyCertArray 52 | CERT_DestroyCertificate 53 | CERT_DestroyCertificateList 54 | CERT_DestroyCertificatePoliciesExtension 55 | CERT_DestroyCertificateRequest 56 | CERT_DestroyCertList 57 | CERT_DestroyName 58 | CERT_DestroyOCSPCertID 59 | CERT_DestroyOCSPRequest 60 | CERT_DestroyOCSPResponse 61 | CERT_DestroyOidSequence 62 | CERT_DestroyUserNotice 63 | CERT_DestroyValidity 64 | CERT_DupCertificate 65 | CERT_DupCertList 66 | CERT_EnableOCSPChecking 67 | CERT_EncodeAltNameExtension 68 | CERT_EncodeAndAddBitStrExtension 69 | CERT_EncodeAuthKeyID 70 | CERT_EncodeBasicConstraintValue 71 | CERT_EncodeCertPoliciesExtension 72 | CERT_EncodeCRLDistributionPoints 73 | CERT_EncodeGeneralName 74 | CERT_EncodeInfoAccessExtension 75 | CERT_EncodeInhibitAnyExtension 76 | CERT_EncodeNoticeReference 77 | CERT_EncodeOCSPRequest 78 | CERT_EncodePolicyConstraintsExtension 79 | CERT_EncodePolicyMappingExtension 80 | CERT_EncodeSubjectKeyID 81 | CERT_EncodeUserNotice 82 | CERT_ExtractPublicKey 83 | CERT_FindCertByName 84 | CERT_FindCRLEntryReasonExten 85 | CERT_FindCRLNumberExten 86 | CERT_FindNameConstraintsExten 87 | CERT_FilterCertListByCANames 88 | CERT_FilterCertListByUsage 89 | CERT_FilterCertListForUserCerts 90 | CERT_FindCertByDERCert 91 | CERT_FindCertByIssuerAndSN 92 | CERT_FindCertByNickname 93 | CERT_FindCertByNicknameOrEmailAddr 94 | CERT_FindCertBySubjectKeyID 95 | CERT_FindCertExtension 96 | CERT_FindCertIssuer 97 | CERT_FindKeyUsageExtension 98 | CERT_FindSMimeProfile 99 | CERT_FindSubjectKeyIDExtension 100 | CERT_FindUserCertByUsage 101 | CERT_FindUserCertsByUsage 102 | CERT_FinishCertificateRequestAttributes 103 | CERT_FinishExtensions 104 | CERT_FormatName 105 | CERT_FreeDistNames 106 | CERT_FreeNicknames 107 | CERT_GetAVATag 108 | CERT_GetCertChainFromCert 109 | CERT_GetCertEmailAddress 110 | CERT_GetCertificateNames 111 | CERT_GetCertificateRequestExtensions 112 | CERT_GetCertIssuerAndSN 113 | CERT_GetCertNicknames 114 | CERT_GetCertTrust 115 | CERT_GetCertUid 116 | CERT_GetClassicOCSPDisabledPolicy 117 | CERT_GetClassicOCSPEnabledHardFailurePolicy 118 | CERT_GetClassicOCSPEnabledSoftFailurePolicy 119 | CERT_GetCommonName 120 | CERT_GetCountryName 121 | CERT_GetDBContentVersion 122 | CERT_GetDefaultCertDB 123 | CERT_GetDomainComponentName 124 | CERT_GetFirstEmailAddress 125 | CERT_GetLocalityName 126 | CERT_GetNextEmailAddress 127 | CERT_GetNextGeneralName 128 | CERT_GetNextNameConstraint 129 | CERT_GetOCSPResponseStatus 130 | CERT_GetOCSPStatusForCertID 131 | CERT_GetOidString 132 | CERT_GetOrgName 133 | CERT_GetOrgUnitName 134 | CERT_GetOCSPAuthorityInfoAccessLocation 135 | CERT_GetPKIXVerifyNistRevocationPolicy 136 | CERT_GetPrevGeneralName 137 | CERT_GetPrevNameConstraint 138 | CERT_GetSlopTime 139 | CERT_GetSSLCACerts 140 | CERT_GetStateName 141 | CERT_GetUsePKIXForValidation 142 | CERT_GetValidDNSPatternsFromCert 143 | CERT_GenTime2FormattedAscii 144 | CERT_Hexify 145 | CERT_ImportCAChain 146 | CERT_ImportCerts 147 | CERT_IsRootDERCert 148 | CERT_IsUserCert 149 | CERT_KeyFromDERCrl 150 | CERT_MakeCANickname 151 | CERT_MergeExtensions 152 | CERT_NameToAscii 153 | CERT_NewCertList 154 | CERT_NewTempCertificate 155 | CERT_NicknameStringsFromCertList 156 | CERT_OpenCertDBFilename 157 | CERT_OCSPCacheSettings 158 | CERT_PKIXVerifyCert 159 | CERT_RemoveCertListNode 160 | CERT_RFC1485_EscapeAndQuote 161 | CERT_SaveSMimeProfile 162 | CERT_SetSlopTime 163 | CERT_SetOCSPFailureMode 164 | CERT_SetOCSPTimeout 165 | CERT_SetUsePKIXForValidation 166 | CERT_StartCertExtensions 167 | CERT_StartCertificateRequestAttributes 168 | CERT_StartCRLEntryExtensions 169 | CERT_StartCRLExtensions 170 | CERT_UncacheCRL 171 | CERT_VerifyCertName 172 | CERT_VerifyCACertForUsage 173 | CERT_VerifyCert 174 | CERT_VerifyCertificate 175 | CERT_VerifyCertificateNow 176 | CERT_VerifyCertNow 177 | CERT_VerifyOCSPResponseSignature 178 | CERT_VerifySignedData 179 | CERT_VerifySignedDataWithPublicKey 180 | CERT_VerifySignedDataWithPublicKeyInfo 181 | NSS_CmpCertChainWCANames 182 | NSS_FindCertKEAType -------------------------------------------------------------------------------- /keyword-setup/openssl-library-ssl.txt: -------------------------------------------------------------------------------- 1 | SSL_CIPHER 2 | SSL_CIPHER_description 3 | SSL_CIPHER_get_bits 4 | SSL_CIPHER_get_name 5 | SSL_CIPHER_get_version 6 | SSL_CTX 7 | SSL_CTX_add_client_CA 8 | SSL_CTX_add_extra_chain_cert 9 | SSL_CTX_add_session 10 | SSL_CTX_check_private_key 11 | SSL_CTX_ctrl 12 | SSL_CTX_flush_sessions 13 | SSL_CTX_free 14 | SSL_CTX_get_app_data 15 | SSL_CTX_get_cert_store 16 | SSL_CTX_get_client_CA_list 17 | SSL_CTX_get_client_cert_cb 18 | SSL_CTX_get_ex_data 19 | SSL_CTX_get_ex_new_index 20 | SSL_CTX_get_info_callback 21 | SSL_CTX_get_quiet_shutdown 22 | SSL_CTX_get_session_cache_mode 23 | SSL_CTX_get_timeout 24 | SSL_CTX_get_verify_callback 25 | SSL_CTX_get_verify_mode 26 | SSL_CTX_load_verify_locations 27 | SSL_CTX_need_tmp_RSA 28 | SSL_CTX_new 29 | SSL_CTX_remove_session 30 | SSL_CTX_sess_accept 31 | SSL_CTX_sess_accept_good 32 | SSL_CTX_sess_accept_renegotiate 33 | SSL_CTX_sess_cache_full 34 | SSL_CTX_sess_cb_hits 35 | SSL_CTX_sess_connect 36 | SSL_CTX_sess_connect_good 37 | SSL_CTX_sess_connect_renegotiate 38 | SSL_CTX_sess_get_cache_size 39 | SSL_CTX_sess_get_get_cb 40 | SSL_CTX_sess_get_new_cb 41 | SSL_CTX_sess_get_remove_cb 42 | SSL_CTX_sess_hits 43 | SSL_CTX_sess_misses 44 | SSL_CTX_sess_number 45 | SSL_CTX_sess_set_cache_size 46 | SSL_CTX_sess_set_get_cb 47 | SSL_CTX_sess_set_new_cb 48 | SSL_CTX_sess_set_remove_cb 49 | SSL_CTX_sess_timeouts 50 | SSL_CTX_sessions 51 | SSL_CTX_set_app_data 52 | SSL_CTX_set_cert_store 53 | SSL_CTX_set_cert_verify_cb 54 | SSL_CTX_set_cipher_list 55 | SSL_CTX_set_client_CA_list 56 | SSL_CTX_set_client_cert_cb 57 | SSL_CTX_set_default_passwd_cb 58 | SSL_CTX_set_default_read_ahead 59 | SSL_CTX_set_default_verify_paths 60 | SSL_CTX_set_ex_data 61 | SSL_CTX_set_info_callback 62 | SSL_CTX_set_msg_callback 63 | SSL_CTX_set_msg_callback_arg 64 | SSL_CTX_set_options 65 | SSL_CTX_set_psk_client_callback 66 | SSL_CTX_set_psk_server_callback 67 | SSL_CTX_set_quiet_shutdown 68 | SSL_CTX_set_session_cache_mode 69 | SSL_CTX_set_ssl_version 70 | SSL_CTX_set_timeout 71 | SSL_CTX_set_tmp_dh 72 | SSL_CTX_set_tmp_dh_callback 73 | SSL_CTX_set_tmp_rsa 74 | SSL_CTX_set_tmp_rsa_callback 75 | SSL_CTX_set_verify 76 | SSL_CTX_use_PrivateKey 77 | SSL_CTX_use_PrivateKey_ASN1 78 | SSL_CTX_use_PrivateKey_file 79 | SSL_CTX_use_RSAPrivateKey 80 | SSL_CTX_use_RSAPrivateKey_ASN1 81 | SSL_CTX_use_RSAPrivateKey_file 82 | SSL_CTX_use_certificate 83 | SSL_CTX_use_certificate_ASN1 84 | SSL_CTX_use_certificate_file 85 | SSL_CTX_use_psk_identity_hint 86 | SSL_METHOD 87 | SSL_SESSION 88 | SSL_SESSION_cmp 89 | SSL_SESSION_free 90 | SSL_SESSION_get_app_data 91 | SSL_SESSION_get_ex_data 92 | SSL_SESSION_get_ex_new_index 93 | SSL_SESSION_get_time 94 | SSL_SESSION_get_timeout 95 | SSL_SESSION_hash 96 | SSL_SESSION_new 97 | SSL_SESSION_print 98 | SSL_SESSION_print_fp 99 | SSL_SESSION_set_app_data 100 | SSL_SESSION_set_ex_data 101 | SSL_SESSION_set_time 102 | SSL_SESSION_set_timeout 103 | SSL_accept 104 | SSL_add_client_CA 105 | SSL_add_dir_cert_subjects_to_stack 106 | SSL_add_file_cert_subjects_to_stack 107 | SSL_alert_desc_string 108 | SSL_alert_desc_string_long 109 | SSL_alert_type_string 110 | SSL_alert_type_string_long 111 | SSL_check_private_key 112 | SSL_clear 113 | SSL_clear_num_renegotiations 114 | SSL_connect 115 | SSL_copy_session_id 116 | SSL_ctrl 117 | SSL_do_handshake 118 | SSL_dup 119 | SSL_dup_CA_list 120 | SSL_free 121 | SSL_get_SSL_CTX 122 | SSL_get_app_data 123 | SSL_get_certificate 124 | SSL_get_cipher 125 | SSL_get_cipher_bits 126 | SSL_get_cipher_list 127 | SSL_get_cipher_name 128 | SSL_get_cipher_version 129 | SSL_get_ciphers 130 | SSL_get_client_CA_list 131 | SSL_get_current_cipher 132 | SSL_get_default_timeout 133 | SSL_get_error 134 | SSL_get_ex_data 135 | SSL_get_ex_data_X509_STORE_CTX_idx 136 | SSL_get_ex_new_index 137 | SSL_get_fd 138 | SSL_get_info_callback 139 | SSL_get_peer_cert_chain 140 | SSL_get_peer_certificate 141 | SSL_get_privatekey 142 | SSL_get_psk_identity 143 | SSL_get_psk_identity_hint 144 | SSL_get_quiet_shutdown 145 | SSL_get_rbio 146 | SSL_get_read_ahead 147 | SSL_get_session 148 | SSL_get_shared_ciphers 149 | SSL_get_shutdown 150 | SSL_get_ssl_method 151 | SSL_get_state 152 | SSL_get_time 153 | SSL_get_timeout 154 | SSL_get_verify_callback 155 | SSL_get_verify_mode 156 | SSL_get_verify_result 157 | SSL_get_version 158 | SSL_get_wbio 159 | SSL_in_accept_init 160 | SSL_in_before 161 | SSL_in_connect_init 162 | SSL_in_init 163 | SSL_is_init_finished 164 | SSL_load_client_CA_file 165 | SSL_load_error_strings 166 | SSL_new 167 | SSL_num_renegotiations 168 | SSL_peek 169 | SSL_pending 170 | SSL_read 171 | SSL_renegotiate 172 | SSL_rstate_string 173 | SSL_rstate_string_long 174 | SSL_session_reused 175 | SSL_set_accept_state 176 | SSL_set_app_data 177 | SSL_set_bio 178 | SSL_set_cipher_list 179 | SSL_set_client_CA_list 180 | SSL_set_connect_state 181 | SSL_set_ex_data 182 | SSL_set_fd 183 | SSL_set_info_callback 184 | SSL_set_msg_callback 185 | SSL_set_msg_callback_arg 186 | SSL_set_options 187 | SSL_set_psk_client_callback 188 | SSL_set_psk_server_callback 189 | SSL_set_quiet_shutdown 190 | SSL_set_read_ahead 191 | SSL_set_rfd 192 | SSL_set_session 193 | SSL_set_shutdown 194 | SSL_set_ssl_method 195 | SSL_set_time 196 | SSL_set_timeout 197 | SSL_set_tmp_rsa_callback 198 | SSL_set_verify 199 | SSL_set_verify_result 200 | SSL_set_wfd 201 | SSL_shutdown 202 | SSL_state 203 | SSL_state_string 204 | SSL_state_string_long 205 | SSL_total_renegotiations 206 | SSL_use_PrivateKey 207 | SSL_use_PrivateKey_ASN1 208 | SSL_use_PrivateKey_file 209 | SSL_use_RSAPrivateKey 210 | SSL_use_RSAPrivateKey_ASN1 211 | SSL_use_RSAPrivateKey_file 212 | SSL_use_certificate 213 | SSL_use_certificate_ASN1 214 | SSL_use_certificate_file 215 | SSL_use_psk_identity_hint 216 | SSL_version 217 | SSL_want 218 | SSL_want_nothing 219 | SSL_want_read 220 | SSL_want_write 221 | SSL_want_x509_lookup 222 | SSL_write 223 | SSLv2_client_method 224 | SSLv2_method 225 | SSLv2_server_method 226 | SSLv3_client_method 227 | SSLv3_method 228 | SSLv3_server_method 229 | TLSv1_client_method 230 | TLSv1_method 231 | TLSv1_server_method 232 | -------------------------------------------------------------------------------- /synonymmapping.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import MySQLdb, pygraph.classes.digraph 4 | from common import * 5 | from database import DB 6 | 7 | map = None 8 | 9 | class KeywordType: 10 | STANDARD = 1 11 | #standard keywords can appear in a commit message, filename/path, or the diffed code. 12 | APICALL = 2 13 | #api keywords can appear in a commit message or diffed code, and represent an API call to a library 14 | # the individual tag does not apply, only the parent tag 15 | MAPPING = 3 16 | #mapping keywords will not appear anywhere in the log, diff, or pathname, but are used to apply tags and apply a parent tag 17 | 18 | def getMap(): 19 | global map 20 | if not map: 21 | conn = DB.getConn() 22 | c = conn.cursor() 23 | 24 | c.execute("SELECT * FROM " + DB.keyword._table) 25 | rows = c.fetchall() 26 | 27 | map = pygraph.classes.digraph.digraph() 28 | 29 | for r in rows: 30 | keyword = r[DB.keyword.keyword].lower() 31 | keyword = keyword.replace(".", "\.") # Any regex metacharacters in the keyword need to be escaped. 32 | 33 | parent = r[DB.keyword.parent] 34 | parent = parent.lower().replace(".", "\.") if parent else None 35 | 36 | type = r[DB.keyword.type] 37 | 38 | if not map.has_node(keyword): 39 | keyword_regex = re.compile('(?<=[^a-zA-Z])' + keyword + '(?![a-zA-Z])') # k is not surrounded by alpha characters. 40 | map.add_node(keyword, [('type', type), ('regex', keyword_regex)]) 41 | else: 42 | pass #We take the first type defined, on the assumption that an APICALL will not also be something else 43 | # And that a MAPPING will not also be standard 44 | 45 | if parent and not map.has_node(parent): 46 | parent_regex = '' if not parent else re.compile('(?<=[^a-zA-Z])' + parent + '(?![a-zA-Z])') 47 | map.add_node(parent, [('type', KeywordType.STANDARD), ('regex', parent_regex)]) 48 | #We define a parent tag as being standard. Really it should be max(existing, type) but that requires a logical definition of 49 | # increasing values of type - which we don't do 50 | if parent: 51 | map.add_edge((keyword, parent), label=type) 52 | 53 | c.execute("SELECT tagname FROM " + DB.repo._table) 54 | rows = c.fetchall() 55 | for r in rows: 56 | keyword = r[0].lower() 57 | parent = 'project-' + keyword 58 | type = KeywordType.APICALL #apply only the project tag, not the non-project tag 59 | 60 | if not map.has_node(keyword): 61 | keyword_regex = re.compile('(?<=[^a-zA-Z])' + keyword + '(?![a-zA-Z])') 62 | map.add_node(keyword, [('type', type), ('regex', keyword_regex)]) 63 | if not map.has_node(parent): 64 | parent_regex = re.compile('(?<=[^a-zA-Z])' + parent + '(?![a-zA-Z])') 65 | map.add_node(parent, [('type', KeywordType.STANDARD), ('regex', parent_regex)]) 66 | map.add_edge((keyword, parent), label=type) 67 | return map 68 | 69 | def getAllChildTags(map, node, sofar=None): 70 | keywords = set() if not sofar else sofar 71 | for e in map.neighbors(node): 72 | #This could potentially apply an APICALL tag, but for that to happen 73 | # an APICALL tag would have to be a child of another. This shouldn't happen 74 | # so we apply all parent tags always. 75 | if e not in keywords: 76 | keywords.add(e) 77 | keywords = keywords.union(getAllChildTags(map, e, keywords)) 78 | return keywords 79 | 80 | 81 | def getTags(commit, diffs): 82 | log = commit.message.lower() 83 | paths = [] 84 | for i in range(len(commit.files)): paths.append(commit.files[i].lower()) 85 | 86 | map = getMap() 87 | keywords = set() 88 | 89 | for k in map.nodes(): 90 | k_type = map.node_attributes(k)[0][1] 91 | 92 | #Don't look for mapping keywords anywhere, they'll be applied by graph travel from another keyword 93 | if k_type == KeywordType.MAPPING: 94 | continue 95 | 96 | kregex = map.node_attributes(k)[1][1] 97 | 98 | if k in log and kregex.search(" " + log): 99 | #Do not apply the base tag for KeywordType.APICALL 100 | if k_type != KeywordType.APICALL: 101 | keywords.add(k) 102 | 103 | #Apply all children tags 104 | keywords = keywords.union(getAllChildTags(map, k)) 105 | #Don't do this expensive check if we added it based on the log message 106 | else: 107 | for d in diffs: 108 | if k in d and kregex.search(" " + d): 109 | #Do not apply the base tag for KeywordType.APICALL 110 | if k_type != KeywordType.APICALL: 111 | keywords.add(k) 112 | 113 | #Apply all children tags 114 | keywords = keywords.union(getAllChildTags(map, k)) 115 | 116 | 117 | #Do not do path search for APICALL keywords 118 | if k_type == KeywordType.APICALL: 119 | continue 120 | 121 | for p in paths: 122 | #Don't check if we already have this keyword 123 | if k not in keywords and k in p and kregex.search(" " + p): 124 | keywords.add(k) 125 | keywords = keywords.union(getAllChildTags(map, k)) 126 | return keywords 127 | 128 | 129 | def projectizeTags(tokens): 130 | map = getMap() 131 | for i in range(len(tokens)): 132 | if map.has_node('project-' + tokens[i]): 133 | tokens[i] = 'project-' + tokens[i] 134 | return tokens 135 | 136 | if __name__ == "__main__": 137 | map = getMap() 138 | for n in map.nodes(): 139 | print n 140 | print "\t", map.neighbors(n) 141 | -------------------------------------------------------------------------------- /databaseQueries.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import MySQLdb, time 4 | 5 | from common import * 6 | from config import Config 7 | from database import DB 8 | from keywordsfilter import * 9 | 10 | from repo import Repo 11 | from commit import Commit 12 | from gitcommit import GitCommit 13 | from svncommit import SVNCommit 14 | 15 | class DBQ: 16 | @staticmethod 17 | def find(query, components): 18 | conn = DB.getConn() 19 | c = conn.cursor() 20 | 21 | c.execute(query, components) 22 | commitrows = c.fetchall() 23 | commitfiles = [] 24 | 25 | if commitrows: 26 | allcommitids = ",".join([str(int(commit[0])) for commit in commitrows]) 27 | 28 | #This is poor practice, but we assured ourselves the value is composed only of ints first 29 | DB.execute(c, "SELECT * from " + DB.commitfile._table + " WHERE commitid IN (" + allcommitids + ")") 30 | commitfiles = c.fetchall() 31 | 32 | DB.execute(c, "SELECT * from " + DB.commitkeyword._table + " WHERE commitid IN (" + allcommitids + ")") 33 | commitkeywords = c.fetchall() 34 | 35 | DB.execute(c, "SELECT commitid, case when length(data) < 307200 then data else 'TOOLARGE' end as data from " + DB.commitdiffs._table + " WHERE commitid IN (" + allcommitids + ")") 36 | commitdata = c.fetchall() 37 | 38 | 39 | commits = [] 40 | for i in commitrows: 41 | r = Repo() 42 | r.loadFromValues(i[DB.commit._numColumns + DB.repo.id], i[DB.commit._numColumns + DB.repo.name], i[DB.commit._numColumns + DB.repo.repotypeid], i[DB.commit._numColumns + DB.repo.url], 43 | i[DB.commit._numColumns + DB.repo.viewlink], i[DB.commit._numColumns + DB.repo.tagname], i[DB.commit._numColumns + DB.repo.tagmaturity]) 44 | 45 | files = [file[DB.commitfile.file] for file in commitfiles 46 | if file[DB.commitfile.commitid] == i[DB.commit.id]] 47 | keywords = [keyword[DB.commitkeyword.keyword] for keyword in commitkeywords 48 | if keyword[DB.commitkeyword.commitid] == i[DB.commit.id]] 49 | data = [cdata[DB.commitdiffs.data] for cdata in commitdata 50 | if cdata[DB.commitdiffs.commitid] == i[DB.commit.id]][0] 51 | 52 | if i[DB.commit._numColumns + DB.repo.repotypeid] == Repo.Type.GIT: 53 | c = GitCommit() 54 | elif i[DB.commit._numColumns + DB.repo.repotypeid] == Repo.Type.SVN: 55 | c = SVNCommit() 56 | else: 57 | c = Commit() 58 | c.loadFromDatabase(r, i, files, keywords, data) 59 | 60 | commits.append(c) 61 | 62 | return commits 63 | 64 | @staticmethod 65 | def findByKeywords(keywords, moreRecentThan=0): 66 | getcommitsSQL = "SELECT c.*, r.* " + \ 67 | "FROM " + DB.commit._table + " c " + \ 68 | "INNER JOIN " + DB.repo._table + " r " + \ 69 | " ON r.id = c.repoid " 70 | 71 | whereClause = " 1=1 " 72 | components = [] 73 | if keywords: 74 | keywordsTree = KeywordsParser(keywords) 75 | whereClause, components = keywordsTree.getEvaluationString('sql') 76 | 77 | 78 | getcommitsSQL += "WHERE " 79 | if moreRecentThan > 0: getcommitsSQL += " c.date > " + str(int(moreRecentThan)) + " AND " 80 | getcommitsSQL += whereClause + " " 81 | getcommitsSQL += "ORDER BY c.date DESC " 82 | 83 | return DBQ.find(getcommitsSQL, components) 84 | 85 | @staticmethod 86 | def findByIDs(project, uniqueid, moreRecentThan=0): 87 | getcommitsSQL = "SELECT c.*, r.* " + \ 88 | "FROM " + DB.commit._table + " c " + \ 89 | "INNER JOIN " + DB.repo._table + " r " + \ 90 | " ON r.id = c.repoid " 91 | 92 | whereClause = " 1=1 " 93 | components = [] 94 | if project and uniqueid: 95 | whereClause += "AND r.tagname = %s AND c.uniqueid = %s " 96 | components = [project, uniqueid] 97 | 98 | getcommitsSQL += "WHERE " 99 | if moreRecentThan > 0: getcommitsSQL += " c.date > " + str(int(moreRecentThan)) + " AND " 100 | getcommitsSQL += whereClause + " " 101 | getcommitsSQL += "ORDER BY c.date DESC " 102 | 103 | return DBQ.find(getcommitsSQL, components) 104 | 105 | @staticmethod 106 | def findByKeywordsAndFulltext(keywords, moreRecentThan=0): 107 | getcommitsSQL = "SELECT c.*, r.* " + \ 108 | "FROM " + DB.commit._table + " c " + \ 109 | "INNER JOIN " + DB.repo._table + " r " + \ 110 | " ON r.id = c.repoid " 111 | 112 | whereClause = " 1=1 " 113 | components = [] 114 | if keywords: 115 | keywordsTree = KeywordsParser(keywords) 116 | whereClause, components = keywordsTree.getEvaluationString('sql') 117 | 118 | getcommitsSQL += "WHERE " 119 | if moreRecentThan > 0: getcommitsSQL += " c.date > " + str(int(moreRecentThan)) + " AND " 120 | getcommitsSQL += whereClause + " " 121 | getcommitsSQL += "ORDER BY c.date DESC " 122 | 123 | prelim_commits = DBQ.find(getcommitsSQL, components) 124 | #This test is done well enough by the database for now. Maybe eventually we'll need to turn it back on 125 | #if keywords and keywordsTree.anyFulltext(): 126 | # final_commits = [] 127 | # 128 | # evalstr, evalcomponents = keywordsTree.getEvaluationString('eval') 129 | # evalstr = evalstr % tuple(evalcomponents) 130 | # 131 | # for c in prelim_commits: 132 | # testResult = eval(evalstr) 133 | # if testResult: 134 | # final_commits.append(c) 135 | # 136 | # return final_commits 137 | #else: 138 | return prelim_commits 139 | @staticmethod 140 | def logTerms(ip, keywords): 141 | insertSQL = "INSERT INTO " + DB.searchqueries._table + "(timestamp, ip, terms) " + \ 142 | "VALUES(%s, INET_ATON(%s), %s) " 143 | conn = DB.getConn() 144 | c = conn.cursor() 145 | 146 | DB.execute(c, insertSQL, (int(time.time()), ip, keywords)) 147 | conn.commit() 148 | -------------------------------------------------------------------------------- /commit.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import time, re, os, MySQLdb, unicodedata, cPickle, zlib 4 | from PyRSS2Gen import RSSItem, Guid 5 | from jinja2 import Markup 6 | 7 | from config import Config 8 | import synonymmapping 9 | from common import * 10 | from database import DB 11 | import gdiff 12 | 13 | re_gitsvn = re.compile('git-svn-id: \w+://.+ \w{4,12}-\w{4,12}-\w{4,12}-\w{4,12}-\w{4,12}') 14 | 15 | svn_diff_header = re.compile('Index: .+\s=+\s-{3} .+\s\+{3} .+\s@@\ [0-9\-, \+]+@@\s+') 16 | svn_diff_newline = re.compile('\\ No newline at end of file\s') 17 | svn_diff_property = re.compile('Property changes on: .+\s_+\sAdded: .+\s\s+[\-\+]\s.+') 18 | svn_diff_deletions = re.compile('^-.+$', re.MULTILINE) 19 | 20 | punctuation = re.compile('[^a-zA-Z0-9 ]') 21 | 22 | class Commit: 23 | repo = None 24 | message = '' 25 | date = 0 26 | files = [] 27 | commitid = -1 28 | changedTexts = None 29 | changedTexts_data = None 30 | diffIsReallyBig = False 31 | 32 | initialized = False 33 | def __init__(self): 34 | self.changedTexts = None 35 | self.diffIsReallyBig = False 36 | 37 | def loadFromSource(self, repo, m, d, files, uid, diffs): 38 | self.initialized = True 39 | 40 | self.repo = repo 41 | self.message = Commit.cleanUpCommitMessage(m) 42 | self.date = d 43 | self.files = files 44 | self.uniqueid = uid 45 | 46 | self.base_paths = self.getBasePath() 47 | self.dbkeywords = self.getSynonyms(diffs) 48 | 49 | self.keywords = set(self.dbkeywords) 50 | self.keywords.add('project-' + repo.tagname) 51 | self.keywords.add('maturity-' + repo.tagmaturity) 52 | 53 | def loadFromDatabase(self, repo, row, files, keywords, data): 54 | self.initialized = True 55 | self.repo = repo 56 | self.commitid = row[DB.commit.id] 57 | self.message = row[DB.commit.message] 58 | self.date = row[DB.commit.date] 59 | self.uniqueid = row[DB.commit.uniqueid] 60 | 61 | self.files = files 62 | self.base_paths = self.getBasePath() 63 | self.dbkeywords = keywords 64 | 65 | self.keywords = set(self.dbkeywords) 66 | self.keywords.add('project-' + repo.tagname) 67 | self.keywords.add('maturity-' + repo.tagmaturity) 68 | 69 | self.changedTexts_data = data 70 | if self.changedTexts_data == 'TOOLARGE': 71 | self.diffIsReallyBig = True 72 | 73 | @staticmethod 74 | def cleanUpCommitMessage(msg): 75 | msg = re_gitsvn.sub('', msg) 76 | return msg.strip() 77 | 78 | def getBasePath(self): 79 | if not self.initialized: 80 | raise Exception("called getBasePath on unitialized Commit object") 81 | 82 | if not self.files: return "" 83 | 84 | paths = set() 85 | for f in self.files: 86 | paths.add(os.path.dirname(f)) 87 | l = [p for p in paths if p] 88 | l.sort() 89 | return l 90 | #Implemented in Child Classes 91 | #Returns an array of google diff structures used for google diff pretty-outputting 92 | def getDiffsArray(self): 93 | pass 94 | #retrieves the metadata the child class needs for getChangedTexts from the populated object 95 | # during creation, this metadata is passed in independently and this is not called 96 | def getChangedTextMetadata(self): 97 | pass 98 | #Returns an array of changed texts used for synonym matching, not google's diff format 99 | def getChangedTexts(self, metadata): 100 | pass 101 | #/Implemented in Child Classes 102 | def _loadChangedTextFromBackingVar(self): 103 | if self.changedTexts != None: 104 | return self.changedTexts 105 | elif self.changedTexts_data == 'TOOLARGE': 106 | self.diffIsReallyBig = True 107 | self.changedTexts = [] 108 | else: 109 | data = zlib.decompress(self.changedTexts_data) 110 | data = cPickle.loads(data) 111 | self.changedTexts = data 112 | return self.changedTexts 113 | def testFulltext(self, fulltext): 114 | #We don't want to fall back on an expensive operation in the subsequent function 115 | if self.changedTexts == None and self.changedTexts_data == None: 116 | raise Exception("called testFulltext prior to changedTexts_data being initialized") 117 | 118 | for d in self.getChangedTexts(None): 119 | if fulltext in d.lower(): return True 120 | 121 | return False 122 | 123 | def getPrettyDiffs(self, htmlize=True): 124 | if not htmlize: 125 | raise Exception("Do not know how to not htmlize prettyDiffs") 126 | 127 | diffs = self.getDiffsArray() 128 | differ = gdiff.diff_match_patch() 129 | 130 | for d in diffs: 131 | differ.diff_cleanupSemantic(d) 132 | str = differ.diff_prettyHtml(d) 133 | if not isinstance(str, unicode): 134 | str = unicode(str, 'utf-8') 135 | else: 136 | str = str.encode('utf-8') 137 | yield Markup(str) 138 | 139 | def getSynonyms(self, diffs): 140 | if not self.initialized: 141 | raise Exception("called getSynonyms on unitialized Commit object") 142 | 143 | keywords = synonymmapping.getTags(self, diffs) 144 | 145 | return keywords 146 | 147 | def save(self): 148 | if not self.initialized: 149 | raise Exception("called save on unitialized Commit object") 150 | 151 | conn = DB.getConn() 152 | c = conn.cursor() 153 | sql = "INSERT INTO " + DB.commit._table + """(repoid, date, message, uniqueid) 154 | VALUES(%s, %s, %s, %s) 155 | ON DUPLICATE KEY UPDATE uniqueid = VALUES(uniqueid)""" 156 | c.execute(sql, (self.repo.id, self.date, self.message, self.uniqueid)) 157 | 158 | if self.commitid <= 0: 159 | self.commitid = conn.insert_id() 160 | 161 | data = self.getChangedTexts(None) 162 | data = cPickle.dumps(data, cPickle.HIGHEST_PROTOCOL) 163 | data = zlib.compress(data) 164 | 165 | 166 | sql = "DELETE FROM " + DB.commitdiffs._table + " WHERE commitid = " + str(self.commitid) 167 | c.execute(sql) 168 | 169 | sql = "INSERT INTO " + DB.commitdiffs._table + "(commitid, data) " 170 | sql += "VALUES(" + str(self.commitid) + ", %s)" 171 | c.execute(sql, [data]) 172 | 173 | if self.files: 174 | sql = "DELETE FROM " + DB.commitfile._table + " WHERE commitid = " + str(self.commitid) 175 | c.execute(sql) 176 | 177 | sql = "INSERT INTO " + DB.commitfile._table + "(commitid, file) " 178 | for f in self.files: 179 | sql += "SELECT " + str(self.commitid) + ", %s UNION " 180 | sql = sql[:-6] 181 | c.execute(sql, self.files) 182 | 183 | if self.dbkeywords: 184 | sql = "DELETE FROM " + DB.commitkeyword._table + " WHERE commitid = " + str(self.commitid) 185 | c.execute(sql) 186 | 187 | sql = "INSERT INTO " + DB.commitkeyword._table + "(commitid, keyword) " 188 | for f in self.dbkeywords: 189 | sql += "SELECT " + str(self.commitid) + ", %s UNION " 190 | sql = sql[:-6] 191 | c.execute(sql, [x for x in self.dbkeywords]) 192 | 193 | sql = "DELETE FROM " + DB.commitwordmap._table + " WHERE commitid = " + str(self.commitid) 194 | c.execute(sql) 195 | 196 | data = self.getChangedTexts(None) 197 | data.append(self.message.lower()) 198 | data.extend([f.lower() for f in self.files]) 199 | data = [punctuation.sub(' ', d) for d in data] 200 | 201 | allwords = set() 202 | for d in data: 203 | dWords = d.split() 204 | dTotalIndex = 0 205 | while dTotalIndex < len(dWords): 206 | words = [] 207 | dThisIndex = 0 208 | sql = "INSERT INTO " + DB.commitwordmap._table + "(commitid, word) " 209 | while dThisIndex < 500 and dTotalIndex < len(dWords): 210 | w = dWords[dThisIndex][:50] 211 | if len(w) > 2 and w not in allwords: 212 | sql += "SELECT " + str(self.commitid) + ", %s UNION " 213 | words.append(w) 214 | allwords.add(w) 215 | dThisIndex += 1 216 | dTotalIndex += 1 217 | sql = sql[:-6] 218 | if words: 219 | DB.execute(c, sql, words) 220 | 221 | conn.commit() 222 | 223 | def getpprint(self, htmlize=False): 224 | if not self.initialized: 225 | raise Exception("called getpprint on unitialized Commit object") 226 | 227 | if htmlize: 228 | process = lambda x : Markup.escape(x) 229 | else: 230 | process = lambda x : x 231 | 232 | eol = "\r\n" 233 | s = "" 234 | s += "Project:\t %s%s" % (self.repo.name, eol) 235 | if htmlize: 236 | s += "Project URL:\t %s%s" % (self.repo.url, self.repo.url, eol) 237 | else: 238 | s += "Project URL:\t %s %s" % (self.repo.url, eol) 239 | s += "Commit Date:\t %s (%s)%s" % (unixToGitDateFormat(self.date), self.date, eol) 240 | s += "Log Message:\t %s%s" % (process(self.message), eol) 241 | s += eol + eol 242 | if self.files: 243 | s += "Files:\t\t %s%s" % (process(self.files[0]), eol) 244 | for p in self.files[1:14]: 245 | s += "\t\t %s%s" % (process(p), eol) 246 | if len(self.files) > 15: 247 | s += "\t\t ...%s" % (eol) 248 | if self.base_paths: 249 | plural = len(self.base_paths) > 1 250 | s += "Base Path%s:\t %s%s" % ("s" if plural else "", process(self.base_paths[0]), eol) 251 | for p in self.base_paths[1:]: 252 | s += "\t\t %s%s" % (process(p), eol) 253 | 254 | s += "Keywords:\t %s%s" % (", ".join(self.keywords), eol) 255 | s += "ID:\t\t %s%s" % (self.uniqueid, eol) 256 | 257 | internallink = Config.rooturl + "/commit/" + self.repo.tagname + "/" + self.uniqueid 258 | if htmlize: 259 | s += "Internal:\t %s%s" % (internallink, internallink, eol) 260 | else: 261 | s += "Internal:\t %s%s" % (internallink, eol) 262 | 263 | if self.repo.viewlink: 264 | externallink = self.repo.viewlink.replace('%ID', self.uniqueid) 265 | if htmlize: 266 | s += "External:\t %s%s" % (externallink, externallink, eol) 267 | else: 268 | s += "External:\t %s%s" % (externallink, eol) 269 | if htmlize: 270 | return Markup(s) 271 | else: 272 | return s 273 | 274 | def pprint(self): 275 | print self.getpprint(False) 276 | 277 | def toRSSItem(self): 278 | title = self.repo.tagname 279 | 280 | if self.message and len(self.message) > 50: title += " - " + str(Markup.escape(self.message[:50])) + "..." 281 | elif self.message: title += " - " + str(Markup.escape(self.message)) 282 | 283 | if self.dbkeywords: title += " - " + ",".join(self.dbkeywords) 284 | 285 | description = "
"
286 |         description += str(self.getpprint(True))
287 |         description += "
" 288 | 289 | if type(title) != unicode: 290 | title = unicode(title, 'utf-8') 291 | if type(description) != unicode: 292 | description = unicode(description, 'utf-8') 293 | title = unicodedata.normalize('NFKD', title).encode('ascii', 'ignore') 294 | description = unicodedata.normalize('NFKD', description).encode('ascii', 'ignore') 295 | 296 | guid = Config.rooturl + "/commit/" + self.repo.tagname + "/" + self.uniqueid 297 | link = '' 298 | if self.repo.viewlink: 299 | link = self.repo.viewlink.replace('%ID', self.uniqueid) 300 | else: 301 | link = guid 302 | 303 | item = RSSItem( 304 | title = title, 305 | link = link, 306 | description = description, 307 | guid = Guid(guid, isPermaLink=0), 308 | pubDate = unixToDatetime(self.date) 309 | ) 310 | return item 311 | 312 | -------------------------------------------------------------------------------- /tags.txt: -------------------------------------------------------------------------------- 1 | project-libgcrypt : crypto-library 2 | project-schleuder : mailinglist 3 | project-sels : mailinglist 4 | project-secure-list-server : mailinglist 5 | project-mmreencrypt : mailinglist 6 | project-shibboleth : mailinglist 7 | project-crypt-ml : mailinglist 8 | 9 | openssl : crypto-library 10 | gnutls : crypto-library 11 | botan : crypto-library 12 | bouncycastle : crypto-library 13 | borzoi : crypto-library 14 | cryptlib : crypto-library 15 | cryptix : crypto-library 16 | libgcrypt : crypto-library 17 | libtomcrypt : crypto-library 18 | nss : crypto-library 19 | 20 | project-onak : keyserver 21 | 22 | project-gnupg : filecrypto 23 | gpg : project-gnupg 24 | project-gnupg-for-android : gpg 25 | 26 | project-gnugpg-for-android : android 27 | project-android-ffmpeg : android 28 | project-android-ffmpeg-java : android 29 | 30 | project-sqlcipher : guardian-project 31 | project-iocipher : guardian-project 32 | project-libsqlfs : guardian-project 33 | project-gibberbot : guardian-project 34 | project-onionkit : guardian-project 35 | project-informacam : guardian-project 36 | project-informacam-server : guardian-project 37 | project-gnupg-for-android : guardian-project 38 | project-storymaker : guardian-project 39 | project-otrfileconverter : guardian-project 40 | project-cacert : guardian-project 41 | project-orweb : guardian-project 42 | project-ProxyMob : guardian-project 43 | project-obscuracam : guardian-project 44 | project-InTheClear : guardian-project 45 | project-android-ffmpeg : guardian-project 46 | project-android-ffmpeg-java : guardian-project 47 | project-odkformparser : guardian-project 48 | project-rss-epub-archiver : guardian-project 49 | 50 | project-aam2mail : remailer 51 | hsub : remailer 52 | esub : remailer 53 | project-nymserv : remailer 54 | pynchon : remailer 55 | project-pynchon : remailer 56 | underhill : remailer 57 | mixminion : remailer 58 | project-mixminion : remailer 59 | mixmaster : remailer 60 | project-mixmaster : remailer 61 | 62 | project-encfs : fde 63 | project-luks : fde 64 | 65 | project-cr-gpg : browser-plugin 66 | project-https-everywhere : browser-plugin 67 | 68 | project-mat : metadata 69 | 70 | project-enigmail : mailcient-plugin 71 | 72 | project-cr-gpg : email-crypto 73 | project-enigmail : email-crypto 74 | penango : email-crypto 75 | smime : email-crypto 76 | 77 | project-tor-debian-polipo : debian 78 | project-tor-debian-tor : debian 79 | 80 | debian : 81 | gentoo : 82 | ubuntu : 83 | osx : 84 | ppc : 85 | slackware : 86 | fedora : 87 | red hat : 88 | rhel : 89 | suse : 90 | windows : 91 | windows xp : windows 92 | windows vista : windows 93 | windows 7 : windows 94 | server 2003 : windows 95 | server 2008 : windows 96 | 97 | pthread : 98 | ctypes : 99 | 100 | valgrind : 101 | oprofile : 102 | doxygen : 103 | 104 | D_FORTIFY_SOURCE : compile-options 105 | fstack-protector-all : compile-options 106 | fwrapv : compile-options 107 | fPIE : compile-options 108 | ssp-buffer-size : compile-options 109 | fno-strict-aliasing : compile-options 110 | fno-strict-overflow : compile-options 111 | relro : compile-options 112 | 113 | design doc : designdoc 114 | 115 | maildir : 116 | 117 | oaep : crypto-padding 118 | oaep : RSA 119 | 120 | plaintext : crypto 121 | ciphertext : crypto 122 | symmetric-crypto : crypto 123 | asymmetric-crypto : crypto 124 | 125 | RSA : asymmetric-crypto 126 | elgamal : asymmetric-crypto 127 | ELG-E : elgamal 128 | ntru : asymmetric-crypto 129 | curve25519 : asymmetric-crypto 130 | privatekey : asymmetric-crypto 131 | publickey : asymmetric-crypto 132 | private-key : asymmetric-crypto 133 | public-key : asymmetric-crypto 134 | 135 | curve25519 : eliptic-curve-crypto 136 | ecdh : eliptic-curve-crypto 137 | ecdhc : eliptic-curve-crypto 138 | ecdsa : eliptic-curve-crypto 139 | 140 | stream-cipher : symmetric-crypto 141 | block-cipher : symmetric-crypto 142 | 143 | aes : block-cipher 144 | blowfish : block-cipher 145 | des : block-cipher 146 | 3des : block-cipher 147 | serpent : block-cipher 148 | twofish : block-cipher 149 | camellia : block-cipher 150 | cast-128 : block-cipher 151 | rc4 : stream-cipher 152 | salsa : stream-cipher 153 | idea : block-cipher 154 | tea : block-cipher 155 | xtea : block-cipher 156 | cast-256 : block-cipher 157 | gost : block-cipher 158 | kasumi : block-cipher 159 | threefish : block-cipher 160 | a5/1 : stream-cipher 161 | a5/2 : stream-cipher 162 | a5/3 : stream-cipher 163 | rijndael : aes 164 | cast5 : block-cipher 165 | 166 | ECB : crypto-mode-of-operation 167 | CBC : crypto-mode-of-operation 168 | PCBC : crypto-mode-of-operation 169 | CFB : crypto-mode-of-operation 170 | OFB : crypto-mode-of-operation 171 | CTR : crypto-mode-of-operation 172 | CCM : crypto-mode-of-operation 173 | GCM : crypto-mode-of-operation 174 | CMAC : crypto-mode-of-operation 175 | CBC-MAC : crypto-mode-of-operation 176 | OMAC : crypto-mode-of-operation 177 | crypto-mode-of-operation : block-cipher 178 | 179 | anonymous : anonymity 180 | alt.anonymous.messages : anonymity 181 | 182 | tlsv1 : tls 183 | tls certificate : tls 184 | ssl certificate : tls 185 | tls certificate : ssl certificate 186 | ssl certificate : tls certificate 187 | tls1 : tls 188 | tlsv1.1 : tls 189 | tls1.1 : tls 190 | tlsv1.2 : tls 191 | tls1.2 : tls 192 | sslv3 : ssl 193 | ssl3 : ssl 194 | sslv2 : ssl 195 | ssl2 : ssl 196 | tls : ssl 197 | ssl : tls 198 | BEAST : ssl 199 | 200 | server_name : tls-extensions 201 | max_fragment_length : tls-extensions 202 | client_certificate_url : tls-extensions 203 | trusted_ca_keys : tls-extensions 204 | truncated_hmac : tls-extensions 205 | status_request : tls-extensions 206 | signature_algorithms : tls-extensions 207 | user_mapping : tls-extensions 208 | client_authz : tls-extensions 209 | server_authz : tls-extensions 210 | cert_type : tls-extensions 211 | elliptic_curves : tls-extensions 212 | ec_point_formats : tls-extensions 213 | srp : tls-extensions 214 | use_srtp : tls-extensions 215 | heartbeat : tls-extensions 216 | renegotiation_info : tls-extensions 217 | SessionTicket : tls-extensions 218 | tls extensions : tls-extensions 219 | tls extension : tls-extensions 220 | tls-extensions : tls 221 | 222 | openpgp : 223 | sexp : openpgp 224 | BEGIN PGP SIGNED MESSAGE : openpgp 225 | BEGIN PGP MESSAGE : openpgp 226 | web of trust : 227 | trust on first use : 228 | 229 | pem : x509 230 | x.509 : x509 231 | public key infrastructure : 232 | certificate authority : 233 | notary : 234 | convergence : 235 | diginotar : 236 | 237 | hmac : 238 | 239 | dane : 240 | dkim : 241 | domain keys : 242 | 243 | ppp : 244 | pptp : 245 | 802.11a : 802.11 246 | 802.11b : 802.11 247 | 802.11g : 802.11 248 | 802.11n : 802.11 249 | tkip : 250 | 802.16 : wimax 251 | 252 | ipv4 : ip 253 | ipv6 : ip 254 | ipsec : ip 255 | sscp : 256 | arp : 257 | rarp : 258 | bgp : 259 | rip : 260 | ndp : 261 | tcp : 262 | udp : 263 | icmp : 264 | ipx : 265 | smb : 266 | cifs : smb 267 | 268 | pop3 : 269 | imap : 270 | soap : 271 | ftp : 272 | sftp : 273 | ssh : 274 | scp : 275 | telnet : 276 | dns : 277 | dnssec : dns 278 | http : 279 | https : ssl 280 | smtp : 281 | starttls : 282 | bittorrent : 283 | dhcp : 284 | finger : 285 | gopher : 286 | irc : 287 | ldap : 288 | mime : 289 | netbios : 290 | nntp : 291 | ntp : 292 | radius : 293 | rsync : 294 | sip : 295 | snmp : 296 | webdav : 297 | xmpp : 298 | bitcoin : 299 | namecoin : 300 | petname : 301 | 302 | 303 | fuse : 304 | 305 | git : 306 | svn : subversion 307 | hg : mercurial 308 | sourcesafe : visualsourcesafe 309 | vss : visualsourcesafe 310 | cvs : 311 | 312 | 313 | message digest : hash-functions 314 | message-digest : hash-functions 315 | messagedigest : hash-functions 316 | md2 : hash-functions 317 | md4 : hash-functions 318 | md5 : hash-functions 319 | sha224 : hash-functions 320 | sha256 : hash-functions 321 | sha384 : hash-functions 322 | sha512 : hash-functions 323 | sha-224 : hash-functions 324 | sha-256 : hash-functions 325 | sha-384 : hash-functions 326 | sha-512 : hash-functions 327 | ripemd : hash-functions 328 | tiger : hash-functions 329 | haval : hash-functions 330 | panama : hash-functions 331 | sha0 : hash-functions 332 | sha1 : hash-functions 333 | sha-0 : hash-functions 334 | sha-1 : hash-functions 335 | whirlpool : hash-functions 336 | 337 | blake : hash-functions 338 | grostl : hash-functions 339 | jh : hash-functions 340 | keccak : hash-functions 341 | skein : hash-functions 342 | 343 | blake : sha3 344 | grostl : sha3 345 | jh : sha3 346 | keccak : sha3 347 | skein : sha3 348 | 349 | sha224 : sha-family 350 | sha256 : sha-family 351 | sha384 : sha-family 352 | sha512 : sha-family 353 | sha-224 : sha-family 354 | sha-256 : sha-family 355 | sha-384 : sha-family 356 | sha-512 : sha-family 357 | sha0 : sha-family 358 | sha1 : sha-family 359 | sha-0 : sha-family 360 | sha-1 : sha-family 361 | 362 | key derivation : key-derivation 363 | KDF : key-derivation 364 | PBKDF2 : key-derivation 365 | KDF : key-derivation 366 | scrypt : key-derivation 367 | bcrypt : key-derivation 368 | 369 | hmac : mac 370 | message authentication code : mac 371 | messageauthenticationcode : mac 372 | message-authentication-code : mac 373 | 374 | project-gibberbot : otr 375 | otr : 376 | socks4 : socks 377 | socka4a : socks 378 | socks5 : socks 379 | socks : proxy 380 | proxies : proxy 381 | 382 | socat : 383 | 384 | tahoe : tahoe-lafs 385 | 386 | project-orweb : project-tor 387 | vidalia : project-tor 388 | torrc : project-tor 389 | 390 | prng : random-number-generator 391 | rng : random-number-generator 392 | random number generator : random-number-generator 393 | psuedonumber generator : random-number-generator 394 | psuedonumber : random-number-generator 395 | lfsr : random-number-generator 396 | linear feedback shift register : random-number-generator 397 | getSecureRandom : random-number-generator 398 | getPseudoRandom : random-number-generator 399 | SecureRandom : random-number-generator 400 | Secure Random : random-number-generator 401 | 402 | torproject : project-tor 403 | project-tor-arm : project-tor 404 | project-tor-bridgedb : project-tor 405 | project-tor-debian-polipo : project-tor 406 | project-tor-debian-tor : project-tor 407 | project-tor-flashproxy : project-tor 408 | project-tor-gettor : project-tor 409 | project-tor-jtorctl : project-tor 410 | project-tor-mat : project-tor 411 | project-tor-metrics-db : project-tor 412 | project-tor-metrics-tasks : project-tor 413 | project-tor-metrics-utils : project-tor 414 | project-tor-metrics-web : project-tor 415 | project-tor-obfsproxy : project-tor 416 | project-tor-orbot : project-tor 417 | project-tor-puppetor : project-tor 418 | project-tor-pytorctl : project-tor 419 | project-tor-thandy : project-tor 420 | project-torbutton : project-tor 421 | project-tor-dnsel : project-tor 422 | project-torsocks : project-tor 423 | project-torspec : project-tor 424 | project-vidalia-plugins : project-vidalia 425 | project-vidalia : project-tor 426 | 427 | bridgedb : tor-bridgedb 428 | flashproxy : tor-flashproxy 429 | gettor : tor-gettor 430 | jtorctl : tor-jtorctl 431 | obfsproxy : tor-obfsproxy 432 | orbot : tor-orbot 433 | puppetor : tor-puppetor 434 | pytorctl : tor-pytorctl 435 | thandy : tor-thandy 436 | dnsel : tor-dnsel 437 | 438 | french : translations 439 | spanish : translations 440 | german : translations 441 | burmese : translations 442 | bulgarian : translations 443 | czech : translations 444 | hebrew : translations 445 | greek : translations 446 | indonesian : translations 447 | korean : translations 448 | dutch : translations 449 | japanese : translations 450 | thai : translations 451 | albanian : translations 452 | vietnamese : translations 453 | chinese : translations 454 | polish : translations 455 | catalan : translations 456 | basque : translations 457 | slovak : translations 458 | i18n : translations 459 | 460 | cookie: cookies 461 | XMLHttpRequest : 462 | Access-Control-Allow-Origin : CORS 463 | Access-Control-Expose-Headers : CORS 464 | Access-Control-Max-Age : CORS 465 | Access-Control-Allow-Credentials : CORS 466 | Access-Control-Allow-Methods : CORS 467 | Access-Control-Allow-Headers : CORS 468 | Access-Control-Request-Method : CORS 469 | Access-Control-Request-Headers : CORS 470 | Strict-Transport-Security : HSTS 471 | Public-Key-Pins : Key-Pinning 472 | TACK : Key-Pinning 473 | WWW-Authenticate : 474 | X-Frame-Options : 475 | X-XSS-Protection : 476 | X-Content-Type-Options : 477 | 478 | 479 | ant : java 480 | nant : 481 | jdk : java 482 | jre : java 483 | jar : java 484 | javac : java 485 | jdb : java 486 | 487 | bluetooth : 488 | rfid : 489 | nfc : 490 | 491 | regex : regular expressions 492 | regular expression : regular expressions 493 | 494 | 495 | SO_BINDTODEVICE : socket 496 | SO_BROADCAST : socket 497 | SO_REUSEADDR : socket 498 | socket_timeout : socket 499 | setsockopt : socket 500 | getsockopt : socket 501 | 502 | ascii : unicode 503 | utf8 : unicode 504 | utf7 : unicode 505 | utf16 : unicode 506 | utf32 : unicode 507 | 508 | fuck : profanity 509 | shit : profanity 510 | crap : profanity 511 | bastard : profanity 512 | asshole : profanity 513 | dumb : profanity 514 | stupid : profanity 515 | 516 | mvc : 517 | mmvc : 518 | 519 | pickle : 520 | cPickle : pickle 521 | 522 | SAML : 523 | oAuth : 524 | OpenId : 525 | 526 | setuid : 527 | setgid : setuid 528 | seteuid : setuid 529 | setegid : setuid 530 | setfsuid : setuid 531 | setreuid : setuid 532 | setregid : setuid 533 | -------------------------------------------------------------------------------- /keywordsfilter.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import sys, re 4 | import ply.yacc as yacc 5 | import ply.lex as lex 6 | 7 | import synonymmapping 8 | from database import DB 9 | 10 | # Lex ================================================================= 11 | reserved = { 12 | 'and' : 'AND', 13 | 'or' : 'OR' 14 | } 15 | tokens = [ 16 | 'NAME', 'MULTINAME', 17 | 'LPAREN','RPAREN' 18 | ] + list(reserved.values()) 19 | t_LPAREN = r'\(' 20 | t_RPAREN = r'\)' 21 | 22 | #Be very, VERY careful about adding symbols to these regexes. 23 | # Adding a single or double quote could allow arbitrary code execution thanks to a eval function 24 | # Any changes must also be made to regex in setup and the regex in _stripIllegalCharacters 25 | def t_MULTINAME(t): 26 | r'"[-_a-zA-Z0-9./]+ [-_a-zA-Z0-9./ ]+"' 27 | t.type = reserved.get(t.value, 'MULTINAME') 28 | return t 29 | 30 | def t_NAME(t): 31 | r'"?[-_a-zA-Z0-9./]+"?' 32 | t.type = reserved.get(t.value, 'NAME') 33 | return t 34 | 35 | t_ignore = " \t\n" 36 | 37 | def t_error(t): 38 | print("Illegal character '%s'" % t.value[0]) 39 | t.lexer.skip(1) 40 | 41 | # Structure =========================================================== 42 | class Tree: 43 | left = None 44 | leftType = 'keyword' 45 | right = None 46 | rightType = 'keyword' 47 | 48 | _keywordphrase = "(SELECT ck.keyword FROM "+ DB.commitkeyword._table +" as ck WHERE ck.commitid = c.id)" 49 | _fulltextphrase_sql = "(SELECT wm.word FROM "+ DB.commitwordmap._table +" as wm WHERE wm.commitid = c.id)" 50 | _fulltextphrase_eval = "c.testFulltext('%s')" 51 | _projectphrase_sql = "r.tagname" 52 | _projectphrase_eval = "c.repo.tagname" 53 | _maturityphrase_sql = "r.maturity" 54 | _maturityphrase_eval = "c.repo.maturity" 55 | 56 | def __init__(self, l, r): 57 | self.left = l 58 | self.right = r 59 | 60 | doLeft = doRight = False 61 | if isinstance(self.left, str) or isinstance(self.left, unicode): 62 | doLeft = True 63 | self.leftType = 'keyword' 64 | else: 65 | self.leftType = 'tree' 66 | 67 | if isinstance(self.right, str) or isinstance(self.right, unicode): 68 | doRight = True 69 | self.rightType = 'keyword' 70 | elif self.right is None: 71 | self.rightType = 'None' 72 | else: 73 | self.rightType = 'tree' 74 | 75 | if not (doLeft or doRight): 76 | return 77 | 78 | if doLeft: self.leftType = 'fulltext' 79 | if doRight: self.rightType = 'fulltext' 80 | for n in synonymmapping.getMap(): 81 | if doLeft and n.__str__() == self.left: 82 | self.leftType = 'keyword' 83 | if doRight and n.__str__() == self.right: 84 | self.rightType = 'keyword' 85 | def __repr__(self): 86 | s = "(" + repr(self.left) + " " + self.mode + " " + repr(self.right) + ")" 87 | return s 88 | def _getEvalNode(self, type, node, nodeType): 89 | evalstring = "" 90 | components = [] 91 | if nodeType == 'keyword': 92 | if node.startswith("project-"): 93 | if type == 'sql': evalstring += " " + self._projectphrase_sql + " = %s" 94 | elif type == 'eval': evalstring += " " + self._projectphrase_eval + " == '%s'" 95 | components.append(node.replace('project-', '')) 96 | elif node.startswith("maturity-"): 97 | if type == 'sql': evalstring += " " + self._maturityphrase_sql + " = %s" 98 | elif type == 'eval': evalstring += " " + self._maturityphrase_eval + " == '%s'" 99 | components.append(node.replace('maturity-', '')) 100 | else: 101 | if type == 'sql': evalstring += "%s IN " + self._keywordphrase 102 | elif type == 'eval': evalstring += "'%s' in " + self._keywordphrase 103 | components.append(node) 104 | elif nodeType == 'fulltext': 105 | if type == 'sql': evalstring += "%s IN " + self._fulltextphrase_sql 106 | elif type == 'eval': evalstring += self._fulltextphrase_eval 107 | components.append(node) 108 | else: 109 | innersql, newcomponents = node.getEvaluationString(type) 110 | 111 | if node.right: evalstring += "(" + innersql + ")" 112 | else: evalstring += innersql 113 | 114 | components.extend(newcomponents) 115 | return evalstring, components 116 | def anyTree(self): 117 | if self.leftType == 'tree': 118 | return True 119 | if self.rightType == 'tree': 120 | return True 121 | return False 122 | def anyFulltext(self): 123 | if self.leftType == 'fulltext': 124 | return True 125 | if self.rightType == 'fulltext': 126 | return True 127 | if self.leftType == 'tree': 128 | if self.left.anyFulltext(): return True 129 | if self.rightType == 'tree': 130 | if self.right.anyFulltext(): return True 131 | 132 | return False 133 | def getEvaluationString(self, type): 134 | 135 | evalstring, components = self._getEvalNode(type, self.left, self.leftType) 136 | #We overload the AndTree to handle the single-node case 137 | if self.right: 138 | evalstring += " " + self.mode + " " 139 | 140 | nextsql, newcomponents = self._getEvalNode(type, self.right, self.rightType) 141 | evalstring += nextsql 142 | components.extend(newcomponents) 143 | return evalstring, components 144 | def printTree(self, indent): 145 | if not self.right: 146 | if self.leftType == 'tree': 147 | self.left.printTree(indent) 148 | else: 149 | print indent + self.left 150 | else: 151 | print indent + self.mode 152 | if self.leftType == 'tree': 153 | self.left.printTree(indent + "\t") 154 | else: 155 | print indent + self.left 156 | if self.rightType == 'tree': 157 | self.right.printTree(indent + "\t") 158 | else: 159 | print indent + self.right 160 | class AndTree(Tree): 161 | mode = "and" 162 | class OrTree(Tree): 163 | mode = "or" 164 | 165 | 166 | # Yacc ================================================================ 167 | 168 | def p_expression_name(p): 169 | '''expression : NAME 170 | | MULTINAME''' 171 | p[0] = AndTree(p[1].replace('"', ''), None) 172 | 173 | def p_expression_names(p): 174 | 'expression : expression expression' 175 | p[0] = AndTree(p[1], p[2]) 176 | 177 | def p_expression_and(p): 178 | 'expression : expression AND expression' 179 | p[0] = AndTree(p[1], p[3]) 180 | 181 | def p_expression_or(p): 182 | 'expression : expression OR expression' 183 | p[0] = OrTree(p[1], p[3]) 184 | 185 | def p_expression_parens(p): 186 | 'expression : LPAREN expression RPAREN' 187 | p[0] = AndTree(p[2], None) 188 | 189 | def p_error(p): 190 | print "Syntax error in input: " + str(p) + "!" 191 | 192 | lex.lex() 193 | yacc.yacc() 194 | 195 | # Interface =========================================================== 196 | unallowed_characters = re.compile('[^-_a-zA-Z0-9./ ()]') 197 | 198 | class KeywordsParser: 199 | @staticmethod 200 | def _isBalanced(keywords): 201 | balancedParens = 0 202 | for c in keywords: 203 | if c == "(": 204 | balancedParens += 1 205 | elif c == ")": 206 | balancedParens -= 1 207 | if balancedParens < 0: 208 | return False 209 | return balancedParens == 0 210 | @staticmethod 211 | def _trimnonsense(tokens): #get rid of beginning or ending combining words, including inside parens 212 | if tokens and tokens[0] in ["and", "or"]: 213 | tokens.pop(0) 214 | if tokens and tokens[-1] in ["and", "or"]: 215 | tokens.pop() 216 | for i in range(len(tokens) - 1): 217 | thistoken = tokens[i] 218 | nexttoken = tokens[i+1] 219 | if thistoken == "(": 220 | if nexttoken in ["and", "or"]: 221 | tokens.pop(i+1) 222 | return KeywordsParser._trimnonsense(tokens) 223 | return tokens 224 | @staticmethod 225 | def _combinenonsense(tokens): #collapse repeated combining words e.g. 'slackware and and sha256' 226 | if len(tokens) > 1: 227 | for i in range(len(tokens)-1): #remove successive combination words 228 | thistoken = tokens[i] 229 | nexttoken = tokens[i+1] 230 | if thistoken in ["and", "or"]: 231 | if nexttoken in ["and", "or"]: 232 | tokens.pop(i+1) 233 | return KeywordsParser._combinenonsense(tokens) 234 | for i in range(len(tokens)-1): #remove empty parens 235 | thistoken = tokens[i] 236 | nexttoken = tokens[i+1] 237 | if thistoken == "(" and nexttoken == ")": 238 | tokens.pop(i) 239 | tokens.pop(i) 240 | return KeywordsParser._combinenonsense(tokens) 241 | return tokens 242 | @staticmethod 243 | def _stripIllegalCharacters(tokens): #Violently remove characters not permitted by t_NAME regex 244 | tokens = unallowed_characters.sub('', tokens) 245 | return tokens 246 | @staticmethod 247 | def _preProcess(keywords): 248 | if not KeywordsParser._isBalanced(keywords): 249 | keywords = keywords.replace("(", "").replace(")", "") 250 | else: 251 | keywords = keywords.replace("(", " ( ").replace(")", " ) ") 252 | keywords = KeywordsParser._stripIllegalCharacters(keywords) 253 | tokens = keywords.lower().split() 254 | tokens = KeywordsParser._combinenonsense(tokens) 255 | tokens = KeywordsParser._trimnonsense(tokens) 256 | tokens = KeywordsParser._combinenonsense(tokens) 257 | 258 | tokens = synonymmapping.projectizeTags(tokens) 259 | 260 | return ' '.join(tokens) 261 | 262 | def __init__(self, keywords): 263 | self.keywords = KeywordsParser._preProcess(keywords) 264 | if self.keywords: 265 | self.result = yacc.parse(self.keywords) 266 | else: 267 | self.result = False 268 | 269 | def getEvaluationString(self, type): 270 | if self.result: 271 | return self.result.getEvaluationString(type) 272 | else: 273 | return ('', []) 274 | def anyFulltext(self): 275 | return self.result.anyFulltext() 276 | def dump(self): 277 | if self.result: 278 | evalstr, evalcomponents = self.result.getEvaluationString('eval') 279 | print "\t", evalstr % tuple(evalcomponents) 280 | 281 | evalstr, evalcomponents = self.result.getEvaluationString('sql') 282 | print "\t", evalstr, evalcomponents 283 | 284 | #self.result.printTree("\t") 285 | else: 286 | print "" 287 | 288 | if __name__ == "__main__": 289 | 290 | testcases = [ 291 | "slackware " 292 | , "tag1 " 293 | , "\"slackware\" " 294 | ,"and " 295 | ,"slackware testcases-git" 296 | ,"slackware or (tag1 and doxygen)" 297 | ,"slackware and testcases-git" 298 | ,"slackware or sha256" 299 | ,"slackware testcases-git maturity-suse" 300 | ,'"slackware testcases-git" maturity-suse' 301 | ,"slackware and and testcases-git maturity-suse" 302 | ,"slackware testcases-git and and maturity-suse" 303 | ,"slackware testcases-git and and maturity-suse and and" 304 | ,"and and slackware testcases-git and and maturity-suse and and" 305 | ,"slackware testcases-git and maturity-suse" 306 | ,"slackware and testcases-git maturity-suse" 307 | ,"slackware and testcases-git or maturity-suse" 308 | ,"or" 309 | ,"slackware testcases-git" 310 | ,"slackware or testcases-git" 311 | ,"slackware or testcases-git" 312 | ,"slackware testcases-git maturity-suse" 313 | ,"slackware or or testcases-git maturity-suse" 314 | ,"slackware testcases-git or or maturity-suse" 315 | ,"slackware testcases-git or or maturity-suse or or" 316 | ,"or or slackware testcases-git or or maturity-suse or or" 317 | ,"slackware testcases-git or maturity-suse" 318 | ,"slackware or testcases-git maturity-suse" 319 | ,"slackware or testcases-git or maturity-suse" 320 | ,"slackware and testcases-git or maturity-suse and tag4" 321 | ,")slackware( " 322 | ,"(and) " 323 | ,"((slackware) testcases-git)" 324 | ,"(slackware and testcases-git)" 325 | ,"(((slackware or sha256)))" 326 | ,"(slackware testcases-git) maturity-suse" 327 | ,"(slackware and and testcases-git) maturity-suse" 328 | ,"slackware (testcases-git and and maturity-suse)" 329 | ,"(slackware testcases-git) and and maturity-suse and and" 330 | ,"and and (slackware testcases-git) and and maturity-suse and and" 331 | ,"(slackware testcases-git) and maturity-suse" 332 | ,"(())()()()(())slackware and testcases-git maturity-suse" 333 | ,"(slackware and testcases-git) or maturity-suse" 334 | ,"slackware and (testcases-git or maturity-suse)" 335 | ,"(slackware or testcases-git) and maturity-suse" 336 | ,"slackware or (testcases-git and maturity-suse)" 337 | ,"(slackware and testcases-git) or (maturity-suse and tag4)" 338 | ,"slackware and (testcases-git or maturity-suse) and tag4" 339 | ,"slackware and (testcases-git or maturity-suse and tag4)" 340 | ,"(slackware and testcases-git or maturity-suse) and tag4" 341 | ,"(slackware and (testcases-git or maturity-suse)) and tag4" 342 | ,"slackware and ((testcases-git or maturity-suse) and tag4)" 343 | 344 | ,"tag1" 345 | ,"tag1' x" 346 | ,"tag1' or 1==1 '" 347 | ] 348 | 349 | 350 | for t in testcases: 351 | tree = KeywordsParser(t) 352 | print t 353 | tree.dump() 354 | print "" 355 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import MySQLdb, argparse, os, re 4 | from database import DB 5 | 6 | 7 | if __name__ == "__main__": 8 | parser = argparse.ArgumentParser() 9 | parser.add_argument('--wipe', dest='wipe', action='store_true', help='Instead of creating tables if they don\'t exist - empty the database.') 10 | parser.add_argument('--populate', dest='populate', action='store_true', help='If a reference data table is created, also populate it with some test data. Mandatory refdata tables will be populated regardless.') 11 | parser.add_argument('--testpopulate', dest='testpopulate', action='store_true', help='Only populate the testcases repo. Use with --populate') 12 | parser.add_argument('--keywords', dest='keywords', action='store_true', help='Only reprocess the kwywords table.') 13 | args = parser.parse_args() 14 | 15 | 16 | 17 | conn = DB.getConn() 18 | 19 | c = conn.cursor() 20 | 21 | if args.wipe: 22 | try: 23 | c.execute("DROP TABLE " + DB.repotype._table) 24 | except: 25 | pass 26 | try: 27 | c.execute("DROP TABLE " + DB.repo._table) 28 | except: 29 | pass 30 | try: 31 | c.execute("DROP TABLE " + DB.keyword._table) 32 | except: 33 | pass 34 | try: 35 | c.execute("DROP TABLE " + DB.commit._table) 36 | except: 37 | pass 38 | try: 39 | c.execute("DROP TABLE " + DB.commitkeyword._table) 40 | except: 41 | pass 42 | try: 43 | c.execute("DROP TABLE " + DB.commitfile._table) 44 | except: 45 | pass 46 | try: 47 | c.execute("DROP TABLE " + DB.commitdiffs._table) 48 | except: 49 | pass 50 | try: 51 | c.execute("DROP TABLE " + DB.searchqueries._table) 52 | except: 53 | pass 54 | 55 | try: 56 | c.execute("DROP TABLE " + DB.commitwordmap._table) 57 | except: 58 | pass 59 | 60 | else: 61 | #repotype._table + """ ------------------------------------------- 62 | c.execute("SHOW TABLES LIKE '" + DB.repotype._table + "'") 63 | r = c.fetchone() 64 | if r: 65 | print "Repo Type Table Exists" 66 | else: 67 | print "Creating Repo Type Table..." 68 | repotype = "CREATE TABLE " + DB.repotype._table + """ 69 | ( 70 | id smallint NOT NULL PRIMARY KEY, 71 | type varchar(10) NOT NULL UNIQUE 72 | ) COLLATE utf8_general_ci ENGINE=innodb; 73 | """ 74 | c.execute(repotype) 75 | 76 | print 'Populating Repo Type...' 77 | repotype = "INSERT INTO " + DB.repotype._table + """(id, type) 78 | SELECT 1, 'svn' UNION 79 | SELECT 2, 'git' UNION 80 | SELECT 3, 'cvs' UNION 81 | SELECT 4, 'rss' UNION 82 | SELECT 5, 'email' UNION 83 | SELECT 6, 'bazaar' UNION 84 | SELECT 7, 'mercurial' UNION 85 | SELECT 8, 'targz' UNION 86 | SELECT 9, 'darcs' 87 | """ 88 | c.execute(repotype) 89 | 90 | #repos._table + """ --------------------------------------------- 91 | c.execute("SHOW TABLES LIKE '" + DB.repo._table + "'") 92 | r = c.fetchone() 93 | if r: 94 | print "Repo Table Exists" 95 | else: 96 | print "Creating Repos Table..." 97 | sql = "CREATE TABLE " + DB.repo._table + """ 98 | ( 99 | id smallint NOT NULL AUTO_INCREMENT PRIMARY KEY, 100 | name varchar(255) NOT NULL, 101 | repotypeid tinyint NOT NULL, 102 | url varchar(255) NOT NULL UNIQUE, 103 | viewlink varchar(512) NULL, 104 | tagname varchar(30) NOT NULL, 105 | maturity varchar(20) NOT NULL 106 | ) COLLATE utf8_general_ci ENGINE=innodb; 107 | """ 108 | c.execute(sql) 109 | 110 | if args.testpopulate: 111 | print 'Populating Repos...' 112 | sql = "INSERT INTO " + DB.repo._table + """(repotypeid, name, url, viewlink, tagname, maturity) 113 | SELECT 2, 'Git Test Cases', 'git://github.com/tomrittervg/Code-Audit-Feed-Test-Cases.git', 'https://github.com/tomrittervg/Code-Audit-Feed-Test-Cases/commit/%ID', 'testcases-git', 'development' UNION 114 | SELECT 1, 'Subversion Test Cases', 'http://code-audit-feed-testcases.googlecode.com/svn/trunk/', 'http://code.google.com/p/code-audit-feed-testcases/source/detail?r=%ID', 'testcases-svn', 'development' """ 115 | 116 | DB.execute(c, sql) 117 | elif args.populate: 118 | print 'Populating Repos...' 119 | sql = "INSERT INTO " + DB.repo._table + """(repotypeid, name, url, viewlink, tagname, maturity) 120 | SELECT 2, 'Crypto.is Docs', 'https://github.com/cryptodotis/crypto.is-docs', 'https://github.com/cryptodotis/crypto.is-docs/commit/%ID', 'crypto.is-docs', 'beta' UNION 121 | SELECT 2, 'Convergence', 'https://github.com/moxie0/Convergence.git', 'https://github.com/moxie0/Convergence/commit/%ID', 'convergence', 'beta' UNION 122 | SELECT 2, 'Obfuscated OpenSSH', 'https://github.com/brl/obfuscated-openssh', 'https://github.com/brl/obfuscated-openssh/commit/%ID', 'obfuscated-openssh', 'stable' UNION 123 | SELECT 1, 'Phantom', 'http://phantom.googlecode.com/svn/trunk/', 'http://code.google.com/p/phantom/source/detail?r=%ID', 'phantom', 'development' UNION 124 | SELECT 8, 'Corkscrew', 'http://www.agroman.net/corkscrew/', NULL, 'corkscrew', 'development' UNION 125 | SELECT 9, 'Tahoe-LAFS', 'http://tahoe-lafs.org/source/tahoe-lafs/trunk/', 'http://tahoe-lafs.org/trac/tahoe-lafs/changeset?old_path=%2Ftrunk&old=%ID&new_path=%2Ftrunk&new=%ID', 'tahoe-lafs', 'beta' UNION 126 | SELECT 2, 'Briar Prototype', 'git://briar.git.sourceforge.net/gitroot/briar/prototype', 'http://briar.git.sourceforge.net/git/gitweb.cgi?p=briar/prototype;a=commitdiff;h=%ID', 'briar', 'development' UNION 127 | SELECT 2, 'Briar Docs', 'git://briar.git.sourceforge.net/gitroot/briar/docs', 'http://briar.git.sourceforge.net/git/gitweb.cgi?p=briar/docs;a=commitdiff;h=%ID', 'briar-docs', 'development' UNION 128 | SELECT 2, 'Metadata Anonymization Toolkit', 'https://git.torproject.org/user/jvoisin/mat.git', 'https://gitweb.torproject.org/user/jvoisin/mat.git/commitdiff/%ID', 'mat', 'beta' UNION 129 | SELECT 2, 'Encounter', 'https://github.com/secYOUre/Encounter.git', 'https://github.com/secYOUre/Encounter/commit/%ID', 'encounter', 'development' UNION 130 | SELECT 2, 'Batphone', 'https://github.com/servalproject/batphone.git', 'https://github.com/servalproject/batphone/commit/%ID', 'serval', 'development' UNION 131 | SELECT 8, 'Haveged', 'http://www.issihosts.com/haveged/', '', 'haveged', 'beta' UNION""" 132 | 133 | #tor 134 | sql += """ 135 | SELECT 2, 'Command-line status monitor for Tor', 'https://git.torproject.org/arm.git', 'https://gitweb.torproject.org/arm.git/commitdiff/%ID', 'tor-arm', 'stable' UNION 136 | SELECT 2, 'BridgeDB', 'https://git.torproject.org/bridgedb.git', 'https://gitweb.torproject.org/bridgedb.git/commitdiff/%ID', 'tor-bridgedb', 'stable' UNION 137 | SELECT 2, 'Tor on Debian', 'https://git.torproject.org/debian/tor.git', 'https://gitweb.torproject.org/debian/tor.git/commitdiff/%ID', 'tor-debian-tor', 'pervasive' UNION 138 | SELECT 2, 'Tor Flashproxy', 'https://git.torproject.org/flashproxy.git', 'https://gitweb.torproject.org/flashproxy.git/commitdiff/%ID', 'tor-flashproxy', 'stable' UNION 139 | SELECT 2, 'GetTor', 'https://git.torproject.org/gettor.git', 'https://gitweb.torproject.org/gettor.git/commitdiff/%ID', 'tor-gettor', 'pervasive' UNION 140 | SELECT 2, 'HTTPS-Everywhere', 'https://git.torproject.org/https-everywhere.git', 'https://gitweb.torproject.org/https-everywhere.git/commitdiff/%ID', 'https-everywhere', 'pervasive' UNION 141 | SELECT 2, 'jtorctl', 'https://git.torproject.org/jtorctl.git', 'https://gitweb.torproject.org/jtorctl.git/commitdiff/%ID', 'tor-jtorctl', 'stable' UNION 142 | SELECT 2, 'Tor Metrics-DB', 'https://git.torproject.org/metrics-db.git', 'https://gitweb.torproject.org/metrics-db.git/commitdiff/%ID', 'tor-metrics-db', 'stable' UNION 143 | SELECT 2, 'Tor Metrics Tasks', 'https://git.torproject.org/metrics-tasks.git', 'https://gitweb.torproject.org/metrics-tasks.git/commitdiff/%ID', 'tor-metrics-tasks', 'stable' UNION 144 | SELECT 2, 'Tor Metrics Utils', 'https://git.torproject.org/metrics-utils.git', 'https://gitweb.torproject.org/metrics-utils.git/commitdiff/%ID', 'tor-metrics-utils', 'stable' UNION 145 | SELECT 2, 'Tor Metrics Web', 'https://git.torproject.org/metrics-web.git', 'https://gitweb.torproject.org/metrics-web.git/commitdiff/%ID', 'tor-metrics-web', 'stable' UNION 146 | SELECT 2, 'Obfsproxy', 'https://git.torproject.org/obfsproxy.git', 'https://gitweb.torproject.org/obfsproxy.git/commitdiff/%ID', 'tor-obfsproxy', 'stable' UNION 147 | SELECT 2, 'Orbot', 'https://git.torproject.org/orbot.git', 'https://gitweb.torproject.org/orbot.git/commitdiff/%ID', 'tor-orbot', 'stable' UNION 148 | SELECT 2, 'Puppetor', 'https://git.torproject.org/puppetor.git', 'https://gitweb.torproject.org/puppetor.git/commitdiff/%ID', 'tor-puppetor', 'stable' UNION 149 | SELECT 2, 'pytorctl', 'https://git.torproject.org/pytorctl.git', 'https://gitweb.torproject.org/pytorctl.git/commitdiff/%ID', 'tor-pytorctl', 'stable' UNION 150 | SELECT 2, 'Thandy', 'https://git.torproject.org/thandy.git', 'https://gitweb.torproject.org/thandy.git/commitdiff/%ID', 'tor-thandy', 'pervasive' UNION 151 | SELECT 2, 'Tor', 'https://git.torproject.org/tor.git', 'https://gitweb.torproject.org/tor.git/commitdiff/%ID', 'tor', 'pervasive' UNION 152 | SELECT 2, 'Torbutton', 'https://git.torproject.org/torbutton.git', 'https://gitweb.torproject.org/torbutton.git/commitdiff/%ID', 'torbutton', 'pervasive' UNION 153 | SELECT 2, 'TorDNSel', 'https://git.torproject.org/tordnsel.git', 'https://gitweb.torproject.org/tordnsel.git/commitdiff/%ID', 'tor-dnsel', 'stable' UNION 154 | SELECT 2, 'TorSOCKS', 'https://git.torproject.org/torsocks.git', 'https://gitweb.torproject.org/torsocks.git/commitdiff/%ID', 'torsocks', 'pervasive' UNION 155 | SELECT 2, 'Tor Spec', 'https://git.torproject.org/torspec.git', 'https://gitweb.torproject.org/torspec.git/commitdiff/%ID', 'torspec', 'pervasive' UNION 156 | SELECT 2, 'Vidalia', 'https://git.torproject.org/vidalia.git', 'https://gitweb.torproject.org/vidalia.git/commitdiff/%ID', 'vidalia', 'pervasive' UNION""" 157 | #crypto libraries 158 | sql += """ 159 | SELECT 2, 'libgcrypt', 'git://git.gnupg.org/libgcrypt.git', 'http://git.gnupg.org/cgi-bin/gitweb.cgi?p=libgcrypt.git;a=commitdiff;h=%ID', 'libgcrypt', 'pervasive' UNION""" 160 | #file crypto 161 | sql += """ 162 | SELECT 2, 'GnuPG', 'git://git.gnupg.org/gnupg.git', 'http://git.gnupg.org/cgi-bin/gitweb.cgi?p=gnupg.git;a=commitdiff;h=%ID', 'gnupg', 'pervasive' UNION 163 | SELECT 2, 'Tomb', 'git://github.com/dyne/Tomb.git', 'https://github.com/dyne/Tomb/commit/%ID', 'tomb', 'stable' UNION 164 | SELECT 2, 'SQLCipher', 'git://github.com/sqlcipher/sqlcipher.git', 'https://github.com/sqlcipher/sqlcipher/commit/%ID', 'sqlcipher', 'stable' UNION""" 165 | #remailer 166 | sql += """ 167 | SELECT 2, 'AAM2Mail', 'https://github.com/crooks/aam2mail.git', 'https://github.com/crooks/aam2mail/commit/%ID', 'aam2mail', 'stable' UNION 168 | SELECT 2, 'nymserv', 'https://github.com/crooks/nymserv', 'https://github.com/crooks/nymserv/commit/%ID', 'nymserv', 'stable' UNION""" 169 | #fde 170 | sql += """ 171 | SELECT 1, 'encfs', 'http://encfs.googlecode.com/svn/trunk/', 'http://code.google.com/p/encfs/source/detail?r=%ID', 'encfs', 'stable' UNION 172 | SELECT 2, 'cryptsetup', 'https://code.google.com/p/cryptsetup/', 'http://code.google.com/p/cryptsetup/source/detail?r=%ID', 'luks', 'pervasive' UNION""" 173 | #keyservers 174 | sql += """ 175 | SELECT 6, 'Onak', 'http://www.earth.li/~noodles/bzr/onak/mainline', NULL, 'onak', 'development' UNION""" 176 | #otr 177 | sql += """ 178 | SELECT 6, 'Python-OTR', 'http://bazaar.launchpad.net/~afflux/python-otr/purepython', NULL, 'python-otr', 'beta' UNION""" 179 | #browser plugins 180 | sql += """ 181 | SELECT 2, 'CR-GPG', 'https://github.com/RC1140/cr-gpg.git', 'https://github.com/RC1140/cr-gpg/commit/%ID', 'cr-gpg', 'development' UNION""" 182 | #mailinglist 183 | sql += """ 184 | SELECT 1, 'SELS', 'https://sels.svn.sourceforge.net/svnroot/sels', NULL, 'sels', 'development' UNION 185 | SELECT 8, 'Secure List Server', 'http://non-gnu.uvt.nl/pub/mailman/', NULL, 'secure-list-server', 'development' UNION 186 | SELECT 2, 'Schleuder', 'git://git.immerda.ch/schleuder.git', NULL, 'schleuder', 'development' UNION 187 | SELECT 8, 'Crypt-ML', 'http://www.synacklabs.net/projects/crypt-ml/', NULL, 'crypt-ml', 'development' UNION 188 | SELECT 3, 'Shibboleth', 'shibboleth.cvs.sourceforge.net', NULL, 'shibboleth', 'development' UNION 189 | SELECT 3, 'Mmreencrypt', 'mmreencrypt.cvs.sourceforge.net', NULL, 'mmreencrypt', 'development' UNION""" 190 | #guardian project 191 | sql += """ 192 | SELECT 2, 'IOCipher', 'https://github.com/guardianproject/IOCipher.git', 'https://github.com/guardianproject/IOCipher/commit/%ID', 'iocipher', 'development' UNION 193 | SELECT 2, 'libsqlfs', 'https://github.com/guardianproject/libsqlfs.git', 'https://github.com/guardianproject/libsqlfs/commit/%ID', 'libsqlfs', 'development' UNION 194 | SELECT 2, 'Gibberbot', 'https://github.com/guardianproject/Gibberbot.git', 'https://github.com/guardianproject/Gibberbot/commit/%ID', 'gibberbot', 'stable' UNION 195 | SELECT 2, 'OnionKit', 'https://github.com/guardianproject/OnionKit.git', 'https://github.com/guardianproject/OnionKit/commit/%ID', 'onionkit', 'beta' UNION 196 | SELECT 2, 'InformaCam', 'https://github.com/guardianproject/InformaCam.git', 'https://github.com/guardianproject/InformaCam/commit/%ID', 'informacam', 'beta' UNION 197 | SELECT 2, 'InformaCam-Server', 'https://github.com/guardianproject/InformaCam-Server.git', 'https://github.com/guardianproject/InformaCam-Server/commit/%ID', 'informacam-server', 'development' UNION 198 | SELECT 2, 'gnupg-for-android', 'https://github.com/guardianproject/gnupg-for-android.git', 'https://github.com/guardianproject/gnupg-for-android/commit/%ID', 'gnupg-for-android', 'development' UNION 199 | SELECT 2, 'Storymaker', 'https://github.com/guardianproject/mrapp.git', 'https://github.com/guardianproject/mrapp/commit/%ID', 'storymaker', 'beta' UNION 200 | SELECT 2, 'otrfileconverter', 'https://github.com/guardianproject/otrfileconverter.git', 'https://github.com/guardianproject/otrfileconverter/commit/%ID', 'otrfileconverter', 'development' UNION 201 | SELECT 2, 'Android cacert keystore', 'https://github.com/guardianproject/cacert.git', 'https://github.com/guardianproject/cacert/commit/%ID', 'cacert', 'stable' UNION 202 | SELECT 2, 'Orweb', 'https://github.com/guardianproject/Orweb.git', 'https://github.com/guardianproject/Orweb/commit/%ID', 'orweb', 'stable' UNION 203 | SELECT 2, 'ProxyMob', 'https://github.com/guardianproject/ProxyMob.git', 'https://github.com/guardianproject/ProxyMob/commit/%ID', 'ProxyMob', 'stable' UNION 204 | SELECT 2, 'ObscuraCam', 'https://github.com/guardianproject/SecureSmartCam.git', 'https://github.com/guardianproject/SecureSmartCam/commit/%ID', 'obscuracam', 'stable' UNION 205 | SELECT 2, 'InTheClear', 'https://github.com/guardianproject/InTheClear.git', 'https://github.com/guardianproject/InTheClear/commit/%ID', 'InTheClear', 'stable' UNION 206 | SELECT 2, 'android-ffmpeg', 'https://github.com/guardianproject/android-ffmpeg.git', 'https://github.com/guardianproject/android-ffmpeg/commit/%ID', 'android-ffmpeg', 'beta' UNION 207 | SELECT 2, 'android-ffmpeg-java', 'https://github.com/guardianproject/android-ffmpeg-java.git', 'https://github.com/guardianproject/android-ffmpeg-java/commit/%ID', 'android-ffmpeg-java', 'development' UNION 208 | SELECT 2, 'ODKFormParser', 'https://github.com/guardianproject/ODKFormParser.git', 'https://github.com/guardianproject/ODKFormParser/commit/%ID', 'odkformparser', 'development' UNION 209 | SELECT 2, 'rss-epub-archiver', 'https://github.com/guardianproject/rss-epub-archiver.git', 'https://github.com/guardianproject/rss-epub-archiver/commit/%ID', 'rss-epub-archiver', 'development'""" 210 | 211 | DB.execute(c, sql) 212 | 213 | #keyword._table + """ --------------------------------------------- 214 | if args.keywords: 215 | try: 216 | c.execute("DROP TABLE " + DB.keyword._table) 217 | except: 218 | pass 219 | c.execute("SHOW TABLES LIKE '" + DB.keyword._table + "'") 220 | r = c.fetchone() 221 | if r: 222 | print "Keyword Table Exists" 223 | else: 224 | print "Creating Keyword Table..." 225 | sql = "CREATE TABLE " + DB.keyword._table + """ 226 | ( 227 | keyword varchar(50) NOT NULL, 228 | parent varchar(50), 229 | type tinyint NOT NULL DEFAULT 1 230 | ) COLLATE utf8_general_ci ENGINE=innodb; 231 | """ 232 | c.execute(sql) 233 | 234 | if args.populate or args.testpopulate or args.keywords: 235 | print 'Populating Keywords...' 236 | 237 | lexpattern = re.compile('^[-_a-zA-Z0-9./ ]+$') 238 | 239 | h = open('tags.txt', 'r') 240 | sql = 'INSERT INTO ' + DB.keyword._table + "(keyword, parent, type)\n" 241 | components = [] 242 | for l in h: 243 | if not l.strip(): continue 244 | parts = l.strip().split(':') 245 | parts = [p.strip() for p in parts if p.strip()] 246 | 247 | if len(parts) > 1: 248 | if not lexpattern.search(parts[0]): 249 | print "Keyword", parts[0], "doesn't match lexer regex - update lexer and setup" 250 | elif not lexpattern.search(parts[1]): 251 | print "Keyword", parts[1], "doesn't match lexer regex - update lexer and setup" 252 | else: 253 | sql += "SELECT %s, %s, 1 UNION\n" 254 | components.extend(parts) 255 | else: 256 | if not lexpattern.search(parts[0]): 257 | print "Keyword", parts[0], "doesn't match lexer regex - update lexer and setup" 258 | else: 259 | sql += "SELECT %s, NULL, 1 UNION\n" 260 | components.append(parts[0]) 261 | sql = sql[0:-6] 262 | DB.execute(c, sql, components) 263 | 264 | 265 | sql = "INSERT INTO " + DB.keyword._table + """(keyword, parent, type) 266 | SELECT 'openssl-library-bio', 'openssl-library', 3 UNION 267 | SELECT 'openssl-library-blowfish', 'openssl-library', 3 UNION 268 | SELECT 'openssl-library-bn', 'openssl-library', 3 UNION 269 | SELECT 'openssl-library-des', 'openssl-library', 3 UNION 270 | SELECT 'openssl-library-dh', 'openssl-library', 3 UNION 271 | SELECT 'openssl-library-dsa', 'openssl-library', 3 UNION 272 | SELECT 'openssl-library-err', 'openssl-library', 3 UNION 273 | SELECT 'openssl-library-evp', 'openssl-library', 3 UNION 274 | SELECT 'openssl-library-hmac', 'openssl-library', 3 UNION 275 | SELECT 'openssl-library-lhash', 'openssl-library', 3 UNION 276 | SELECT 'openssl-library-md5', 'openssl-library', 3 UNION 277 | SELECT 'openssl-library-mdc2', 'openssl-library', 3 UNION 278 | SELECT 'openssl-library-pem', 'openssl-library', 3 UNION 279 | SELECT 'openssl-library-rand', 'openssl-library', 3 UNION 280 | SELECT 'openssl-library-rc4', 'openssl-library', 3 UNION 281 | SELECT 'openssl-library-ripemd', 'openssl-library', 3 UNION 282 | SELECT 'openssl-library-rsa', 'openssl-library', 3 UNION 283 | SELECT 'openssl-library-sha', 'openssl-library', 3 UNION 284 | SELECT 'openssl-library-ssl', 'openssl-library', 3 UNION 285 | SELECT 'openssl-library-threads', 'openssl-library', 3 UNION 286 | SELECT 'openssl-library-x509', 'openssl-library', 3 UNION 287 | SELECT 'openssl-library', NULL, 3 288 | 289 | 290 | """ 291 | c.execute(sql) 292 | 293 | #sql = "INSERT INTO " + DB.keyword._table + """(keyword, parent) 294 | #SELECT CONCAT('project-', r.tagname), NULL 295 | #FROM """ + DB.repo._table + " as r" 296 | #c.execute(sql) 297 | 298 | for f in os.listdir('keyword-setup'): 299 | h = open('keyword-setup/' + f, 'r') 300 | t = f.replace('.txt', '').replace("'", "") #rudimentary escaping here 301 | 302 | sql = 'INSERT INTO ' + DB.keyword._table + "(keyword, parent, type)\n" 303 | components = [] 304 | for l in h: 305 | sql += "SELECT %s, '" + t + "', 2 UNION\n" 306 | components.append(l.strip()) 307 | sql = sql[0:-6] 308 | DB.execute(c, sql, components) 309 | 310 | 311 | #commit._table + """ ---------------------------------------- 312 | c.execute("SHOW TABLES LIKE '" + DB.commit._table + "'") 313 | r = c.fetchone() 314 | if r: 315 | print "Commit Table Exists" 316 | else: 317 | print "Creating Commit Table..." 318 | sql = "CREATE TABLE " + DB.commit._table + """ 319 | ( 320 | id int NOT NULL AUTO_INCREMENT PRIMARY KEY, 321 | repoid tinyint NOT NULL, 322 | date int NOT NULL, 323 | message text, 324 | uniqueid varchar(64) NOT NULL, 325 | UNIQUE (repoid, uniqueid) 326 | ) COLLATE utf8_general_ci ENGINE=innodb; 327 | """ 328 | c.execute(sql) 329 | #commitfile._table + """ ---------------------------------------- 330 | c.execute("SHOW TABLES LIKE '" + DB.commitfile._table + "'") 331 | r = c.fetchone() 332 | if r: 333 | print "Commit File Table Exists" 334 | else: 335 | print "Creating Commit Table..." 336 | sql = "CREATE TABLE " + DB.commitfile._table + """ 337 | ( 338 | commitid int NOT NULL, 339 | file varchar(512) 340 | ) COLLATE utf8_general_ci ENGINE=innodb; 341 | """ 342 | c.execute(sql) 343 | #commitkeyword._table + """ ---------------------------------------- 344 | c.execute("SHOW TABLES LIKE '" + DB.commitkeyword._table + "'") 345 | r = c.fetchone() 346 | if r: 347 | print "Commit Keyword Table Exists" 348 | else: 349 | print "Creating Commit Keywords Table..." 350 | sql = "CREATE TABLE " + DB.commitkeyword._table + """ 351 | ( 352 | commitid int NOT NULL, 353 | keyword varchar(50) NOT NULL, 354 | PRIMARY KEY(commitid, keyword) 355 | ) COLLATE utf8_general_ci ENGINE=innodb; 356 | """ 357 | c.execute(sql) 358 | #commitdiffs._table + """ ---------------------------------------- 359 | c.execute("SHOW TABLES LIKE '" + DB.commitdiffs._table + "'") 360 | r = c.fetchone() 361 | if r: 362 | print "Commit Diffs Table Exists" 363 | else: 364 | print "Creating Commit Diffs Table..." 365 | sql = "CREATE TABLE " + DB.commitdiffs._table + """ 366 | ( 367 | commitid int NOT NULL, 368 | data LONGBLOB NOT NULL, 369 | PRIMARY KEY(commitid) 370 | ) COLLATE utf8_general_ci ENGINE=innodb; 371 | """ 372 | c.execute(sql) 373 | #commitwordmap._table + """ ---------------------------------------- 374 | c.execute("SHOW TABLES LIKE '" + DB.commitwordmap._table + "'") 375 | r = c.fetchone() 376 | if r: 377 | print "Commit Wordmap Table Exists" 378 | else: 379 | print "Creating Commit Wordmap Table..." 380 | sql = "CREATE TABLE " + DB.commitwordmap._table + """ 381 | ( 382 | commitid int NOT NULL, 383 | word varchar(50) NOT NULL, 384 | PRIMARY KEY(commitid, word) 385 | ) COLLATE utf8_general_ci ENGINE=innodb; 386 | """ 387 | c.execute(sql) 388 | #searchqueries._table + """ ---------------------------------------- 389 | c.execute("SHOW TABLES LIKE '" + DB.searchqueries._table + "'") 390 | r = c.fetchone() 391 | if r: 392 | print "Search Queries Table Exists" 393 | else: 394 | print "Creating Search Queries Table..." 395 | sql = "CREATE TABLE " + DB.searchqueries._table + """ 396 | ( 397 | timestamp int NOT NULL, 398 | ip int NOT NULL, 399 | terms varchar(512) NOT NULL 400 | ) COLLATE utf8_general_ci ENGINE=innodb; 401 | """ 402 | DB.execute(c, sql) 403 | 404 | 405 | conn.commit() 406 | -------------------------------------------------------------------------------- /templates/commit.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | Commit Details 4 | 37 | 38 | 39 |
40 |

Commit

41 |
42 | 
{{ commit.getpprint(true)}}
43 | 44 | Diffs 45 |
46 | {% if commit.diffIsReallyBig %} 47 | Diff is too large to display. 48 | {% else %} 49 | {% for d in commit.getPrettyDiffs(True) %} 50 | {{ d }} 51 |
52 | {% endfor %} 53 | {% endif %} 54 | 55 | 56 | 57 | 58 | -------------------------------------------------------------------------------- /templates/searchresults.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | Search Results 4 | 39 | 40 | 41 |
42 |

Search Results

RSS Feed For This Search Search Again

43 | 44 |
45 | {% if commits %} 46 | {% for c in commits %} 47 | Commit 48 |
49 | 
{{ c.getpprint(true)}}
50 |
51 | {% endfor %} 52 | {% else %} 53 | No results found. Search again. 54 | {% endif %} 55 |
56 | 57 | 58 | 59 | --------------------------------------------------------------------------------