├── .gitattributes
├── DB-search.pl
├── DeepCpf1_Code
    ├── Analysis of indel frequency
    │   ├── analyser_v3.exe
    │   ├── analyser_v3.py
    │   ├── extractor_v3.exe
    │   └── extractor_v3.py
    ├── DeepCpf1.py
    ├── input_example.txt
    ├── output_example.txt
    └── weights
    │   ├── DeepCpf1_weights.h5
    │   └── Seq_deepCpf1_weights.h5
├── PL-search.pl
├── RD-build.pl
├── README.md
├── Rule_Set_2_scoring_v1
    ├── PaxHeader
    │   ├── README_v1.pdf
    │   ├── analysis
    │   └── saved_models
    ├── README_v1.pdf
    ├── analysis
    │   ├── PaxHeader
    │   │   ├── features
    │   │   ├── load_data.py
    │   │   ├── load_data.pyc
    │   │   ├── local_multiprocessing.py
    │   │   ├── local_multiprocessing.pyc
    │   │   ├── metrics.py
    │   │   ├── metrics.pyc
    │   │   ├── model_comparison.py
    │   │   ├── model_comparison.pyc
    │   │   ├── models
    │   │   ├── predict.py
    │   │   ├── predict.pyc
    │   │   ├── rs2_score_calculator.py
    │   │   ├── util.py
    │   │   └── util.pyc
    │   ├── features
    │   │   ├── PaxHeader
    │   │   │   ├── __init__.py
    │   │   │   ├── __init__.pyc
    │   │   │   ├── featurization.py
    │   │   │   ├── featurization.pyc
    │   │   │   ├── microhomology.py
    │   │   │   └── microhomology.pyc
    │   │   ├── __init__.pyc
    │   │   ├── featurization.py
    │   │   ├── featurization.pyc
    │   │   ├── microhomology.py
    │   │   └── microhomology.pyc
    │   ├── load_data.py
    │   ├── load_data.pyc
    │   ├── local_multiprocessing.py
    │   ├── local_multiprocessing.pyc
    │   ├── metrics.py
    │   ├── metrics.pyc
    │   ├── model_comparison.py
    │   ├── model_comparison.pyc
    │   ├── models
    │   │   ├── DNN.py
    │   │   ├── DNN.pyc
    │   │   ├── GP.py
    │   │   ├── PaxHeader
    │   │   │   ├── DNN.py
    │   │   │   ├── DNN.pyc
    │   │   │   ├── GP.py
    │   │   │   ├── __init__.py
    │   │   │   ├── __init__.pyc
    │   │   │   ├── baselines.py
    │   │   │   ├── ensembles.py
    │   │   │   ├── ensembles.pyc
    │   │   │   ├── gpy_ssk.py
    │   │   │   ├── regression.py
    │   │   │   ├── regression.pyc
    │   │   │   └── ssk.py
    │   │   ├── __init__.pyc
    │   │   ├── baselines.py
    │   │   ├── ensembles.py
    │   │   ├── ensembles.pyc
    │   │   ├── gpy_ssk.py
    │   │   ├── regression.py
    │   │   ├── regression.pyc
    │   │   └── ssk.py
    │   ├── predict.py
    │   ├── predict.pyc
    │   ├── rs2_score_calculator.py
    │   ├── util.py
    │   └── util.pyc
    └── saved_models
    │   ├── PaxHeader
    │       ├── V3_model_full.pickle
    │       └── V3_model_nopos.pickle
    │   ├── V3_model_full.pickle
    │   └── V3_model_nopos.pickle
├── UD-build.pl
├── cfd-score-calculator.py
├── mismatch_score.pkl
├── pam_scores.pkl
├── seqmap-1.0.12-linux-64
└── sgRNA_reformatting.pl


/.gitattributes:
--------------------------------------------------------------------------------
1 | *.py linguist-language=perl
2 | *.pl linguist-language=perl
3 | *.pyc linguist-language=perl
4 | *.pkl linguist-language=perl
5 | *.pickle linguist-language=perl
6 | 


--------------------------------------------------------------------------------
/DB-search.pl:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/perl
  2 | use strict;
  3 | use warnings;
  4 | use Getopt::Long;
  5 | use File::Find;
  6 | use File::Path;
  7 | use Cwd;
  8 | 
  9 | local $SIG{__WARN__} = sub {
 10 | 	my $message = shift;
 11 | 	die $message;
 12 | };
 13 | 
 14 | my ($List, $RefDB, $UserDB, $Path, $opt_help,$label);
 15 | my $time = time();
 16 | 
 17 | GetOptions(	
 18 | 			'g=s' => \$List,
 19 | 			'i=s' => \$RefDB,
 20 | 			'u:s' => \$UserDB,
 21 | 			'o=s' => \$Path,
 22 | 			'h!'  => \$opt_help,		#Help message
 23 | 			'l:s' => \$label,
 24 |           );
 25 | 
 26 | my $dir_default = getcwd;             #default output
 27 | $Path ||= $dir_default;
 28 | my $dir =$Path;
 29 | my $errmsg = "Use '$0 -h' for quick help; for more information, please see README.";
 30 | $label ||= "DB_search";
 31 | my $helpmsg = qq{
 32 | =============== CRISPR-Local ===============
 33 | 
 34 | --- Usage ---
 35 | e.g.,
 36 | 	perl $0 -g ZmB73_query_gene.list -i ZmB73.reference.database.txt -o /your_dir/
 37 | 	
 38 | 	or
 39 | 
 40 | 	perl $0 -g ZmB73_query_gene.list -i ZmB73.reference.database.txt -u ZmC01.gene.sgRNA.db.alignment.txt -o /your_dir/
 41 | 
 42 | --- Options ---
 43 | 
 44 | 	-g <string>	:Query gene list file
 45 | 	-i <string>	:Reference sgRNA database (RD)
 46 | 	-u <string>	:User's sgRNA database (UD)
 47 | 	-o <string>	:Output path (defualt: $dir_default)
 48 | 	-l <label>	:Name prefix for output file (default:DB_search)
 49 | 
 50 | 	-h		:show this help
 51 | };
 52 | err( $helpmsg ) if $opt_help;
 53 | 
 54 | err( "ERROR - No query gene list file provided.\n$errmsg" ) if !$List;
 55 | err( "ERROR - No reference database file provided.\n$errmsg" ) if !$RefDB;
 56 | 
 57 | print "\n  Welcome to CRISPR-Local\n";
 58 | print " Program DB_search: a local tool for search best target of query gene.\n";
 59 | print "  ---------------------------------------------------------\n";
 60 | print " Version   : 3.0"."\n";
 61 | print " Copyright : Free software"."\n";
 62 | print " Author    : Jiamin Sun"."\n";
 63 | print " Email     : sunjm0824\@webmail.hzau.edu.cn"."\n\n";
 64 | 
 65 | my $local_time;
 66 | $local_time = localtime();
 67 | print "# Today : $local_time\n\n";
 68 | print "# Program DB-search: Search the reference and user's database to select the sgRNA with high on-target score, low off-target score.\n";
 69 | 
 70 | print "# Reading gene list file.\n";
 71 | open (LIST,$List) or die "Can't open $List for reading!\n";
 72 | my %gene_list;
 73 | while (<LIST>) {
 74 | 	chomp;
 75 | 	$gene_list{$_}=undef;
 76 | }
 77 | my @list = keys %gene_list;
 78 | my $list = @list;
 79 | print "# There are $list gene in your list.\n";
 80 | 
 81 | my(%RD_sgRNA,%UD_sgRNA);
 82 | print "# Reading reference database file.\n";
 83 | open (SGRNA,$RefDB) or die $!;
 84 | while (<SGRNA>) {
 85 | 	chomp;
 86 | 	my($gene,$pos,$seq,$score,$OT_1,$OT_2,$OT_3,$OT_4,$OT_score)=(split /\t/,$_)[0,1,2,3,4,5,6,7,-1];
 87 | 	if (exists $gene_list{$gene}) {
 88 | 		$RD_sgRNA{$gene}{$seq}.="$gene\t$pos\t$seq\t$score\t$OT_1\t$OT_2\t$OT_3\t$OT_4\t$OT_score\n";
 89 | 	}
 90 | }
 91 | close SGRNA;
 92 | my @RD_valid = sort keys %RD_sgRNA;
 93 | my $RD_num = @RD_valid;
 94 | my %gene_left = %gene_list;
 95 | print "# There are $RD_num available gene in the reference database.\n";
 96 | if ($list > $RD_num) {
 97 | 	foreach my $key (@RD_valid) {
 98 | 		delete $gene_left{$key};
 99 | 	}
100 | 	open (LEFT,">$dir/$label\_Invalid_gene_RD.list") or die "Can't open $label\_Invalid_gene_RD.list for writing!\n";
101 | 	foreach my $key (sort keys %gene_left) {
102 | 		print LEFT "$key\n";
103 | 	}
104 | 	print "# The invalid gene is output to $label\_Invalid_gene_RD.list, please add annotation information of these invalid gene.\n";
105 | 	close LEFT;
106 | }
107 | 
108 | if (!$UserDB) {
109 | 	print "# Output reference database search result.\n";
110 | 	open (OUT,">$dir/$label\_result_RD.txt") or die "Can't open $label\_result_RD.txt for writing!\n";
111 | 	foreach my $RD_valid (@RD_valid) {
112 | 		my @rs2 = sort keys %{$RD_sgRNA{$RD_valid}};
113 | 		for (my $i=0;$i<=$#rs2;$i++) {
114 | 			print OUT "$RD_sgRNA{$RD_valid}{$rs2[$i]}";
115 | 		}
116 | 	}
117 | 	close OUT;
118 | 	print "# Your job is done, check result.\n# END\n\n";
119 | 	my $oo = time() - $time;
120 | 	print "Total time consumption is $oo second.\nDone!\n";
121 | 	exit(0);
122 | }
123 | 
124 | if ($UserDB =~ /gene.sgRNA.db.alignment.txt$/) {
125 | 	print "# Reading User's alignment database file.\n";
126 | 	open (UDB,$UserDB) or die;
127 | 	while (<UDB>) {
128 | 		chomp;
129 | 		my($gene,$pos,$num,$score,$seq) = split /\t/,$_;
130 | 		if (exists $gene_list{$gene}) {
131 | 			$UD_sgRNA{$gene}{$seq}.="$_\n";
132 | 		}
133 | 	}
134 | 	close UDB;
135 | 	my @UD_valid = sort keys %UD_sgRNA;
136 | 	my $UD_num = @UD_valid;
137 | 	%gene_left = %gene_list;
138 | 	print "# There are $UD_num available gene in the user's alignment database.\n";
139 | 	if ($list > $UD_num) {
140 | 		foreach my $key (@UD_valid) {
141 | 			delete $gene_left{$key};
142 | 		}
143 | 		open (LEFT,">$dir/$label\_Invalid_gene_UD.list") or die "Can't open $label\_Invalid_gene_UD.list for writing!\n";
144 | 		foreach my $key (sort keys %gene_left) {
145 | 			print LEFT "$key\n";
146 | 		}
147 | 		close LEFT;
148 | 		print "# The invalid gene is output to $label\_Invalid_gene_UD.list.\n";
149 | 	}
150 | 	print "# Find out the 'RO', 'UO' and 'BO' sgRNA.\n";
151 | 	my @inter = grep {$RD_sgRNA{$_}} @UD_valid;
152 | 	my %merge = map {$_ => 1} @RD_valid,@UD_valid;
153 | 	my @merge = sort keys (%merge);
154 | 	my @RDC_gene = grep {!$RD_sgRNA{$_}} @merge;
155 | 	my @UDC_gene = grep {!$UD_sgRNA{$_}} @merge;
156 | 	open (REF,">$dir/$label\_result_RO.txt");
157 | 	open (USER,">$dir/$label\_result_UO.txt");
158 | 	open (CORNA,">$dir/$label\_result_BO.txt");
159 | 	print "# Output the 'RO', 'UO' and 'BO' sgRNA result.\n";
160 | 	foreach my $RDC_gene (@RDC_gene) {
161 | 		my @rs2 = sort keys %{$UD_sgRNA{$RDC_gene}};
162 | 		for (my $i=0;$i<=$#rs2;$i++) {
163 | 			print USER "$UD_sgRNA{$RDC_gene}{$rs2[$i]}";
164 | 		}
165 | 	}
166 | 	foreach my $UDC_gene (@UDC_gene) {
167 | 		my @rs2 = sort keys %{$RD_sgRNA{$UDC_gene}};
168 | 		for (my $i=0;$i<=$#rs2;$i++) {
169 | 			print REF "$RD_sgRNA{$UDC_gene}{$rs2[$i]}";
170 | 		}
171 | 	}
172 | 	foreach my $inter (sort @inter) {
173 | 		my @RD_seq = sort keys %{$RD_sgRNA{$inter}};
174 | 		my @UD_seq = sort keys %{$UD_sgRNA{$inter}};
175 | 		my @inter_seq = grep {$RD_sgRNA{$inter}{$_}} @UD_seq;
176 | 		my %merge_seq = map {$_ => 1} @RD_seq,@UD_seq;
177 | 		my @merge_seq = sort keys (%merge_seq);
178 | 		my @RDC_seq = grep {!$RD_sgRNA{$inter}{$_}} @merge_seq;
179 | 		my @UDC_seq = grep {!$UD_sgRNA{$inter}{$_}} @merge_seq;
180 | 
181 | 		for (my $i=0;$i<=$#RDC_seq;$i++) {
182 | 			print USER "$UD_sgRNA{$inter}{$RDC_seq[$i]}";
183 | 		}
184 | 		for (my $i=0;$i<=$#UDC_seq;$i++) {
185 | 			print REF "$RD_sgRNA{$inter}{$UDC_seq[$i]}";
186 | 		}
187 | 		for (my $i=0;$i<=$#inter_seq;$i++) {
188 | 			print CORNA "$RD_sgRNA{$inter}{$inter_seq[$i]}";
189 | 		}
190 | 	}
191 | 	close REF;close USER;close CORNA;
192 | }elsif ($UserDB =~ /intergenic.sgRNA.db.alignment.txt$/) {
193 | 	print "# Reading User's intergenic alignment database file.\n";
194 | 	my %seq_num;
195 | 	open (UDB,$UserDB) or die;
196 | 	open (INTER,">$dir/$label\_intergenic_result_BO.txt");
197 | 	open (REF,">$dir/$label\_intergenic_result_RO.txt");
198 | 	foreach my $RD_valid (@RD_valid) {
199 | 		foreach my $seq (keys %{$RD_sgRNA{$RD_valid}}){
200 | 			$seq_num{$seq}=0;
201 | 		}
202 | 	}
203 | 	while (<UDB>) {
204 | 		chomp;
205 | 		my ($num,$inter_seq) = (split /\t/,$_)[2,-1];
206 | 		if (exists $seq_num{$inter_seq}) {
207 | 			$seq_num{$inter_seq}+=$num;
208 | 		}
209 | 	}
210 | 	close UDB;
211 | 	print "# Find out the 'RO' and 'BO' sgRNA.\n";
212 | 	foreach my $RD_valid (@RD_valid) {
213 | 		foreach my $key (keys %{$RD_sgRNA{$RD_valid}}) {
214 | 			if ($seq_num{$key} > 0) {
215 | 				print INTER "$RD_sgRNA{$RD_valid}{$key}";
216 | 			}else{
217 | 				print REF "$RD_sgRNA{$RD_valid}{$key}";
218 | 			}
219 | 		}
220 | 	}
221 | 	close INTER;close REF;
222 | }elsif ($UserDB =~ /gene.sgRNA.db.fastq.txt$/) {
223 | 	print "# Reading User's fastq reads database file.\n";
224 | 	my %seq_num;
225 | 	open (UDB,$UserDB) or die;
226 | 	open (INTER,">$dir/$label\_result_BO.txt");
227 | 	open (REF,">$dir/$label\_result_RO.txt");
228 | 	foreach my $RD_valid (@RD_valid) {
229 | 		foreach my $seq (keys %{$RD_sgRNA{$RD_valid}}){
230 | 			$seq_num{$seq}=0;
231 | 		}
232 | 	}
233 | 	while (<UDB>) {
234 | 		chomp;
235 | 		my ($num,$inter_seq) = (split /\t/,$_)[1,-1];
236 | 		if (exists $seq_num{$inter_seq}) {
237 | 			$seq_num{$inter_seq}+=$num;
238 | 		}
239 | 	}
240 | 	close UDB;
241 | 	print "# Find out the 'RO' and 'BO' sgRNA.\n";
242 | 	foreach my $RD_valid (@RD_valid) {
243 | 		foreach my $key (keys %{$RD_sgRNA{$RD_valid}}) {
244 | 			if ($seq_num{$key} > 0) {
245 | 				print INTER "$RD_sgRNA{$RD_valid}{$key}";
246 | 			}else{
247 | 				print REF "$RD_sgRNA{$RD_valid}{$key}";
248 | 			}
249 | 		}
250 | 	}
251 | 	close INTER;close REF;
252 | }elsif ($UserDB =~ /gene.sgRNA.db.fasta.txt$/) {
253 | 	print "# Reading User's fasta sequence database file.\n";
254 | 	my %seq_num;
255 | 	open (UDB,$UserDB) or die;
256 | 	open (INTER,">$dir/$label\_result_BO.txt");
257 | 	open (REF,">$dir/$label\_result_RO.txt");
258 | 	foreach my $RD_valid (@RD_valid) {
259 | 		foreach my $seq (keys %{$RD_sgRNA{$RD_valid}}){
260 | 			$seq_num{$seq}=0;
261 | 		}
262 | 	}
263 | 	while (<UDB>) {
264 | 		chomp;
265 | 		my ($inter_seq) = (split /\t/,$_)[-1];
266 | 		if (exists $seq_num{$inter_seq}) {
267 | 			$seq_num{$inter_seq}++;
268 | 		}
269 | 	}
270 | 	close UDB;
271 | 	print "# Find out the 'RO' and 'BO' sgRNA.\n";
272 | 	foreach my $RD_valid (@RD_valid) {
273 | 		foreach my $key (keys %{$RD_sgRNA{$RD_valid}}) {
274 | 			if ($seq_num{$key} > 0) {
275 | 				print INTER "$RD_sgRNA{$RD_valid}{$key}";
276 | 			}else{
277 | 				print REF "$RD_sgRNA{$RD_valid}{$key}";
278 | 			}
279 | 		}
280 | 	}
281 | 	close INTER;close REF;
282 | }
283 | print "# Your job is done, check result.\n# END\n\n";
284 | my $oo = time() - $time;
285 | print "Total time consumption is $oo second.\nDone!\n";
286 | sub err {
287 | 	my ( $msg ) = @_;
288 | 	print "\n$msg\n\n";
289 | 	die "\n";
290 | }
291 | 
292 | 


--------------------------------------------------------------------------------
/DeepCpf1_Code/Analysis of indel frequency/analyser_v3.exe:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sunjiamin0824/CRISPR-Local/c6e80b2fd8ee4065c0655edf9f5206bf8cc316e2/DeepCpf1_Code/Analysis of indel frequency/analyser_v3.exe


--------------------------------------------------------------------------------
/DeepCpf1_Code/Analysis of indel frequency/analyser_v3.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/python
  2 | #-*- coding: utf-8 -*-
  3 | __author__ = 'forestkeep21@naver.com'
  4 | 
  5 | import re
  6 | import sys
  7 | import os
  8 | 
  9 | from Levenshtein import editops
 10 | # 해당 라이브러리 도큐먼트
 11 | # https://rawgit.com/ztane/python-Levenshtein/master/docs/Levenshtein.html
 12 | 
 13 | BASE_DIR = os.path.dirname(sys.executable)
 14 | 
 15 | #for debug
 16 | # BASE_DIR = os.path.dirname(os.path.realpath(__file__))
 17 | 
 18 | 
 19 | def seq_validator(data):
 20 |     # 문자열이 시퀀스인지 판별. 판별 끝난 후 비교를 쉽게 하기 위해 모두 대문자로 변환한다.
 21 |     m = re.findall(r'^[A|a|T|t|C|c|G|g]+$', data)
 22 |     return m[0].upper() if m else None
 23 | 
 24 | 
 25 | def count_line_in_file(file_name):
 26 |     # 파일내 라인 카운터
 27 |     count = 0
 28 |     for line in open(file_name, 'r'):
 29 |         count += 1
 30 |     return count
 31 | 
 32 | 
 33 | def do(input_file_name, backward_target_length, dest_folder_path):
 34 |     cur_cnt = 0
 35 |     target_cnt = count_line_in_file(input_file_name)
 36 |     final_results = {}
 37 | 
 38 |     for target_set in open(input_file_name, 'r'):
 39 |         # 인풋 파일은 이름 : 와일드시퀀스 : 타겟 의 형태를 띈다.
 40 |         tmp = target_set.split(':')
 41 |         file_name_no_ext = tmp[0].strip()
 42 |         file_name = '{}.txt'.format(file_name_no_ext)
 43 |         wild_seq = tmp[1].strip()
 44 |         target = tmp[2].strip()
 45 | 
 46 |         # 타겟 검사
 47 |         target = seq_validator(target)
 48 |         if not target:
 49 |             continue
 50 | 
 51 |         # 와일드 시퀀스 검사
 52 |         wild_seq = seq_validator(wild_seq)
 53 |         if not wild_seq:
 54 |             continue
 55 | 
 56 |         # 결과 저장용 폴더 생성
 57 |         result_folder_name = os.path.join(BASE_DIR, 'analyse_results')
 58 |         if not os.path.exists(result_folder_name):
 59 |                 os.makedirs(result_folder_name)
 60 | 
 61 |         try:
 62 |             # 결과 임시 저장 dict
 63 |             result = {
 64 |                 'total_cnt': count_line_in_file(os.path.join(dest_folder_path, file_name)),
 65 |                 'mutated_cnt': 0,
 66 |                 'mutated_rates': 0.0,
 67 |                 'mutated_dict': {}
 68 |             }
 69 | 
 70 |             for line in open(os.path.join(dest_folder_path, file_name), 'r'):
 71 |                 # 대상 시퀀스 검사
 72 |                 line = seq_validator(line)
 73 |                 if not line:
 74 |                     continue
 75 | 
 76 |                 # 와일드 시퀀스와 타겟을 이용하여 와일드 시퀀스에서 타겟의 시작, 종료 위치를 파악한다. editops에서 사용.
 77 |                 target_start_pos_in_wild = int(wild_seq.find(target))
 78 |                 target_end_pos = target_start_pos_in_wild + len(target)
 79 | 
 80 |                 # 와일드 시퀀스를 기준으로 대상 시퀀스와 비교하여 레벤슈타인 유사도 측정에서 editops를 뽑아낸다.
 81 |                 # editops는 (변형방법, 와일드시퀀스 기준 위치, 대상시퀀스 기준 위치) 의 형태로 결과가 나온다.
 82 |                 # 예를 들어, editops('test', 'teaasz') 의 경우 [('insert', 2, 2), ('insert', 2, 3), ('replace', 3, 5)]
 83 |                 # 1번 인덱스 : 삽입이 와일드시퀀스 기준 2번째, 대상시퀀스 기준 2번째에서 발생
 84 |                 # 2번 인덱스 : 삽입이 와일드시퀀스 기준 2번째, 대상시퀀스 기준 3번째에서 발생
 85 |                 # 3번 인덱스 : 교체가 와일드시퀀스 기준 3번째, 대상시퀀스 기준 5번째에서 발생
 86 |                 # 때문에 와일드시퀀스에서 타겟의 위치만 정확히 파악한다면 대상시퀀스에서 변형이 어느부분에 일어났는지
 87 |                 # 몰라도 사용자가 지정한 위치에서의 변형 여부를 충분히 잡아낼 수 있다.
 88 |                 for mutation_info in editops(wild_seq, line):
 89 |                     # 사용자 지정 위치 검사(타겟의 뒤에서부터 backward_target_length 번째까지)
 90 |                     if target_end_pos - int(backward_target_length) <= mutation_info[1] <= target_end_pos:
 91 |                         # 교체는 변형으로 치지 않는다.
 92 |                         if mutation_info[0] != 'replace':
 93 |                             # 여기까지 왔다면 변형으로 쳐서 카운트+1
 94 |                             result['mutated_cnt'] += 1
 95 |                             # 변형된 대상시퀀스를 결과 출력을 위해 저장하고 동일 시퀀스 갯수 조사를 위해 카운팅한다.
 96 |                             if line not in result['mutated_dict'].keys():
 97 |                                 result['mutated_dict'][line] = 1
 98 |                             else:
 99 |                                 result['mutated_dict'][line] += 1
100 |                             break
101 | 
102 |             # 변형 퍼센티지 계산
103 |             try:
104 |                 result['mutated_rates'] = float(result['mutated_cnt']) / result['total_cnt'] * 100
105 |             except:
106 |                 result['mutated_rates'] = 0
107 | 
108 |             # 각 결과값 저장.
109 |             with open(os.path.join(result_folder_name, file_name), 'w') as f:
110 |                 for mutated_seq, cnt in result['mutated_dict'].items():
111 |                     f.write('{} X {}\n'.format(mutated_seq, cnt))
112 |                 f.write('--------\n')
113 |                 f.write('mutation rates : {} %'.format(result['mutated_rates']))
114 | 
115 |         except Exception as e:
116 |             print e
117 |             print file_name, ' not found.'
118 |             pass
119 |         else:
120 |             # 문제 없다면 결과물을 모은다.
121 |             final_results[file_name_no_ext] = result
122 | 
123 |         # 타겟 하나 분석 종료 카운트+1
124 |         cur_cnt += 1
125 |         # 진행율 화면 표시
126 |         progress_percentage = float(cur_cnt) / target_cnt * 100
127 |         print '{} % done'.format(progress_percentage)
128 | 
129 |         # 최종 결과물 파일 저장.
130 |         with open(os.path.join(result_folder_name, 'result_info.txt'), 'w') as f:
131 |             for name, data in final_results.items():
132 |                 f.write('{} : {} : {}/{}\n'.format(name, data['mutated_rates'], data['mutated_cnt'], data['total_cnt']))
133 | 
134 | if __name__ == '__main__':
135 |     print u'Input file name with extension: '
136 |     # 이름 : 와일드시퀀스 : 타겟 으로 구성된 파일을 입력받는다.
137 |     input_file_name = raw_input()
138 |     input_file_name = os.path.join(BASE_DIR, input_file_name)
139 | 
140 |     if not os.path.isfile(input_file_name):
141 |         print u'File Not Found. Check it is in same folder'
142 |         raise
143 | 
144 |     print u'Input length to check mutation from backward of target: '
145 |     # 사용자 지정 위치를 입력받는다. 타겟의 제일 위에서부터 ~번째이다.
146 |     backward_target_length = raw_input()
147 | 
148 |     print u'Input result folder name: '
149 |     # 추출기가 뽑아낸 대상시퀀스들이 모여있는 폴더 이름 입력.
150 |     dest_folder_name = raw_input()
151 |     dest_folder_name = os.path.join(BASE_DIR, dest_folder_name)
152 | 
153 |     if not os.path.isdir(dest_folder_name):
154 |         print u'Folder Not Found'
155 |         raise
156 | 
157 |     # 분석시작
158 |     do(input_file_name, backward_target_length, dest_folder_name)
159 | 
160 |     print u'Well done. Press any key'
161 |     raw_input()
162 | 
163 | 


--------------------------------------------------------------------------------
/DeepCpf1_Code/Analysis of indel frequency/extractor_v3.exe:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sunjiamin0824/CRISPR-Local/c6e80b2fd8ee4065c0655edf9f5206bf8cc316e2/DeepCpf1_Code/Analysis of indel frequency/extractor_v3.exe


--------------------------------------------------------------------------------
/DeepCpf1_Code/Analysis of indel frequency/extractor_v3.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/python
  2 | #-*- coding: utf-8 -*-
  3 | __author__ = 'forestkeep21@naver.com'
  4 | 
  5 | import re
  6 | import sys
  7 | import os
  8 | 
  9 | BASE_DIR = os.path.dirname(sys.executable)
 10 | 
 11 | #for debug
 12 | # BASE_DIR = os.path.dirname(os.path.realpath(__file__))
 13 | 
 14 | 
 15 | def seq_validator(data):
 16 |     m = re.findall(r'^[A|a|T|t|C|c|G|g]+$', data)
 17 |     return m[0] if m else None
 18 | 
 19 | 
 20 | def count_line_in_file(file_name):
 21 |     count = 0
 22 |     for line in open(file_name, 'r'):
 23 |         count += 1
 24 |     return count
 25 | 
 26 | 
 27 | def do(src_file_name, dest_file_name):
 28 |     # 프로그램 진행율을 계산하기 위해 파일의 라인수를 센다.
 29 |     src_line_cnt = count_line_in_file(src_file_name)
 30 |     if src_line_cnt == 0:
 31 |         print u'File Not Found'
 32 |         raise
 33 |     current_cnt = 0
 34 |     extracted_line_index = []
 35 | 
 36 |     # 결과가 저장될 폴더 지정
 37 |     result_folder_name = os.path.join(BASE_DIR, 'results')
 38 |     # 추출할 시퀸스가 있는 파일을 읽어온다.
 39 |     data = [line.strip() for line in open(dest_file_name, 'r') if seq_validator(line)]
 40 |     # 바코드가 있는 파일을 읽어온다.
 41 |     barcode_data = [line for line in open(src_file_name, 'r')]
 42 | 
 43 |     # 결과가 저장될 폴더가 없다면 하나 생성
 44 |     if not os.path.exists(result_folder_name):
 45 |         os.makedirs(result_folder_name)
 46 | 
 47 |     try:
 48 |         # 읽어온 바코드를 속도를 위해 모두 메모리에 올려놓고 분석을 시작한다.
 49 |         for barcode in barcode_data:
 50 |             # 바코드셋은 :를 구분자로 앞은 파일명, 뒤는 바코드로 되어있다.
 51 |             barcode_set = barcode.split(':')
 52 |             if len(barcode_set) < 2:
 53 |                 continue
 54 |             # 파일명에서 화이트 스페이스 삭제
 55 |             file_name = barcode_set[0].strip()
 56 |             # 바코드가 valid한지 검증
 57 |             barcode = seq_validator(barcode_set[1].strip())
 58 | 
 59 |             used_data = []
 60 |             # 대상이 되는 시퀸스들을 하나하나 분석한다.
 61 |             for line in data:
 62 |                 # 대상 시퀸스 valid 검증
 63 |                 line = seq_validator(line)
 64 |                 if line is None:
 65 |                     continue
 66 | 
 67 |                 # 비교를 위해 바코드, 대상 시퀸스 둘다 소문자로 변환하여 바코드가 대상 시퀸스 내에 존재하는지 검사
 68 |                 if barcode.lower() in line.lower():
 69 |                     # 존재한다면 대상 시퀸스는 이제 필요없으므로 추후 메모리에서 제거하기 위해 따로 보관한다.
 70 |                     used_data.append(line)
 71 | 
 72 |             # 결과가 저장될 파일명 지정
 73 |             file_name = os.path.join(result_folder_name, '{}.txt'.format(file_name))
 74 |             # 결과 파일 쓰기 시작
 75 |             with open(file_name, 'w') as f:
 76 |                 # 추출된 대상 시퀸스들을 파일에 쓴다.
 77 |                 for datum in used_data:
 78 |                     f.write('{}\n'.format(datum))
 79 | 
 80 |             # 파일에 전부 옮겨담았다면 메모리에 올라간 전체 대상 시퀸스들에서 파일에 쓴 대상 시퀸스를 뺀다.
 81 |             [data.remove(used_datum) for used_datum in used_data]
 82 | 
 83 |             # 프로그램 진행율 계산 부분
 84 |             current_cnt += 1
 85 |             progress_percentage = (float(current_cnt) / src_line_cnt * 100)
 86 |             print u'{} %'.format(progress_percentage)
 87 | 
 88 |     except Exception as e:
 89 |         print e
 90 |         print u'Extraction Failure.'
 91 |         raise
 92 | 
 93 |     try:
 94 |         # 모든 바코드의 분석이 종료되었다면 총 결과파일을 쓴다. 총 결과 파일명 지정
 95 |         result_info_file_name = os.path.join(result_folder_name, 'result_info.txt')
 96 |         with open(result_info_file_name, 'w') as f:
 97 |             # 각 개별 결과 파일을 열어서
 98 |             for line in open(src_file_name, 'r'):
 99 |                 barcode_set = line.split(':')
100 |                 file_name = barcode_set[0].strip()
101 |                 # 라인 수를 센다음에
102 |                 count = count_line_in_file(os.path.join(BASE_DIR, result_folder_name, '{}.txt'.format(file_name)))
103 |                 # 총 결과 파일에 파일명 : 라인수 형식으로 쓴다.
104 |                 f.write('{} : {}\n'.format(file_name, count))
105 |     except Exception as e:
106 |         print e
107 |         print u'Extraction has been done. But Making a result-info.txt is failed.'
108 |         raise
109 | 
110 | if __name__ == "__main__":
111 |     print u'Input barcode file name with extension: '
112 |     src_file_name = raw_input()
113 |     src_file_name = os.path.join(BASE_DIR, src_file_name)
114 | 
115 |     if not os.path.isfile(src_file_name):
116 |         print u'File Not Found. Check it is in same folder'
117 |         raise
118 | 
119 |     print u'Input sequence file name with extension: '
120 |     dest_file_name = raw_input()
121 |     dest_file_name = os.path.join(BASE_DIR, dest_file_name)
122 | 
123 |     if not os.path.isfile(dest_file_name):
124 |         print u'File Not Found. Check it is in same folder'
125 |         raise
126 | 
127 |     do(src_file_name, dest_file_name)
128 | 
129 |     print u'Well done. Press any key'
130 |     raw_input()
131 | 
132 | 


--------------------------------------------------------------------------------
/DeepCpf1_Code/DeepCpf1.py:
--------------------------------------------------------------------------------
  1 | from numpy import *
  2 | import sys;  
  3 | 
  4 | from keras.models import Model
  5 | from keras.layers import Input
  6 | from keras.layers.merge import Multiply
  7 | from keras.layers.core import Dense, Dropout, Activation, Flatten
  8 | from keras.layers.convolutional import Convolution1D, AveragePooling1D
  9 | 
 10 | def main():
 11 | #    print "Usage: python DeepCpf1.py input.txt output.txt"
 12 | #    print "input.txt must include 3 columns with single header row"
 13 | #    print "\t1st column: sequence index"
 14 | #    print "\t2nd column: 34bp target sequence"
 15 | #    print "\t3rd column: binary chromain information of the target sequence\n"        
 16 | 
 17 | #    print "DeepCpf1 currently requires python=2.7.12, theano=1.0.1, keras=2.1.5"
 18 | #    print "DeepCpf1 available on GitHub requires pre-obtained binary chromatin information (DNase-seq narraow peak data from ENCODE)"
 19 | #    print "DeepCpf1 web tool, available at http://data.snu.ac.kr/DeepCpf1, provides entire pipeline including binary chromatin accessibility for 125 cell lines\n"    
 20 |     
 21 |     if len(sys.argv) < 3:
 22 |         sys.exit()
 23 |     
 24 | #    print "Building models"
 25 |     Seq_deepCpf1_Input_SEQ = Input(shape=(34,4))
 26 |     Seq_deepCpf1_C1 = Convolution1D(80, 5, activation='relu')(Seq_deepCpf1_Input_SEQ)
 27 |     Seq_deepCpf1_P1 = AveragePooling1D(2)(Seq_deepCpf1_C1)
 28 |     Seq_deepCpf1_F = Flatten()(Seq_deepCpf1_P1)
 29 |     Seq_deepCpf1_DO1= Dropout(0.3)(Seq_deepCpf1_F)
 30 |     Seq_deepCpf1_D1 = Dense(80, activation='relu')(Seq_deepCpf1_DO1)
 31 |     Seq_deepCpf1_DO2= Dropout(0.3)(Seq_deepCpf1_D1)
 32 |     Seq_deepCpf1_D2 = Dense(40, activation='relu')(Seq_deepCpf1_DO2)
 33 |     Seq_deepCpf1_DO3= Dropout(0.3)(Seq_deepCpf1_D2)
 34 |     Seq_deepCpf1_D3 = Dense(40, activation='relu')(Seq_deepCpf1_DO3)
 35 |     Seq_deepCpf1_DO4= Dropout(0.3)(Seq_deepCpf1_D3)
 36 |     Seq_deepCpf1_Output = Dense(1, activation='linear')(Seq_deepCpf1_DO4)
 37 |     Seq_deepCpf1 = Model(inputs=[Seq_deepCpf1_Input_SEQ], outputs=[Seq_deepCpf1_Output])
 38 |     
 39 |     DeepCpf1_Input_SEQ = Input(shape=(34,4))
 40 |     DeepCpf1_C1 = Convolution1D(80, 5, activation='relu')(DeepCpf1_Input_SEQ)
 41 |     DeepCpf1_P1 = AveragePooling1D(2)(DeepCpf1_C1)
 42 |     DeepCpf1_F = Flatten()(DeepCpf1_P1)
 43 |     DeepCpf1_DO1= Dropout(0.3)(DeepCpf1_F)
 44 |     DeepCpf1_D1 = Dense(80, activation='relu')(DeepCpf1_DO1)
 45 |     DeepCpf1_DO2= Dropout(0.3)(DeepCpf1_D1)
 46 |     DeepCpf1_D2 = Dense(40, activation='relu')(DeepCpf1_DO2)
 47 |     DeepCpf1_DO3= Dropout(0.3)(DeepCpf1_D2)
 48 |     DeepCpf1_D3_SEQ = Dense(40, activation='relu')(DeepCpf1_DO3)
 49 |     
 50 |     DeepCpf1_Input_CA = Input(shape=(1,))
 51 |     DeepCpf1_D3_CA = Dense(40, activation='relu')(DeepCpf1_Input_CA)
 52 |     DeepCpf1_M = Multiply()([DeepCpf1_D3_SEQ, DeepCpf1_D3_CA])
 53 |     
 54 |     DeepCpf1_DO4= Dropout(0.3)(DeepCpf1_M)
 55 |     DeepCpf1_Output = Dense(1, activation='linear')(DeepCpf1_DO4)
 56 |     DeepCpf1 = Model(inputs=[DeepCpf1_Input_SEQ, DeepCpf1_Input_CA], outputs=[DeepCpf1_Output])
 57 |     
 58 | #    print "Loading weights for the models"
 59 |     Seq_deepCpf1.load_weights('DeepCpf1_Code/weights/Seq_deepCpf1_weights.h5')
 60 |     DeepCpf1.load_weights('DeepCpf1_Code/weights/DeepCpf1_weights.h5')
 61 |     
 62 | #    print "Loading test data"
 63 |     FILE = open(sys.argv[1], "r")
 64 |     data = FILE.readlines()
 65 |     SEQ, CA = PREPROCESS(data)
 66 |     FILE.close()
 67 |     
 68 | #    print "Predicting on test data"
 69 |     Seq_deepCpf1_SCORE = Seq_deepCpf1.predict([SEQ], batch_size=50, verbose=0)
 70 |     DeepCpf1_SCORE = DeepCpf1.predict([SEQ, CA], batch_size=50, verbose=0) * 3
 71 |     
 72 | #    print "Saving to " + sys.argv[2]
 73 |     OUTPUT = open(sys.argv[2], "w")
 74 |     for l in range(len(data)):
 75 |         if l == 0:
 76 |             OUTPUT.write(data[l].strip())
 77 |             OUTPUT.write("\tSeq-deepCpf1 Score\tDeepCpf1 Score\n")
 78 |         else:
 79 |             OUTPUT.write(data[l].strip())
 80 |             OUTPUT.write("\t%f\t%f\n" % (Seq_deepCpf1_SCORE[l-1], DeepCpf1_SCORE[l-1]))
 81 |     OUTPUT.close()
 82 |     
 83 | def PREPROCESS(lines):
 84 |     data_n = len(lines) - 1
 85 |     SEQ = zeros((data_n, 34, 4), dtype=int)
 86 |     CA = zeros((data_n, 1), dtype=int)
 87 |     
 88 |     for l in range(1, data_n+1):
 89 |         data = lines[l].split('\t')
 90 |         seq = data[1]
 91 |         for i in range(34):
 92 |             if seq[i] in "Aa":
 93 |                 SEQ[l-1, i, 0] = 1
 94 |             elif seq[i] in "Cc":
 95 |                 SEQ[l-1, i, 1] = 1
 96 |             elif seq[i] in "Gg":
 97 |                 SEQ[l-1, i, 2] = 1
 98 |             elif seq[i] in "Tt":
 99 |                 SEQ[l-1, i, 3] = 1
100 |         CA[l-1,0] = int(data[2])*100
101 | 
102 |     return SEQ, CA
103 | 
104 | if __name__ == '__main__':
105 |         main()
106 |         
107 | 


--------------------------------------------------------------------------------
/DeepCpf1_Code/input_example.txt:
--------------------------------------------------------------------------------
1 | Target number	34 bp target sequence (4 bp + PAM + 23 bp protospacer + 3 bp)		"Chromatin accessibility (1= DNase I hypersensitive sites, 0 = Dnase I non-sensitive sites)"	
2 | 1	TGACTTTGAATGGAGTCGTGAGCGCAAGAACGCT		1
3 | 2	GTTATTTGAGCAATGCCACTTAATAAACATGTAA		0


--------------------------------------------------------------------------------
/DeepCpf1_Code/output_example.txt:
--------------------------------------------------------------------------------
1 | Target number	34 bp target sequence (4 bp + PAM + 23 bp protospacer + 3 bp)		"Chromatin accessibility (1= DNase I hypersensitive sites, 0 = Dnase I non-sensitive sites)"	Seq-deepCpf1 Score	DeepCpf1 Score
2 | 1	TGACTTTGAATGGAGTCGTGAGCGCAAGAACGCT		1	55.699318	46.077488
3 | 2	GTTATTTGAGCAATGCCACTTAATAAACATGTAA		0	53.469837	7.932923
4 | 
5 | 


--------------------------------------------------------------------------------
/DeepCpf1_Code/weights/DeepCpf1_weights.h5:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sunjiamin0824/CRISPR-Local/c6e80b2fd8ee4065c0655edf9f5206bf8cc316e2/DeepCpf1_Code/weights/DeepCpf1_weights.h5


--------------------------------------------------------------------------------
/DeepCpf1_Code/weights/Seq_deepCpf1_weights.h5:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sunjiamin0824/CRISPR-Local/c6e80b2fd8ee4065c0655edf9f5206bf8cc316e2/DeepCpf1_Code/weights/Seq_deepCpf1_weights.h5


--------------------------------------------------------------------------------
/PL-search.pl:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/perl
  2 | use strict;
  3 | use warnings;
  4 | use Getopt::Long;
  5 | use File::Find;
  6 | use File::Path;
  7 | use Cwd;
  8 | 
  9 | local $SIG{__WARN__} = sub {
 10 | 	my $message = shift;
 11 | 	die $message;
 12 | };
 13 | 
 14 | my ($List, $RefDB, $UserDB, $Path, $opt_help,$label);
 15 | my $time = time();
 16 | GetOptions(	
 17 | 			'g=s' => \$List,
 18 | 			'i=s' => \$RefDB,
 19 | 			'u:s' => \$UserDB,
 20 | 			'o=s' => \$Path,
 21 | 			'h!'  => \$opt_help,		#Help message
 22 | 			'l:s' => \$label,
 23 |           );
 24 | 
 25 | my $dir_default = getcwd;             #default output
 26 | $Path ||= $dir_default;
 27 | my $dir =$Path;
 28 | $label ||= "PL-search";
 29 | my $errmsg = "Use '$0 -h' for quick help; for more information, please see README.";
 30 | my $helpmsg = qq{
 31 | =============== CRISPR-Local ===============
 32 | 
 33 | --- Usage ---
 34 | e.g.,
 35 | 	perl $0 -g ZmB73_paralogous_gene.list -i ZmB73.reference.database.txt -o /your_dir/
 36 | 
 37 | 	or
 38 | 
 39 | 	perl $0 -g ZmB73_paralogous_gene.list -i ZmB73.reference.database.txt -u ZmC01.gene.sgRNA.db.alignment.txt -o /your_dir/
 40 | 
 41 | 	
 42 | --- Options ---
 43 | 
 44 | 	-h: show this help and exit
 45 | 
 46 | 	-g <string>	:Paralog gene list
 47 | 	-i <string>	:Reference sgRNA database (RD)
 48 | 	-u <string>	:User's sgRNA database (UD)
 49 | 	-o <string>	:Output path (defualt: $dir_default)
 50 | 	-l <label>	:Name prefix for output file (default:PL_search)
 51 | };
 52 | err( $helpmsg ) if $opt_help;
 53 | 
 54 | err( "ERROR - No paralog gene list file provided.\n$errmsg" ) if !$List;
 55 | err( "ERROR - No reference database file provided.\n$errmsg" ) if !$RefDB;
 56 | 
 57 | print "\n  Welcome to CRISPR-Local\n";
 58 | print " Program PL_search: a local tool for search exclusive and commom target for paralogous gene pair.\n";
 59 | print "  ---------------------------------------------------------\n";
 60 | print " Version   : 1.0"."\n";
 61 | print " Copyright : Free software"."\n";
 62 | print " Author    : Jiamin Sun"."\n";
 63 | print " Email     : sunjm0824\@webmail.hzau.edu.cn"."\n\n";
 64 | 
 65 | my $local_time;
 66 | $local_time = localtime();
 67 | print "# Today : $local_time\n\n";
 68 | print "# Program PL-search: a local tool for search exclusive and commom target for paralogous gene pair.\n";
 69 | 
 70 | print "# Reading paralog gene list file.\n";
 71 | open (LIST,$List) or die "Can't open $List for reading!\n";
 72 | 
 73 | my %gene_list;my %gene_pair;my(%RD_sgRNA,%RD_pos);
 74 | 
 75 | while (<LIST>) {
 76 | 	chomp;
 77 | 	my @gene = (split ",",$_);
 78 | 	foreach my $gene (@gene) {
 79 | 		$gene_list{$gene} = undef;
 80 | 	}
 81 | 	$gene_pair{$_} = undef;
 82 | }
 83 | print "# Reading reference database file.\n";
 84 | open (SGRNA,$RefDB) or die $!;
 85 | while (<SGRNA>) {
 86 | 	chomp;
 87 | 	my($gene,$pos,$seq,$score,$OT_1,$OT_2,$OT_3,$OT_4,$OT_score)=(split /\t/,$_)[0,1,2,3,4,5,6,7,-1];
 88 | 	if (exists $gene_list{$gene}) {
 89 | 		$RD_pos{"$gene:$seq"}.="$gene:$pos:$score;";
 90 | 		$RD_sgRNA{$gene}{$seq}="$gene\t$pos\t$seq\t$score\t$OT_1\t$OT_2\t$OT_3\t$OT_4\t$OT_score";
 91 | 	}
 92 | }
 93 | close SGRNA;
 94 | 
 95 | my %Inter_hash;
 96 | foreach my $gene_pair (sort keys %gene_pair) {
 97 | 	my @pair = (split ",",$gene_pair);
 98 | 	my @inter_seq = keys %{$RD_sgRNA{$pair[0]}};
 99 | 	for (my $i=0;$i<=$#pair;$i++) {
100 | 		@inter_seq = grep {$RD_sgRNA{$pair[$i]}{$_}} @inter_seq;
101 | 	}
102 | 	foreach my $inter (@inter_seq) {
103 | 		$Inter_hash{$gene_pair}{$inter} = 1;
104 | 		foreach my $paralog (@pair) {
105 | 			delete $RD_sgRNA{$paralog}{$inter};
106 | 		}
107 | 	}
108 | }
109 | if (!$UserDB) {
110 | 	print "# Finding the commom targets...\n";
111 | 	open (OUT,">$dir/$label\_common_targets.txt") or die "Can't open $label\_common_targets.txt for writing!\n";
112 | 	foreach my $gene_pair (sort keys %gene_pair) {
113 | 		my @pair = (split ",",$gene_pair);
114 | 		my @inter_seq = keys %{$Inter_hash{$gene_pair}};
115 | 		foreach my $inter (@inter_seq) {
116 | 			print OUT "$inter";
117 | 			foreach my $paralog (@pair) {
118 | 				print OUT "\t".$RD_pos{"$paralog:$inter"};
119 | 			}
120 | 			print OUT "\n";
121 | 		}
122 | 		print OUT "\n";
123 | 	}
124 | 	close OUT;
125 | 	print "# Finding the exclusive targets...\n";
126 | 	open (OUT,">$dir/$label\_exclusive_targets.txt") or die "Can't open $label\_exclusive_targets.txt for writing!\n";
127 | 	foreach my $gene_pair (sort keys %gene_pair) {
128 | 		my @pair = (split ",",$gene_pair);
129 | 		for (my $i=0;$i<=$#pair;$i++) {
130 | 			my @RD_seq = sort keys %{$RD_sgRNA{$pair[$i]}};
131 | 			for (my $j=0;$j<=$#RD_seq;$j++) {
132 | 				print OUT "$RD_sgRNA{$pair[$i]}{$RD_seq[$j]}\n";
133 | 			}
134 | 		}
135 | 	}
136 | 	close OUT;
137 | 	print "# Your job is done, check result.\n# END\n\n";
138 | 	my $oo = time() - $time;
139 | 	print "Total time consumption is $oo second.\nDone!\n";
140 | 	exit(0);
141 | }
142 | 
143 | my %UD_merge;my %UD_score;
144 | if ($UserDB =~ /gene.sgRNA.db.alignment.txt$/) {
145 | 	print "# Reading User's database file.\n";
146 | 	open (UDB,$UserDB) or die;
147 | 	while (<UDB>) {
148 | 		chomp;
149 | 		my($gene,$num,$seq) = (split /\t/,$_)[0,2,-1];
150 | 		if (exists $gene_list{$gene}) {
151 | 			$UD_score{$gene}{$seq}+=$num;
152 | 		}
153 | 	}
154 | 	close UDB;
155 | 	print "# Finding the commom targets...\n";
156 | 	open (OUT,">$dir/$label\_common_targets.txt") or die "Can't open $label\_common_targets.txt for writing!\n";
157 | 	foreach my $gene_pair (sort keys %gene_pair) {
158 | 		my @pair = (split ",",$gene_pair);
159 | 		my @inter_seq = keys %{$Inter_hash{$gene_pair}};
160 | 		for (my $i=0;$i<=$#pair;$i++) {
161 | 			my @UD_seq = keys %{$UD_score{$pair[$i]}};
162 | 			$UD_merge{$_} = 1, foreach @UD_seq;
163 | 		}
164 | 		foreach my $inter (@inter_seq) {
165 | 			print OUT "$inter";
166 | 			if (exists $UD_merge{$inter}) {
167 | 				print OUT "\tUD";
168 | 			}else{
169 | 				print OUT "\tRD";
170 | 			}
171 | 			foreach my $paralog (@pair) {
172 | 				print OUT "\t".$RD_pos{"$paralog:$inter"};
173 | 			}
174 | 			print OUT "\n";
175 | 		}
176 | 		print OUT "\n";
177 | 	}
178 | 	close OUT;
179 | 
180 | 	print "# Finding the exclusive targets...\n";
181 | 	open (OUT,">$dir/$label\_exclusive_targets.txt") or die "Can't open $label\_exclusive_targets.txt for writing!\n";
182 | 	foreach my $gene_pair (sort keys %gene_pair) {
183 | 		my @pair = (split ",",$gene_pair);
184 | 		for (my $i=0;$i<=$#pair;$i++) {
185 | 			my @RD_seq = sort keys %{$RD_sgRNA{$pair[$i]}};
186 | 			my %UD;
187 | 			foreach my $RD_seq (@RD_seq) {
188 | 				if (exists $UD_score{$pair[$i]}{$RD_seq}) {
189 | 					$UD{$pair[$i]}{$RD_seq} = "UD\t$UD_score{$pair[$i]}{$RD_seq}";
190 | 				}else{
191 | 					$UD{$pair[$i]}{$RD_seq} = "RD\t0";
192 | 				}
193 | 			}
194 | 			for (my $j=0;$j<=$#RD_seq;$j++) {
195 | 				print OUT "$RD_sgRNA{$pair[$i]}{$RD_seq[$j]}\t$UD{$pair[$i]}{$RD_seq[$j]}\n";
196 | 			}
197 | 		}
198 | 	}
199 | 	close OUT;
200 | }elsif ($UserDB =~ /(intergenic.sgRNA.db.alignment.txt)|(gene.sgRNA.db.fastq.txt)/) {
201 | 	print "# Reading User's database file.\n";
202 | 	open (UDB,$UserDB) or die;
203 | 	while (<UDB>) {
204 | 		chomp;
205 | 		my($num,$seq) = (split /\t/,$_)[-3,-1];
206 | 		$UD_score{$seq}+=$num;
207 | 	}
208 | 	close UDB;
209 | 	print "# Finding the commom targets...\n";
210 | 	open (OUT,">$dir/$label\_common_targets.txt") or die "Can't open $label\_common_targets.txt for writing!\n";
211 | 	foreach my $gene_pair (sort keys %gene_pair) {
212 | 		my @pair = (split ",",$gene_pair);
213 | 		my @inter_seq = keys %{$Inter_hash{$gene_pair}};
214 | 		foreach my $inter (@inter_seq) {
215 | 			print OUT "$inter";
216 | 			if (exists $UD_score{$inter}) {
217 | 				print OUT "\tUD";
218 | 			}else{
219 | 				print OUT "\tRD";
220 | 			}
221 | 			foreach my $paralog (@pair) {
222 | 				print OUT "\t".$RD_pos{"$paralog:$inter"};
223 | 			}
224 | 			print OUT "\n";
225 | 		}
226 | 		print OUT "\n";
227 | 	}
228 | 	close OUT;
229 | 
230 | 	print "# Finding the exclusive targets...\n";
231 | 	open (OUT,">$dir/$label\_exclusive_targets.txt") or die "Can't open $label\_exclusive_targets.txt for writing!\n";
232 | 	foreach my $gene_pair (sort keys %gene_pair) {
233 | 		my @pair = (split ",",$gene_pair);
234 | 		for (my $i=0;$i<=$#pair;$i++) {
235 | 			my @RD_seq = sort keys %{$RD_sgRNA{$pair[$i]}};
236 | 			my %UD;
237 | 			foreach my $RD_seq (@RD_seq) {
238 | 				if (exists $UD_score{$pair[$i]}{$RD_seq}) {
239 | 					$UD{$pair[$i]}{$RD_seq} = "UD\t$UD_score{$pair[$i]}{$RD_seq}";
240 | 				}else{
241 | 					$UD{$pair[$i]}{$RD_seq} = "RD\t0";
242 | 				}
243 | 			}
244 | 			for (my $j=0;$j<=$#RD_seq;$j++) {
245 | 				print OUT "$RD_sgRNA{$pair[$i]}{$RD_seq[$j]}\t$UD{$pair[$i]}{$RD_seq[$j]}\n";
246 | 			}
247 | 		}
248 | 	}
249 | 	close OUT;
250 | }elsif ($UserDB =~ /gene.sgRNA.db.fasta.txt/) {
251 | 	print "# Reading User's database file.\n";
252 | 	open (UDB,$UserDB) or die;
253 | 	while (<UDB>) {
254 | 		chomp;
255 | 		my($num,$seq) = (split /\t/,$_)[-3,-1];
256 | 		$UD_score{$seq}++;
257 | 	}
258 | 	close UDB;
259 | 	print "# Finding the commom targets...\n";
260 | 	open (OUT,">$dir/$label\_common_targets.txt") or die "Can't open $label\_common_targets.txt for writing!\n";
261 | 	foreach my $gene_pair (sort keys %gene_pair) {
262 | 		my @pair = (split ",",$gene_pair);
263 | 		my @inter_seq = keys %{$Inter_hash{$gene_pair}};
264 | 		foreach my $inter (@inter_seq) {
265 | 			print OUT "$inter";
266 | 			if (exists $UD_score{$inter}) {
267 | 				print OUT "\tUD";
268 | 			}else{
269 | 				print OUT "\tRD";
270 | 			}
271 | 			foreach my $paralog (@pair) {
272 | 				print OUT "\t".$RD_pos{"$paralog:$inter"};
273 | 			}
274 | 			print OUT "\n";
275 | 		}
276 | 		print OUT "\n";
277 | 	}
278 | 	close OUT;
279 | 
280 | 	print "# Finding the exclusive targets...\n";
281 | 	open (OUT,">$dir/$label\_exclusive_targets.txt") or die "Can't open $label\_exclusive_targets.txt for writing!\n";
282 | 	foreach my $gene_pair (sort keys %gene_pair) {
283 | 		my @pair = (split ",",$gene_pair);
284 | 		for (my $i=0;$i<=$#pair;$i++) {
285 | 			my @RD_seq = sort keys %{$RD_sgRNA{$pair[$i]}};
286 | 			my %UD;
287 | 			foreach my $RD_seq (@RD_seq) {
288 | 				if (exists $UD_score{$pair[$i]}{$RD_seq}) {
289 | 					$UD{$pair[$i]}{$RD_seq} = "UD\t$UD_score{$pair[$i]}{$RD_seq}";
290 | 				}else{
291 | 					$UD{$pair[$i]}{$RD_seq} = "RD\t0";
292 | 				}
293 | 			}
294 | 			for (my $j=0;$j<=$#RD_seq;$j++) {
295 | 				print OUT "$RD_sgRNA{$pair[$i]}{$RD_seq[$j]}\t$UD{$pair[$i]}{$RD_seq[$j]}\n";
296 | 			}
297 | 		}
298 | 	}
299 | 	close OUT;
300 | }
301 | 
302 | print "# Your job is done, check result.\n# END\n\n";
303 | my $oo = time() - $time;
304 | print "Total time consumption is $oo second.\nDone!\n";
305 | 
306 | sub err {
307 | 	my ( $msg ) = @_;
308 | 	print "\n$msg\n\n";
309 | 	die "\n";
310 | }
311 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | CRISPR-Local
  2 | ============
  3 | 
  4 | Please forward any question and suggestion about CRISPR-Local to: 
  5 | ## sunjiamin0824@qq.com
  6 | The original email address 'sunjm0824@webmail.hzau.edu.cn' has been deactivated.
  7 | 
  8 | ## A local single-guide RNA (sgRNA) design tool for non-reference plant genomes.
  9 | ### Motivation:
 10 | For genome editing experiments, it is critical to design a reliable single-guide RNA (sgRNA). However, most existing tools are limited to reference genome and do not consider variations in genetic background. Taking this variation into account is especially important for plant genome editing studies, since the transformed lines are rarely the sequenced reference genomes and the presence of large diversity between them.
 11 | 
 12 | ### Highlights:
 13 | 
 14 | 
 15 | 
 16 | 
 17 | * Designing sgRNAs suitable for non-reference varieties. In the plants for gene editing, the genome of the material and the reference genome are quite different. Leading to a fact that the sgRNAs designed according to the reference genome can be not well used for other materials. 
 18 | * Supporting various PAM. PAM motifs for Cas9, Cpf1 or Custom are provided. For Cas9, the length of protospacer (20nt) and PAM sequence (NGG) are fixed. The length of protospacer and PAM sequence can be set by users for the other two. Simple rules are:
 19 | 	*  Cas9	:On-target: 20 nt protospacer + NGG, off-target: 20 nt + NRG
 20 | 	*  Cpf1	:On-target: TTTN/TTN + 23/24/25 nt protospacer
 21 | 	*  Custom	:On-target: 15-25 nt protospacer + custom PAM sequence
 22 | * Screening for sgRNAs that are capable of simultaneously targeting multiple genes.
 23 | * Adopting the “one-for-all” strategy to build a whole-genome sgRNA database with high efficiency.
 24 | * Running offline, with both command-line and graphical user interface modes, and the ability to export multiple formats for further batch analysis or visualization.
 25 | * A substantial pre-stored sgRNA database for 71 public plant reference genomes. CRISPR-Local has been applied to 71 public plant genomes, for both CRISPR/Cas9 (with NGG PAM) and CRISPR/cpf1 (with TTV(A/G/C) and TTTV PAM), which can be directly searched or downloaded from http://crispr.hzau.edu.cn/CRISPR-Local/.
 26 | 
 27 | ### A General View of CRISPR-Local
 28 | * UD-build could build another modified sgRNA database based on the additional sequencing data on non-reference lines, and be called as User’s sgRNA database (UD). Both aligned reads and unmapped reads are used to screen suitable sgRNA, the difference is that each aligned read can be assigned to one gene according to its alignment position. User can summit their own sequence data in bam/sam/fasta/fastq format.
 29 | * Program DB-search would compare the RD with the UD, and the results consist of three parts: The sgRNAs present exclusively in RD (RD only, RO) or UD (UD only, UO), or both (BO). Generally, the sgRNAs from BO is preferential, and cautions should be required for UO, and RO is strongly not recommended especially when the supplied NGS data is adequate enough. 
 30 | * PL-search is typically designed for editing multiple paralogs to address redundancy concerns, like knocking out multiple key genes for several pathways determining one phenotype. This step follows the function of above DB-search mode that distinguish targets to RO, UO or BO. Moreover, PL-search will additionally partition the targets to common (suits to all candidates) and exclusive (individually matched) according to any submitted list of genes.
 31 | 
 32 | ### Prerequisites:
 33 | The following software and libraries are additionally required: Seqmap (version: 1.0.12), Samtools (>1.3.1), Python (>2.7) with the scikit-learn (0.16.1), biopython, pandas, keras, numpy, and scipy libraries, and Perl (>5.10) with the Parallel::ForkManager, File::Basename, Getopt::Long, Data::Dumper, Cwd modules. Most of the installation steps are fully automatic using a simple command line on a Linux system. 
 34 | 
 35 | ### 1.Prepare CRISPR-Local input (fasta/genome) files
 36 | (1) Reference genome sequences with fasta file, can be downloaded from Ensembl Plants, NCBI or other sources.
 37 |   * Before performing CRISPR-Local program, multiple fasta files need be combined into one big fasta file by using cat command.
 38 |  
 39 | (2) The reference annotation file in GFF3 format, can also be downloaded from Ensembl Plants.
 40 |   * The chromosome names (sequence headers) in the FASTA reference genome file must be the same as the first column of GFF3 annotation file.
 41 | 
 42 |   * Example GFF3 file
 43 | ```
 44 | ##gff-version 3
 45 | #!genome-build  Pmarinus_7.0
 46 | #!genome-version Pmarinus_7.0
 47 | #!genome-date 2011-01
 48 | #!genebuild-last-updated 2013-04
 49 | GL476399 Pmarinus_7.0 supercontig 1 4695893 . . . ID=supercontig:GL476399;Alias=scaffold_71
 50 | GL476399 ensembl gene 2596494 2601138 . + . ID=gene:ENSPMAG00000009070;Name=TRYPA3;biotype=protein_coding;description=Trypsinogen A1%3B Trypsinogen a3%3B Uncharacterized protein [Source:UniProtKB/TrEMBL%3BAcc:O42608];logic_name=ensembl;version=1
 51 | GL476399 ensembl transcript 2596494 2601138 . + . ID=transcript:ENSPMAT00000010026;Name=TRYPA3-201;Parent=gene:ENSPMAG00000009070;biotype=protein_coding;version=1
 52 | GL476399 ensembl exon 2596494 2596538 . + . Name=ENSPMAE00000087923;Parent=transcript:ENSPMAT00000010026;constitutive=1;ensembl_end_phase=1;ensembl_phase=-1;rank=1;version=1
 53 | GL476399 ensembl exon 2598202 2598361 . + . Name=ENSPMAE00000087929;Parent=transcript:ENSPMAT00000010026;constitutive=1;ensembl_end_phase=2;ensembl_phase=1;rank=2;version=1
 54 | GL476399 ensembl exon 2599023 2599282 . + . Name=ENSPMAE00000087937;Parent=transcript:ENSPMAT00000010026;constitutive=1;ensembl_end_phase=1;ensembl_phase=2;rank=3;version=1
 55 | GL476399 ensembl exon 2599814 2599947 . + . Name=ENSPMAE00000087952;Parent=transcript:ENSPMAT00000010026;constitutive=1;ensembl_end_phase=0;ensembl_phase=1;rank=4;version=1
 56 | GL476399 ensembl exon 2600895 2601138 . + . Name=ENSPMAE00000087966;Parent=transcript:ENSPMAT00000010026;constitutive=1;ensembl_end_phase=-1;ensembl_phase=0;rank=5;version=1
 57 | GL476399 ensembl CDS 2596499 2596538 . + 0 ID=CDS:ENSPMAP00000009982;Parent=transcript:ENSPMAT00000010026
 58 | GL476399 ensembl CDS 2598202 2598361 . + 2 ID=CDS:ENSPMAP00000009982;Parent=transcript:ENSPMAT00000010026
 59 | GL476399 ensembl CDS 2599023 2599282 . + 1 ID=CDS:ENSPMAP00000009982;Parent=transcript:ENSPMAT00000010026
 60 | GL476399 ensembl CDS 2599814 2599947 . + 2 ID=CDS:ENSPMAP00000009982;Parent=transcript:ENSPMAT00000010026
 61 | GL476399 ensembl CDS 2600895 2601044 . + 0 ID=CDS:ENSPMAP00000009982;Parent=transcript:ENSPMAT00000010026
 62 | ```
 63 | 	
 64 | ### 2. How to run CRISPR-Local
 65 | 
 66 | ####  (1) program RD-build:
 67 | ```
 68 | Parameters
 69 | --- Required ---
 70 | 	
 71 | 	-m <string>	:Given specific PAM for sgRNA design : Cas9, Cpf1 or Custom (default: Cas9)
 72 | 
 73 | 	  Cas9		:On-target: 20 nt protospacer + NGG, off-target: 20 nt + NRG
 74 | 	  Cpf1		:On-target: TTTN/TTN + 23/24/25 nt protospacer
 75 | 	  Custom	:On-target: 15-25 nt protospacer + custom PAM sequence
 76 | 
 77 | 	-i <string>	:reference genome sequence file (In *.fasta format)
 78 | 	-g <string>	:reference genome annotation file (In *.gff3 format)
 79 | 
 80 | --- Options ---
 81 | 
 82 | 	-o <string>	:Output path (default: current directory)
 83 | 	-l <string>	:Name prefix for output file (default: CRISPR)
 84 | 	-p <int>	:The number of process to use (default: 1)
 85 | 
 86 | 	For Cas9:
 87 | 
 88 | 	 -U <int>	:An integer specifying the number of base pairs expanding 5'-end of exon boundary (default: 0, >15 is not allowed)
 89 | 	 -D <int>	:An integer specifying the number of base pairs expanding 3'-end of exon boundary (default: 0, >3 is not allowed)
 90 | 
 91 | 	For Cpf1 or Custom:
 92 | 
 93 | 	  -x <int>	:Cpf1: Length of spacer: between 23 to 25 (default: 24 nt);
 94 | 			:Custom: Length of spacer: between 15 to 25 (default: 20 nt);
 95 | 	  -t <string>	:Cpf1: Type of PAM sequence: TTX or TTTX (X: One of A C G T R Y M K S W H B V D N; default: TTTN)
 96 | 			:Custom: Type of PAM sequence (default: NGG)
 97 | 
 98 | 	-h: show this help
 99 | ```
100 | ##### (i) Designing sgRNA for Cas9:
101 | 
102 | * In cas9 mode, this script: 
103 | 	* (a) Screening all possible on-target sgRNAs (with NGG PAM type) and scoring them with the Rule set 2 algorithm (Doench et al., 2016) for each exon, and screening all potential off-target sites (with NRG PAM type); 
104 | 	* (b) Retrieving potential off-target sites against each candidate sgRNA by using SeqMap program (Jiang and Wong, 2008) under a default maximum mismatch of 4; 
105 | 	* (c) Appling the cutting frequency determination (CFD) score (Doench et al., 2016) to predict the effects of each off-target site, the highest CFD score for each sgRNA is retained, and all the genome-wide results were exported as the RD.
106 | 
107 | * In addition, the number of base pairs expanding 5'- and 3'-end of exon boundary can be assigned to indicate if the sgRNA can be accepted when they are partially overlapped with intron while the likely editing positions are still located in exon to knock out likewise.
108 | 
109 | ```
110 | Example:
111 | ----------------------------
112 | perl RD-build.pl -m cas9 -i Reference_Genome.fa -g Reference_annotation.gff3 -o /opt/your_dir/ -l Label -U 15 -D 3 -p 8
113 |     
114 | * This command will generate a reference database file and a log file.
115 | 
116 | Example of reference database files:
117 | ----------------------------
118 | 	
119 | TAIR10.reference.database.txt
120 | 
121 | AT1G01010  1:+3855  CGTTGAAGTAGCCATCAGCG+AGG  0.7909  AT1G06430  1:+1962511	CGATGAAGCAGCCATCTGCA+CAG  4  AT1G01010.1.exon1;  [3760:3631:283:224:95];   0.0214
122 | AT1G01010  1:-3849  TGATGGCTACTTCAACGTCG+CGG  0.7724  AT1G01740  1:+274664	CTTTGGCTACTTCAACATCG+CAG  4  AT1G01010.1.exon1;  [3760:3631:283:64:-65];	  0.0932
123 | AT1G01010  1:+4118  GTTGAGGTCAAGGACCAGTG+GGG  0.7487  AT1G51035  1:+18917578	GTTCAGTTCATGAACCAGTG+CAG  4  AT1G01010.1.exon2;  [3760:3996:281:122:358];  0.0223
124 | AT1G01010  1:-5499  TTCACCGTGTTGGTGGATGG+AGG  0.7036  AT1G31080  1:+11092019	GTCACCGTCTTGGTGGATCC+CGG  4  AT1G01010.1.exon6;  [3760:5439:461:400:2079]; 0.0990
125 | AT1G01010  1:+4102  GCTTACCGGAGAATCTGTTG+AGG  0.7031  AT1G04680  1:+1307049	GCTAACCGGAGAAACCGTTA+GAG  4  AT1G01010.1.exon2;  [3760:3996:281:106:342];  0.0478
126 | AT1G01010  1:+3690  CAGAGAGCGAGAGAGATCGA+CGG  0.7005  AT1G30540  1:+10817050	AAGAGAGAGAGAGAGAGAGA+GAG  4  AT1G01010.1.exon1;  [3760:3631:283:59:-70];	  0.0107
127 | 
128 | 
129 | * There are 11 columns in the RD file and their corresponding meanings are listed below:
130 | 
131 | Column 1:	The name of gene where the sgRNA located.
132 | Column 2:	The chromosome and the coordinate of the start position of the sgRNA.
133 | Column 3:	The sequence of sgRNA(23nt).
134 | Column 4:	The on-target score of the sgRNA. 
135 | Column 5:	The name of off-target gene with the highest CFD score.
136 | Column 6:	The chromosome and the coordinate of the start position of the off-target site with the highest CFD score.
137 | Column 7:	The sequence of off-target site.
138 | Column 8:	The number of mismatches between sgRNA and off-target site.
139 | Column 9:	The name of exon where the sgRNA located(split by ;).
140 | Column 10:	The number that split by ":" means "TSS position", "exon start position", "length of exon", "relative position of sgRNA against exon" and "relative position of sgRNA against TSS", respectively.
141 | Column 11:	The highest CFD score between sgRNA and all off-target sites.
142 | ```
143 | ##### (ii) Cpf1
144 | * Cpf1 is a single RNA-guided endonuclease lacking tracrRNA, and it utilizes a T-rich PAM (5'-TTN or 5'-TTTN). Each mature crRNA begins with 19 nt of the direct repeat followed by 23¨C25 nt of the spacer sequence (Zetsche and Gootenberg et al.,2015);
145 | * In cpf1 mode, this script: 
146 | 	* (a) Screening all possible on-target sgRNAs (with TTX or TTTX PAM type, X represents any base including A C G T R Y M K S W H B V D N),and screening all potential off-target sites (with TTX or TTTX PAM type). The type of PAM and length of protospacer are determined by user;
147 | 	* (b) Scoring all sgRNAs with the method raised by Kim et al (2017);
148 | 	* (c) Retrieving potential off-target sites against each candidate sgRNA by using SeqMap program under a default maximum mismatch of 4; 
149 | 	* (d) Summarizing the off-target results, and all the genome-wide results were exported as the RD.
150 | ```
151 | Example:
152 | ----------------------------
153 | perl RD-build.pl -m cpf1 -i Reference_Genome.fa -g Reference_annotation.gff3 -o /opt/your_dir/ -l Label -x 24 -t TTTV -p 8
154 | 
155 | * This command will generate a reference database file and a log file.
156 | 
157 | Example of reference database files:
158 | ----------------------------
159 | 	
160 | TAIR10.reference.database.txt
161 | 
162 | AT1G67220	1:-25147036	TTTC+CTCGGTTTGAATCTTTCCTTTGTT	35.156925	5	U1	0,1,4,0,0	NA	AT1G67220.1.exon1;	[25145587:25145587:2181:731:731];	NA
163 | AT1G67220	1:-25147059	TTTC+TTCTCATAGTTCAAAGACCTTTCC	56.483639	5	R1	0,2,3,0,0	NA	AT1G67220.1.exon1;	[25145587:25145587:2181:708:708];	NA
164 | AT1G67220	1:-25147095	TTTC+ATCGGCTCAACAATATCCACACCA	55.160500	8	R0	2,3,2,1,0	AT1G67220(1:-25146972);AT1G67220(1:-25147302);	AT1G67220.1.exon1;	[25145587:25145587:2181:672:672];	NA
165 | AT1G67220	1:-25147164	TTTC+ATTGGCTCAACAATAACCACATCA	48.010044	8	R3	0,0,0,7,1	NA	AT1G67220.1.exon1;	[25145587:25145587:2181:603:603];	NA
166 | AT1G67220	1:-25147173	TTTA+TTACATTTCATTGGCTCAACAATA	42.615669	0	NM	0,0,0,0,0	NA	AT1G67220.1.exon1;	[25145587:25145587:2181:594:594];	NA
167 | AT1G67220	1:-25147233	TTTC+ATTGGCTCAACAATATTCACACCA	41.623928	8	R2	0,0,3,4,1	NA	AT1G67220.1.exon1;	[25145587:25145587:2181:534:534];	NA
168 | AT1G67220	1:-25147302	TTTC+ATCGGCTCAACAATATCCACACCA	55.647942	8	R0	2,3,2,1,0	AT1G67220(1:-25146972);AT1G67220(1:-25147095);	AT1G67220.1.exon1;	[25145587:25145587:2181:465:465];	NA
169 | * There are 11 columns in the RD file and their corresponding meanings are listed below:
170 | 
171 | Column 1:	The name of gene where the sgRNA located.
172 | Column 2:	The chromosome and the coordinate of the start position of the sgRNA.
173 | Column 3:	The sequence of sgRNA.
174 | Column 4:	The on-target score of the sgRNA. 
175 | Column 5:	The number of off-target sites.
176 | Column 6:	Type of match between sgRNA and off-target sites (NM:no match found; U0:Best match found was a unique exact match; U1:Best match found was a unique 1-error match; U2:Best match found was a unique 2-error match... R0:Multiple exact matches found; R1:Multiple 1-error matches found, no exact matches; R2:Multiple 2-error matches found, no exact or 1-error matches.)
177 | Column 7:	The number of exact, 1-error, 2-error, 3-error and 4-error matches found.
178 | Column 8:	The gene and position in which exact match was found. (If there is no exact match, then denoted by NA)
179 | Column 9:	The name of exon where the sgRNA located(split by ;).
180 | Column 10:	The number that split by ":" means "TSS position", "exon start position", "length of exon", "relative position of sgRNA against exon" and "relative position of sgRNA against TSS", respectively.
181 | Column 11:	The highest off-target score between sgRNA and all off-target sites.(There is no available off-target scoring method for Cpf1 sgRNA, denoted by NA)
182 | ```
183 | ####  (2) Program UD-build (if necessary):
184 | 
185 | * This script use to accept user's data to build user's sgRNA database.
186 | * According to the contents and format of the data file, different methods were used to design sgRNAs and build database.
187 | * Program UD-build supports input alignment file in Bam or Sam format, and raw reads file in fasta/fa/fasta.gz/fa.gz or fq/fastq/fq.gz/fastq.gz format.
188 | * If your file is Bam or Sam format, the annotation file in GFF3 format is needed to be specified.
189 | 
190 |  ```    
191 | Parameters
192 | --- Required ---
193 | 
194 | 	-m <string>	:Given specific PAM for sgRNA design : Cas9, Cpf1 or Custom (default: Cas9)
195 | 
196 | 	  Cas9		:On-target: 20 nt protospacer + NGG, off-target: 20 nt + NRG
197 | 	  Cpf1		:On-target: TTTN/TTN + 23/24/25 nt protospacer
198 | 	  Custom	:On-target: 15-25 nt protospacer + custom PAM sequence
199 | 
200 | 	-i <string>	:User's sequence file (In bam, fastq or fasta format)
201 | 	-g <string>	:Genome annotation file (This parameter is required only if the format of sequence file is bam)
202 | 
203 | --- Options ---
204 | 
205 | 	-o <string>	:Output path (default: current directory)
206 | 	-p <int>	:The number of process to use (default:1)
207 | 
208 | 	-h: show this help
209 | 	
210 | 	For Cpf1 or Custom mode:
211 | 
212 | 	  -x <int>	:Cpf1: Length of spacer: between 23 to 25 (default: 24 nt);
213 | 			:Custom: Length of spacer: between 15 to 25 (default: 20 nt);
214 | 	  -t <string>	:Cpf1: Type of PAM sequence: TTX or TTTX (X: One of A C G T R Y M K S W H B V D N; default: TTTN)
215 | 			:Custom: Type of PAM sequence (default: NGG)
216 | 
217 | Example:
218 | ----------------------------
219 | For Cas9 mode:
220 | 
221 | 	perl UD-build.pl -m cas9 -i Your_data.bam -g Annotation.gff3 -o /your_dir/ -p 10
222 | 
223 | For Cpf1 mode:
224 | 
225 | 	perl UD-build.pl -m cpf1 -i Your_data.fasta -o /your_dir/ -p 10 -x 24 -t TTTV
226 | 
227 | For Custom mode:
228 | 
229 | 	perl UD-build.pl -m custom -i Your_data.fastq -o /your_dir/ -t NRG -p 10 -x 20
230 | 
231 | * Two user's sgRNA database files would be generated when the input file is Bam or Sam format. 
232 | 
233 | Example of user's database files:
234 | ----------------------------
235 | ZmC01.gene.sgRNA.db.alignment.txt
236 |       
237 | Zm00001d022658  1:+4991138      5       0.2965  ATTCTGATTATATAGATATT+AGG
238 | Zm00001d022658  1:+4991139      1       0.2965  ATTCTGATTATATAGATATT+AGG
239 | Zm00001d022658  1:+4991231      4       0.5081  TGTTAGCCATGACATGTTTG+AGG
240 | Zm00001d022658  1:+4991232      4       0.5198  GTTAGCCATGACATGTTTGA+GGG
241 | Zm00001d022658  1:+4991233      4       0.6922  TTAGCCATGACATGTTTGAG+GGG
242 | 
243 | ZmC01.intergenic.sgRNA.db.alignment.txt
244 | 
245 | Intergenic      1:+1024980      1       0.2012  CAGTCGTTGCCAAGCGTTCT+TGG
246 | Intergenic      1:+1025011      2       0.4447  ACCAGCAAGCAGCGCACCAC+CGG
247 | Intergenic      1:+1025033      2       0.6557  GCAAGCAGCGCACCACCACA+AGG
248 | Intergenic      1:+1025039      2       0.5222  AGCGCACCACCACAAGGTTG+CGG
249 | Intergenic      1:+1025057      2       0.5021  TGCGGCTTCGAGCACTGCAC+CGG
250 | Intergenic      1:+1025080      1       0.5388  CAAGCAGCGCACCACCAGCA+AGG
251 | Intergenic      1:+1025085      1       0.5927  AGCGCACCACCAGCAAGGAG+TGG
252 | 
253 | 
254 | * There are 5 columns in the UD file and their meanings are listed below:
255 | 
256 | Column 1:	The name of gene where the sgRNA located ("Intergenic" means sgRNA locate in intergenic region).
257 | Column 2:	The chromosome and the coordinate of the start position of the sgRNA.
258 | Column 3:	The number of reads that contains this sgRNA.
259 | Column 4:	The on-target score of the sgRNA.
260 | Column 5:	The sequence of sgRNA (23nt).
261 | 
262 | * A user's sgRNA database file would be generated when the input file is fasta or fastq format. 
263 | 
264 | Example of user's database files:
265 | ----------------------------
266 | ZmC01.gene.sgRNA.db.fastq.txt 
267 | 
268 | ZmC01        3       0.5809  ACAAACAGAGGTCTAAAGCA+AGG
269 | ZmC01        5       0.4756  ACAAACAGCCGGTGAAGCTC+CGG
270 | ZmC01        1       0.4811  ACAAACATTACCTTGTTGAG+AGG
271 | ZmC01        1       0.5456  ACAAACCTGCTCTCAGGGGT+GGG
272 | ZmC01        1       0.5662  ACAAACCTTTCTGTTCTGAT+GGG
273 | ZmC01        2       0.7371  ACAAACGCATGATACATAGG+TGG
274 | ZmC01        16      0.4073  ACAAACGGCCGGCGGCAGCT+AGG
275 | 
276 | Column 1:	The ID of sgRNA.
277 | Column 2:	The number of reads that contains this sgRNA.
278 | Column 3:	The on-target score of the sgRNA. 
279 | Column 4:	The sequence of sgRNA(23nt).
280 | 
281 | ZmC01.gene.sgRNA.db.fasta.txt
282 |         
283 | chr1    +1124   0.1066  TAATCAAATAAATAAGTTTA+TGG
284 | chr1    +1279   0.3327  AGTAATACATTCTTATAAAA+TGG
285 | chr1    +1428   0.4202  GAGTCAGTGTCGTTATGTTA+TGG
286 | chr1    +1501   0.4973  TTACAAGGGAAGTCCCCAAT+TGG
287 | chr1    +1568   0.2559  AATCTTCTAATTACTGTATA+TGG
288 | chr1    +1620   0.3354  GTGGCCAAGGTTCCGTCATT+TGG
289 | chr1    +1699   0.3337  ACATCTATCTCCATATGATA+TGG
290 | 
291 | Column 1:	The ID of reads.
292 | Column 2:	The coordinate of the start position of the sgRNA.
293 | Column 3:	The on-target score of the sgRNA. 
294 | Column 4:	The sequence of sgRNA(23nt).
295 | ```
296 | #### (3) Program DB-search:
297 | 
298 | * User could provide a gene list file in TXT format, one gene per line, the program DB-search would search and compare RD and UD.
299 | * If you did not run the program UD-build, you can specify only the RD.
300 | * DB-search select the important columns in the database file and finally output the result.
301 | ```
302 | Parameters
303 | --- Required ---
304 | 
305 | 	-g <string>	:Query gene list file
306 | 	-i <string>	:Reference sgRNA database (RD)
307 | 
308 | --- Optional ---
309 | 
310 | 	-u <string>	:User's sgRNA database (UD)
311 | 	-o <string>	:Output path (defualt: current directory)
312 | 	-l <label>	:Name prefix for output file (default:DB_search)
313 | 
314 | 	-h		:show this help
315 | 
316 | Example:
317 |      ----------------------------
318 |      perl DB-search.pl -g ZmB73_query_gene.list -i ZmB73.reference.database.txt -u ZmC01.gene.sgRNA.db.alignment.txt -o /your_dir/ -l label
319 | 
320 |      Output of this command
321 |      Invalid_gene_RD.list	(Includes genes that do not exist in RD)
322 |      label_result_RO.txt	(The RD-specific sgRNAs)
323 |      label_result_UO.txt	(The UD-specific sgRNAs)
324 |      label_result_BO.txt	(The common sgRNAs)
325 | 
326 |      Zm00001d001775  2:-547319       0.6568  0.0728  CGGGCGCATCATGCGCCGCG+CGG
327 |      Zm00001d001775  2:-547292       0.6551  0.0458  GGAGAACGGAAAGATCGCTA+GGG
328 |      Zm00001d001775  2:+547304       0.6542  0.0543  TTCCGTTCTCCGTCACCGCG+CGG
329 |      Zm00001d001792  2:+1029153      0.7932  0.0457  GCCGGAGTACTCGAGCAGCG+CGG
330 |      Zm00001d001792  2:+1027126      0.7213  0.1346  GAAAGTGAGATACAAGCCAG+TGG
331 |      Zm00001d001792  2:+1028960      0.7101  0.0478  CGGCAGGTGATGAGTCCTCG+GGG
332 |      
333 |     * User could select the sgRNA with high on-target score (column 3) and low off-target score (column 4).
334 | ```   
335 | #### (4) Program PL-search:
336 |     
337 | * PL-search is a local tool for search exclusive and common target for paralogous gene pair.
338 | * Example of paralog gene list:
339 | ```
340 | Zm00001d049540,Zm00001d024543
341 | Zm00001d048890,Zm00001d029176
342 | Zm00001d031930,Zm00001d018487,Zm00001d003312
343 | Zm00001d036877,Zm00001d046535
344 | ```
345 | ```     
346 | Parameters
347 | --- Required ---
348 | 
349 | 	-g <string>	:Paralog gene list
350 | 	-i <string>	:Reference sgRNA database (RD)
351 | 
352 | --- Optional ---
353 | 
354 | 	-u <string>	:User's sgRNA database (UD)
355 | 	-o <string>	:Output path (defualt: current directory)
356 | 	-l <label>	:Name prefix for output file (default:PL_search)
357 | 
358 | 	-h: show this help and exit
359 | 
360 | Example:
361 |      ----------------------------
362 |       perl PL-search -g ZmB73_paralog_gene.list -i ZmB73.reference.database.txt -u ZmC01.gene.sgRNA.db.alignment.txt -o /your_dir/ -l label
363 |       
364 |       Output of this command
365 |       label_common_targets.txt
366 |       label_exclusive_targets.txt
367 |       
368 |       label_common_targets.txt
369 |       
370 |       GAAAATGTTGCCCATCGATA+TGG UD      Zm00001d031930:1:-207131656:0.428795    Zm00001d018487:5:-221689986:0.428795    Zm00001d003312:2:+39761372:0.428795
371 |       AAGAAAGGGCTGCCCATTCT+TGG UD      Zm00001d031930:1:-207131394:0.351515    Zm00001d018487:5:-221689268:0.351515    Zm00001d003312:2:+39761819:0.351515
372 |       CTCCTCCGGCAGCGGGAGCT+GGG UD      Zm00001d036877:6:+105440097:0.354041    Zm00001d046535:9:-94504945:0.354041
373 |       CGCCACCCAGCTCCCGCTGC+CGG UD      Zm00001d036877:6:-105440102:0.340637    Zm00001d046535:9:+94504940:0.340637
374 |       TGCAGGCGGCGTACATCCTG+TGG UD      Zm00001d036877:6:-105440202:0.487230    Zm00001d046535:9:+94504840:0.487230
375 |       CCCGCTGCCGGAGGAGTTCC+TGG UD      Zm00001d036877:6:-105440090:0.158463    Zm00001d046535:9:+94504952:0.163690
376 |       
377 |       Column 1: The sequence of sgRNA.
378 |       Column 2:	The sgRNAs exist in UD are indicated with "UD" mark.
379 |       Column 3,4,5:	The gene ID; the coordinate o sgRNA; the on-target score of the sgRNA. 
380 |       
381 |       label_exclusive_targets.txt
382 |       
383 |       Zm00001d031930  1:-207131270    0.230513        TGGAGTGGATCCAGCTATTA+TGG Zm00001d028694  1:+43348332     TGTAGTTGATCCAGGTACTA+CAG 4       0.0016  RD      0
384 |       Zm00001d031930  1:+207131844    0.139577        TTCCCAAGCAATGGAATTTA+TGG Zm00001d024478  10:-73139286    TTTACAAGCAGTGGAACTTA+AGG 4       0.2656  UD      1
385 |       Zm00001d018487  5:+221690821    0.782208        GAACCTTGAGAACAACAGTG+AGG Zm00001d027293  1:+2033102      TTCCCTTGACAACAACAGTG+CGG 4       0.1247  RD      0
386 |       Zm00001d018487  5:-221690848    0.776871        CTTCAAGGACACCTACCCAG+AGG Zm00001d023582  10:+10766178    TTTCAAGCACCCCTTCCCAG+TGG 4       0.0492  UD      312
387 |       Zm00001d018487  5:+221688910    0.748847        GCAGCAATAGACAAACTGAG+AGG Zm00001d024647  10:+82039457    GAAGCTATAGACAAGCTCAG+GAG 4       0.0417  UD      303
388 |       Zm00001d018487  5:+221690357    0.733564        GTAGCCCAGATGAACACTGG+CGG Zm00001d025888  10:+133099566   GTAGGCAAAATGAACTCTGG+TAG 4       0.0000  UD      357
389 |       
390 |       Column 1-11: As stated above.
391 |       Column 12 : The sgRNAs exist in UD are indicated with "UD" mark, else indicated with "RD" mark.
392 |       Column 13 : The frequency of sgRNA in UD.
393 | ```
394 | 
395 | For other technical issues about CRISPR-P 2.0, please contact: liuhao1122@webmail.hzau.edu.cn
396 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/PaxHeader/README_v1.pdf:
--------------------------------------------------------------------------------
1 | 18 gid=1594166068
2 | 18 uid=1248751364
3 | 20 ctime=1447951994
4 | 20 atime=1447951994
5 | 23 SCHILY.dev=16777220
6 | 23 SCHILY.ino=24334127
7 | 18 SCHILY.nlink=1
8 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/PaxHeader/analysis:
--------------------------------------------------------------------------------
1 | 18 gid=1594166068
2 | 18 uid=1248751364
3 | 20 ctime=1447952234
4 | 20 atime=1447952235
5 | 38 LIBARCHIVE.creationtime=1431967336
6 | 23 SCHILY.dev=16777220
7 | 23 SCHILY.ino=24334091
8 | 19 SCHILY.nlink=17
9 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/PaxHeader/saved_models:
--------------------------------------------------------------------------------
1 | 18 gid=1594166068
2 | 18 uid=1248751364
3 | 20 ctime=1447951994
4 | 20 atime=1447952249
5 | 23 SCHILY.dev=16777220
6 | 23 SCHILY.ino=24334128
7 | 18 SCHILY.nlink=4
8 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/README_v1.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sunjiamin0824/CRISPR-Local/c6e80b2fd8ee4065c0655edf9f5206bf8cc316e2/Rule_Set_2_scoring_v1/README_v1.pdf


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/PaxHeader/features:
--------------------------------------------------------------------------------
1 | 18 gid=1594166068
2 | 18 uid=1248751364
3 | 20 ctime=1447951994
4 | 20 atime=1447951994
5 | 38 LIBARCHIVE.creationtime=1447431915
6 | 23 SCHILY.dev=16777220
7 | 23 SCHILY.ino=24334093
8 | 18 SCHILY.nlink=8
9 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/PaxHeader/load_data.py:
--------------------------------------------------------------------------------
1 | 18 gid=1594166068
2 | 18 uid=1248751364
3 | 20 ctime=1447951994
4 | 20 atime=1447951994
5 | 23 SCHILY.dev=16777220
6 | 23 SCHILY.ino=24334100
7 | 18 SCHILY.nlink=1
8 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/PaxHeader/load_data.pyc:
--------------------------------------------------------------------------------
1 | 18 gid=1594166068
2 | 18 uid=1248751364
3 | 20 ctime=1447951994
4 | 20 atime=1447951994
5 | 23 SCHILY.dev=16777220
6 | 23 SCHILY.ino=24334101
7 | 18 SCHILY.nlink=1
8 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/PaxHeader/local_multiprocessing.py:
--------------------------------------------------------------------------------
1 | 18 gid=1594166068
2 | 18 uid=1248751364
3 | 20 ctime=1447951994
4 | 20 atime=1447951994
5 | 23 SCHILY.dev=16777220
6 | 23 SCHILY.ino=24334102
7 | 18 SCHILY.nlink=1
8 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/PaxHeader/local_multiprocessing.pyc:
--------------------------------------------------------------------------------
1 | 18 gid=1594166068
2 | 18 uid=1248751364
3 | 20 ctime=1447951994
4 | 20 atime=1447951994
5 | 23 SCHILY.dev=16777220
6 | 23 SCHILY.ino=24334103
7 | 18 SCHILY.nlink=1
8 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/PaxHeader/metrics.py:
--------------------------------------------------------------------------------
1 | 18 gid=1594166068
2 | 18 uid=1248751364
3 | 20 ctime=1447951994
4 | 20 atime=1447951994
5 | 23 SCHILY.dev=16777220
6 | 23 SCHILY.ino=24334104
7 | 18 SCHILY.nlink=1
8 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/PaxHeader/metrics.pyc:
--------------------------------------------------------------------------------
1 | 18 gid=1594166068
2 | 18 uid=1248751364
3 | 20 ctime=1447951994
4 | 20 atime=1447951994
5 | 23 SCHILY.dev=16777220
6 | 23 SCHILY.ino=24334105
7 | 18 SCHILY.nlink=1
8 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/PaxHeader/model_comparison.py:
--------------------------------------------------------------------------------
1 | 18 gid=1594166068
2 | 18 uid=1248751364
3 | 20 ctime=1447951994
4 | 20 atime=1447951994
5 | 23 SCHILY.dev=16777220
6 | 23 SCHILY.ino=24334106
7 | 18 SCHILY.nlink=1
8 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/PaxHeader/model_comparison.pyc:
--------------------------------------------------------------------------------
1 | 18 gid=1594166068
2 | 18 uid=1248751364
3 | 20 ctime=1447951994
4 | 20 atime=1447951994
5 | 23 SCHILY.dev=16777220
6 | 23 SCHILY.ino=24334107
7 | 18 SCHILY.nlink=1
8 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/PaxHeader/models:
--------------------------------------------------------------------------------
1 | 18 gid=1594166068
2 | 18 uid=1248751364
3 | 20 ctime=1447951994
4 | 20 atime=1447951994
5 | 38 LIBARCHIVE.creationtime=1447431915
6 | 23 SCHILY.dev=16777220
7 | 23 SCHILY.ino=24334108
8 | 19 SCHILY.nlink=14
9 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/PaxHeader/predict.py:
--------------------------------------------------------------------------------
1 | 18 gid=1594166068
2 | 18 uid=1248751364
3 | 20 ctime=1447951994
4 | 20 atime=1447951994
5 | 23 SCHILY.dev=16777220
6 | 23 SCHILY.ino=24334121
7 | 18 SCHILY.nlink=1
8 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/PaxHeader/predict.pyc:
--------------------------------------------------------------------------------
1 | 18 gid=1594166068
2 | 18 uid=1248751364
3 | 20 ctime=1447951994
4 | 20 atime=1447951994
5 | 23 SCHILY.dev=16777220
6 | 23 SCHILY.ino=24334122
7 | 18 SCHILY.nlink=1
8 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/PaxHeader/rs2_score_calculator.py:
--------------------------------------------------------------------------------
1 | 18 gid=1594166068
2 | 18 uid=1248751364
3 | 20 ctime=1447951994
4 | 20 atime=1447952094
5 | 23 SCHILY.dev=16777220
6 | 23 SCHILY.ino=24334123
7 | 18 SCHILY.nlink=1
8 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/PaxHeader/util.py:
--------------------------------------------------------------------------------
1 | 18 gid=1594166068
2 | 18 uid=1248751364
3 | 20 ctime=1447951994
4 | 20 atime=1447951994
5 | 23 SCHILY.dev=16777220
6 | 23 SCHILY.ino=24334124
7 | 18 SCHILY.nlink=1
8 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/PaxHeader/util.pyc:
--------------------------------------------------------------------------------
1 | 18 gid=1594166068
2 | 18 uid=1248751364
3 | 20 ctime=1447951994
4 | 20 atime=1447951994
5 | 23 SCHILY.dev=16777220
6 | 23 SCHILY.ino=24334125
7 | 18 SCHILY.nlink=1
8 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/features/PaxHeader/__init__.py:
--------------------------------------------------------------------------------
1 | 18 gid=1594166068
2 | 18 uid=1248751364
3 | 20 ctime=1447951994
4 | 20 atime=1447951994
5 | 23 SCHILY.dev=16777220
6 | 23 SCHILY.ino=24334094
7 | 18 SCHILY.nlink=1
8 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/features/PaxHeader/__init__.pyc:
--------------------------------------------------------------------------------
1 | 18 gid=1594166068
2 | 18 uid=1248751364
3 | 20 ctime=1447951994
4 | 20 atime=1447951994
5 | 23 SCHILY.dev=16777220
6 | 23 SCHILY.ino=24334095
7 | 18 SCHILY.nlink=1
8 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/features/PaxHeader/featurization.py:
--------------------------------------------------------------------------------
1 | 18 gid=1594166068
2 | 18 uid=1248751364
3 | 20 ctime=1447951994
4 | 20 atime=1447951994
5 | 23 SCHILY.dev=16777220
6 | 23 SCHILY.ino=24334096
7 | 18 SCHILY.nlink=1
8 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/features/PaxHeader/featurization.pyc:
--------------------------------------------------------------------------------
1 | 18 gid=1594166068
2 | 18 uid=1248751364
3 | 20 ctime=1447951994
4 | 20 atime=1447951994
5 | 23 SCHILY.dev=16777220
6 | 23 SCHILY.ino=24334097
7 | 18 SCHILY.nlink=1
8 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/features/PaxHeader/microhomology.py:
--------------------------------------------------------------------------------
1 | 18 gid=1594166068
2 | 18 uid=1248751364
3 | 20 ctime=1447951994
4 | 20 atime=1447951994
5 | 23 SCHILY.dev=16777220
6 | 23 SCHILY.ino=24334098
7 | 18 SCHILY.nlink=1
8 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/features/PaxHeader/microhomology.pyc:
--------------------------------------------------------------------------------
1 | 18 gid=1594166068
2 | 18 uid=1248751364
3 | 20 ctime=1447951994
4 | 20 atime=1447951994
5 | 23 SCHILY.dev=16777220
6 | 23 SCHILY.ino=24334099
7 | 18 SCHILY.nlink=1
8 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/features/__init__.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sunjiamin0824/CRISPR-Local/c6e80b2fd8ee4065c0655edf9f5206bf8cc316e2/Rule_Set_2_scoring_v1/analysis/features/__init__.pyc


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/features/featurization.py:
--------------------------------------------------------------------------------
  1 | import pandas
  2 | import time
  3 | import numpy as np
  4 | import Bio.SeqUtils as SeqUtil
  5 | import Bio.Seq as Seq
  6 | import util
  7 | import sys
  8 | import Bio.SeqUtils.MeltingTemp as Tm
  9 | import pickle
 10 | import itertools
 11 | import sklearn
 12 | 
 13 | def featurize_data(data, learn_options, Y, gene_position):
 14 |     '''
 15 |     assumes that data contains the 30mer
 16 |     returns set of features from which one can make a kernel for each one
 17 |     '''
 18 | 
 19 |     #print "Constructing features..."
 20 |     t0 = time.time()
 21 | 
 22 |     feature_sets = {}
 23 | 
 24 |     if learn_options["nuc_features"]:
 25 |         # spectrum kernels (position-independent) and weighted degree kernels (position-dependent)
 26 |         get_all_order_nuc_features(data['30mer'], feature_sets, learn_options, learn_options["order"], max_index_to_use=30)
 27 | 
 28 |     if learn_options["gc_features"]:
 29 |         gc_above_10, gc_below_10, gc_count = gc_features(data)
 30 |         feature_sets['gc_above_10'] = pandas.DataFrame(gc_above_10)
 31 |         feature_sets['gc_below_10'] = pandas.DataFrame(gc_below_10)
 32 |         feature_sets['gc_count'] = pandas.DataFrame(gc_count)
 33 | 
 34 |     if learn_options["include_gene_position"]:
 35 |         # gene_position_columns = ["Amino Acid Cut position", "Percent Peptide", "Nucleotide cut position"]
 36 |         # gene_position_columns = ["Percent Peptide", "Nucleotide cut position"]
 37 | 
 38 |         for set in gene_position.columns:
 39 |             set_name = set
 40 |             feature_sets[set_name] = pandas.DataFrame(gene_position[set])
 41 |         feature_sets["Percent Peptide <50%"] = feature_sets["Percent Peptide"] < 50
 42 | 
 43 |     if learn_options["include_gene_effect"]:
 44 |         print "including gene effect"
 45 |         gene_names = Y['Target gene']
 46 |         enc = sklearn.preprocessing.OneHotEncoder()
 47 |         label_encoder = sklearn.preprocessing.LabelEncoder()
 48 |         label_encoder.fit(gene_names)
 49 |         one_hot_genes = np.array(enc.fit_transform(label_encoder.transform(gene_names)[:, None]).todense())
 50 |         feature_sets["gene effect"] = pandas.DataFrame(one_hot_genes,
 51 |                                                        columns=["gene_%d" % i for i in range(one_hot_genes.shape[1])], index=gene_names.index)
 52 | 
 53 |     if learn_options['include_known_pairs']:
 54 |         feature_sets['known pairs'] = pandas.DataFrame(Y['test'])
 55 | 
 56 |     if learn_options["include_NGGX_interaction"]:
 57 |         feature_sets["NGGX"] = NGGX_interaction_feature(data)
 58 | 
 59 |     if learn_options["include_Tm"]:
 60 |         feature_sets["Tm"] = Tm_feature(data)
 61 | 
 62 |     if learn_options["include_sgRNAscore"]:
 63 |         feature_sets["sgRNA Score"] = pandas.DataFrame(data["sgRNA Score"])
 64 | 
 65 |     if learn_options["include_drug"]:
 66 |         # feature_sets["drug"] = pandas.DataFrame(data["drug"])
 67 |         drug_names = Y.index.get_level_values('drug').tolist()
 68 |         enc = sklearn.preprocessing.OneHotEncoder()
 69 |         label_encoder = sklearn.preprocessing.LabelEncoder()
 70 |         label_encoder.fit(drug_names)
 71 |         one_hot_drugs = np.array(enc.fit_transform(label_encoder.transform(drug_names)[:, None]).todense())
 72 |         feature_sets["drug"] = pandas.DataFrame(one_hot_drugs, columns=["drug_%d" % i for i in range(one_hot_drugs.shape[1])], index=drug_names)
 73 | 
 74 |     if learn_options['include_strand']:
 75 |         feature_sets['Strand effect'] = (pandas.DataFrame(data['Strand']) == 'sense')*1
 76 | 
 77 |     if learn_options["include_gene_feature"]:
 78 |         feature_sets["gene features"] = gene_feature(Y, data, learn_options)
 79 | 
 80 |     if learn_options["include_gene_guide_feature"] > 0:
 81 |         tmp_feature_sets = gene_guide_feature(Y, data, learn_options)
 82 |         for key in tmp_feature_sets:
 83 |             feature_sets[key] = tmp_feature_sets[key]
 84 | 
 85 |     if learn_options["include_microhomology"]:
 86 |         feature_sets["microhomology"] = get_micro_homology_features(Y['Target gene'], learn_options, data)
 87 | 
 88 |     t1 = time.time()
 89 |     #print "\t\tElapsed time for constructing features is %.2f seconds" % (t1-t0)
 90 | 
 91 |     check_feature_set_dimensions(feature_sets)
 92 | 
 93 |     if learn_options['normalize_features']:
 94 |         feature_sets = normalize_feature_sets(feature_sets)
 95 |         check_feature_set_dimensions(feature_sets)
 96 | 
 97 |     return feature_sets
 98 | 
 99 | 
100 | def check_feature_set_dimensions(feature_sets):
101 |     '''
102 |     Ensure the # of people is the same in each feature set
103 |     '''
104 |     N = None
105 |     for ft in feature_sets.keys():
106 |         N2 = feature_sets[ft].shape[0]
107 |         if N is None:
108 |             N = N2
109 |         else:
110 |             assert N == N2, "# of individuals do not match up across feature sets"
111 | 
112 | 
113 | def NGGX_interaction_feature(data):
114 |     '''
115 |     assuming 30-mer, grab the NGGX _ _ positions, and make a one-hot
116 |     encoding of the NX nucleotides yielding 4x4=16 features
117 |     '''
118 |     sequence = data['30mer'].values
119 |     feat_NX = pandas.DataFrame()
120 |     # check that GG is where we think
121 |     for seq in sequence:
122 |         if seq[25:27] != "GG":
123 |             raise Exception("expected GG but found %s" % seq[25:27])
124 |         NX = seq[24]+seq[27]
125 |         NX_onehot = nucleotide_features(NX,order=2, include_pos_independent=False, max_index_to_use=2, prefix="NGGX")
126 |         # NX_onehot[:] = np.random.rand(NX_onehot.shape[0]) ##TESTING RANDOM FEATURE
127 |         feat_NX = pandas.concat([feat_NX, NX_onehot], axis=1)
128 |     return feat_NX.T
129 | 
130 | 
131 | def get_all_order_nuc_features(data, feature_sets, learn_options, maxorder, max_index_to_use, prefix=""):
132 |     for order in range(1, maxorder+1):
133 |         #print "\t\tconstructing order %s features" % order
134 |         nuc_features_pd, nuc_features_pi = apply_nucleotide_features(data, order, learn_options["num_proc"],
135 |                                                                      include_pos_independent=True, max_index_to_use=max_index_to_use, prefix=prefix)
136 |         feature_sets['%s_nuc_pd_Order%i' % (prefix, order)] = nuc_features_pd
137 |         if learn_options['include_pi_nuc_feat']:
138 |             feature_sets['%s_nuc_pi_Order%i' % (prefix, order)] = nuc_features_pi
139 |         #print "\t\t\t\t\t\t\tdone"
140 | 
141 | 
142 | def countGC(s):
143 |     '''
144 |     GC content for only the 20mer, as per the Doench paper/code
145 |     '''
146 |     assert len(s) == 30, "seems to assume 30mer"
147 |     return len(s[5:25].replace('A', '').replace('T', ''))
148 | 
149 | 
150 | def SeqUtilFeatures(data):
151 |     '''
152 |     assuming '30-mer'is a key
153 |     get melting temperature features from:
154 |         0-the 30-mer ("global Tm")
155 |         1-the Tm (melting temperature) of the DNA:RNA hybrid from positions 16 - 20 of the sgRNA, i.e. the 5nts immediately proximal of the NGG PAM
156 |         2-the Tm of the DNA:RNA hybrid from position 8 - 15 (i.e. 8 nt)
157 |         3-the Tm of the DNA:RNA hybrid from position 3 - 7  (i.e. 5 nt)
158 |     '''
159 |     sequence = data['30mer'].values
160 |     num_features = 1
161 |     featarray = np.ones((sequence.shape[0], num_features))
162 |     for i, seq in enumerate(sequence):
163 |         assert len(seq) == 30, "seems to assume 30mer"
164 |         featarray[i, 0] = SeqUtil.molecular_weight(str(seq))
165 | 
166 |     feat = pandas.DataFrame(pandas.DataFrame(featarray))
167 |     return feat
168 | 
169 | 
170 | def organism_feature(data):
171 |     '''
172 |     Human vs. mouse
173 |     '''
174 |     organism = np.array(data['Organism'].values)
175 |     feat = pandas.DataFrame(pandas.DataFrame(featarray))
176 |     import ipdb; ipdb.set_trace()
177 |     return feat
178 | 
179 | 
180 | def get_micro_homology_features(gene_names, learn_options, X):
181 |     # originally was flipping the guide itself as necessary, but now flipping the gene instead
182 | 
183 |     print "building microhomology features"
184 |     feat = pandas.DataFrame(index=X.index)
185 |     feat["mh_score"] = ""
186 |     feat["oof_score"] = ""
187 | 
188 |     #with open(r"tmp\V%s_gene_mismatches.csv" % learn_options["V"],'wb') as f:
189 |     if True:
190 |         # number of nulceotides to take to the left and right of the guide
191 |         k_mer_length_left = 9
192 |         k_mer_length_right = 21
193 |         for gene in gene_names.unique():
194 |             gene_seq = Seq.Seq(util.get_gene_sequence(gene)).reverse_complement()
195 |             guide_inds = np.where(gene_names.values == gene)[0]
196 |             print "getting microhomology for all %d guides in gene %s" % (len(guide_inds), gene)
197 |             for j, ps in enumerate(guide_inds):
198 |                 guide_seq = Seq.Seq(X['30mer'][ps])
199 |                 strand = X['Strand'][ps]
200 |                 if strand=='sense':              
201 |                     gene_seq = gene_seq.reverse_complement()
202 |                 # figure out the sequence to the left and right of this guide, in the gene
203 |                 ind = gene_seq.find(guide_seq)                        
204 |                 if ind==-1:
205 |                     gene_seq = gene_seq.reverse_complement()
206 |                     ind = gene_seq.find(guide_seq)                        
207 |                     #assert ind != -1, "still didn't work"
208 |                     #print "shouldn't get here"
209 |                 else:
210 |                     #print "all good"
211 |                     pass
212 |                 #assert ind != -1, "could not find guide in gene"
213 |                 if ind==-1:
214 |                     #print "***could not find guide %s for gene %s" % (str(guide_seq), str(gene))
215 |                     #if.write(str(gene) + "," + str(guide_seq))
216 |                     mh_score = 0
217 |                     oof_score = 0
218 |                 else:
219 |                     #print "worked"
220 | 
221 |                     assert gene_seq[ind:(ind+len(guide_seq))]==guide_seq, "match not right"
222 |                     left_win = gene_seq[(ind - k_mer_length_left):ind]
223 |                     right_win = gene_seq[(ind + len(guide_seq)):(ind + len(guide_seq) + k_mer_length_right)]
224 | 
225 |                     #if strand=='antisense':
226 |                     #    # it's arbitrary which of sense and anti-sense we flip, we just want
227 |                     #    # to keep them in the same relative alphabet/direction
228 |                     #    left_win = left_win.reverse_complement()
229 |                     #    right_win = right_win.reverse_complement()
230 |                     assert len(left_win.tostring())==k_mer_length_left
231 |                     assert len(right_win.tostring())==k_mer_length_right
232 |                                     
233 |                     sixtymer = str(left_win) + str(guide_seq) + str(right_win)
234 |                     assert len(sixtymer)==60, "should be of length 60"
235 |                     mh_score, oof_score = microhomology.compute_score(sixtymer)
236 | 
237 |                 feat.ix[ps,"mh_score"] = mh_score
238 |                 feat.ix[ps,"oof_score"] = oof_score
239 |             print "computed microhomology of %s" % (str(gene))           
240 |     
241 |     return pandas.DataFrame(feat, dtype='float')
242 | 
243 | 
244 | def local_gene_seq_features(gene_names, learn_options, X):
245 |     
246 |     print "building local gene sequence features"
247 |     feat = pandas.DataFrame(index=X.index)
248 |     feat["gene_left_win"] = ""
249 |     feat["gene_right_win"] = ""
250 | 
251 |     # number of nulceotides to take to the left and right of the guide
252 |     k_mer_length = learn_options['include_gene_guide_feature']
253 |     for gene in gene_names.unique():
254 |         gene_seq = Seq.Seq(util.get_gene_sequence(gene)).reverse_complement()
255 |         for ps in np.where(gene_names.values==gene)[0]:
256 |             guide_seq = Seq.Seq(X['30mer'][ps])
257 |             strand = X['Strand'][ps]
258 |             if strand=='sense':
259 |                 guide_seq = guide_seq.reverse_complement()
260 |                 #gene_seq = gene_seq.reverse_complement()
261 |             # figure out the sequence to the left and right of this guide, in the gene
262 |             ind = gene_seq.find(guide_seq)
263 |             if ind ==-1:
264 |                 #gene_seq = gene_seq.reverse_complement()
265 |                 #ind = gene_seq.find(guide_seq)
266 |                 assert ind != -1, "could not find guide in gene"
267 |             assert gene_seq[ind:(ind+len(guide_seq))]==guide_seq, "match not right"
268 |             left_win = gene_seq[(ind - k_mer_length):ind]
269 |             right_win = gene_seq[(ind + len(guide_seq)):(ind + len(guide_seq) + k_mer_length)]
270 | 
271 |             if strand=='antisense':
272 |                 # it's arbitrary which of sense and anti-sense we flip, we just want
273 |                 # to keep them in the same relative alphabet/direction
274 |                 left_win = left_win.reverse_complement()
275 |                 right_win = right_win.reverse_complement()
276 |             assert not left_win.tostring()=="", "k_mer_context, %s, is too large" % k_mer_length
277 |             assert not left_win.tostring()=="", "k_mer_context, %s, is too large" % k_mer_length
278 |             assert len(left_win)==len(right_win), "k_mer_context, %s, is too large" % k_mer_length
279 |             feat.ix[ps,"gene_left_win"] = left_win.tostring()
280 |             feat.ix[ps,"gene_right_win"] = right_win.tostring()
281 |         print "featurizing local context of %s" % (gene)
282 | 
283 |     feature_sets = {}
284 |     get_all_order_nuc_features(feat["gene_left_win"], feature_sets, learn_options, learn_options["order"], max_index_to_use=sys.maxint, prefix="gene_left_win")
285 |     get_all_order_nuc_features(feat["gene_right_win"], feature_sets, learn_options, learn_options["order"], max_index_to_use=sys.maxint, prefix="gene_right_win")
286 |     return feature_sets
287 | 
288 | def gene_feature(Y, X, learn_options):
289 |     '''
290 |     Things like the sequence of the gene, the DNA Tm of the gene, etc.
291 |     '''
292 | 
293 |     gene_names = Y['Target gene']
294 | 
295 |     gene_length = np.zeros((gene_names.values.shape[0], 1))
296 |     gc_content = np.zeros((gene_names.shape[0], 1))
297 |     temperature = np.zeros((gene_names.shape[0], 1))
298 |     molecular_weight = np.zeros((gene_names.shape[0], 1))
299 | 
300 |     for gene in gene_names.unique():
301 |         seq = util.get_gene_sequence(gene)
302 |         gene_length[gene_names.values==gene] = len(seq)
303 |         gc_content[gene_names.values==gene] = SeqUtil.GC(seq)
304 |         temperature[gene_names.values==gene] = Tm.Tm_staluc(seq, rna=False)
305 |         molecular_weight[gene_names.values==gene] = SeqUtil.molecular_weight(seq, 'DNA')
306 | 
307 |     all = np.concatenate((gene_length, gc_content, temperature, molecular_weight), axis=1)
308 |     df = pandas.DataFrame(data=all, index=gene_names.index, columns=['gene length',
309 |                                                                      'gene GC content',
310 |                                                                      'gene temperature',
311 |                                                                      'gene molecular weight'])
312 |     return df
313 | 
314 | def gene_guide_feature(Y, X, learn_options):
315 |     #features, which are related to parts of the gene-local to the guide, and
316 |     #possibly incorporating the guide or interactions with it
317 | 
318 |     #expensive, so pickle if necessary
319 |     gene_file = r"..\data\gene_seq_feat_V%s_km%s.ord%s.pickle" % (learn_options['V'], learn_options['include_gene_guide_feature'], learn_options['order'])
320 | 
321 |     if False: #os.path.isfile(gene_file): #while debugging, comment out
322 |         print "loading local gene seq feats from file %s" % gene_file
323 |         with open(gene_file, "rb") as f: feature_sets = pickle.load(f)
324 |     else:
325 |         feature_sets = local_gene_seq_features(Y['Target gene'], learn_options, X)
326 |         print "writing local gene seq feats to file %s" % gene_file
327 |         with open(gene_file, "wb") as f: pickle.dump(feature_sets, f)
328 | 
329 |     return feature_sets
330 | 
331 | 
332 | def gc_cont(seq):
333 |     return (seq.count('G') + seq.count('C'))/float(len(seq))
334 | 
335 | 
336 | 
337 | def Tm_feature(data):
338 |     '''
339 |     assuming '30-mer'is a key
340 |     get melting temperature features from:
341 |         0-the 30-mer ("global Tm")
342 |         1-the Tm (melting temperature) of the DNA:RNA hybrid from positions 16 - 20 of the sgRNA, i.e. the 5nts immediately proximal of the NGG PAM
343 |         2-the Tm of the DNA:RNA hybrid from position 8 - 15 (i.e. 8 nt)
344 |         3-the Tm of the DNA:RNA hybrid from position 3 - 7  (i.e. 5 nt)
345 |     '''
346 |     sequence = data['30mer'].values
347 |     featarray = np.ones((sequence.shape[0],4))
348 |     for i, seq in enumerate(sequence):
349 |         if seq[25:27]!="GG":
350 |             raise Exception("expected GG but found %s" % seq[25:27])
351 |         rna = False
352 |         featarray[i,0] = Tm.Tm_staluc(seq, rna=rna)        #30mer Tm
353 |         featarray[i,1] = Tm.Tm_staluc(seq[20:25], rna=rna) #5nts immediately proximal of the NGG PAM
354 |         featarray[i,2] = Tm.Tm_staluc(seq[12:20], rna=rna)   #8-mer
355 |         featarray[i,3] = Tm.Tm_staluc(seq[7:12], rna=rna)      #5-mer
356 | 
357 |     feat = pandas.DataFrame(featarray, index=data.index, columns=["Tm global_%s" % rna, "5mer_end_%s" %rna, "8mer_middle_%s" %rna, "5mer_start_%s" %rna])
358 | 
359 |     return feat
360 | 
361 | def gc_features(data):
362 |     gc_count = data['30mer'].apply(countGC)
363 |     gc_count.name = 'GC count'
364 |     gc_above_10 = (gc_count > 10)*1
365 |     gc_above_10.name = 'GC > 10'
366 |     gc_below_10 = (gc_count < 10)*1
367 |     gc_below_10.name = 'GC < 10'
368 |     return gc_above_10, gc_below_10, gc_count
369 | 
370 | 
371 | 
372 | def normalize_features(data,axis):
373 |     '''
374 |     input: Pandas.DataFrame of dtype=np.float64 array, of dimensions
375 |     mean-center, and unit variance each feature
376 |     '''
377 |     data -= data.mean(axis)
378 |     data /= data.std(axis)
379 |     # remove rows with NaNs
380 |     data = data.dropna(1)
381 |     if np.any(np.isnan(data.values)): raise Exception("found NaN in normalized features")
382 |     return data
383 | 
384 | def apply_nucleotide_features(seq_data_frame, order, num_proc, include_pos_independent, max_index_to_use, prefix=""):
385 | 
386 |     fast = True
387 | 
388 |     if fast:
389 |         feat_pd = seq_data_frame.apply(nucleotide_features, args=(order, include_pos_independent, max_index_to_use, prefix, 'pos_dependent'))
390 |         feat_pi = seq_data_frame.apply(nucleotide_features, args=(order, include_pos_independent, max_index_to_use, prefix, 'pos_independent'))
391 |     else:
392 |         # old, much slower code
393 |         feat_pd = pandas.DataFrame()
394 |         feat_pi = pandas.DataFrame()
395 |         for s in seq_data_frame.values:
396 |             assert not (s==''), "string is empty"
397 |             feat_pd_tmp, feat_pi_tmp = nucleotide_features(s, order, include_pos_independent, max_index_to_use, prefix=prefix)
398 |             feat_pd = pandas.concat([feat_pd,feat_pd_tmp], axis=1)
399 |             feat_pi = pandas.concat([feat_pi,feat_pi_tmp], axis=1)
400 |         
401 |         feat_pd = feat_pd.T
402 |         feat_pi = feat_pi.T  
403 |     
404 |     return feat_pd, feat_pi
405 | 
406 | 
407 | 
408 | def nucleotide_features(s, order, include_pos_independent, max_index_to_use, prefix="", feature_type='all'):
409 |     '''
410 |     compute position-specific order-mer features for the 4-letter alphabet
411 |     (e.g. for a sequence of length 30, there are 30*4 single nucleotide features
412 |           and (30-1)*4^2=464 double nucleotide features
413 |     '''
414 |     if max_index_to_use is not None:
415 |         s = s[:max_index_to_use]
416 |     #assert(len(s)==30, "length not 30")
417 |     #s = s[:30] #cut-off at thirty to clean up extra data that they accidentally left in, and were instructed to ignore in this way
418 |     raw_alphabet = ['A', 'T', 'C', 'G']
419 |     alphabet = ["".join(i) for i in itertools.product(raw_alphabet, repeat=order)]
420 |     features_pos_dependent = np.zeros(len(alphabet)*(len(s)-(order-1)))
421 |     features_pos_independent = np.zeros(np.power(len(raw_alphabet),order))
422 | 
423 | 
424 |     #for position in range(0, len(s)-order, 1): JENN 9/4/2014 failing when len(s)=2
425 |     for position in range(0, len(s)-order+1, 1):
426 |         nucl = s[position:position+order]
427 |         features_pos_dependent[alphabet.index(nucl)+(position*len(alphabet))] = 1.0
428 |         features_pos_independent[alphabet.index(nucl)] += 1.0
429 |     index_dependent = ['%s_pd.Order%d_P%d' % (prefix, order,i) for i in range(len(features_pos_dependent))]
430 | 
431 |     if feature_type == 'all' or feature_type == 'pos_independent':
432 |         if include_pos_independent:
433 |             index_independent = ['%s_pi.Order%d_P%d' % (prefix, order,i) for i in range(len(features_pos_independent))]
434 |             if feature_type == 'all':
435 |                 return pandas.Series(features_pos_dependent,index=index_dependent), pandas.Series(features_pos_independent,index=index_independent)
436 |             else:
437 |                 return pandas.Series(features_pos_independent, index=index_independent)
438 |     
439 |     if np.any(np.isnan(features_pos_dependent)): raise Exception("found nan features in features_pos_dependent")
440 |     if np.any(np.isnan(features_pos_independent)): raise Exception("found nan features in features_pos_independent")
441 |     
442 |     return pandas.Series(features_pos_dependent, index=index_dependent)
443 | 
444 | 
445 | 
446 | def normalize_feature_sets(feature_sets):
447 |     '''
448 |     zero-mean, unit-variance each feature within each set
449 |     '''
450 | 
451 |     print "Normalizing features..."
452 |     t1 = time.time()
453 | 
454 |     new_feature_sets = {}
455 |     for set in feature_sets:
456 |          new_feature_sets[set] = normalize_features(feature_sets[set],axis=0)
457 |          if np.any(np.isnan(new_feature_sets[set].values)):
458 |              raise Exception("found Nan feature values in set=%s" % set)
459 | 
460 |     t2 = time.time()
461 |     print "\t\tElapsed time for normalizing features is %.2f seconds" % (t2-t1)
462 | 
463 |     return new_feature_sets
464 | 
465 | 
466 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/features/featurization.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sunjiamin0824/CRISPR-Local/c6e80b2fd8ee4065c0655edf9f5206bf8cc316e2/Rule_Set_2_scoring_v1/analysis/features/featurization.pyc


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/features/microhomology.py:
--------------------------------------------------------------------------------
  1 | #Supplementary Figure 3  |  Source code for assigning a score to a hypothetical deletion 
  2 | #pattern associated with microhomology 
  3 | # ------------------------------------------
  4 | # comes from the Supplementary Info of the paper, in pdf form, copied here, but refactored to make a function
  5 | #    rather than to write it to file
  6 | # also see their web server version: http://www.rgenome.net/mich-calculator/ where they say:
  7 | # Insert one or more query sequences (A, G, T, C only) flanking the same length at a cleavage site (100bp or less, 60~80bp recommended).
  8 | 
  9 | from math import exp       
 10 | from re import findall 
 11 |  
 12 | def compute_score(seq, tmpfile1="1.before removing duplication.txt", tmpfile2="2.all microhomology patterns.txt", verbose=False):
 13 |     length_weight=20.0 
 14 |     left=30        # Insert the position expected to be broken. 
 15 |     right=len(seq)-int(left) 
 16 |     #print 'length of seq = '+str(len(seq)) 
 17 |      
 18 |     file_temp=open(tmpfile1, "w") 
 19 |     for k in range(2,left)[::-1]: 
 20 |             for j in range(left,left+right-k+1): 
 21 |                     for i in range(0,left-k+1): 
 22 |                             if seq[i:i+k]==seq[j:j+k]: 
 23 |                                     length=j-i 
 24 |                                     file_temp.write(seq[i:i+k]+'\t'+str(i)+'\t'+str(i+k)+'\t'+str(j)+'\t'+str(j+k)+'\t'+str(length)+'\n') 
 25 |     file_temp.close() 
 26 |      
 27 |     ### After searching out all microhomology patterns, duplication should be removed!! 
 28 |     f1=open(tmpfile1, "r") 
 29 |     s1=f1.read() 
 30 |      
 31 |     f2=open(tmpfile2, "w") #After removing duplication 
 32 |     f2.write(seq+'\t'+'microhomology\t'+'deletion length\t'+'score of a pattern\n') 
 33 |      
 34 |     if s1!="": 
 35 |             list_f1=s1.strip().split('\n') 
 36 |             sum_score_3=0 
 37 |             sum_score_not_3=0 
 38 |      
 39 |             for i in range(len(list_f1)): 
 40 |                     n=0 
 41 |                     score_3=0 
 42 |                     score_not_3=0 
 43 |                     line=list_f1[i].split('\t') 
 44 |                     scrap=line[0] 
 45 |                     left_start=int(line[1]) 
 46 |                     left_end=int(line[2]) 
 47 |                     right_start=int(line[3]) 
 48 |                     right_end=int(line[4]) 
 49 |                     length=int(line[5]) 
 50 |      
 51 |                     for j in range(i): 
 52 |                             line_ref=list_f1[j].split('\t') 
 53 |                             left_start_ref=int(line_ref[1]) 
 54 |                             left_end_ref=int(line_ref[2]) 
 55 |                             right_start_ref=int(line_ref[3]) 
 56 |                             right_end_ref=int(line_ref[4]) 
 57 |      
 58 |                             if (left_start >= left_start_ref) and (left_end <= left_end_ref) and (right_start >= right_start_ref) and (right_end <= right_end_ref): 
 59 |                                     if (left_start - left_start_ref)==(right_start - right_start_ref) and (left_end - left_end_ref)==(right_end - right_end_ref): 
 60 |                                             n+=1 
 61 |                             else: pass 
 62 |                            
 63 |                     if n == 0: 
 64 |                             if (length % 3)==0: 
 65 |                                     length_factor = round(1/exp((length)/(length_weight)),3) 
 66 |                                     num_GC=len(findall('G',scrap))+len(findall('C',scrap)) 
 67 |                                     score_3=100*length_factor*((len(scrap)-num_GC)+(num_GC*2)) 
 68 |                                      
 69 |                             elif (length % 3)!=0: 
 70 |                                     length_factor = round(1/exp((length)/(length_weight)),3) 
 71 |                                     num_GC=len(findall('G',scrap))+len(findall('C',scrap)) 
 72 |                                     score_not_3=100*length_factor*((len(scrap)-num_GC)+(num_GC*2)) 
 73 |      
 74 |                             f2.write(seq[0:left_end]+'-'*length+seq[right_end:]+'\t'+scrap+'\t'+str(length)+'\t'+str(100*length_factor*((len(scrap)-num_GC)+(num_GC*2)))+'\n') 
 75 |                     sum_score_3+=score_3 
 76 |                     sum_score_not_3+=score_not_3 
 77 |      
 78 |             mh_score = sum_score_3+sum_score_not_3
 79 |             oof_score = (sum_score_not_3)*100/(sum_score_3+sum_score_not_3)
 80 |             if verbose:
 81 |                 print 'Microhomology score = ' + str(mh_score) 
 82 |                 print 'Out-of-frame score = ' + str(oof_score) 
 83 |     f1.close() 
 84 |     f2.close()
 85 |     return mh_score, oof_score
 86 | 
 87 | if __name__ == '__main__':
 88 |     seq='GGAGGAAGGGCCTGAGTCCGAGCAGAAGAAGAAGGGCTCCCATCACATCAACCGGTGGCG'    # The length of sequence is recommend within 60~80 bases. 
 89 | 
 90 |     tmpfile1 = "1.before removing duplication.txt"
 91 |     tmpfile2 = "2.all microhomology patterns.txt"
 92 |     
 93 |     mh_score, oof_score = compute_score(seq, tmpfile1=tmpfile1, tmpfile2=tmpfile2, verbose=True)
 94 | 
 95 |     # The row of output file is consist of (full sequence, microhomology scrap, deletion length, score of pattern). 
 96 | 
 97 |     #correct output is
 98 |     #Microhomology score = 4662.9
 99 |     #Out-of-frame score = 50.7473889639
100 |     #GGAGGAAGGGCCTGAGTCCGAGCAGAAGAAGAAGGGCTCCCATCACATCAACCGGTGGCG    
101 |     
102 |     print seq  


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/features/microhomology.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sunjiamin0824/CRISPR-Local/c6e80b2fd8ee4065c0655edf9f5206bf8cc316e2/Rule_Set_2_scoring_v1/analysis/features/microhomology.pyc


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/load_data.py:
--------------------------------------------------------------------------------
  1 | import pandas
  2 | import util
  3 | import matplotlib.pyplot as plt
  4 | import scipy as sp
  5 | import scipy.stats
  6 | import numpy as np
  7 | 
  8 | def from_custom_file(data_file, learn_options):
  9 |     # use semantics of when we load V2 data
 10 |     print "Loading inputs to predict from %s" % data_file        
 11 |     data = pandas.read_csv(data_file) 
 12 |            
 13 |     mandatory_columns = ['30mer', 'Target gene', 'Percent Peptide', 'Amino Acid Cut position']
 14 |     for col in mandatory_columns:
 15 |         assert col in data.columns, "inputs for prediction must include these columns: %s" % mandatory_columns
 16 |            
 17 |     Xdf = pandas.DataFrame(data)
 18 |     Xdf['30mercopy'] = Xdf['30mer']
 19 |     Xdf = Xdf.set_index(['30mer', 'Target gene'])
 20 |     Xdf['30mer'] = Xdf['30mercopy']
 21 |     Xdf.index.names = ['Sequence', 'Target']                 
 22 |     Xdf['drug']= ['dummydrug%s' % i for i in range(Xdf.shape[0])]
 23 |     Xdf = Xdf.set_index('drug', append=True)
 24 |             
 25 |     Y = None        
 26 |     gene_position = Xdf[['Percent Peptide', 'Amino Acid Cut position']]
 27 |     target_genes =  np.unique(Xdf.index.levels[1])
 28 |     
 29 |     learn_options = set_V2_target_names(learn_options)
 30 | 
 31 |     return Xdf, Y, gene_position, target_genes
 32 |     
 33 | 
 34 | def from_file(data_file, learn_options, data_file2=None):
 35 |     if learn_options["V"] == 1:  # from Nature Biotech paper
 36 | 
 37 |         print "loading V%d data" % learn_options["V"]
 38 | 
 39 |         assert not learn_options["weighted"] is not None, "not supported for V1 data"
 40 |         annotations, gene_position, target_genes, Xdf, Y = read_V1_data(data_file, learn_options)
 41 | 
 42 |         learn_options['binary target name'] = 'average threshold'
 43 |         learn_options['rank-transformed target name'] = 'average rank'
 44 |         learn_options['raw target name'] = 'average activity'
 45 | 
 46 |         # NF: not sure why the line below was uncommented
 47 |         # gene_position, selected_ind, target_genes, Xdf, Y = extract_by_organism("mouse", Xdf, Y, gene_position)
 48 | 
 49 |     elif learn_options["V"] == 2:  # from Nov 2014, hot off the machines
 50 |         Xdf, drugs_to_genes, target_genes, Y, gene_position = read_V2_data(data_file, learn_options)
 51 | 
 52 |         # check that data is consistent with sgRNA score
 53 |         xx = Xdf['sgRNA Score'].values
 54 |         yy = Y['score_drug_gene_rank'].values
 55 |         rr,pp = sp.stats.pearsonr(xx, yy)
 56 |         assert rr > 0, "data processing has gone wrong as correlation with previous predictions is negative"
 57 | 
 58 |         learn_options = set_V2_target_names(learn_options)
 59 | 
 60 |     elif learn_options["V"] == 3:  # merge of V1 and V2
 61 |         # these are relative to the V2 data, and V1 will be made to automatically match
 62 |         learn_options['binary target name'] = 'score_drug_gene_threshold'
 63 |         learn_options['rank-transformed target name'] = 'score_drug_gene_rank'
 64 |         learn_options['raw target name'] = None
 65 | 
 66 |         Xdf, Y, gene_position, target_genes = mergeV1_V2(data_file, data_file2, learn_options)
 67 | 
 68 |     # truncate down to 30--some data sets gave us more.
 69 |     Xdf["30mer"] = Xdf["30mer"].apply(lambda x: x[0:30])
 70 | 
 71 |     return Xdf, Y, gene_position, target_genes
 72 | 
 73 | 
 74 | def set_V2_target_names(learn_options):
 75 |     if 'binary target name' not in learn_options.keys():
 76 |         learn_options['binary target name'] = 'score_drug_gene_threshold'
 77 |     if 'rank-transformed target name' not in learn_options.keys():
 78 |         learn_options['rank-transformed target name'] = 'score_drug_gene_rank'
 79 |     learn_options['raw target name'] = 'score'
 80 |     return learn_options
 81 | 
 82 | 
 83 | def combine_organisms(human_data, mouse_data):
 84 |     # 'Target' is the column name, 'CD13' are some rows in that column
 85 |     # xs slices through the pandas data frame to return another one
 86 |     cd13 = human_data.xs('CD13', level='Target', drop_level=False)
 87 |     # y_names are column names, cd13 is a pandas object
 88 |     X_CD13, Y_CD13 = util.get_data(cd13, y_names=['NB4 CD13', 'TF1 CD13'])
 89 |     cd33 = human_data.xs('CD33', level='Target', drop_level=False)
 90 |     X_CD33, Y_CD33 = util.get_data(cd33, y_names=['MOLM13 CD33', 'TF1 CD33', 'NB4 CD33'])
 91 |     cd15 = human_data.xs('CD15', level='Target', drop_level=False)
 92 |     X_CD15, Y_CD15 = util.get_data(cd15, y_names=['MOLM13 CD15'])
 93 | 
 94 |     mouse_X = pandas.DataFrame()
 95 |     mouse_Y = pandas.DataFrame()
 96 |     for k in mouse_data.index.levels[1]:
 97 |         # is k the gene
 98 |         X, Y = util.get_data(mouse_data.xs(k, level='Target', drop_level=False), ["On-target Gene"], target_gene=k, organism='mouse')
 99 |         mouse_X = pandas.concat([mouse_X, X], axis=0)
100 |         mouse_Y = pandas.concat([mouse_Y, Y], axis=0)
101 | 
102 |     X = pandas.concat([X_CD13, X_CD15, X_CD33, mouse_X], axis=0)
103 |     Y = pandas.concat([Y_CD13, Y_CD15, Y_CD33, mouse_Y], axis=0)
104 | 
105 |     return X, Y
106 | 
107 | 
108 | def read_V1_data(data_file, learn_options):
109 |     if data_file is None:
110 |         data_file = "../data/07-23 Data for Jennifer.xlsx"
111 |     human_data = pandas.read_excel(data_file, sheetname=0, index_col=[0, 1])
112 |     mouse_data = pandas.read_excel(data_file, sheetname=1, index_col=[0, 1])
113 |     Xdf, Y = combine_organisms(human_data, mouse_data)
114 | 
115 |     # get position within each gene, then join and re-order
116 |     # note that 11 missing guides we were told to ignore
117 |     annotations = pandas.read_csv(r"..\data\AML_EL4_PercentRank_0725.txt", delimiter='\t', index_col=[0, 4])
118 |     annotations.index.names = Xdf.index.names
119 |     gene_position = pandas.merge(Xdf, annotations, how="inner", left_index=True, right_index=True)
120 |     gene_position = util.impute_gene_position(gene_position)
121 |     gene_position = gene_position[['Amino Acid Cut position', 'Nucleotide cut position', 'Percent Peptide']]
122 |     Y = Y.loc[gene_position.index]
123 |     Xdf = Xdf.loc[gene_position.index]
124 | 
125 |     Y['test'] = 1  # for bookeeping to keep consistent with V2 which uses this for "extra pairs"
126 | 
127 |     target_genes = Y['Target gene'].unique()
128 | 
129 |     Y.index.names = ['Sequence', 'Target gene']
130 | 
131 |     assert Xdf.index.equals(Y.index), "The index of Xdf is different from the index of Y (this can cause inconsistencies/random performance later on)"
132 | 
133 |     if learn_options is not None and learn_options["flipV1target"]:
134 |         print "************************************************************************"
135 |         print "*****************MATCHING DOENCH CODE (DEBUG MODE)**********************"
136 |         print "************************************************************************"
137 |         # normally it is: Y['average threshold'] = Y['average rank'] > 0.8, where 1s are good guides, 0s are not
138 |         Y['average threshold'] = Y['average rank'] < 0.2  # 1s are bad guides
139 |         print "press c to continue"
140 |         import ipdb
141 |         ipdb.set_trace()
142 | 
143 |     return annotations, gene_position, target_genes, Xdf, Y
144 | 
145 | 
146 | def read_V2_data(data_file, learn_options=None, verbose=True):
147 |     if data_file is None:
148 |         data_file = "../data/11-15-2014 DeepXPR results_processed.xlsx"
149 | 
150 |     # to compare
151 |     # import predict as pr; a1, g1, t1, X1, Y1 = pr.data_setup()
152 |     # a1.index.names
153 | 
154 |     data = pandas.read_excel(data_file, sheetname="ResultsFiltered", skiprows=range(0, 6+1), index_col=[0, 4])
155 |     # grab data relevant to each of three drugs, which exludes some genes
156 |     # note gene MED12 has two drugs, all others have at most one
157 |     Xdf = pandas.DataFrame()
158 | 
159 |     # This comes from the "Pairs" tab in their excel sheet,
160 |     # note HPRT/HPRT1 are same thing, and also PLX_2uM/PLcX_2uM
161 |     known_pairs = {'AZD_200nM':  ['CCDC101', 'MED12', 'TADA2B', 'TADA1'],
162 |                    '6TG_2ug/mL': ['HPRT1'],
163 |                    'PLX_2uM':    ['CUL3', 'NF1', 'NF2', 'MED12']}
164 | 
165 |     drugs_to_genes = {'AZD_200nM':  ['CCDC101', 'MED12', 'TADA2B', 'TADA1'],
166 |                       '6TG_2ug/mL': ['HPRT1'],
167 |                       'PLX_2uM':    ['CUL3', 'NF1', 'NF2', 'MED12']}
168 | 
169 |     if learn_options is not None:
170 |         assert not (learn_options['extra pairs'] and learn_options['all pairs']), "extra pairs and all pairs options (in learn_options) can't be active simultaneously."
171 | 
172 |         if learn_options['extra pairs']:
173 |             drugs_to_genes['AZD_200nM'].extend(['CUL3', 'NF1', 'NF2'])
174 |         elif learn_options['all pairs']:
175 |             drugs_to_genes['AZD_200nM'].extend(['HPRT1', 'CUL3', 'NF1', 'NF2'])
176 |             drugs_to_genes['PLX_2uM'].extend(['HPRT1', 'CCDC101', 'TADA2B', 'TADA1'])
177 |             drugs_to_genes['6TG_2ug/mL'].extend(['CCDC101', 'MED12', 'TADA2B', 'TADA1', 'CUL3', 'NF1', 'NF2'])
178 | 
179 |     count = 0
180 |     for drug in drugs_to_genes.keys():
181 |         genes = drugs_to_genes[drug]
182 |         for g in genes:
183 |             Xtmp = data.copy().xs(g, level='Target gene', drop_level=False)
184 |             Xtmp['drug'] = drug
185 |             Xtmp['score'] = Xtmp[drug].copy()  # grab the drug results that are relevant for this gene
186 | 
187 |             if g in known_pairs[drug]:
188 |                 Xtmp['test'] = 1.
189 |             else:
190 |                 Xtmp['test'] = 0.
191 | 
192 |             count = count + Xtmp.shape[0]
193 |             Xdf = pandas.concat([Xdf, Xtmp], axis=0)
194 |             if verbose:
195 |                 print "Loaded %d samples for gene %s \ttotal number of samples: %d" % (Xtmp.shape[0], g, count)
196 | 
197 |     # create new index that includes the drug
198 |     Xdf = Xdf.set_index('drug', append=True)
199 | 
200 |     Y = pandas.DataFrame(Xdf.pop("score"))
201 |     Y.columns.names = ["score"]
202 | 
203 |     test_gene = pandas.DataFrame(Xdf.pop('test'))
204 |     target = pandas.DataFrame(Xdf.index.get_level_values('Target gene').values, index=Y.index, columns=["Target gene"])
205 |     Y = pandas.concat((Y, target, test_gene), axis=1)
206 |     target_genes = Y['Target gene'].unique()
207 |     gene_position = Xdf[["Percent Peptide", "Amino Acid Cut position"]].copy()
208 | 
209 |     # convert to ranks for each (gene, drug combo)
210 |     # flip = True
211 |     y_rank = pandas.DataFrame()
212 |     y_threshold = pandas.DataFrame()
213 |     y_quant = pandas.DataFrame()
214 |     for drug in drugs_to_genes.keys():
215 |         gene_list = drugs_to_genes[drug]
216 |         for gene in gene_list:
217 |             ytmp = pandas.DataFrame(Y.xs((gene, drug), level=["Target gene", "drug"], drop_level=False)['score'])
218 |             y_ranktmp, y_rank_raw, y_thresholdtmp, y_quanttmp = util.get_ranks(ytmp, thresh=0.8, prefix="score_drug_gene", flip=False)
219 |             # np.unique(y_rank.values-y_rank_raw.values)
220 |             y_rank = pandas.concat((y_rank, y_ranktmp), axis=0)
221 |             y_threshold = pandas.concat((y_threshold, y_thresholdtmp), axis=0)
222 |             y_quant = pandas.concat((y_quant, y_quanttmp), axis=0)
223 | 
224 |     yall = pandas.concat((y_rank, y_threshold, y_quant), axis=1)
225 |     Y = pandas.merge(Y, yall, how='inner', left_index=True, right_index=True)
226 | 
227 |     # convert also by drug only, irrespective of gene
228 |     y_rank = pandas.DataFrame()
229 |     y_threshold = pandas.DataFrame()
230 |     y_quant = pandas.DataFrame()
231 |     for drug in drugs_to_genes.keys():
232 |         ytmp = pandas.DataFrame(Y.xs(drug, level="drug", drop_level=False)['score'])
233 |         y_ranktmp, y_rank_raw, y_thresholdtmp, y_quanttmp = util.get_ranks(ytmp, thresh=0.8, prefix="score_drug", flip=False)
234 |         # np.unique(y_rank.values-y_rank_raw.values)
235 |         y_rank = pandas.concat((y_rank, y_ranktmp), axis=0)
236 |         y_threshold = pandas.concat((y_threshold, y_thresholdtmp), axis=0)
237 |         y_quant = pandas.concat((y_quant, y_quanttmp), axis=0)
238 | 
239 |     yall = pandas.concat((y_rank, y_threshold, y_quant), axis=1)
240 |     Y = pandas.merge(Y, yall, how='inner', left_index=True, right_index=True)
241 | 
242 |     PLOT = False
243 |     if PLOT:
244 |         # to better understand, try plotting something like:
245 |         labels = ["score", "score_drug_gene_rank", "score_drug_rank", "score_drug_gene_threshold", "score_drug_threshold"]
246 | 
247 |         for label in labels:
248 |             plt.figure()
249 |             plt.plot(Xdf['sgRNA Score'].values, Y[label].values, '.')
250 |             r, pearp = sp.stats.pearsonr(Xdf['sgRNA Score'].values.flatten(), Y[label].values.flatten())
251 |             plt.title(label + ' VS pred. score, $r$=%0.2f (p=%0.2e)' % (r, pearp))
252 |             plt.xlabel("sgRNA prediction score")
253 |             plt.ylabel(label)
254 | 
255 |     gene_position = util.impute_gene_position(gene_position)
256 | 
257 |     if learn_options is not None and learn_options["weighted"] == "variance":
258 |         print "computing weights from replicate variance..."
259 |         # compute the variance across replicates so can use it as a weight
260 |         data = pandas.read_excel(data_file, sheetname="Normalized", skiprows=range(0, 6+1), index_col=[0, 4])
261 |         data.index.names = ["Sequence", "Target gene"]
262 | 
263 |         experiments = {}
264 |         experiments['AZD_200nM'] = ['Deep 25', 'Deep 27', 'Deep 29 ', 'Deep 31']
265 |         experiments['6TG_2ug/mL'] = ['Deep 33', 'Deep 35', 'Deep 37', 'Deep 39']
266 |         experiments['PLX_2uM'] = ['Deep 49', 'Deep 51', 'Deep 53', 'Deep 55']
267 | 
268 |         variance = None
269 |         for drug in drugs_to_genes.keys():
270 |             data_tmp = data.iloc[data.index.get_level_values('Target gene').isin(drugs_to_genes[drug])][experiments[drug]]
271 |             data_tmp["drug"] = drug
272 |             data_tmp = data_tmp.set_index('drug', append=True)
273 |             data_tmp["variance"] = np.var(data_tmp.values, axis=1)
274 |             if variance is None:
275 |                 variance = data_tmp["variance"].copy()
276 |             else:
277 |                 variance = pandas.concat((variance, data_tmp["variance"]), axis=0)
278 | 
279 |         orig_index = Y.index.copy()
280 |         Y = pandas.merge(Y, pandas.DataFrame(variance), how="inner", left_index=True, right_index=True)
281 |         Y = Y.ix[orig_index]
282 |         print "done."
283 | 
284 |     # Make sure to keep this check last in this function
285 |     assert Xdf.index.equals(Y.index), "The index of Xdf is different from the index of Y (this can cause inconsistencies/random performance later on)"
286 | 
287 |     return Xdf, drugs_to_genes, target_genes, Y, gene_position
288 | 
289 | 
290 | def mergeV1_V2(data_file, data_file2, learn_options):
291 |     '''
292 |     ground_truth_label, etc. are taken to correspond to the V2 data, and then the V1 is appropriately matched
293 |     based on semantics
294 |     '''
295 |     assert not learn_options['include_strand'], "don't currently have 'Strand' column in V1 data"
296 | 
297 |     annotations, gene_position1, target_genes1, Xdf1, Y1 = read_V1_data(data_file, learn_options)
298 |     Xdf2, drugs_to_genes, target_genes2, Y2, gene_position2 = read_V2_data(data_file2)
299 | 
300 |     Y1.rename(columns={'average rank': learn_options["rank-transformed target name"]}, inplace=True)
301 |     Y1.rename(columns={'average threshold': learn_options["binary target name"]}, inplace=True)
302 | 
303 |     # rename columns, and add a dummy "drug" to V1 so can join the data sets
304 |     Y1["drug"] = ["nodrug" for x in range(Y1.shape[0])]
305 |     Y1 = Y1.set_index('drug', append=True)
306 |     Y1.index.names = ['Sequence', 'Target gene', 'drug']
307 | 
308 |     Y_cols_to_keep = np.unique(['Target gene', 'test', 'score_drug_gene_rank', 'score_drug_gene_threshold'])
309 | 
310 |     Y1 = Y1[Y_cols_to_keep]
311 |     Y2 = Y2[Y_cols_to_keep]
312 | 
313 |     Xdf1["drug"] = ["nodrug" for x in range(Xdf1.shape[0])]
314 |     Xdf1 = Xdf1.set_index('drug', append=True)
315 | 
316 |     X_cols_to_keep = ['30mer', 'Strand']
317 |     Xdf1 = Xdf1[X_cols_to_keep]
318 |     Xdf2 = Xdf2[X_cols_to_keep]
319 | 
320 |     gene_position1["drug"] = ["nodrug" for x in range(gene_position1.shape[0])]
321 |     gene_position1 = gene_position1.set_index('drug', append=True)
322 |     gene_position1.index.names = ['Sequence', 'Target gene', 'drug']
323 |     cols_to_keep = [u'Percent Peptide', u'Amino Acid Cut position']
324 |     gene_position1 = gene_position1[cols_to_keep]
325 |     gene_position2 = gene_position2[cols_to_keep]
326 | 
327 |     Y = pandas.concat((Y1, Y2), axis=0)
328 |     Xdf = pandas.concat((Xdf1, Xdf2), axis=0)
329 |     gene_position = pandas.concat((gene_position1, gene_position2))
330 | 
331 |     # target_genes = target_genes1 + target_genes2
332 |     target_genes = np.concatenate((target_genes1, target_genes2))
333 | 
334 |     save_to_file = False
335 |     if save_to_file:
336 |         Y.index.names = ['Sequence', 'Target', 'drug']
337 |         XandY = pandas.merge(Xdf, Y, how="inner", left_index=True, right_index=True)
338 |         gene_position_tmp = gene_position.copy()
339 |         gene_position_tmp.index.names = ['Sequence', 'Target', 'drug']
340 |         XandY = pandas.merge(XandY, gene_position_tmp, how="inner", left_index=True, right_index=True)
341 |         # truncate to 30mers
342 |         XandY["30mer"] = XandY["30mer"].apply(lambda x: x[0:30])
343 |         XandY.to_csv(r'D:\Source\CRISPR\data\tmp\V3.csv')
344 |         import ipdb
345 |         ipdb.set_trace()
346 | 
347 |     return Xdf, Y, gene_position, target_genes
348 | 
349 | 
350 | def get_V1_genes(data_file=None):
351 |     annotations, gene_position, target_genes, Xdf, Y = read_V1_data(data_file, learn_options=None)
352 |     return target_genes
353 | 
354 | 
355 | def get_V2_genes(data_file=None):
356 |     Xdf, drugs_to_genes, target_genes, Y, gene_position = read_V2_data(data_file, verbose=False)
357 |     return target_genes
358 | 
359 | 
360 | def get_V3_genes(data_fileV1=None, data_fileV2=None):
361 |     target_genes = np.concatenate((get_V1_genes(data_fileV1), get_V2_genes(data_fileV2)))
362 |     return target_genes
363 | 
364 | 
365 | def get_mouse_genes(data_file=None):
366 |     annotations, gene_position, target_genes, Xdf, Y = read_V1_data(data_file, learn_options=None)
367 |     return Xdf[Xdf['Organism'] == 'mouse']['Target gene'].unique()
368 | 
369 | 
370 | def get_human_genes(data_file=None):
371 |     annotations, gene_position, target_genes, Xdf, Y = read_V1_data(data_file, learn_options=None)
372 |     mouse_genes = Xdf[Xdf['Organism'] == 'mouse']['Target gene'].unique()
373 |     all_genes = get_V3_genes(None, None)  # TODO this needs to support specifying file names (!= 'None')
374 |     return np.setdiff1d(all_genes, mouse_genes)
375 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/load_data.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sunjiamin0824/CRISPR-Local/c6e80b2fd8ee4065c0655edf9f5206bf8cc316e2/Rule_Set_2_scoring_v1/analysis/load_data.pyc


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/local_multiprocessing.py:
--------------------------------------------------------------------------------
 1 | import multiprocessing
 2 | import numpy as np
 3 | 
 4 | 
 5 | def configure(num_jobs=8, TEST=False, subtract=0, num_proc=None, num_thread_per_proc=None):
 6 |     '''
 7 |     num_jobs is typically the # of genes we are parallelizing over
 8 |     '''
 9 |     if num_proc is None:
10 |         num_proc = multiprocessing.cpu_count() - subtract
11 | 
12 |     if num_jobs > num_proc:
13 |         num_jobs = num_proc
14 | 
15 |     if num_thread_per_proc is None:
16 |         num_thread_per_proc = int(np.floor(num_proc/num_jobs))
17 | 
18 |     if TEST:
19 |         num_jobs = 1
20 |         num_thread_per_proc = 1
21 | 
22 |     try:
23 |         import mkl
24 |         mkl.set_num_threads(num_thread_per_proc)    
25 |     except ImportError:
26 |         print "MKL not available, so I'm not adjusting the number of threads"
27 | 
28 |     print "Launching %d jobs with %d MKL threads each" % (num_jobs, num_thread_per_proc)
29 | 
30 |     return num_jobs
31 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/local_multiprocessing.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sunjiamin0824/CRISPR-Local/c6e80b2fd8ee4065c0655edf9f5206bf8cc316e2/Rule_Set_2_scoring_v1/analysis/local_multiprocessing.pyc


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/metrics.py:
--------------------------------------------------------------------------------
  1 | """
  2 | from https://gist.github.com/bwhite/3726239
  3 | 
  4 | Information Retrieval metrics
  5 | 
  6 | Useful Resources:
  7 | http://www.cs.utexas.edu/~mooney/ir-course/slides/Evaluation.ppt
  8 | http://www.nii.ac.jp/TechReports/05-014E.pdf
  9 | http://www.stanford.edu/class/cs276/handouts/EvaluationNew-handout-6-per.pdf
 10 | http://hal.archives-ouvertes.fr/docs/00/72/67/60/PDF/07-busa-fekete.pdf
 11 | Learning to Rank for Information Retrieval (Tie-Yan Liu)
 12 | """
 13 | import numpy as np
 14 | import scipy as sp
 15 | import scipy.stats
 16 | 
 17 | 
 18 | def mean_reciprocal_rank(rs):
 19 |     """Score is reciprocal of the rank of the first relevant item
 20 | 
 21 |     First element is 'rank 1'.  Relevance is binary (nonzero is relevant).
 22 | 
 23 |     Example from http://en.wikipedia.org/wiki/Mean_reciprocal_rank
 24 |     >>> rs = [[0, 0, 1], [0, 1, 0], [1, 0, 0]]
 25 |     >>> mean_reciprocal_rank(rs)
 26 |     0.61111111111111105
 27 |     >>> rs = np.array([[0, 0, 0], [0, 1, 0], [1, 0, 0]])
 28 |     >>> mean_reciprocal_rank(rs)
 29 |     0.5
 30 |     >>> rs = [[0, 0, 0, 1], [1, 0, 0], [1, 0, 0]]
 31 |     >>> mean_reciprocal_rank(rs)
 32 |     0.75
 33 | 
 34 |     Args:
 35 |         rs: Iterator of relevance scores (list or numpy) in rank order
 36 |             (first element is the first item)
 37 | 
 38 |     Returns:
 39 |         Mean reciprocal rank
 40 |     """
 41 |     rs = (np.asarray(r).nonzero()[0] for r in rs)
 42 |     return np.mean([1. / (r[0] + 1) if r.size else 0. for r in rs])
 43 | 
 44 | 
 45 | def r_precision(r):
 46 |     """Score is precision after all relevant documents have been retrieved
 47 | 
 48 |     Relevance is binary (nonzero is relevant).
 49 | 
 50 |     >>> r = [0, 0, 1]
 51 |     >>> r_precision(r)
 52 |     0.33333333333333331
 53 |     >>> r = [0, 1, 0]
 54 |     >>> r_precision(r)
 55 |     0.5
 56 |     >>> r = [1, 0, 0]
 57 |     >>> r_precision(r)
 58 |     1.0
 59 | 
 60 |     Args:
 61 |         r: Relevance scores (list or numpy) in rank order
 62 |             (first element is the first item)
 63 | 
 64 |     Returns:
 65 |         R Precision
 66 |     """
 67 |     r = np.asarray(r) != 0
 68 |     z = r.nonzero()[0]
 69 |     if not z.size:
 70 |         return 0.
 71 |     return np.mean(r[:z[-1] + 1])
 72 | 
 73 | 
 74 | def precision_at_k(r, k):
 75 |     """Score is precision @ k
 76 | 
 77 |     Relevance is binary (nonzero is relevant).
 78 | 
 79 |     >>> r = [0, 0, 1]
 80 |     >>> precision_at_k(r, 1)
 81 |     0.0
 82 |     >>> precision_at_k(r, 2)
 83 |     0.0
 84 |     >>> precision_at_k(r, 3)
 85 |     0.33333333333333331
 86 |     >>> precision_at_k(r, 4)
 87 |     Traceback (most recent call last):
 88 |         File "<stdin>", line 1, in ?
 89 |     ValueError: Relevance score length < k
 90 | 
 91 | 
 92 |     Args:
 93 |         r: Relevance scores (list or numpy) in rank order
 94 |             (first element is the first item)
 95 | 
 96 |     Returns:
 97 |         Precision @ k
 98 | 
 99 |     Raises:
100 |         ValueError: len(r) must be >= k
101 |     """
102 |     assert k >= 1
103 |     r = np.asarray(r)[:k] != 0
104 |     if r.size != k:
105 |         raise ValueError('Relevance score length < k')
106 |     return np.mean(r)
107 | 
108 | 
109 | def average_precision(r):
110 |     """Score is average precision (area under PR curve)
111 | 
112 |     Relevance is binary (nonzero is relevant).
113 | 
114 |     >>> r = [1, 1, 0, 1, 0, 1, 0, 0, 0, 1]
115 |     >>> delta_r = 1. / sum(r)
116 |     >>> sum([sum(r[:x + 1]) / (x + 1.) * delta_r for x, y in enumerate(r) if y])
117 |     0.7833333333333333
118 |     >>> average_precision(r)
119 |     0.78333333333333333
120 | 
121 |     Args:
122 |         r: Relevance scores (list or numpy) in rank order
123 |             (first element is the first item)
124 | 
125 |     Returns:
126 |         Average precision
127 |     """
128 |     r = np.asarray(r) != 0
129 |     out = [precision_at_k(r, k + 1) for k in range(r.size) if r[k]]
130 |     if not out:
131 |         return 0.
132 |     return np.mean(out)
133 | 
134 | 
135 | def mean_average_precision(rs):
136 |     """Score is mean average precision
137 | 
138 |     Relevance is binary (nonzero is relevant).
139 | 
140 |     >>> rs = [[1, 1, 0, 1, 0, 1, 0, 0, 0, 1]]
141 |     >>> mean_average_precision(rs)
142 |     0.78333333333333333
143 |     >>> rs = [[1, 1, 0, 1, 0, 1, 0, 0, 0, 1], [0]]
144 |     >>> mean_average_precision(rs)
145 |     0.39166666666666666
146 | 
147 |     Args:
148 |         rs: Iterator of relevance scores (list or numpy) in rank order
149 |             (first element is the first item)
150 | 
151 |     Returns:
152 |         Mean average precision
153 |     """
154 |     return np.mean([average_precision(r) for r in rs])
155 | 
156 | 
157 | def dcg_at_k(r, k, method=0):
158 |     """Score is discounted cumulative gain (dcg)
159 | 
160 |     Relevance is positive real values.  Can use binary
161 |     as the previous methods.
162 | 
163 |     Example from
164 |     http://www.stanford.edu/class/cs276/handouts/EvaluationNew-handout-6-per.pdf
165 |     >>> r = [3, 2, 3, 0, 0, 1, 2, 2, 3, 0]
166 |     >>> dcg_at_k(r, 1)
167 |     3.0
168 |     >>> dcg_at_k(r, 1, method=1)
169 |     3.0
170 |     >>> dcg_at_k(r, 2)
171 |     5.0
172 |     >>> dcg_at_k(r, 2, method=1)
173 |     4.2618595071429155
174 |     >>> dcg_at_k(r, 10)
175 |     9.6051177391888114
176 |     >>> dcg_at_k(r, 11)
177 |     9.6051177391888114
178 | 
179 |     Args:
180 |         r: Relevance scores (list or numpy) in rank order
181 |             (first element is the first item)
182 |         k: Number of results to consider
183 |         method: If 0 then weights are [1.0, 1.0, 0.6309, 0.5, 0.4307, ...]
184 |                 If 1 then weights are [1.0, 0.6309, 0.5, 0.4307, ...]
185 | 
186 |     Returns:
187 |         Discounted cumulative gain
188 |     """
189 |     r = np.asfarray(r)[:k]
190 |     if r.size:
191 |         if method == 0:
192 |             return r[0] + np.sum(r[1:] / np.log2(np.arange(2, r.size + 1)))
193 |         elif method == 1:
194 |             return np.sum(r / np.log2(np.arange(2, r.size + 2)))
195 |         else:
196 |             raise ValueError('method must be 0 or 1.')
197 |     return 0.
198 | 
199 | 
200 | def ndcg_at_k(r, k, method=0):
201 |     """Score is normalized discounted cumulative gain (ndcg)
202 | 
203 |     Relevance is positive real values.  Can use binary
204 |     as the previous methods.
205 | 
206 |     Example from
207 |     http://www.stanford.edu/class/cs276/handouts/EvaluationNew-handout-6-per.pdf
208 |     >>> r = [3, 2, 3, 0, 0, 1, 2, 2, 3, 0]
209 |     >>> ndcg_at_k(r, 1)
210 |     1.0
211 |     >>> r = [2, 1, 2, 0]
212 |     >>> ndcg_at_k(r, 4)
213 |     0.9203032077642922
214 |     >>> ndcg_at_k(r, 4, method=1)
215 |     0.96519546960144276
216 |     >>> ndcg_at_k([0], 1)
217 |     0.0
218 |     >>> ndcg_at_k([1], 2)
219 |     1.0
220 | 
221 |     Args:
222 |         r: Relevance scores (list or numpy) in rank order
223 |             (first element is the first item)
224 |         k: Number of results to consider
225 |         method: If 0 then weights are [1.0, 1.0, 0.6309, 0.5, 0.4307, ...]
226 |                 If 1 then weights are [1.0, 0.6309, 0.5, 0.4307, ...]
227 | 
228 |     Returns:
229 |         Normalized discounted cumulative gain
230 |     """
231 |     dcg_max = dcg_at_k(sorted(r, reverse=True), k, method)
232 |     if not dcg_max:
233 |         return 0.
234 |     return dcg_at_k(r, k, method) / dcg_max
235 | 
236 | ## ------------------------------------------------------------------------------------
237 | ## custom stuff from us to avoid problem with ties
238 | 
239 | def ndcg_at_k_ties(labels, predictions, k, method=0):
240 |     '''
241 |     See 2008 McSherry et al on how to efficiently compute NDCG with ties
242 |     labels are ground truth
243 |     '''
244 | 
245 |     labels = labels.copy()
246 | 
247 |     # NDCG requires the ground truth labels to be > 0 (the predictions can be negative)
248 |     # if np.any(labels < 0):
249 |     #     labels += np.abs(labels.min() + 1e-5)
250 | 
251 |     # this is the one with ties:
252 |     dcg = dcg_at_k_ties(labels, predictions, k, method=method)
253 | 
254 |     # this one is the vanilla computation that ignores ties (and should match dcg_at_k_ties when no ties are present):
255 |     # highest-to-lowest of the true labels (i.e. best first)
256 |     dcg_max = dcg_at_k(sorted(labels, reverse=True), k, method)
257 |     # NOTE: I have checked that dcg_at_k_ties and dcg_at_k match when there are no ties, or ties in the labels
258 | 
259 |     ndcg = dcg / dcg_max
260 | 
261 |     if not dcg_max:
262 |         return 0.
263 |     #assert ndcg >= (0.0 - 1e-8) and ndcg <= (1.0 + 1e-8), "ndcg should be between 0 and 1"
264 | 
265 |     return ndcg
266 | 
267 | def dcg_at_k_ties(labels, predictions, k, method=0):
268 |     '''
269 |     See 2008 McSherry et al on how to efficiently compute NDCG (method=0 here) with ties
270 |     labels are what the "ground truth" judges assign
271 |     predictions are the algorithm predictions corresponding to each label
272 |     Also, http://en.wikipedia.org/wiki/Discounted_cumulative_gain for basic defns
273 |     '''
274 | 
275 |     assert len(labels) == len(predictions), "labels and predictions should be of same length"
276 |     assert k <= len(labels), "k should be <= len(labels)"
277 |     # order both labels and preds so that they are in order of decreasing predictive score
278 |     sorted_ind = np.argsort(predictions)[::-1]
279 |     predictions = predictions[sorted_ind]
280 |     labels = labels[sorted_ind]
281 | 
282 |     def gain(label, method=0):
283 |         if method==0:
284 |             return label
285 |         elif method==1:
286 |             return 2**label-1.0
287 | 
288 |     if method==0:
289 |         discount_factors = get_discount_factors(labels)
290 |     elif method==1:
291 |         raise Exception("need to implement: log_2(i+1)")
292 |     assert len(discount_factors) == len(labels), "discount factors has wrong length"
293 | 
294 |     #step through, in current order (of decreasing predictions), accumulating tied gains (which may be singletons)
295 |     ii = 0
296 |     dcg = 0.0
297 |     while (ii < k):
298 |         current_pred = predictions[ii]
299 |         current_gain = gain(labels[ii])
300 |         # intializing the tied cumulative variables
301 |         cum_tied_gain = current_gain
302 |         cum_tied_disc = discount_factors[ii]
303 |         num_ties = 1
304 |         ii += 1
305 |         # count number of ties
306 |         while (ii<len(predictions) and predictions[ii]==current_pred):  #while tied
307 |             num_ties += 1.0
308 |             cum_tied_gain += gain(labels[ii])
309 |             if ii < k: cum_tied_disc += discount_factors[ii]
310 |             ii += 1
311 |         #if len(np.unique(predictions))==1:  import ipdb; ipdb.set_trace()
312 |         avg_gain = cum_tied_gain/num_ties
313 |         dcg += avg_gain*cum_tied_disc
314 |         assert not np.isnan(dcg), "found nan dcg"
315 |     assert not np.isnan(dcg), "found nan dcg"
316 |     return dcg
317 | 
318 | def get_discount_factors(labels):
319 |     ii_range = np.arange(len(labels)) + 1
320 |     discount_factors = np.concatenate((np.array([1.0]), 1.0/np.log2(ii_range[1:])))
321 |     return discount_factors
322 | 
323 | def rank_data(r, rground):
324 |     # we checked this heavily, and is correct, e.g. rground will go from largest rank to smallest
325 |     r = sp.stats.mstats.rankdata(r)
326 |     rground = sp.stats.mstats.rankdata(rground)
327 |     assert np.sum(r)==np.sum(rground), "ranks should add up to the same"
328 |     return r, rground
329 | 
330 | #todo: look at rank_data for bug, look at highest alpha predictions for ties
331 | def dcg_alt(relevances, rank=20):
332 |     relevances = np.asarray(relevances)[:rank]
333 |     n_relevances = len(relevances)
334 |     if n_relevances == 0:
335 |         return 0.
336 |     discounts = np.log2(np.arange(n_relevances) + 2)
337 |     return np.sum(relevances / discounts)
338 | 
339 | def ndcg_alt(relevances, rank=20):
340 |     best_dcg = dcg_alt(sorted(relevances, reverse=True), rank)
341 |     if best_dcg == 0:
342 |         return 0.
343 |     return dcg_alt(relevances, rank) / best_dcg
344 | 
345 | if __name__ == "__main__":
346 |     # # e.g. where all predictions are the same
347 |     # labels = np.arange(30)
348 |     # predictions = np.ones(30)
349 | 
350 |     # discount_factors = get_discount_factors(labels)
351 |     # avg_label = np.mean(labels)
352 |     # avg_label_vec = avg_label*np.ones((len(labels),1))
353 | 
354 |     # for k in range(10):
355 |     #     # one way
356 |     #     dcg1 = np.dot(discount_factors[0:k,None].T, avg_label_vec[0:k])[0][0]
357 |     #     # another way
358 |     #     dcg2 = np.sum(discount_factors[0:k])*avg_label
359 |     #     # using our function
360 |     #     dcg3 = dcg_at_k_ties(labels,predictions,k)
361 | 
362 |     #     print "%f, %f, %f" % (dcg1, dcg2, dcg3)
363 |     #     assert(np.abs(dcg1 - dcg2) < 1e-8)
364 |     #     assert(np.abs(dcg2 - dcg3) < 1e-8)
365 |     # print "check out ok for case with all ties in predictions"
366 | 
367 |     truth = np.array([3, 4, 2, 1, 0, 0, 0])
368 |     pred1 = np.array([3, 4, 2, 1, 0, 0, 0])
369 |     pred2 = np.array([2, 1, 3, 4, 5, 6, 7])
370 | 
371 |     print ndcg_alt(truth[np.argsort(pred2)[::-1]], 5)
372 |     print ndcg_at_k(truth[np.argsort(pred2)[::-1]], 5, method=1)
373 |     print ndcg_at_k(truth[np.argsort(pred2)[::-1]], 5, method=0)
374 | 
375 |     print ndcg_at_k_ties(truth, pred2, 5, method=1)
376 |     print ndcg_at_k_ties(truth, pred2, 5, method=0)
377 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/metrics.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sunjiamin0824/CRISPR-Local/c6e80b2fd8ee4065c0655edf9f5206bf8cc316e2/Rule_Set_2_scoring_v1/analysis/metrics.pyc


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/model_comparison.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sunjiamin0824/CRISPR-Local/c6e80b2fd8ee4065c0655edf9f5206bf8cc316e2/Rule_Set_2_scoring_v1/analysis/model_comparison.pyc


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/models/DNN.py:
--------------------------------------------------------------------------------
 1 | import numpy as np
 2 | import scipy as sp
 3 | import sklearn
 4 | 
 5 | def DNN_on_fold(feature_sets, train, test, y, y_all, X, dim, dimsum, learn_options):
 6 |     import theanets
 7 |     from sklearn.metrics import accuracy_score
 8 |     y = np.array(y_all[learn_options['DNN target variable']].values, dtype=float)
 9 |     y_train, X_train = y[train][:, None], X[train]
10 |     y_test, X_test = y[test][:, None], X[test]
11 | 
12 |     num_hidden_layers = [1]#, 2, 3]
13 |     num_units = [2]#, 5, 8, 10, 15, 20, 25, 30, 40, 50, 60]
14 |     accuracies = np.zeros((len(num_hidden_layers), len(num_units)))
15 |     best_score = None
16 |     best_model = None
17 | 
18 |     for i, hl in enumerate(num_hidden_layers):
19 |         for j, nu in enumerate(num_units):
20 |             architecture = np.zeros((2+hl,))
21 |             architecture[0] = X_train.shape[1]
22 |             architecture[-1] = 1#len(np.unique(y_train))
23 |             architecture[1:-1] = [nu for l in range(hl)]
24 | 
25 |             if learn_options["cv"] == "stratified":
26 |                 label_encoder = sklearn.preprocessing.LabelEncoder()
27 |                 label_encoder.fit(y_all['Target gene'].values[train])
28 |                 gene_classes = label_encoder.transform(y_all['Target gene'].values[train])
29 |                 n_folds = len(np.unique(gene_classes))
30 |                 cv = sklearn.cross_validation.StratifiedKFold(gene_classes, n_folds=n_folds, shuffle=True)
31 |             elif learn_options["cv"]=="gene":
32 |                 gene_list = np.unique(y_all['Target gene'].values[train])
33 |                 cv = []
34 |                 for gene in gene_list:
35 |                     cv.append(get_train_test(gene, y_all[train]))
36 |                 n_folds = len(cv)
37 | 
38 | 
39 |             for train_ind, valid_ind in cv:
40 |                 # e = theanets.Experiment(
41 |                 #     theanets.Classifier,
42 |                 #     layers=architecture,
43 |                 #     train_batches=32,
44 |                 #     # patience=100,
45 |                 #     # tied_weights=False,
46 |                 #     )
47 | 
48 |                 e = theanets.Experiment(
49 |                     theanets.Regressor,
50 |                     layers=architecture,
51 |                     train_batches=32,
52 |                     # patience=100,
53 |                     # tied_weights=False,
54 |                     )
55 | 
56 |                 e.run((X_train[train_ind], y_train[train_ind]), (X_train[valid_ind], y_train[valid_ind]))
57 |                 pred = e.network.predict(X_train[valid_ind])
58 | 
59 |                 accuracies[i, j] += sp.stats.spearmanr(pred.flatten(), y_train[valid_ind].flatten())[0]
60 | 
61 |             accuracies[i, j] = accuracies[i, j]/float(n_folds)
62 | 
63 | 
64 |             if best_score is None or accuracies[i, j] > best_score:
65 |                 best_score = accuracies[i, j]
66 |                 best_model = copy.deepcopy(e)
67 |                 print "DNN with %d hidden layers and %d units, accuracy: %.4f   *" % (hl, nu, accuracies[i,j])
68 |             else:
69 |                 print "DNN with %d hidden layers and %d units, accuracy: %.4f" % (hl, nu, accuracies[i,j])
70 | 
71 |     best_model.run((X_train, y_train), (X_test, y_test))
72 |     y_pred = best_model.network.predict(X[test])
73 | 
74 |     return y_pred, None


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/models/DNN.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sunjiamin0824/CRISPR-Local/c6e80b2fd8ee4065c0655edf9f5206bf8cc316e2/Rule_Set_2_scoring_v1/analysis/models/DNN.pyc


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/models/GP.py:
--------------------------------------------------------------------------------
  1 | import numpy as np
  2 | import matplotlib.pyplot as plt
  3 | 
  4 | 
  5 | def gp_on_fold(feature_sets, train, test, y, y_all, inputs, dim, dimsum, learn_options):
  6 |     import GPy
  7 |     from models.gpy_ssk import WeightedDegree
  8 | 
  9 |     sequences = np.array([str(x) for x in y_all.index.get_level_values(0).tolist()])
 10 | 
 11 |     kern = WeightedDegree(1, sequences, d=learn_options['kernel degree'], active_dims=[0])
 12 |     X = np.arange(len(train))[:, None]
 13 | 
 14 |     current_dim = 1
 15 | 
 16 |     if 'gc_count' in feature_sets:
 17 |         kern += GPy.kern.RBF(1, active_dims=[current_dim], name='GC_rbf')
 18 |         X = np.concatenate((X, feature_sets['gc_count'].values), axis=1)
 19 |         current_dim += 1
 20 |         assert X.shape[1] == current_dim
 21 | 
 22 |     if 'drug' in feature_sets:
 23 |         Q = feature_sets['drug'].values.shape[1]
 24 |         kern += GPy.kern.Linear(Q, active_dims=range(current_dim, current_dim+Q), name='drug_lin')
 25 |         X = np.concatenate((X, feature_sets['drug'].values), axis=1)
 26 |         current_dim += Q
 27 |         assert X.shape[1] == current_dim
 28 | 
 29 |     if 'gene effect' in feature_sets:
 30 |         Q = feature_sets['gene effect'].values.shape[1]
 31 |         kern += GPy.kern.Linear(Q, active_dims=range(current_dim, current_dim+Q), name='gene_lin')
 32 |         X = np.concatenate((X, feature_sets['gene effect'].values), axis=1)
 33 |         current_dim += Q
 34 |         assert X.shape[1] == current_dim
 35 | 
 36 |     if "Percent Peptide" in feature_sets:
 37 |         Q = feature_sets['Percent Peptide'].values.shape[1]
 38 |         kern += GPy.kern.RBF(Q, active_dims=range(current_dim, current_dim+Q), name='percent_pept')
 39 |         X = np.concatenate((X, feature_sets['Percent Peptide'].values), axis=1)
 40 |         current_dim += Q
 41 |         assert X.shape[1] == current_dim
 42 | 
 43 |     if "Nucleotide cut position" in feature_sets:
 44 |         Q = feature_sets['Nucleotide cut position'].values.shape[1]
 45 |         kern += GPy.kern.RBF(Q, active_dims=range(current_dim, current_dim+Q), name='nucleo_cut')
 46 |         X = np.concatenate((X, feature_sets['Nucleotide cut position'].values), axis=1)
 47 |         current_dim += Q
 48 |         assert X.shape[1] == current_dim
 49 | 
 50 |     if "Strand effect" in feature_sets:
 51 |         Q = feature_sets['Strand effect'].values.shape[1]
 52 |         kern += GPy.kern.Linear(Q, active_dims=range(current_dim, current_dim+Q), name='strand')
 53 |         X = np.concatenate((X, feature_sets['Strand effect'].values), axis=1)
 54 |         current_dim += Q
 55 |         assert X.shape[1] == current_dim
 56 | 
 57 |     if "NGGX" in feature_sets:
 58 |         Q = feature_sets['NGGX'].values.shape[1]
 59 |         kern += GPy.kern.Linear(Q, active_dims=range(current_dim, current_dim+Q), name='NGGX')
 60 |         X = np.concatenate((X, feature_sets['NGGX'].values), axis=1)
 61 |         current_dim += Q
 62 |         assert X.shape[1] == current_dim
 63 | 
 64 |     if "TM" in feature_sets:
 65 |         Q = feature_sets['TM'].values.shape[1]
 66 |         kern += GPy.kern.RBF(Q, ARD=True, active_dims=range(current_dim, current_dim+Q), name='TM')
 67 |         X = np.concatenate((X, feature_sets['TM'].values), axis=1)
 68 |         current_dim += Q
 69 |         assert X.shape[1] == current_dim
 70 | 
 71 |     if "gene features" in feature_sets:
 72 |         Q = feature_sets['gene features'].values.shape[1]
 73 |         kern += GPy.kern.Linear(Q, ARD=True, active_dims=range(current_dim, current_dim+Q), name='genefeat')
 74 |         X = np.concatenate((X, feature_sets['gene features'].values), axis=1)
 75 |         current_dim += Q
 76 |         assert X.shape[1] == current_dim
 77 | 
 78 |     kern += GPy.kern.Bias(X.shape[1])
 79 | 
 80 |     if learn_options['warpedGP']:
 81 |         m = GPy.models.WarpedGP(X[train], y[train], kernel=kern)
 82 |     else:
 83 |         m = GPy.models.GPRegression(X[train], y[train], kernel=kern)
 84 | 
 85 | 
 86 |     if False: # debug plots
 87 |         m.optimize_restarts(3, messages=1)
 88 |         y_pred, y_uncert = m.predict(X[test])
 89 | 
 90 |         plt.figure('residuals')
 91 |         plt.title('residuals')
 92 |         plt.hist(y_pred.flatten()-y[test].flatten(), bins=50)
 93 | 
 94 |         plt.figure('predictions')
 95 |         plt.title('predictions')
 96 |         plt.errorbar(y_pred.flatten(), y[test].flatten(), yerr=np.sqrt(y_uncert.flatten()), fmt='o')
 97 |         plt.xlabel('prediction')
 98 |         plt.ylabel('truth')
 99 | 
100 |         plt.figure('kernel')
101 |         plt.title('kernel')
102 |         plt.imshow(m.kern.K(X,X))
103 |         print m
104 |         print "%.3f variance explained" % (m.Gaussian_noise.variance/y[train].var())
105 |         import ipdb; ipdb.set_trace()
106 |         plt.close('all')
107 |     else:
108 |         m.optimize_restarts(3)
109 |         # m.optimize(messages=1)
110 |         y_pred, y_uncert = m.predict(X[test])
111 | 
112 |     # TODO add offset such that low scores are around 0 (not -4 or so)
113 | 
114 |     return y_pred, m[:]
115 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/models/PaxHeader/DNN.py:
--------------------------------------------------------------------------------
1 | 18 gid=1594166068
2 | 18 uid=1248751364
3 | 20 ctime=1447951994
4 | 20 atime=1447951994
5 | 23 SCHILY.dev=16777220
6 | 23 SCHILY.ino=24334112
7 | 18 SCHILY.nlink=1
8 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/models/PaxHeader/DNN.pyc:
--------------------------------------------------------------------------------
1 | 18 gid=1594166068
2 | 18 uid=1248751364
3 | 20 ctime=1447951994
4 | 20 atime=1447951994
5 | 23 SCHILY.dev=16777220
6 | 23 SCHILY.ino=24334113
7 | 18 SCHILY.nlink=1
8 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/models/PaxHeader/GP.py:
--------------------------------------------------------------------------------
1 | 18 gid=1594166068
2 | 18 uid=1248751364
3 | 20 ctime=1447951994
4 | 20 atime=1447951994
5 | 23 SCHILY.dev=16777220
6 | 23 SCHILY.ino=24334116
7 | 18 SCHILY.nlink=1
8 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/models/PaxHeader/__init__.py:
--------------------------------------------------------------------------------
1 | 18 gid=1594166068
2 | 18 uid=1248751364
3 | 20 ctime=1447951994
4 | 20 atime=1447951994
5 | 23 SCHILY.dev=16777220
6 | 23 SCHILY.ino=24334109
7 | 18 SCHILY.nlink=1
8 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/models/PaxHeader/__init__.pyc:
--------------------------------------------------------------------------------
1 | 18 gid=1594166068
2 | 18 uid=1248751364
3 | 20 ctime=1447951994
4 | 20 atime=1447951994
5 | 23 SCHILY.dev=16777220
6 | 23 SCHILY.ino=24334110
7 | 18 SCHILY.nlink=1
8 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/models/PaxHeader/baselines.py:
--------------------------------------------------------------------------------
1 | 18 gid=1594166068
2 | 18 uid=1248751364
3 | 20 ctime=1447951994
4 | 20 atime=1447951994
5 | 23 SCHILY.dev=16777220
6 | 23 SCHILY.ino=24334111
7 | 18 SCHILY.nlink=1
8 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/models/PaxHeader/ensembles.py:
--------------------------------------------------------------------------------
1 | 18 gid=1594166068
2 | 18 uid=1248751364
3 | 20 ctime=1447951994
4 | 20 atime=1447951994
5 | 23 SCHILY.dev=16777220
6 | 23 SCHILY.ino=24334114
7 | 18 SCHILY.nlink=1
8 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/models/PaxHeader/ensembles.pyc:
--------------------------------------------------------------------------------
1 | 18 gid=1594166068
2 | 18 uid=1248751364
3 | 20 ctime=1447951994
4 | 20 atime=1447951994
5 | 23 SCHILY.dev=16777220
6 | 23 SCHILY.ino=24334115
7 | 18 SCHILY.nlink=1
8 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/models/PaxHeader/gpy_ssk.py:
--------------------------------------------------------------------------------
1 | 18 gid=1594166068
2 | 18 uid=1248751364
3 | 20 ctime=1447951994
4 | 20 atime=1447951994
5 | 23 SCHILY.dev=16777220
6 | 23 SCHILY.ino=24334117
7 | 18 SCHILY.nlink=1
8 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/models/PaxHeader/regression.py:
--------------------------------------------------------------------------------
1 | 18 gid=1594166068
2 | 18 uid=1248751364
3 | 20 ctime=1447951994
4 | 20 atime=1447951994
5 | 23 SCHILY.dev=16777220
6 | 23 SCHILY.ino=24334118
7 | 18 SCHILY.nlink=1
8 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/models/PaxHeader/regression.pyc:
--------------------------------------------------------------------------------
1 | 18 gid=1594166068
2 | 18 uid=1248751364
3 | 20 ctime=1447951994
4 | 20 atime=1447951994
5 | 23 SCHILY.dev=16777220
6 | 23 SCHILY.ino=24334119
7 | 18 SCHILY.nlink=1
8 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/models/PaxHeader/ssk.py:
--------------------------------------------------------------------------------
1 | 18 gid=1594166068
2 | 18 uid=1248751364
3 | 20 ctime=1447951994
4 | 20 atime=1447951994
5 | 23 SCHILY.dev=16777220
6 | 23 SCHILY.ino=24334120
7 | 18 SCHILY.nlink=1
8 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/models/__init__.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sunjiamin0824/CRISPR-Local/c6e80b2fd8ee4065c0655edf9f5206bf8cc316e2/Rule_Set_2_scoring_v1/analysis/models/__init__.pyc


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/models/baselines.py:
--------------------------------------------------------------------------------
 1 | import numpy as np
 2 | import sklearn
 3 | from sklearn.svm import LinearSVC
 4 | from sklearn.linear_model import LogisticRegression
 5 | import sklearn.linear_model
 6 | 
 7 | 
 8 | def mean_on_fold(feature_sets, train, test, y, y_all, inputs, dim, dimsum, learn_options):
 9 |     return np.ones((test.sum(), 1))*y[train].mean(), None
10 | 
11 | 
12 | def random_on_fold(feature_sets, train, test, y, y_all, inputs, dim, dimsum, learn_options):
13 |     return np.random.randn(test.sum(), 1), None
14 | 
15 | 
16 | def doench_on_fold(feature_sets, train, test, y, y_all, X, dim, dimsum, learn_options):
17 |     auto_class_weight = None  # 'auto'/None
18 |     verbose = False
19 |     penalty = [0.005*pow(1.15, x) for x in range(0, 45)]  # ian's code:  tvals = [0.005*pow(1.15,x) for x in range(0,45)]
20 |     y_bin = y_all[learn_options['binary target name']].values[:, None]
21 | 
22 |     label_encoder = sklearn.preprocessing.LabelEncoder()
23 |     label_encoder.fit(y_all['Target gene'].values[train])
24 |     gene_classes = label_encoder.transform(y_all['Target gene'].values[train])
25 |     cv = sklearn.cross_validation.StratifiedKFold(gene_classes, n_folds=10, shuffle=True)
26 | 
27 |     best_penalty = None
28 | 
29 |     cv_results = np.zeros((10, len(penalty)))
30 | 
31 |     for j, split in enumerate(cv):
32 |         train_inner, test_inner = split
33 |         for i, c in enumerate(penalty):
34 |             # fit an L1-penalized SVM classifier
35 |             clf = LinearSVC(penalty='l1', C=c, dual=False, class_weight=auto_class_weight)
36 |             clf.fit(X[train][train_inner], y_bin[train][train_inner].flatten())
37 | 
38 |             # pass features with non-zero coeff to Logistic with l2 penalty (original code?)
39 |             non_zero_coeff = (clf.coef_ != 0.0)
40 | 
41 |             if np.all(non_zero_coeff is False):
42 |                 # if all are zero, turn one on so as to be able to run the code.
43 |                 non_zero_coeff[0] = True
44 | 
45 |             clf = LogisticRegression(penalty='l2', class_weight=auto_class_weight)
46 |             clf.fit(X[train][train_inner][:, non_zero_coeff.flatten()], y[train][train_inner].flatten())
47 |             y_test = clf.predict_proba(X[train][test_inner][:, non_zero_coeff.flatten()])[:, 1]
48 | 
49 |             fpr, tpr, _ = sklearn.metrics.roc_curve(y_bin[train][test_inner], y_test)
50 |             assert np.nan not in fpr, "found nan fpr"
51 |             assert np.nan not in tpr, "found nan tpr"
52 |             roc_auc = sklearn.metrics.auc(fpr, tpr)
53 |             if verbose:
54 |                 print j, i, roc_auc
55 |             cv_results[j][i] = roc_auc
56 | 
57 |     best_penalty = penalty[np.argmax(np.mean(cv_results, axis=0))]
58 |     print "best AUC for penalty: ", np.median(cv_results, axis=0)
59 |     clf = LinearSVC(penalty='l1', C=best_penalty, dual=False, class_weight=auto_class_weight)
60 |     clf.fit(X[train], y_bin[train].flatten())
61 |     non_zero_coeff = (clf.coef_ != 0.0)
62 | 
63 |     clf = LogisticRegression(penalty='l2', class_weight=auto_class_weight)
64 |     clf.fit(X[train][:, non_zero_coeff.flatten()], y[train].flatten())
65 |     y_pred = clf.predict_proba(X[test][:, non_zero_coeff.flatten()])[:, 1:2]
66 | 
67 |     return y_pred, clf
68 | 
69 | 
70 | def sgrna_from_doench_on_fold(feature_sets, train, test, y, y_all, X, dim, dimsum, learn_options):
71 |     assert len(feature_sets.keys()) == 1, "should only use sgRNA Score here"
72 |     assert feature_sets.keys()[0] == "sgRNA Score"
73 |     y_pred = X[test][:, 0]
74 |     return y_pred, None
75 | 
76 |     
77 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/models/ensembles.py:
--------------------------------------------------------------------------------
  1 | import numpy as np
  2 | import sklearn.linear_model
  3 | import sklearn.ensemble as en
  4 | from sklearn.grid_search import GridSearchCV
  5 | import sklearn
  6 | from sklearn.linear_model import LinearRegression
  7 | import scipy as sp
  8 | from regression import linreg_on_fold
  9 | import sklearn
 10 | import sklearn.tree as tree
 11 | from sklearn import svm
 12 | from sklearn.grid_search import GridSearchCV
 13 | 
 14 | 
 15 | def spearman_scoring(clf, X, y):
 16 |     y_pred = clf.predict(X).flatten()
 17 |     return sp.stats.spearmanr(y_pred, y.flatten())[0]
 18 | 
 19 | 
 20 | def adaboost_on_fold(feature_sets, train, test, y, y_all, X, dim, dimsum, learn_options):
 21 |     '''
 22 |     AdaBoostRegressor from scikitlearn.
 23 |     '''
 24 | 
 25 |     if learn_options['adaboost_version'] == 'python':
 26 |         if not learn_options['adaboost_CV']:
 27 |             clf = en.GradientBoostingRegressor(loss=learn_options['adaboost_loss'], learning_rate=learn_options['adaboost_learning_rate'],
 28 |                                                n_estimators=learn_options['adaboost_n_estimators'],
 29 |                                                alpha=learn_options['adaboost_alpha'],
 30 |                                                subsample=1.0, min_samples_split=2, min_samples_leaf=1, max_depth=learn_options['adaboost_max_depth'],
 31 |                                                init=None, random_state=None, max_features=None,
 32 |                                                verbose=0, max_leaf_nodes=None, warm_start=False)
 33 | 
 34 |             clf.fit(X[train], y[train].flatten())
 35 |             y_pred = clf.predict(X[test])[:, None]
 36 |         else:
 37 |             print "Adaboost with GridSearch"
 38 |             from sklearn.grid_search import GridSearchCV
 39 |             param_grid = {'learning_rate': [0.1, 0.05, 0.01],
 40 |                           'max_depth': [4, 5, 6, 7],
 41 |                           'min_samples_leaf': [5, 7, 10, 12, 15],
 42 |                           'max_features': [1.0, 0.5, 0.3, 0.1]}
 43 | 
 44 | 
 45 |             label_encoder = sklearn.preprocessing.LabelEncoder()
 46 |             label_encoder.fit(y_all['Target gene'].values[train])
 47 |             gene_classes = label_encoder.transform(y_all['Target gene'].values[train])
 48 |             n_folds = len(np.unique(gene_classes))
 49 |             cv = sklearn.cross_validation.StratifiedKFold(gene_classes, n_folds=n_folds, shuffle=True)
 50 | 
 51 |             est = en.GradientBoostingRegressor(loss=learn_options['adaboost_loss'], n_estimators=learn_options['adaboost_n_estimators'])
 52 |             clf = GridSearchCV(est, param_grid, n_jobs=20, verbose=1, cv=cv, scoring=spearman_scoring, iid=False).fit(X[train], y[train].flatten())
 53 |             print clf.best_params_
 54 |             y_pred = clf.predict(X[test])[:, None]
 55 |     else:
 56 |         raise NotImplementedError
 57 | 
 58 |     return y_pred, clf
 59 | 
 60 | 
 61 | def LASSOs_ensemble_on_fold(feature_sets, train, test, y, y_all, X, dim, dimsum, learn_options):
 62 |     train_indices = np.where(train)[0]
 63 |     sel = len(train_indices)*0.10
 64 |     permuted_ind = np.random.permutation(train_indices)
 65 |     valid_indices = permuted_ind[:sel]
 66 |     train_indices = permuted_ind[sel:]
 67 |     train_sub = np.zeros_like(train, dtype=bool)
 68 |     valid_sub = np.zeros_like(train, dtype=bool)
 69 |     train_sub[train_indices] = True
 70 |     valid_sub[valid_indices] = True
 71 | 
 72 |     validations = np.zeros((len(valid_indices), len(feature_sets.keys())))
 73 |     predictions = np.zeros((test.sum(), len(feature_sets.keys())))
 74 | 
 75 |     for i, feature_name in enumerate(feature_sets.keys()):
 76 |         X_feature = feature_sets[feature_name].values
 77 |         y_pred, m = linreg_on_fold(feature_sets, train_sub, valid_sub, y, y_all, X_feature, dim, dimsum, learn_options)
 78 |         predictions[:, i] = m.predict(X_feature[test]).flatten()
 79 |         validations[:, i] = y_pred.flatten()
 80 | 
 81 |     clf = LinearRegression()
 82 |     clf.fit(validations, y[valid_sub])
 83 |     y_pred = clf.predict(predictions)
 84 | 
 85 |     return y_pred, None
 86 | 
 87 | 
 88 | def randomforest_on_fold(feature_sets, train, test, y, y_all, X, dim, dimsum, learn_options):
 89 |     '''
 90 |     RandomForestRegressor from scikitlearn.
 91 |     '''
 92 |     clf = en.RandomForestRegressor(oob_score=True)
 93 |     clf.fit(X[train], y[train][:, 0])
 94 |     y_pred = clf.predict(X[test])[:, None]
 95 |     return y_pred, clf
 96 | 
 97 | 
 98 | def decisiontree_on_fold(feature_sets, train, test, y, y_all, X, dim, dimsum, learn_options):
 99 |     '''
100 |     DecisionTreeRegressor from scikitlearn.
101 |     '''
102 |     clf = tree.DecisionTreeRegressor()
103 |     clf.fit(X[train], y[train][:, 0])
104 |     y_pred = clf.predict(X[test])[:, None]
105 |     return y_pred, clf
106 | 
107 | 
108 | def linear_stacking(y_train, X_train, X_test):
109 |     clf = sklearn.linear_model.LinearRegression()
110 |     clf.fit(X_train, y_train)
111 |     y_pred = clf.predict(X_test)
112 |     return y_pred.flatten()
113 | 
114 | 
115 | def pairwise_majority_voting(y):
116 |     N = y.shape[0]
117 |     y_pred = np.zeros((N, N))
118 |     for i in range(N):
119 |         for j in range(N):
120 |             if i == j:
121 |                 continue
122 | 
123 |             y_pred[i, j] = (y[i] > y[j]).sum() > y.shape[1]/2
124 | 
125 |     return y_pred.sum(1)/y_pred.sum(1).max()
126 | 
127 | 
128 | def median(y):
129 |     return np.median(y, axis=1)
130 | 
131 | 
132 | def GBR_stacking(y_train, X_train, X_test):
133 |     param_grid = {'learning_rate': [0.1, 0.05, 0.01],
134 |                   'max_depth': [2, 3, 4, 5],  # [2, 3, 4, 6],
135 |                   'min_samples_leaf': [1, 2, 3],  # ,5, 7],
136 |                   'max_features': [1.0, 0.5, 0.3, 0.1]}
137 | 
138 |     est = en.GradientBoostingRegressor(loss='ls', n_estimators=100)
139 |     clf = GridSearchCV(est, param_grid, n_jobs=3, verbose=1, cv=20, scoring=spearman_scoring).fit(X_train, y_train.flatten())
140 |     # clf.fit(X_train, y_train.flatten())
141 |     return clf.predict(X_test)
142 | 
143 | 
144 | def GP_stacking(y_train, X_train, X_test):
145 |     import GPy
146 |     m = GPy.models.SparseGPRegression(X_train, y_train, num_inducing=20, kernel=GPy.kern.RBF(X_train.shape[1]))
147 |     m.optimize('bfgs', messages=0)
148 |     y_pred = m.predict(X_test)[0]
149 |     return y_pred.flatten()
150 | 
151 | 
152 | def SVM_stacking(y_train, X_train, X_test):
153 |     parameters = {'kernel': ('linear', 'rbf'), 'C': np.linspace(1, 10, 10), 'gamma': np.linspace(1e-3, 1., 10)}
154 |     svr = svm.SVR()
155 |     clf = GridSearchCV(svr, parameters, n_jobs=3, verbose=1, cv=10, scoring=spearman_scoring)
156 |     clf.fit(X_train, y_train.flatten())
157 |     return clf.predict(X_test)
158 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/models/ensembles.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sunjiamin0824/CRISPR-Local/c6e80b2fd8ee4065c0655edf9f5206bf8cc316e2/Rule_Set_2_scoring_v1/analysis/models/ensembles.pyc


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/models/gpy_ssk.py:
--------------------------------------------------------------------------------
 1 | from ssk_cython import *
 2 | import GPy
 3 | from GPy.util.caching import Cache_this
 4 | 
 5 | class WeightedDegree(GPy.kern.Kern):
 6 |     def __init__(self, input_dim, strings, d=4, variance=1., active_dims=None, name='weighted degree'):
 7 |         super(WeightedDegree, self).__init__(input_dim, active_dims, name)
 8 |         self.variance = GPy.core.parameterization.Param('variance', variance, GPy.core.parameterization.transformations.Logexp())
 9 |         self.link_parameters(self.variance)
10 |         self.strings = strings
11 |         self.string_kernel = cython_WD_K(self.strings.tolist(), self.strings.tolist(), d=d)
12 | 
13 |     def K(self, X, X2=None):
14 |         if X2 is None:
15 |             X2 = X
16 |             
17 |         Xind = np.asarray(X, dtype=int).flatten()
18 |         X2ind = np.asarray(X2, dtype=int).flatten()
19 |         K = self.string_kernel[Xind][:, X2ind]
20 | 
21 |         return self.variance * K
22 | 
23 |     def Kdiag(self, X):
24 |         Xind = np.asarray(X, dtype=int).flatten()
25 |         K = self.string_kernel[Xind][:, Xind]
26 | 
27 |         return self.variance * K.diagonal()
28 | 
29 |     def update_gradients_full(self, dL_dK, X, X2=None):
30 |         if X2 is None:
31 |             X2 = X
32 | 
33 |         Xind = np.asarray(X, dtype=int).flatten()
34 |         X2ind = np.asarray(X2, dtype=int).flatten()
35 |         K = self.string_kernel[Xind][:, X2ind]
36 | 
37 |         self.variance.gradient = np.einsum('ij,ij', dL_dK, K)
38 | 
39 |     def update_gradients_diag(self, dL_dKdiag, X):
40 |         Xind = np.asarray(X, dtype=int).flatten()
41 |         K = self.string_kernel[Xind][:, Xind]
42 | 
43 |         self.variance.gradient = np.einsum('i,i', dL_dKdiag, K)
44 | 
45 | 
46 | 
47 | if __name__ == '__main__':
48 |     x1 = 'ATCGATCG'
49 |     x2 = 'ATCGATCG'
50 |     x3 = 'ATCGATCC'
51 |     x4 = 'ATCGATAA'
52 |     x5 = 'ANNNNNNN'
53 |     X = np.arange(5)[:, None]
54 |     y = np.random.randn(5, 1)
55 |     kern = WeightedDegree(1, np.array([x1, x2, x3, x4, x5]))
56 |     m = GPy.models.GPRegression(X, y, kernel=kern)
57 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/models/regression.py:
--------------------------------------------------------------------------------
  1 | import numpy as np
  2 | import sklearn
  3 | from sklearn.linear_model import ARDRegression, LinearRegression
  4 | from sklearn.metrics import roc_curve, auc
  5 | import sklearn.linear_model
  6 | import util
  7 | import metrics as ranking_metrics
  8 | import predict
  9 | 
 10 | 
 11 | def ARDRegression_on_fold(feature_sets, train, test, y, y_all, X, dim, dimsum, learn_options):
 12 |     '''
 13 |     '''
 14 |     clf = ARDRegression()
 15 |     clf.fit(X[train], y[train][:, 0])
 16 |     y_pred = clf.predict(X[test])[:, None]
 17 |     return y_pred, clf
 18 | 
 19 | 
 20 | def train_linreg_model(alpha, l1r, learn_options, fold, X, y, y_all):
 21 |     '''
 22 |     fold is something like train_inner (boolean array specifying what is in the fold)
 23 |     '''
 24 |     if learn_options["penalty"] == "L2":
 25 |         clf = sklearn.linear_model.Ridge(alpha=alpha, fit_intercept=True, normalize=True, copy_X=True, max_iter=None, tol=0.001, solver='auto')
 26 |         weights = get_weights(learn_options, fold, y, y_all)
 27 |         clf.fit(X[fold], y[fold], sample_weight=weights)
 28 |     elif learn_options["penalty"] == 'EN' or learn_options["penalty"] == 'L1':
 29 |         if learn_options["loss"] == "squared":
 30 |             clf = sklearn.linear_model.ElasticNet(alpha=alpha, l1_ratio=l1r, fit_intercept=True, normalize=True, max_iter=3000)
 31 |         elif learn_options["loss"] == "huber":
 32 |             clf = sklearn.linear_model.SGDRegressor('huber', epsilon=0.7, alpha=alpha,
 33 |                                                     l1_ratio=l1r, fit_intercept=True, n_iter=10,
 34 |                                                     penalty='elasticnet', shuffle=True)
 35 |         clf.fit(X[fold], y[fold])
 36 |     return clf
 37 | 
 38 | 
 39 | def logreg_on_fold(feature_sets, train, test, y, y_all, X, dim, dimsum, learn_options):
 40 |     '''
 41 |     (L1/L2 penalized) logistic reggresion using scikitlearn
 42 |     '''
 43 | 
 44 |     assert len(np.unique(y)) <= 2, "if using logreg need binary targets"
 45 |     assert learn_options["weighted"] is None, "cannot do weighted Log reg"
 46 |     assert learn_options['feature_select'] is False, "cannot do feature selection yet in logistic regression--see linreg_on_fold to implement"
 47 | 
 48 |     cv, n_folds = set_up_folds(learn_options, y_all.iloc[train])
 49 | 
 50 |     assert learn_options['penalty'] == "L1" or learn_options['penalty'] == "L2", "can only use L1 or L2 with logistic regression"
 51 | 
 52 |     performance = np.zeros((len(learn_options["alpha"]), 1))
 53 |     # degenerate_pred = np.zeros((len(learn_options["alpha"])))
 54 |     for train_inner, test_inner in cv:
 55 |         for i, alpha in enumerate(learn_options["alpha"]):
 56 |             clf = sklearn.linear_model.LogisticRegression(penalty=learn_options['penalty'], dual=False,
 57 |                                                           fit_intercept=True, class_weight='auto', tol=0.001, C=1.0/alpha)
 58 |             clf.fit(X[train][train_inner], y[train][train_inner].flatten())
 59 |             tmp_pred = clf.predict(X[train][test_inner])
 60 | 
 61 |             if learn_options["training_metric"] == "AUC":
 62 |                 fpr, tpr, _ = roc_curve(y_all[learn_options["ground_truth_label"]][train][test_inner], tmp_pred)
 63 |                 assert ~np.any(np.isnan(fpr)), "found nan fpr"
 64 |                 assert ~np.any(np.isnan(tpr)), "found nan tpr"
 65 |                 tmp_auc = auc(fpr, tpr)
 66 |                 performance[i] += tmp_auc
 67 |             else:
 68 |                 raise Exception("can only use AUC metric for cv with classification")
 69 | 
 70 |     performance /= n_folds
 71 | 
 72 |     max_score_ind = np.where(performance == np.nanmax(performance))
 73 |     assert max_score_ind != len(performance), "enlarge alpha range as hitting max boundary"
 74 | 
 75 |     # in the unlikely event of tied scores, take the first one.
 76 |     if len(max_score_ind[0]) > 1:
 77 |         max_score_ind = [max_score_ind[0][0], max_score_ind[1][0]]
 78 | 
 79 |     best_alpha = learn_options["alpha"][max_score_ind[0]]
 80 | 
 81 |     print "\t\tbest alpha is %f from range=%s" % (best_alpha, learn_options["alpha"][[0, -1]])
 82 |     max_perf = np.nanmax(performance)
 83 | 
 84 |     if max_perf < 0.0:
 85 |         raise Exception("performance is negative")
 86 | 
 87 |     print "\t\tbest performance is %f" % np.nanmax(performance)
 88 | 
 89 |     clf = sklearn.linear_model.LogisticRegression(penalty=learn_options['penalty'],
 90 |                                                   dual=False, fit_intercept=True, class_weight='auto',
 91 |                                                   tol=0.001, C=1.0/best_alpha)
 92 |     clf.fit(X[train], y[train].flatten())
 93 |     y_pred = clf.predict(X[test])
 94 |     y_pred = y_pred[:, None]
 95 |     return y_pred, clf
 96 | 
 97 | 
 98 | def linreg_on_fold(feature_sets, train, test, y, y_all, X, dim, dimsum, learn_options):
 99 |     '''
100 |     linreg using scikitlearn, using more standard regression models with penalization requiring
101 |     nested-cross-validation
102 |     '''
103 | 
104 |     if learn_options["weighted"] is not None and (learn_options["penalty"] != "L2" or learn_options["method"] != "linreg"):
105 |         raise NotImplementedError("weighted prediction not implemented for any methods by L2 at the moment")
106 | 
107 |     cv, n_folds = set_up_folds(learn_options, y_all.iloc[train])
108 | 
109 |     if learn_options['penalty'] == "L1":
110 |         l1_ratio = [1.0]
111 |     elif learn_options['penalty'] == "L2":
112 |         l1_ratio = [0.0]
113 |     elif learn_options['penalty'] == "EN":  # elastic net
114 |         l1_ratio = np.linspace(0.0, 1.0, 20)
115 | 
116 |     performance = np.zeros((len(learn_options["alpha"]), len(l1_ratio)))
117 |     degenerate_pred = np.zeros((len(learn_options["alpha"])))
118 |     for train_inner, test_inner in cv:
119 |         for i, alpha in enumerate(learn_options["alpha"]):
120 |             for j, l1r in enumerate(l1_ratio):
121 |                 clf = train_linreg_model(alpha, l1r, learn_options, train_inner, X[train], y[train], y_all.iloc[train])
122 |                 if learn_options["feature_select"]:
123 |                     clf, tmp_pred = feature_select(clf, learn_options, test_inner, train_inner, X[train], y[train])
124 |                 else:
125 |                     tmp_pred = clf.predict(X[train][test_inner])
126 | 
127 |                 if learn_options["training_metric"] == "AUC":
128 |                     fpr, tpr, _ = roc_curve(y_all[learn_options["ground_truth_label"]][train][test_inner], tmp_pred)
129 |                     assert ~np.any(np.isnan(fpr)), "found nan fpr"
130 |                     assert ~np.any(np.isnan(tpr)), "found nan tpr"
131 |                     tmp_auc = auc(fpr, tpr)
132 |                     performance[i, j] += tmp_auc
133 | 
134 |                 elif learn_options['training_metric'] == 'spearmanr':
135 |                     spearman = util.spearmanr_nonan(y_all[learn_options['ground_truth_label']][train][test_inner], tmp_pred.flatten())[0]
136 |                     performance[i, j] += spearman
137 | 
138 |                 elif learn_options['training_metric'] == 'score':
139 |                     performance[i, j] += clf.score(X[test_inner], y_all[learn_options['ground_truth_label']][train][test_inner])
140 | 
141 |                 elif learn_options["training_metric"] == "NDCG":
142 |                     assert "thresh" not in learn_options["ground_truth_label"], "for NDCG must not use thresholded ranks, but pure ranks"
143 | 
144 |                     # sorted = tmp_pred[np.argsort(y_all[ground_truth_label].values[test_inner])[::-1]].flatten()
145 |                     # sortedgt = np.sort(y_all[ground_truth_label].values[test_inner])[::-1].flatten()
146 |                     # tmp_perf = ranking_metrics.ndcg_at_k_ties(sorted, learn_options["NDGC_k"], sortedgt)
147 |                     tmp_truth = y_all[learn_options["ground_truth_label"]].values[train][test_inner].flatten()
148 |                     tmp_perf = ranking_metrics.ndcg_at_k_ties(tmp_truth, tmp_pred.flatten(), learn_options["NDGC_k"])
149 |                     performance[i, j] += tmp_perf
150 | 
151 |                     degenerate_pred_tmp = len(np.unique(tmp_pred)) < len(tmp_pred)/2.0
152 |                     degenerate_pred[i] += degenerate_pred_tmp
153 | 
154 |                     # sanity checking metric wrt ties, etc.
155 |                     # rmse = np.sqrt(np.mean((tmp_pred - tmp_truth)**2))
156 |                     tmp_pred_r, tmp_truth_r = ranking_metrics.rank_data(tmp_pred, tmp_truth)
157 |                     # rmse_r = np.sqrt(np.mean((tmp_pred_r-tmp_truth_r)**2))
158 | 
159 |     performance /= n_folds
160 | 
161 |     max_score_ind = np.where(performance == np.nanmax(performance))
162 |     assert max_score_ind != len(performance), "enlarge alpha range as hitting max boundary"
163 |     # assert degenerate_pred[max_score_ind[0][0]]==0, "found degenerate predictions at max score"
164 | 
165 |     # in the unlikely event of tied scores, take the first one.
166 |     if len(max_score_ind[0]) > 1:
167 |         max_score_ind = [max_score_ind[0][0], max_score_ind[1][0]]
168 | 
169 |     best_alpha, best_l1r = learn_options["alpha"][max_score_ind[0]], l1_ratio[max_score_ind[1]]
170 | 
171 |     print "\t\tbest alpha is %f from range=%s" % (best_alpha, learn_options["alpha"][[0, -1]])
172 |     if learn_options['penalty'] == "EN":
173 |         print "\t\tbest l1_ratio is %f from range=%s" % (best_l1r, l1_ratio[[0, -1]])
174 |     max_perf = np.nanmax(performance)
175 | 
176 |     if max_perf < 0.0:
177 |         raise Exception("performance is negative")
178 | 
179 |     print "\t\tbest performance is %f" % max_perf
180 | 
181 |     clf = train_linreg_model(best_alpha, l1r, learn_options, train, X, y, y_all)
182 |     if learn_options["feature_select"]:
183 |         raise Exception("untested in a long time, should double check")
184 |         clf, y_pred = feature_select(clf, learn_options, test, train, X, y)
185 |     else:
186 |         y_pred = clf.predict(X[test])
187 | 
188 |     if learn_options["penalty"] != "L2":
189 |         y_pred = y_pred[:, None]
190 | 
191 |     return y_pred, clf
192 | 
193 | 
194 | def feature_select(clf, learn_options, test_inner, train_inner, X, y):
195 |     assert not learn_options["weighted"] is not None, "cannot currently do feature selection with weighted regression"
196 |     assert learn_options["loss"] is not "huber", "won't use huber loss function with feature selection"
197 |     non_zero_coeff = (clf.coef_ != 0.0)
198 |     if non_zero_coeff.sum() > 0:
199 |         clf = LinearRegression()
200 |         clf.fit(X[train_inner][:, non_zero_coeff.flatten()], y[train_inner])
201 |         tmp_pred = clf.predict(X[test_inner][:, non_zero_coeff.flatten()])
202 |     else:
203 |         tmp_pred = np.ones_like(test_inner)
204 |     return clf, tmp_pred
205 | 
206 | 
207 | def get_weights(learn_options, fold, y, y_all):
208 |     '''
209 |     fold is an object like train_inner which is boolean for which indexes are in the fold
210 |     '''
211 |     weights = None
212 |     if learn_options["weighted"] == "variance":
213 |         weights = 1.0/y_all["variance"].values[fold]
214 |     elif learn_options["weighted"] == "ndcg":
215 |         # DCG: r[0] + np.sum(r[1:] / np.log2(np.arange(2, r.size + 1)))
216 |         N = len(fold)
217 |         r = np.ones(N)
218 |         discount = np.concatenate((np.array([r[0]]), r[1:] / np.log2(np.arange(2, r.size + 1))))[::1]
219 |         ind = np.argsort(y[fold], axis=0).flatten()
220 |         weights = np.ones(len(ind))
221 |         weights[ind] = discount
222 |     elif learn_options["weighted"] == "rank":
223 |         N = len(y[fold])
224 |         inverse_ranks = (np.arange(N) + 1.0)[::-1]
225 |         ind = np.argsort(y[fold], axis=0).flatten()
226 |         weights = np.ones(len(ind))
227 |         weights[ind] = inverse_ranks
228 |     elif learn_options["weighted"] == "score":
229 |         N = len(y[fold])
230 |         score = y[fold] + np.abs(np.min(y[fold]))
231 |         ind = np.argsort(y[fold], axis=0).flatten()
232 |         weights = np.ones(len(ind))
233 |         weights[ind] = score
234 |     elif learn_options["weighted"] == "random":
235 |         N = len(y[fold])
236 |         weights = np.random.rand(N)
237 |     elif learn_options["weighted"] is not None:
238 |         raise Exception("invalid weighted type, %s" % learn_options["weighted"])
239 |     # plt.plot(weights, y[train_inner],'.')
240 |     return weights
241 | 
242 | 
243 | def set_up_folds(learn_options, y):
244 |     if learn_options["cv"] == "stratified":
245 |         label_encoder = sklearn.preprocessing.LabelEncoder()
246 |         label_encoder.fit(y['Target gene'].values)
247 |         gene_classes = label_encoder.transform(y['Target gene'].values)
248 |         n_folds = len(np.unique(gene_classes))
249 |         cv = sklearn.cross_validation.StratifiedKFold(gene_classes, n_folds=n_folds, shuffle=True)
250 |     elif learn_options["cv"] == "gene":
251 |         gene_list = np.unique(y['Target gene'].values)
252 |         cv = []
253 |         for gene in gene_list:
254 |             cv.append(predict.get_train_test(gene, y))
255 |         n_folds = len(cv)
256 |     return cv, n_folds
257 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/models/regression.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sunjiamin0824/CRISPR-Local/c6e80b2fd8ee4065c0655edf9f5206bf8cc316e2/Rule_Set_2_scoring_v1/analysis/models/regression.pyc


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/models/ssk.py:
--------------------------------------------------------------------------------
 1 | import numpy as np
 2 | from ssk_cython import *
 3 | 
 4 | def weighted_degree_kxx(x, xp, d):
 5 |     assert len(x) == len(xp)
 6 |     l = len(x)
 7 |     sim = 0
 8 | 
 9 |     for k in range(1, d):
10 |         beta_k = 2*(d-k+1)/float((d*(d+1)))
11 |         sim_k = 0
12 |         for i in range(1, l-k+1):
13 |             if x[i:i+k] == xp[i:i+k]:
14 |                 sim_k += 1.
15 |         sim += beta_k * sim_k
16 | 
17 |     return sim
18 | 
19 | def WD_K(sequences, d=4, cython=False):
20 |     num_sequences = len(sequences)
21 |     K = np.zeros((num_sequences, num_sequences))
22 | 
23 |     for i in range(num_sequences):
24 |         for j in range(num_sequences):
25 |             if cython:
26 |                 K[i, j] = cython_weighted_degree_kxx(sequences[i], sequences[j], d=d)
27 |             else:
28 |                 K[i, j] = weighted_degree_kxx(sequences[i], sequences[j], d=d)
29 | 
30 |     return K
31 | 
32 | if __name__ == '__main__':
33 |     x1 = 'ATCGATCG'
34 |     x2 = 'ATCGATCG'
35 |     x3 = 'ATCGATCC'
36 |     x4 = 'ATCGATAA'
37 |     x5 = 'ANNNNNNN'
38 |     K = WD_K([x1, x2, x3, x4, x5])
39 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/predict.py:
--------------------------------------------------------------------------------
  1 | import numpy as np
  2 | import copy
  3 | import util
  4 | import time
  5 | import metrics as ranking_metrics
  6 | import models.regression
  7 | import models.DNN
  8 | import multiprocessing
  9 | import sys
 10 | import sklearn
 11 | from sklearn.metrics import roc_curve, auc
 12 | import sklearn.metrics
 13 | import sklearn.cross_validation
 14 | 
 15 | 
 16 | def fill_in_truth_and_predictions(truth, predictions, fold, y_all, y_pred, learn_options, test):
 17 |     truth[fold]['ranks'] = np.hstack((truth[fold]['ranks'],
 18 |                                       y_all[learn_options['rank-transformed target name']].values[test].flatten()))
 19 | 
 20 |     truth[fold]['thrs'] = np.hstack((truth[fold]['thrs'],
 21 |                                      y_all[learn_options['binary target name']].values[test].flatten()))
 22 | 
 23 |     if 'raw_target_name' in learn_options.keys():
 24 |         truth[fold]['raw'] = np.hstack((truth[fold]['raw'],
 25 |                                         y_all[learn_options['raw target name']].values[test].flatten()))
 26 | 
 27 |     predictions[fold] = np.hstack((predictions[fold], y_pred.flatten()))
 28 | 
 29 |     return truth, predictions
 30 | 
 31 | 
 32 | def construct_filename(learn_options, TEST):
 33 |     filename = "V%s" % learn_options["V"]
 34 | 
 35 |     if TEST:
 36 |         filename = "TEST."
 37 | 
 38 |     filename += learn_options["method"]
 39 |     filename += '.order%d' % learn_options["order"]
 40 |     # try:
 41 |     #     learn_options["target_name"] = ".%s" % learn_options["target_name"].split(" ")[1]
 42 |     # except:
 43 |     #     pass
 44 |     filename += learn_options["target_name"]
 45 |     if learn_options["method"] == "GPy":
 46 |         pass
 47 |         # filename += ".R%d" % opt_options['num_restarts']
 48 |         # filename += ".K%s" % learn_options['kerntype']
 49 |         # if learn_options.has_key('degree'):
 50 |         #     filename += "d%d" % learn_options['degree']
 51 |         # if learn_options['warped']:
 52 |         #     filename += ".Warp"
 53 |     elif learn_options["method"] == "linreg":
 54 |         filename += "." + learn_options["penalty"]
 55 |     filename += "." + learn_options["cv"]
 56 | 
 57 |     if learn_options["training_metric"] == "NDCG":
 58 |         filename += ".NDGC_%d" % learn_options["NDGC_k"]
 59 |     elif learn_options["training_metric"] == "AUC":
 60 |         filename += ".AUC"
 61 |     elif learn_options["training_metric"] == 'spearmanr':
 62 |         filename += ".spearman"
 63 | 
 64 |     print "filename = %s" % filename
 65 |     return filename
 66 | 
 67 | def print_summary(global_metric, results, learn_options, feature_sets, flags):
 68 |     print "\nSummary:"
 69 |     print learn_options
 70 |     print "\t\tglobal %s=%.2f" % (learn_options['metric'], global_metric)
 71 |     print "\t\tmedian %s across folds=%.2f" % (learn_options['metric'], np.median(results[0]))
 72 |     print "\t\torder=%d" % learn_options["order"]
 73 |     if learn_options.has_key('kerntype'): "\t\tkern type = %s" % learn_options['kerntype']
 74 |     if learn_options.has_key('degree'): print "\t\tdegree=%d" % learn_options['degree']
 75 |     print "\t\ttarget_name=%s" % learn_options["target_name"]
 76 | 
 77 |     for k in flags.keys():
 78 |         print '\t\t' + k + '=' + str(learn_options[k])
 79 | 
 80 |     print "\t\tfeature set:"
 81 |     for set in feature_sets.keys():
 82 |         print "\t\t\t%s" % set
 83 |     print "\t\ttotal # features=%d" % results[4]
 84 | 
 85 | def extract_fpr_tpr_for_fold(aucs, fold, i, predictions, truth, y_binary, test, y_pred):
 86 |     assert len(np.unique(y_binary))<=2, "if using AUC need binary targets"
 87 |     fpr, tpr, _ = roc_curve(y_binary[test], y_pred)
 88 |     roc_auc = auc(fpr, tpr)
 89 |     aucs.append(roc_auc)
 90 | 
 91 | def extract_NDCG_for_fold(metrics, fold, i, predictions, truth, y_ground_truth, test, y_pred, learn_options):
 92 |     NDCG_fold = ranking_metrics.ndcg_at_k_ties(y_ground_truth[test].flatten(), y_pred.flatten(), learn_options["NDGC_k"])
 93 |     metrics.append(NDCG_fold)
 94 | 
 95 | def extract_spearman_for_fold(metrics, fold, i, predictions, truth, y_ground_truth, test, y_pred, learn_options):
 96 |     spearman = util.spearmanr_nonan(y_ground_truth[test].flatten(), y_pred.flatten())[0]
 97 |     assert not np.isnan(spearman), "found nan spearman"
 98 |     metrics.append(spearman)
 99 | 
100 | def get_train_test(test_gene, y_all, train_genes=None):
101 |     # this is a bit convoluted because the train_genes+test_genes may not add up to all genes
102 |     # for e.g. when we load up V3, but then use only V2, etc.
103 |     not_test = (y_all.index.get_level_values('Target gene').values != test_gene)
104 |     if train_genes is not None:
105 |         in_train_genes = np.zeros(not_test.shape, dtype=bool)
106 |         for t_gene in train_genes:
107 |             in_train_genes = np.logical_or(in_train_genes, (y_all.index.get_level_values('Target gene').values == t_gene))
108 |         train = np.logical_and(not_test, in_train_genes)
109 |     else:
110 |         train = not_test
111 |     #y_all['test'] as to do with extra pairs in V2
112 |     test = (y_all.index.get_level_values('Target gene').values== test_gene) * (y_all['test'].values == 1.)
113 | 
114 |     # convert to indices
115 |     test = np.where(test == True)[0]
116 |     train = np.where(train == True)[0]
117 |     return train, test
118 | 
119 | def cross_validate(y_all, feature_sets, learn_options=None, TEST=False, train_genes=None, CV=True):
120 |     '''
121 |     feature_sets is a dictionary of "set name" to pandas.DataFrame
122 |     one set might be single-nucleotide, position-independent features of order X, for e.g.
123 |     Method: "GPy" or "linreg"
124 |     Metric: NDCG (learning to rank metric, Normalized Discounted Cumulative Gain); AUC
125 |     Output: cv_score_median, gene_rocs
126 |     '''
127 | 
128 |     allowed_methods = ["GPy", "linreg", "AdaBoostRegressor",
129 |                        "DecisionTreeRegressor", "RandomForestRegressor",
130 |                        "ARDRegression", "GPy_fs", "mean", "random", "DNN", "lasso_ensemble", "doench", "logregL1", "sgrna_from_doench"]
131 |     assert learn_options["method"] in allowed_methods,"invalid method: %s" % learn_options["method"]
132 |     assert learn_options["method"] == "linreg" and learn_options['penalty'] == 'L2' or learn_options["weighted"] is None, "weighted only works with linreg L2 right now"
133 | 
134 |     # construct filename from options
135 |     filename = construct_filename(learn_options, TEST)
136 | 
137 |     print "Cross-validating genes..."
138 |     t2 = time.time()
139 | 
140 |     y = np.array(y_all[learn_options["target_name"]].values[:,None],dtype=np.float64)
141 | 
142 |     # concatenate feature sets in to one nparray, and get dimension of each
143 |     inputs, dim, dimsum, feature_names = util.concatenate_feature_sets(feature_sets)
144 | 
145 |     if not CV:
146 |         assert learn_options['cv'] == 'gene', 'Can only use gene-CV when CV is False (I need to use all of the genes and stratified complicates that)'
147 | 
148 |     # set-up for cross-validation
149 |     ## for outer loop, the one Doench et al use genes for
150 |     if learn_options["cv"] == "stratified":
151 |         assert not learn_options['extra pairs'], "can't use extra pairs with stratified CV, need to figure out how to properly account for genes affected by two drugs"
152 |         label_encoder = sklearn.preprocessing.LabelEncoder()
153 |         label_encoder.fit(y_all['Target gene'].values)
154 |         gene_classes = label_encoder.transform(y_all['Target gene'].values)
155 |         if learn_options['train_genes'] is not None and learn_options["test_genes"] is not None:
156 |             n_folds = len(learn_options["test_genes"])
157 |         else:
158 |             n_folds = len(learn_options['all_genes'])
159 |         cv = sklearn.cross_validation.StratifiedKFold(gene_classes, n_folds=n_folds, shuffle=True, indices=True)
160 |         fold_labels = ["fold%d" % i for i in range(1,n_folds+1)]
161 |         if learn_options['num_genes_remove_train'] is not None:
162 |             raise NotImplementedException()
163 |     elif learn_options["cv"]=="gene":
164 |         cv = []
165 | 
166 |         if not CV:
167 |             train_test_tmp = get_train_test('dummy', y_all) # get train, test split using a dummy gene
168 |             train_tmp, test_tmp = train_test_tmp
169 |             # not a typo, using training set to test on as well, just for this case. Test set is not used
170 |             # for internal cross-val, etc. anyway.
171 |             train_test_tmp = (train_tmp, train_tmp)
172 |             cv.append(train_test_tmp)
173 |             fold_labels = learn_options['all_genes']
174 | 
175 |         elif learn_options['train_genes'] is not None and learn_options["test_genes"] is not None:
176 |             assert learn_options['train_genes'] is not None and learn_options['test_genes'] is not None, "use both or neither"
177 |             for i, gene in enumerate(learn_options['test_genes']):
178 |                 cv.append(get_train_test(gene, y_all, learn_options['train_genes']))
179 |             fold_labels = learn_options["test_genes"]
180 |             # if train and test genes are seperate, there should be only one fold
181 |             train_test_disjoint = set.isdisjoint(set(learn_options["train_genes"].tolist()), set(learn_options["test_genes"].tolist()))
182 | 
183 |         else:
184 |             for i, gene in enumerate(learn_options['all_genes']):
185 |                 train_test_tmp = get_train_test(gene, y_all)
186 |                 cv.append(train_test_tmp)
187 |             fold_labels = learn_options['all_genes']
188 | 
189 | 
190 |         if learn_options['num_genes_remove_train'] is not None:
191 |             for i, (train,test) in enumerate(cv):
192 |                 unique_genes = np.random.permutation(np.unique(np.unique(y_all['Target gene'][train])))
193 |                 genes_to_keep = unique_genes[0:len(unique_genes) - learn_options['num_genes_remove_train']]
194 |                 guides_to_keep = []
195 |                 filtered_train = []
196 |                 for j, gene in enumerate(y_all['Target gene']):
197 |                     if j in train and gene in genes_to_keep:
198 |                         filtered_train.append(j)
199 |                 cv_i_orig = copy.deepcopy(cv[i])
200 |                 cv[i] = (filtered_train, test)
201 |                 if learn_options['num_genes_remove_train']==0:
202 |                     assert np.all(cv_i_orig[0]==cv[i][0])
203 |                     assert np.all(cv_i_orig[1]==cv[i][1])
204 |                 print "# train/train after/before is %s, %s" % (len(cv[i][0]), len(cv_i_orig[0]))
205 |                 print "# test/test after/before is %s, %s" % (len(cv[i][1]), len(cv_i_orig[1]))
206 |     else:
207 |         raise Exception("invalid cv options given: %s" % learn_options["cv"])
208 | 
209 |     cv = [c for c in cv] #make list from generator, so can subset for TEST case
210 |     if TEST:
211 |         ind_to_use = [0]#[0,1]
212 |         cv = [cv[i] for i in ind_to_use]
213 |         fold_labels = [fold_labels[i] for i in ind_to_use]
214 | 
215 |     truth =  dict([(t, dict([(m, np.array([])) for m in ['raw', 'ranks', 'thrs']])) for t in fold_labels])
216 |     predictions =  dict([(t, np.array([])) for t in fold_labels])
217 | 
218 |     m = {}
219 |     metrics = []
220 | 
221 |     #do the cross-validation
222 |     num_proc = learn_options["num_proc"]
223 |     if num_proc>1:
224 |         num_proc = np.min([num_proc,len(cv)])
225 |         print "using multiprocessing with %d procs--one for each fold" % num_proc
226 |         jobs = []
227 |         pool = multiprocessing.Pool(processes=num_proc)
228 |         for i,fold in enumerate(cv):
229 |             train,test = fold
230 |             print "working on fold %d of %d, with %d train and %d test" % (i, len(cv), len(train), len(test))
231 |             if learn_options["method"]=="GPy":
232 |                 job = pool.apply_async(models.GP.gp_on_fold, args=(feature_sets, train, test, y, y_all, inputs, dim, dimsum, learn_options))
233 |             elif learn_options["method"]=="linreg":
234 |                 job = pool.apply_async(models.regression.linreg_on_fold, args=(feature_sets, train, test, y, y_all, inputs, dim, dimsum, learn_options))
235 |             elif learn_options["method"]=="logregL1":
236 |                 job = pool.apply_async(models.regression.logreg_on_fold, args=(feature_sets, train, test, y, y_all, inputs, dim, dimsum, learn_options))
237 |             elif learn_options["method"]=="AdaBoostRegressor":
238 |                  job = pool.apply_async(models.ensembles.adaboost_on_fold, args=(feature_sets, train, test, y, y_all, inputs, dim, dimsum, learn_options))
239 |             elif learn_options["method"]=="DecisionTreeRegressor":
240 |                 job = pool.apply_async(models.ensembles.decisiontree_on_fold, args=(feature_sets, train, test, y, y_all, inputs, dim, dimsum, learn_options))
241 |             elif learn_options["method"]=="RandomForestRegressor":
242 |                 job = pool.apply_async(models.ensembles.randomforest_on_fold, args=(feature_sets, train, test, y, y_all, inputs, dim, dimsum, learn_options))
243 |             elif learn_options["method"]=="ARDRegression":
244 |                 job = pool.apply_async(models.regression.ARDRegression_on_fold, args=(feature_sets, train, test, y, y_all, inputs, dim, dimsum, learn_options))
245 |             elif learn_options["method"] == "random":
246 |                 job = pool.apply_async(models.baselines.random_on_fold, args=(feature_sets, train, test, y, y_all, inputs, dim, dimsum, learn_options))
247 |             elif learn_options["method"] == "mean":
248 |                 job = pool.apply_async(models.baselines.mean_on_fold, args=(feature_sets, train, test, y, y_all, inputs, dim, dimsum, learn_options))
249 |             elif learn_options["method"] == "DNN":
250 |                 job = pool.apply_async(models.DNN.DNN_on_fold, args=(feature_sets, train, test, y, y_all, inputs, dim, dimsum, learn_options))
251 |             elif learn_options["method"] == "lasso_ensemble":
252 |                 job = pool.apply_async(models.ensembles.LASSOs_ensemble_on_fold, args=(feature_sets, train, test, y, y_all, inputs, dim, dimsum, learn_options))
253 |             elif learn_options["method"] == "doench":
254 |                     job = pool.apply_async(models.baselines.doench_on_fold, args=(feature_sets, train, test, y, y_all, inputs, dim, dimsum, learn_options))
255 |             elif learn_options["method"] == "sgrna_from_doench":
256 |                     job = pool.apply_async(models.baselines.sgrna_from_doench_on_fold, args=(feature_sets, train, test, y, y_all, inputs, dim, dimsum, learn_options))
257 | 
258 |             else:
259 |                 raise Exception("did not find method=%s" % learn_options["method"])
260 |             jobs.append(job)
261 |         pool.close()
262 |         pool.join()
263 |         for i,fold in enumerate(cv):#i in range(0,len(jobs)):
264 |             y_pred, m[i] = jobs[i].get()
265 |             train,test = fold
266 | 
267 |             if learn_options["training_metric"]=="AUC":
268 |                 extract_fpr_tpr_for_fold(metrics, fold_labels[i], i, predictions, truth, y_all[learn_options["ground_truth_label"]].values, test, y_pred)
269 |             elif learn_options["training_metric"]=="NDCG":
270 |                 extract_NDCG_for_fold(metrics, fold_labels[i], i, predictions, truth, y_all[learn_options["ground_truth_label"]].values, test, y_pred, learn_options)
271 |             elif learn_options["training_metric"] == 'spearmanr':
272 |                  extract_spearman_for_fold(metrics, fold_labels[i], i, predictions, truth, y_all[learn_options["ground_truth_label"]].values, test, y_pred, learn_options)
273 |             else:
274 |                 raise Exception("invalid 'training_metric' in learn_options: %s" % learn_options["training_metric"])
275 | 
276 |             truth, predictions = fill_in_truth_and_predictions(truth, predictions, fold_labels[i], y_all, y_pred, learn_options, test)
277 | 
278 |         pool.terminate()
279 | 
280 |     else:
281 |         # non parallel version
282 |         for i,fold in enumerate(cv):
283 |             train,test = fold
284 |             if learn_options["method"]=="GPy":
285 |                 y_pred, m[i] = gp_on_fold(models.GP.feature_sets, train, test, y, y_all, inputs, dim, dimsum, learn_options)
286 |             elif learn_options["method"]=="linreg":
287 |                 y_pred, m[i] = models.regression.linreg_on_fold(feature_sets, train, test, y, y_all, inputs, dim, dimsum, learn_options)
288 |             elif learn_options["method"]=="logregL1":
289 |                 y_pred, m[i] = models.regression.logreg_on_fold(feature_sets, train, test, y, y_all, inputs, dim, dimsum, learn_options)
290 |             elif learn_options["method"]=="AdaBoostRegressor":
291 |                  y_pred, m[i] = models.ensembles.adaboost_on_fold(feature_sets, train, test, y, y_all, inputs, dim, dimsum, learn_options)
292 |             elif learn_options["method"]=="DecisionTreeRegressor":
293 |                  y_pred, m[i] = models.ensembles.decisiontree_on_fold(feature_sets, train, test, y, y_all, inputs, dim, dimsum, learn_options)
294 |             elif learn_options["method"]=="RandomForestRegressor":
295 |                  y_pred, m[i] = models.ensembles.randomforest_on_fold(feature_sets, train, test, y, y_all, inputs, dim, dimsum, learn_options)
296 |             elif learn_options["method"]=="ARDRegression":
297 |                  y_pred, m[i] = models.regression.ARDRegression_on_fold(feature_sets, train, test, y, y_all, inputs, dim, dimsum, learn_options)
298 |             elif learn_options["method"]=="GPy_fs":
299 |                  y_pred, m[i] = models.GP.gp_with_fs_on_fold(feature_sets, train, test, y, y_all, inputs, dim, dimsum, learn_options)
300 |             elif learn_options["method"] == "random":
301 |                  y_pred, m[i] = models.baselines.random_on_fold(feature_sets, train, test, y, y_all, inputs, dim, dimsum, learn_options)
302 |             elif learn_options["method"] == "mean":
303 |                  y_pred, m[i] = models.baselines.mean_on_fold(feature_sets, train, test, y, y_all, inputs, dim, dimsum, learn_options)
304 |             elif learn_options["method"] == "DNN":
305 |                  y_pred, m[i] = models.DNN.DNN_on_fold(feature_sets, train, test, y, y_all, inputs, dim, dimsum, learn_options)
306 |             elif learn_options["method"] == "lasso_ensemble":
307 |                  y_pred, m[i] = models.ensembles.LASSOs_ensemble_on_fold(feature_sets, train, test, y, y_all, inputs, dim, dimsum, learn_options)
308 |             elif learn_options["method"] == "doench":
309 |                  y_pred, m[i] = models.baselines.doench_on_fold(feature_sets, train, test, y, y_all, inputs, dim, dimsum, learn_options)
310 |             elif learn_options["method"] == "sgrna_from_doench":
311 |                  y_pred, m[i] = models.baselines.sgrna_from_doench_on_fold(feature_sets, train, test, y, y_all, inputs, dim, dimsum, learn_options)
312 |             else:
313 |                 raise Exception("invalid method found: %s" % learn_options["method"])
314 | 
315 |             if learn_options["training_metric"]=="AUC":
316 |                 # fills in truth and predictions
317 |                 extract_fpr_tpr_for_fold(metrics, fold_labels[i], i, predictions, truth, y_all[learn_options['ground_truth_label']].values, test, y_pred)
318 |             elif learn_options["training_metric"]=="NDCG":
319 |                 extract_NDCG_for_fold(metrics, fold_labels[i], i, predictions, truth, y_all[learn_options["ground_truth_label"]].values, test, y_pred, learn_options)
320 |             elif learn_options["training_metric"] == 'spearmanr':
321 |                 extract_spearman_for_fold(metrics, fold_labels[i], i, predictions, truth, y_all[learn_options["ground_truth_label"]].values, test, y_pred, learn_options)
322 | 
323 |             truth, predictions = fill_in_truth_and_predictions(truth, predictions, fold_labels[i], y_all, y_pred, learn_options, test)
324 | 
325 |             print "\t\tRMSE: ", np.sqrt(((y_pred - y[test])**2).mean())
326 |             print "\t\tSpearman correlation: ", util.spearmanr_nonan(y[test], y_pred)[0]
327 |             print "\t\tfinished fold/gene %i of %i" % (i, len(fold_labels))
328 | 
329 | 
330 |     cv_median_metric =[np.median(metrics)]
331 |     gene_pred = [(truth, predictions)]
332 |     print "\t\tmedian %s across gene folds: %.3f" % (learn_options["training_metric"], cv_median_metric[-1])
333 | 
334 |     t3 = time.time()
335 |     print "\t\tElapsed time for cv is %.2f seconds" % (t3-t2)
336 |     return metrics, gene_pred, fold_labels, m, dimsum, filename, feature_names
337 | 
338 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/predict.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sunjiamin0824/CRISPR-Local/c6e80b2fd8ee4065c0655edf9f5206bf8cc316e2/Rule_Set_2_scoring_v1/analysis/predict.pyc


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/rs2_score_calculator.py:
--------------------------------------------------------------------------------
 1 | #Calculates the Rule set 2 score for the given 30-mer
 2 | #Input: 1. 30mer sgRNA+context sequence, NNNN[sgRNA sequence]NGGNNN
 3 | #       2. Amino acid cut position, for full model prediction only
 4 | #       3. Percent peptide, for full model prediction only
 5 | #Output: Rule set 2 score
 6 | 
 7 | import pandas as pd
 8 | import csv, argparse, sys
 9 | import pickle
10 | import model_comparison
11 | import os
12 | def get_parser():
13 |     parser = argparse.ArgumentParser()
14 |     parser.add_argument('--input', type=str, help='input 30-mer sgRNA fasta file')
15 |     parser.add_argument('--output',	type=str, help='output 20-mer sgRNA-scored fasta file')
16 |     parser.add_argument('--aa-cut', type=int, default=-1, help='Amino acid cut position of sgRNA')
17 |     parser.add_argument('--per-peptide', type=float, default=-1, help='Percentage of protein cut by sgRNA')
18 |     return parser
19 | if __name__ == '__main__':
20 |     args = get_parser().parse_args()
21 |     input = args.input
22 |     output = args.output
23 |     aa_cut = args.aa_cut
24 |     per_peptide = args.per_peptide
25 |     model_file_1 = 'Rule_Set_2_scoring_v1/saved_models/V3_model_nopos.pickle'
26 |     model_file_2 = 'Rule_Set_2_scoring_v1/saved_models/V3_model_full.pickle'
27 |     if (aa_cut == -1) or (per_peptide == -1):
28 |         model_file = model_file_1
29 |     else:
30 |         model_file = model_file_2
31 |     try:
32 |         with open(model_file, 'rb') as f:
33 |             model= pickle.load(f)    
34 |     except:
35 |         raise Exception("could not find model stored to file %s" % model_file)
36 |     i = open(input,'r')
37 |     o = open(output,'w')
38 |     seq={}
39 |     for line in i:
40 |         if line.startswith('>'):
41 |                 name = line.replace('\n','')
42 |         else:
43 |                 seq[name]=line.replace('\n','')
44 |                 score = model_comparison.predict(seq[name], aa_cut, per_peptide, model=model)
45 |                 o.write(name + '#%.6f'% (score) + '\n' + seq[name][4:24] + '\n')
46 |     i.close()
47 |     o.close()
48 |     os.remove(input)
49 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/analysis/util.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sunjiamin0824/CRISPR-Local/c6e80b2fd8ee4065c0655edf9f5206bf8cc316e2/Rule_Set_2_scoring_v1/analysis/util.pyc


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/saved_models/PaxHeader/V3_model_full.pickle:
--------------------------------------------------------------------------------
1 | 18 gid=1594166068
2 | 18 uid=1248751364
3 | 20 ctime=1447951994
4 | 20 atime=1447951994
5 | 23 SCHILY.dev=16777220
6 | 23 SCHILY.ino=24334129
7 | 18 SCHILY.nlink=1
8 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/saved_models/PaxHeader/V3_model_nopos.pickle:
--------------------------------------------------------------------------------
1 | 18 gid=1594166068
2 | 18 uid=1248751364
3 | 20 ctime=1447951994
4 | 20 atime=1447951994
5 | 23 SCHILY.dev=16777220
6 | 23 SCHILY.ino=24334130
7 | 18 SCHILY.nlink=1
8 | 


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/saved_models/V3_model_full.pickle:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sunjiamin0824/CRISPR-Local/c6e80b2fd8ee4065c0655edf9f5206bf8cc316e2/Rule_Set_2_scoring_v1/saved_models/V3_model_full.pickle


--------------------------------------------------------------------------------
/Rule_Set_2_scoring_v1/saved_models/V3_model_nopos.pickle:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sunjiamin0824/CRISPR-Local/c6e80b2fd8ee4065c0655edf9f5206bf8cc316e2/Rule_Set_2_scoring_v1/saved_models/V3_model_nopos.pickle


--------------------------------------------------------------------------------
/cfd-score-calculator.py:
--------------------------------------------------------------------------------
 1 | #Calculates the Cutting Frequency Determination score
 2 | #Requirements: 1. Pickle file with mismatch scores in working directory
 3 | #              2. Pickle file containing PAM scores in working directory 
 4 | #Input: 1. 23mer WT sgRNA sequence
 5 | #       2. 23mer Off-target sgRNA sequence
 6 | #Output: CFD score
 7 | import pickle
 8 | import argparse
 9 | import re
10 | import numpy as np
11 | import os
12 | 
13 | def get_parser():
14 |     parser = argparse.ArgumentParser(description='Calculates CFD score')
15 |     parser.add_argument('--input', type=str, help='WT and Off-target 23mer sgRNA sequence file.')
16 |     parser.add_argument('--output', type=str, help='CFD score result file.')
17 |     return parser
18 | 
19 | #Reverse complements a given string
20 | def revcom(s):
21 |     basecomp = {'A': 'T', 'C': 'G', 'G': 'C', 'T': 'A','U':'A'}
22 |     letters = list(s[::-1])
23 |     letters = [basecomp[base] for base in letters]
24 |     return ''.join(letters)
25 | 
26 | #Unpickle mismatch scores and PAM scores
27 | def get_mm_pam_scores():
28 |     try:
29 |         mm_scores = pickle.load(open('mismatch_score.pkl','rb'))
30 |         pam_scores = pickle.load(open('pam_scores.pkl','rb'))
31 |         return (mm_scores,pam_scores)
32 |     except: 
33 |         raise Exception("Could not find file with mismatch scores or PAM scores")
34 | 
35 | #Calculates CFD score
36 | def calc_cfd(wt,sg,pam):
37 |     mm_scores,pam_scores = get_mm_pam_scores()
38 |     score = 1
39 |     sg = sg.replace('T','U')
40 |     wt = wt.replace('T','U')
41 |     s_list = list(sg)
42 |     wt_list = list(wt)
43 |     for i,sl in enumerate(s_list):
44 |         if wt_list[i] == sl:
45 |             score*=1
46 |         else:
47 |             key = 'r'+wt_list[i]+':d'+revcom(sl)+','+str(i+1)
48 |             score*= mm_scores[key]
49 |     score*=pam_scores[pam]
50 |     return (score)
51 | 
52 | if __name__ == '__main__':
53 | 	args = get_parser().parse_args()
54 | 	mm_scores,pam_scores = get_mm_pam_scores()
55 | 	input = args.input
56 | 	output = args.output
57 | 	i = open(input,'r')
58 | 	o = open(output,'w')
59 | 	for line in i:
60 | 		line = line.replace('\n','')
61 | 		line_list=line.split('\t')
62 | 		if line_list[0]==line_list[4] and line_list[7] == 0:continue
63 | 		off = line_list[6]
64 | 		wt = line_list[2]
65 | 		pam = off[-2:]
66 | 		sg = off[:-3]
67 | 		cfd_score = calc_cfd(wt,sg,pam)
68 | 		CFD = '%.4f'%(cfd_score)
69 | 		o.write(line + '\t' + CFD + '\n')
70 | 	i.close()
71 | 	o.close()
72 | 	os.remove(input)
73 | 
74 | 
75 | 
76 | 


--------------------------------------------------------------------------------
/mismatch_score.pkl:
--------------------------------------------------------------------------------
  1 | (dp0
  2 | S'rU:dT,12'
  3 | p1
  4 | F0.8
  5 | sS'rU:dT,13'
  6 | p2
  7 | F0.692307692
  8 | sS'rU:dC,5'
  9 | p3
 10 | F0.64
 11 | sS'rG:dA,14'
 12 | p4
 13 | F0.26666666699999997
 14 | sS'rG:dG,19'
 15 | p5
 16 | F0.448275862
 17 | sS'rG:dG,18'
 18 | p6
 19 | F0.47619047600000003
 20 | sS'rG:dG,15'
 21 | p7
 22 | F0.272727273
 23 | sS'rG:dG,14'
 24 | p8
 25 | F0.428571429
 26 | sS'rG:dG,17'
 27 | p9
 28 | F0.235294118
 29 | sS'rG:dG,16'
 30 | p10
 31 | F0.0
 32 | sS'rC:dC,20'
 33 | p11
 34 | F0.058823529000000006
 35 | sS'rG:dT,20'
 36 | p12
 37 | F0.9375
 38 | sS'rG:dG,13'
 39 | p13
 40 | F0.42105263200000004
 41 | sS'rG:dG,12'
 42 | p14
 43 | F0.529411765
 44 | sS'rU:dC,6'
 45 | p15
 46 | F0.571428571
 47 | sS'rU:dG,14'
 48 | p16
 49 | F0.28571428600000004
 50 | sS'rU:dT,18'
 51 | p17
 52 | F0.666666667
 53 | sS'rA:dG,13'
 54 | p18
 55 | F0.21052631600000002
 56 | sS'rA:dG,12'
 57 | p19
 58 | F0.263157895
 59 | sS'rA:dG,11'
 60 | p20
 61 | F0.4
 62 | sS'rA:dG,10'
 63 | p21
 64 | F0.333333333
 65 | sS'rA:dA,19'
 66 | p22
 67 | F0.538461538
 68 | sS'rA:dA,18'
 69 | p23
 70 | F0.5
 71 | sS'rA:dG,15'
 72 | p24
 73 | F0.272727273
 74 | sS'rA:dG,14'
 75 | p25
 76 | F0.214285714
 77 | sS'rA:dA,15'
 78 | p26
 79 | F0.2
 80 | sS'rA:dA,14'
 81 | p27
 82 | F0.533333333
 83 | sS'rA:dA,17'
 84 | p28
 85 | F0.133333333
 86 | sS'rA:dA,16'
 87 | p29
 88 | F0.0
 89 | sS'rA:dA,11'
 90 | p30
 91 | F0.307692308
 92 | sS'rA:dA,10'
 93 | p31
 94 | F0.882352941
 95 | sS'rA:dA,13'
 96 | p32
 97 | F0.3
 98 | sS'rA:dA,12'
 99 | p33
100 | F0.333333333
101 | sS'rG:dA,13'
102 | p34
103 | F0.3
104 | sS'rG:dA,12'
105 | p35
106 | F0.384615385
107 | sS'rG:dA,11'
108 | p36
109 | F0.384615385
110 | sS'rG:dA,10'
111 | p37
112 | F0.8125
113 | sS'rG:dA,17'
114 | p38
115 | F0.25
116 | sS'rG:dA,16'
117 | p39
118 | F0.0
119 | sS'rG:dA,15'
120 | p40
121 | F0.14285714300000002
122 | sS'rG:dA,6'
123 | p41
124 | F0.666666667
125 | sS'rG:dG,20'
126 | p42
127 | F0.428571429
128 | sS'rG:dA,19'
129 | p43
130 | F0.666666667
131 | sS'rG:dA,18'
132 | p44
133 | F0.666666667
134 | sS'rU:dC,4'
135 | p45
136 | F0.625
137 | sS'rG:dT,12'
138 | p46
139 | F0.933333333
140 | sS'rG:dT,13'
141 | p47
142 | F0.923076923
143 | sS'rU:dG,11'
144 | p48
145 | F0.666666667
146 | sS'rC:dA,3'
147 | p49
148 | F0.6875
149 | sS'rC:dA,2'
150 | p50
151 | F0.9090909090000001
152 | sS'rC:dA,1'
153 | p51
154 | F1.0
155 | sS'rC:dA,7'
156 | p52
157 | F0.8125
158 | sS'rC:dA,6'
159 | p53
160 | F0.9285714290000001
161 | sS'rC:dA,5'
162 | p54
163 | F0.636363636
164 | sS'rC:dA,4'
165 | p55
166 | F0.8
167 | sS'rC:dA,9'
168 | p56
169 | F0.875
170 | sS'rC:dA,8'
171 | p57
172 | F0.875
173 | sS'rU:dT,6'
174 | p58
175 | F0.8666666670000001
176 | sS'rA:dG,20'
177 | p59
178 | F0.22727272699999998
179 | sS'rG:dT,18'
180 | p60
181 | F0.692307692
182 | sS'rU:dG,10'
183 | p61
184 | F0.533333333
185 | sS'rG:dT,19'
186 | p62
187 | F0.7142857140000001
188 | sS'rG:dA,20'
189 | p63
190 | F0.7
191 | sS'rC:dT,20'
192 | p64
193 | F0.5
194 | sS'rU:dC,2'
195 | p65
196 | F0.84
197 | sS'rG:dG,10'
198 | p66
199 | F0.4
200 | sS'rC:dA,17'
201 | p67
202 | F0.46666666700000003
203 | sS'rC:dA,16'
204 | p68
205 | F0.307692308
206 | sS'rC:dA,15'
207 | p69
208 | F0.066666667
209 | sS'rC:dA,14'
210 | p70
211 | F0.7333333329999999
212 | sS'rC:dA,13'
213 | p71
214 | F0.7
215 | sS'rC:dA,12'
216 | p72
217 | F0.538461538
218 | sS'rC:dA,11'
219 | p73
220 | F0.307692308
221 | sS'rC:dA,10'
222 | p74
223 | F0.9411764709999999
224 | sS'rG:dG,11'
225 | p75
226 | F0.428571429
227 | sS'rU:dC,20'
228 | p76
229 | F0.176470588
230 | sS'rG:dG,3'
231 | p77
232 | F0.384615385
233 | sS'rC:dA,19'
234 | p78
235 | F0.46153846200000004
236 | sS'rC:dA,18'
237 | p79
238 | F0.642857143
239 | sS'rU:dG,17'
240 | p80
241 | F0.705882353
242 | sS'rU:dG,16'
243 | p81
244 | F0.666666667
245 | sS'rU:dG,15'
246 | p82
247 | F0.272727273
248 | sS'rG:dG,2'
249 | p83
250 | F0.692307692
251 | sS'rU:dG,13'
252 | p84
253 | F0.7894736840000001
254 | sS'rU:dG,12'
255 | p85
256 | F0.947368421
257 | sS'rG:dA,9'
258 | p86
259 | F0.533333333
260 | sS'rG:dA,8'
261 | p87
262 | F0.625
263 | sS'rG:dA,7'
264 | p88
265 | F0.571428571
266 | sS'rG:dG,5'
267 | p89
268 | F0.7857142859999999
269 | sS'rG:dA,5'
270 | p90
271 | F0.3
272 | sS'rG:dA,4'
273 | p91
274 | F0.363636364
275 | sS'rG:dA,3'
276 | p92
277 | F0.5
278 | sS'rG:dA,2'
279 | p93
280 | F0.636363636
281 | sS'rG:dA,1'
282 | p94
283 | F1.0
284 | sS'rG:dG,4'
285 | p95
286 | F0.529411765
287 | sS'rG:dG,1'
288 | p96
289 | F0.7142857140000001
290 | sS'rA:dC,9'
291 | p97
292 | F0.666666667
293 | sS'rG:dG,7'
294 | p98
295 | F0.6875
296 | sS'rG:dT,5'
297 | p99
298 | F0.8666666670000001
299 | sS'rU:dT,20'
300 | p100
301 | F0.5625
302 | sS'rC:dC,15'
303 | p101
304 | F0.05
305 | sS'rC:dC,14'
306 | p102
307 | F0.0
308 | sS'rC:dC,17'
309 | p103
310 | F0.058823529000000006
311 | sS'rC:dC,16'
312 | p104
313 | F0.153846154
314 | sS'rC:dC,11'
315 | p105
316 | F0.25
317 | sS'rC:dC,10'
318 | p106
319 | F0.38888888899999996
320 | sS'rC:dC,13'
321 | p107
322 | F0.13636363599999998
323 | sS'rC:dC,12'
324 | p108
325 | F0.444444444
326 | sS'rC:dA,20'
327 | p109
328 | F0.3
329 | sS'rC:dC,19'
330 | p110
331 | F0.125
332 | sS'rC:dC,18'
333 | p111
334 | F0.133333333
335 | sS'rA:dA,1'
336 | p112
337 | F1.0
338 | sS'rA:dA,3'
339 | p113
340 | F0.705882353
341 | sS'rA:dA,2'
342 | p114
343 | F0.727272727
344 | sS'rA:dA,5'
345 | p115
346 | F0.363636364
347 | sS'rA:dA,4'
348 | p116
349 | F0.636363636
350 | sS'rA:dA,7'
351 | p117
352 | F0.4375
353 | sS'rA:dA,6'
354 | p118
355 | F0.7142857140000001
356 | sS'rA:dA,9'
357 | p119
358 | F0.6
359 | sS'rA:dA,8'
360 | p120
361 | F0.428571429
362 | sS'rU:dG,20'
363 | p121
364 | F0.090909091
365 | sS'rC:dC,9'
366 | p122
367 | F0.6190476189999999
368 | sS'rC:dC,8'
369 | p123
370 | F0.642857143
371 | sS'rU:dT,10'
372 | p124
373 | F0.857142857
374 | sS'rU:dT,11'
375 | p125
376 | F0.75
377 | sS'rU:dT,16'
378 | p126
379 | F0.9090909090000001
380 | sS'rU:dT,17'
381 | p127
382 | F0.533333333
383 | sS'rU:dT,14'
384 | p128
385 | F0.6190476189999999
386 | sS'rU:dT,15'
387 | p129
388 | F0.578947368
389 | sS'rC:dC,1'
390 | p130
391 | F0.913043478
392 | sS'rU:dT,3'
393 | p131
394 | F0.7142857140000001
395 | sS'rC:dC,3'
396 | p132
397 | F0.5
398 | sS'rC:dC,2'
399 | p133
400 | F0.695652174
401 | sS'rC:dC,5'
402 | p134
403 | F0.6
404 | sS'rC:dC,4'
405 | p135
406 | F0.5
407 | sS'rC:dC,7'
408 | p136
409 | F0.470588235
410 | sS'rC:dC,6'
411 | p137
412 | F0.5
413 | sS'rU:dT,4'
414 | p138
415 | F0.47619047600000003
416 | sS'rU:dT,8'
417 | p139
418 | F0.8
419 | sS'rU:dT,9'
420 | p140
421 | F0.9285714290000001
422 | sS'rA:dC,19'
423 | p141
424 | F0.375
425 | sS'rA:dC,18'
426 | p142
427 | F0.4
428 | sS'rA:dC,17'
429 | p143
430 | F0.176470588
431 | sS'rA:dC,16'
432 | p144
433 | F0.192307692
434 | sS'rA:dC,15'
435 | p145
436 | F0.65
437 | sS'rA:dC,14'
438 | p146
439 | F0.46666666700000003
440 | sS'rA:dC,13'
441 | p147
442 | F0.6521739129999999
443 | sS'rA:dC,12'
444 | p148
445 | F0.7222222220000001
446 | sS'rA:dC,11'
447 | p149
448 | F0.65
449 | sS'rA:dC,10'
450 | p150
451 | F0.5555555560000001
452 | sS'rU:dC,7'
453 | p151
454 | F0.588235294
455 | sS'rC:dT,8'
456 | p152
457 | F0.65
458 | sS'rC:dT,9'
459 | p153
460 | F0.857142857
461 | sS'rC:dT,6'
462 | p154
463 | F0.9285714290000001
464 | sS'rC:dT,7'
465 | p155
466 | F0.75
467 | sS'rC:dT,4'
468 | p156
469 | F0.842105263
470 | sS'rC:dT,5'
471 | p157
472 | F0.571428571
473 | sS'rC:dT,2'
474 | p158
475 | F0.727272727
476 | sS'rC:dT,3'
477 | p159
478 | F0.8666666670000001
479 | sS'rC:dT,1'
480 | p160
481 | F1.0
482 | sS'rA:dC,8'
483 | p161
484 | F0.7333333329999999
485 | sS'rU:dT,1'
486 | p162
487 | F1.0
488 | sS'rU:dC,3'
489 | p163
490 | F0.5
491 | sS'rU:dC,1'
492 | p164
493 | F0.956521739
494 | sS'rU:dT,2'
495 | p165
496 | F0.846153846
497 | sS'rU:dG,19'
498 | p166
499 | F0.275862069
500 | sS'rG:dT,14'
501 | p167
502 | F0.75
503 | sS'rG:dT,15'
504 | p168
505 | F0.9411764709999999
506 | sS'rG:dT,16'
507 | p169
508 | F1.0
509 | sS'rG:dT,17'
510 | p170
511 | F0.933333333
512 | sS'rG:dT,10'
513 | p171
514 | F0.933333333
515 | sS'rG:dT,11'
516 | p172
517 | F1.0
518 | sS'rA:dG,9'
519 | p173
520 | F0.571428571
521 | sS'rA:dG,8'
522 | p174
523 | F0.428571429
524 | sS'rA:dG,7'
525 | p175
526 | F0.4375
527 | sS'rA:dG,6'
528 | p176
529 | F0.454545455
530 | sS'rA:dG,5'
531 | p177
532 | F0.5
533 | sS'rA:dG,4'
534 | p178
535 | F0.352941176
536 | sS'rA:dG,3'
537 | p179
538 | F0.428571429
539 | sS'rA:dG,2'
540 | p180
541 | F0.7857142859999999
542 | sS'rA:dG,1'
543 | p181
544 | F0.857142857
545 | sS'rU:dT,5'
546 | p182
547 | F0.5
548 | sS'rG:dT,2'
549 | p183
550 | F0.846153846
551 | sS'rA:dC,3'
552 | p184
553 | F0.611111111
554 | sS'rA:dC,20'
555 | p185
556 | F0.764705882
557 | sS'rG:dT,1'
558 | p186
559 | F0.9
560 | sS'rG:dT,6'
561 | p187
562 | F1.0
563 | sS'rG:dT,7'
564 | p188
565 | F1.0
566 | sS'rG:dT,4'
567 | p189
568 | F0.9
569 | sS'rC:dT,19'
570 | p190
571 | F0.428571429
572 | sS'rG:dG,9'
573 | p191
574 | F0.538461538
575 | sS'rG:dG,8'
576 | p192
577 | F0.615384615
578 | sS'rG:dT,8'
579 | p193
580 | F1.0
581 | sS'rG:dT,9'
582 | p194
583 | F0.642857143
584 | sS'rU:dG,18'
585 | p195
586 | F0.428571429
587 | sS'rU:dT,7'
588 | p196
589 | F0.875
590 | sS'rG:dG,6'
591 | p197
592 | F0.681818182
593 | sS'rA:dA,20'
594 | p198
595 | F0.6
596 | sS'rU:dC,9'
597 | p199
598 | F0.6190476189999999
599 | sS'rA:dG,17'
600 | p200
601 | F0.176470588
602 | sS'rU:dC,8'
603 | p201
604 | F0.7333333329999999
605 | sS'rA:dG,16'
606 | p202
607 | F0.0
608 | sS'rA:dG,19'
609 | p203
610 | F0.20689655199999998
611 | sS'rG:dT,3'
612 | p204
613 | F0.75
614 | sS'rU:dG,3'
615 | p205
616 | F0.428571429
617 | sS'rU:dG,2'
618 | p206
619 | F0.857142857
620 | sS'rU:dG,1'
621 | p207
622 | F0.857142857
623 | sS'rA:dG,18'
624 | p208
625 | F0.19047619
626 | sS'rU:dG,7'
627 | p209
628 | F0.6875
629 | sS'rU:dG,6'
630 | p210
631 | F0.9090909090000001
632 | sS'rU:dG,5'
633 | p211
634 | F1.0
635 | sS'rU:dG,4'
636 | p212
637 | F0.647058824
638 | sS'rU:dG,9'
639 | p213
640 | F0.923076923
641 | sS'rU:dG,8'
642 | p214
643 | F1.0
644 | sS'rU:dC,19'
645 | p215
646 | F0.25
647 | sS'rU:dC,18'
648 | p216
649 | F0.333333333
650 | sS'rU:dC,13'
651 | p217
652 | F0.260869565
653 | sS'rU:dC,12'
654 | p218
655 | F0.5
656 | sS'rU:dC,11'
657 | p219
658 | F0.4
659 | sS'rU:dC,10'
660 | p220
661 | F0.5
662 | sS'rU:dC,17'
663 | p221
664 | F0.117647059
665 | sS'rU:dC,16'
666 | p222
667 | F0.346153846
668 | sS'rU:dC,15'
669 | p223
670 | F0.05
671 | sS'rU:dC,14'
672 | p224
673 | F0.0
674 | sS'rC:dT,10'
675 | p225
676 | F0.8666666670000001
677 | sS'rC:dT,11'
678 | p226
679 | F0.75
680 | sS'rC:dT,12'
681 | p227
682 | F0.7142857140000001
683 | sS'rC:dT,13'
684 | p228
685 | F0.384615385
686 | sS'rC:dT,14'
687 | p229
688 | F0.35
689 | sS'rC:dT,15'
690 | p230
691 | F0.222222222
692 | sS'rC:dT,16'
693 | p231
694 | F1.0
695 | sS'rC:dT,17'
696 | p232
697 | F0.46666666700000003
698 | sS'rC:dT,18'
699 | p233
700 | F0.538461538
701 | sS'rA:dC,2'
702 | p234
703 | F0.8
704 | sS'rA:dC,1'
705 | p235
706 | F1.0
707 | sS'rA:dC,7'
708 | p236
709 | F0.705882353
710 | sS'rA:dC,6'
711 | p237
712 | F0.7142857140000001
713 | sS'rA:dC,5'
714 | p238
715 | F0.72
716 | sS'rA:dC,4'
717 | p239
718 | F0.625
719 | sS'rU:dT,19'
720 | p240
721 | F0.28571428600000004
722 | s.


--------------------------------------------------------------------------------
/pam_scores.pkl:
--------------------------------------------------------------------------------
 1 | (dp0
 2 | S'AA'
 3 | p1
 4 | F0.0
 5 | sS'AC'
 6 | p2
 7 | F0.0
 8 | sS'GT'
 9 | p3
10 | F0.016129031999999998
11 | sS'AG'
12 | p4
13 | F0.25925925899999996
14 | sS'CC'
15 | p5
16 | F0.0
17 | sS'CA'
18 | p6
19 | F0.0
20 | sS'CG'
21 | p7
22 | F0.107142857
23 | sS'TT'
24 | p8
25 | F0.0
26 | sS'GG'
27 | p9
28 | F1.0
29 | sS'GC'
30 | p10
31 | F0.022222222000000003
32 | sS'AT'
33 | p11
34 | F0.0
35 | sS'GA'
36 | p12
37 | F0.06944444400000001
38 | sS'TG'
39 | p13
40 | F0.038961038999999996
41 | sS'TA'
42 | p14
43 | F0.0
44 | sS'TC'
45 | p15
46 | F0.0
47 | sS'CT'
48 | p16
49 | F0.0
50 | s.


--------------------------------------------------------------------------------
/seqmap-1.0.12-linux-64:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sunjiamin0824/CRISPR-Local/c6e80b2fd8ee4065c0655edf9f5206bf8cc316e2/seqmap-1.0.12-linux-64


--------------------------------------------------------------------------------
/sgRNA_reformatting.pl:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/perl
 2 | use strict;
 3 | use warnings;
 4 | die  "Usage:perl $0 <Designing_mode> <GFF_index> <All_match_file> <Outfile>\n" unless (@ARGV == 4);
 5 | 
 6 | open (INDEX,"<$ARGV[1]") or die;
 7 | my %exon_list_gene;
 8 | while (<INDEX>) {
 9 | 	chomp;
10 | 	my($gene,$exon)=split /\t/,$_;
11 | 	$exon_list_gene{$exon}=$gene;
12 | }
13 | close INDEX;
14 | open (SEQMAP,"<$ARGV[2]") or die;
15 | readline SEQMAP;
16 | open (OUTPUT,">$ARGV[3]") or die;
17 | my $sgRNA_score;my $sgRNA_pam;my $sgRNA_pos;my $sgRNA_info;
18 | my($OT_gene,$OT_pos,$OT_pam);
19 | my %sgRNA_POS;my %sgRNA;
20 | while (<SEQMAP>) {
21 | 	chomp;
22 | 	my($OT_id,undef,$OT_seq,$sgRNA_id,$sgRNA_seq,$mismatch) = split("\t",$_); 
23 | 	my $Exon = $sgRNA_id;
24 | 	$Exon =~ s/\#\[.+//;
25 | 	if (exists $exon_list_gene{$Exon}) {
26 | 		($OT_gene,$OT_pos,$OT_pam) = split ("\#",$OT_id);
27 | 		($sgRNA_info,$sgRNA_pos,$sgRNA_pam,$sgRNA_score) = (split "\#",$sgRNA_id)[1,2,3,4];
28 | 		if ($ARGV[0] eq "cpf1") {
29 | 			$OT_seq = $OT_pam.$OT_seq;
30 | 			$sgRNA_seq = $sgRNA_pam.$sgRNA_seq;
31 | 		}else{
32 | 			$OT_seq = $OT_seq.$OT_pam;
33 | 			$sgRNA_seq = $sgRNA_seq.$sgRNA_pam;
34 | 		}
35 | 		$sgRNA_POS{"$exon_list_gene{$Exon}:$sgRNA_pos"}{$Exon}=$sgRNA_info;
36 | 		$sgRNA{"$exon_list_gene{$Exon}:$sgRNA_pos"}{$OT_pos}="$exon_list_gene{$Exon}\t$sgRNA_pos\t$sgRNA_seq\t$sgRNA_score\t$OT_gene\t$OT_pos\t$OT_seq\t$mismatch";
37 | 	}
38 | }
39 | close SEQMAP;
40 | my %Exon_list;my %Position_list;
41 | foreach my $key1 (sort keys %sgRNA_POS) {
42 | 	foreach	my $key2 (sort keys %{$sgRNA_POS{$key1}}){
43 | 		$Exon_list{$key1}.="$key2;";
44 | 		$Position_list{$key1}.="$sgRNA_POS{$key1}{$key2};";
45 | 	}
46 | 	foreach my $key3 (sort keys %{$sgRNA{$key1}}) {
47 | 		print OUTPUT "$sgRNA{$key1}{$key3}\t$Exon_list{$key1}\t$Position_list{$key1}\n";
48 | 	}
49 | }
50 | close OUTPUT;
51 | system("rm $ARGV[2]");
52 | 


--------------------------------------------------------------------------------