├── thirdparty
    ├── interleaved-split
    ├── interleave
    ├── Chunker
    ├── Trimmer.cpp
    ├── Splitter.cpp
    ├── SeqChunker-sed
    ├── SeqChunker-dd
    └── SeqChunker-perl
├── src
    ├── BlasrAdapter.h
    ├── parsingargs.h
    ├── HALC.h
    ├── parsingargs.cpp
    └── BlasrAdapter.cpp
├── Makefile
├── Readme.md
└── runHALC.py


/thirdparty/interleaved-split:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env perl
 2 | use warnings;
 3 | use strict;
 4 | 
 5 | use Pod::Usage;
 6 | 
 7 | =head1 SYNOPSIS
 8 | 
 9 |   cat interleaved.fq | interleaved_split 1>lib_1.fq 2>lib_2.fq;
10 | 
11 | =cut
12 | 
13 | pod2usage if @ARGV;
14 | 
15 | my ($i, $h, $s, $d, $q) = (0);
16 | 
17 | while($h=<>,$s=<>,$d=<>,$q=<>){
18 |     $i++; 
19 |     print {$i%2 ? *STDOUT : *STDERR} $h,$s,$d,$q
20 | }
21 | 
22 | =head1 AUTHOR
23 | 
24 | Thomas Hackl S<thomas.hackl@uni-wuerzburg.de>
25 | 
26 | =cut
27 | 


--------------------------------------------------------------------------------
/thirdparty/interleave:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env perl
 2 | use warnings;
 3 | use strict;
 4 | 
 5 | use Pod::Usage;
 6 | 
 7 | =head1 SYNOPSIS
 8 | 
 9 |   interleave <A_1.fq> <A_2.fq> [B_1.fq B_2.fq ...] > interleaved.fq";
10 | 
11 | =cut
12 | 
13 | my @fq = @ARGV;
14 | pod2usage unless @fq;
15 | die "uneven number of libraries!" if @fq%2;
16 | 
17 | while(@fq){
18 |     my $rf = shift(@fq);
19 |     my $mf = shift(@fq);
20 |     open(my $r, $rf) or die "$!: $rf";
21 |     open(my $m, $mf) or die "$!: $mf";
22 |     
23 |     while(defined(my $h = <$r>)){
24 | 	print $h, scalar <$r>, scalar <$r>, scalar <$r>;
25 | 	print scalar <$m>, scalar <$m>, scalar <$m>, scalar <$m>;
26 |     }
27 | 
28 |     close $rf;
29 |     close $mf;
30 | }
31 | 
32 | =head1 AUTHOR
33 | 
34 | Thomas Hackl S<thomas.hackl@uni-wuerzburg.de>
35 | 
36 | =cut
37 | 


--------------------------------------------------------------------------------
/src/BlasrAdapter.h:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * BlasrAdapter.h
 3 |  *
 4 |  *  Created on: Dec 2, 2015
 5 |  *      Author: llx
 6 |  */
 7 | 
 8 | #ifndef BLASRADAPTER_H_
 9 | #define BLASRADAPTER_H_
10 | #include <iostream>
11 | #include <fstream>
12 | 
13 | #include "HALC.h"
14 | 
15 | inline bool ComFileindex(Ccutpoint first, Ccutpoint second)
16 | {
17 | 	return first.fileindex < second.fileindex;
18 | }
19 | 
20 | class BlasrAdapter
21 | {
22 | private:
23 | 	int mythreshold;
24 | 	char * contigfilename;
25 | 	std::string outputpath;
26 | 	std::string prefix;
27 | public:
28 | 	BlasrAdapter(int threshold , char * contigfile, std::string outpath);
29 | 	bool RunAdapter(std::ifstream &infile);
30 | 	bool ChangeCutPoints();
31 | 	bool ChangeCutPointsReverse();
32 | 	bool GetNewBlasrFile(std::ifstream &infile);
33 | 	bool ReSortCutPoints();
34 | };
35 | 
36 | 
37 | 
38 | 
39 | 
40 | #endif /* BLASRADAPTER_H_ */
41 | 


--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
 1 | # FLAGS
 2 | CXXFLAGS = -fopenmp#-Wall  # put compiler settings here
 3 | # put linker settings here
 4 | CXX	= g++ -std=c++11 -g -O0
 5 | RM  	= rm -f 
 6 | MV	= mv
 7 | CP	= cp
 8 | 
 9 | all:HALC Splitter Trimmer bin instbin
10 | 
11 | HALC:src/HALC.o src/BlasrAdapter.o src/parsingargs.o
12 | 	$(CXX) $(CXXFLAGS) src/HALC.o src/BlasrAdapter.o src/parsingargs.o -o HALC
13 | 
14 | HALC.o:src/HALC.cpp src/HALC.h
15 | 	$(CXX) $(CXXFLAGS) -c src/HALC.cpp
16 | 
17 | BlasrAdapter.o:src/BlasrAdapter.cpp src/BlasrAdapter.h
18 | 	$(CXX) $(CXXFLAGS) -c src/BlasrAdapter.cpp
19 | 
20 | parsingargs.o:src/parsingargs.cpp src/parsingargs.h
21 | 	$(CXX) $(CXXFLAGS) -c src/parsingargs.cpp
22 | 
23 | Splitter:thirdparty/Splitter.o
24 | 	$(CXX) thirdparty/Splitter.o -o Splitter
25 | 
26 | Splitter.o:thirdparty/Splitter.cpp
27 | 	$(CXX) -c thirdparty/Splitter.cpp
28 | 
29 | Trimmer:thirdparty/Trimmer.o
30 | 	$(CXX) thirdparty/Trimmer.o -o Trimmer
31 | 
32 | Trimmer.o:thirdparty/Trimmer.cpp
33 | 	$(CXX) -c thirdparty/Trimmer.cpp
34 | 
35 | bin:		
36 | 	mkdir bin
37 | 
38 | instbin:
39 | 	$(MV) HALC Splitter Trimmer bin
40 | 	$(CP) thirdparty/Chunker thirdparty/SeqChunker-dd thirdparty/SeqChunker-perl thirdparty/SeqChunker-sed thirdparty/interleave thirdparty/interleaved-split bin
41 | 
42 | clean:
43 | 	$(RM) src/HALC.o src/BlasrAdapter.o src/parsingargs.o thirdparty/Splitter.o thirdparty/Trimmer.o
44 | 
45 | purge:		clean
46 | 	$(RM) -r bin/
47 | 
48 | 


--------------------------------------------------------------------------------
/thirdparty/Chunker:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | # Author:
 4 | #   Frank Foerster, frank.foerster@biozentrum.uni-wuerzburg.de
 5 | #   Simon Pfaff, simon.pfaff@stud-mail.uni-wuerzburg.de
 6 | #   Thomas Hackl, thomas.hackl@uni-wuerzburg.de
 7 | #
 8 | # Last Modified: Thomas Hackl, 2014-09-10
 9 | 
10 | # Version 0.4
11 | # reliable detect binary folder with compability to MacOSX
12 | function expand_link {
13 |         FILE="$1"
14 | 
15 |         TRACKING_FILE=$(mktemp)
16 | 
17 |         echo "$FILE" >"$TRACKING_FILE"
18 | 
19 |         NEW_FILE=$(readlink "$FILE");
20 | 
21 |         while [ "$?" -eq 0 ]
22 |         do
23 |                 echo "$NEW_FILE" >>"$TRACKING_FILE"
24 | 
25 |                 # check for duplicates
26 |                 NUM_DUPLICATES=$(cat "$TRACKING_FILE" | sort | uniq -d | wc -l)
27 | 
28 |                 if [ "$NUM_DUPLICATES" -gt 0 ]
29 |                 then
30 |                         break
31 |                 fi
32 | 
33 |                 FILE=$(dirname "$FILE")/"$NEW_FILE"
34 |                 echo "$FILE" >&2
35 |                 NEW_FILE=$(readlink "$FILE")
36 |         done
37 | 
38 |         rm "$TRACKING_FILE"
39 | 
40 |         echo "FILE should be located at '$FILE'" >&2
41 | 
42 |         echo "$FILE"
43 | }
44 | 
45 | Bin="$(dirname "$(expand_link "$0")")"
46 | 
47 | # check if perl exists, than use the perl version
48 | if [ ! -z $(which perl) ]; then
49 | 
50 |     perl "$Bin/SeqChunker-perl" $@
51 | 
52 | else 
53 |     # use the bash only or the sed based version of SeqChunker
54 | 
55 |     #SeqChunker-dd need Bash Version 4 or higher, fallback to slower sed version
56 | 
57 |     VER=`bash --version | grep -Eo [0-9][.][0-9] | cut -d. -f1`
58 | 
59 |     if [ "$VER" -ge 4 ]; then
60 | 
61 | 	bash "$Bin/SeqChunker-dd" $@
62 |     
63 |     else
64 | 
65 | 	bash "$Bin/SeqChunker-sed" $@
66 | 	echo "#Bash Version older than 4.0, using SeqChunker-sed"
67 |     fi;
68 | fi;
69 | 


--------------------------------------------------------------------------------
/Readme.md:
--------------------------------------------------------------------------------
 1 | ### LATEST NEWS
 2 | The HALC paper is accepted for publication in BMC Bioinformatics!
 3 | 
 4 | ### Overview
 5 | HALC is software that makes error correction for long reads with high throughput.
 6 | 
 7 | ### Copy right
 8 | HALC is under the [Artistic License 2.0](http://opensource.org/licenses/Artistic-2.0).
 9 | 
10 | ### Short manual
11 | 1. System requirements
12 | 
13 |    HALC is suitable for 32-bit or 64-bit machines with Linux operating systems. At least 4GB of system memory is recommended for correcting larger data sets.
14 | 
15 | 2. Installation
16 | 
17 |    Aligner [BLASR](https://github.com/PacificBiosciences/blasr) and error correction software [LoRDEC](http://www.atgc-montpellier.fr/lordec/) (only for -ordinary mode) are required to run HALC.
18 |    * The source files in 'src' and 'thirdparty' folders can be compiled to generate a 'bin' folder by running Makefile: `make all`.
19 |    * Put BLASR, LoRDEC and the 'bin' folder to your $PATH: `export PATH=PATH2BLASR:$PATH` , `export PATH=PATH2LoRDEC:$PATH` and `export PATH=PATH2bin:$PATH`, respectively.
20 | 
21 | 3. Inputs
22 |    * Long reads in FASTA format.
23 |    * Contigs assembled from the corresponding short reads in FASTA format.
24 |    * The initial short reads in FASTA format (only for -ordinary mode; obtained with `cat left_reads.fa >short_reads.fa` and then `cat right_reads.fa >>short_reads.fa`).
25 | 
26 | 4. Using AlignGraph
27 | 
28 |    ```
29 |    runHALC.py long_reads.fa contigs.fa [-options|-options]
30 |    ```
31 | 
32 |    <p>Options (default value):<br>
33 |    -o/-ordinary short_reads.fa (yes)<br>
34 |    Ordinary mode utilizing repeats to make correction. The error correction software LoRDEC and the initial short reads are required to refine the repeat corrected regions. It is exclusive with the -repeat-free option.<br>
35 |    -r/-repeat-free (no)<br>
36 |    Repeat-free mode without utilizing repeats to make correction. It is exclusive with the -ordinary option.<br>
37 |    -b/-boundary n (4)<br>
38 |    Maximum boundary difference to split the subcontigs.<br>
39 |    -a/-accurate (yes)<br>
40 |    Accurate construction of the contig graph.<br>
41 |    -c/-coverage n (auto)<br>
42 |    Expected coverage on contigs. If not specified, it can be automatically calculated.<br>
43 |    -w/-width n (4)<br>
44 |    Maximum width of the dynamic programming table.<br>
45 |    -k/-kmer n (25)<br>
46 |    Kmer length for LoRDEC refinement.<br>
47 |    -t/-threads n (auto)<br>
48 |    Number of threads for one process to create. It is automatically set to the number of computing cores.<br>
49 |    -l/-log (no)<br>
50 |    System log to print.</p>
51 |    
52 | 5. Outputs
53 |    * Error corrected full long reads.
54 |    * Error corrected trimmed long reads.
55 |    * Error corrected split long reads.
56 | 
57 | ### Chinese name
58 |    HALC's Chinese name is 浩克.
59 | 
60 | 
61 | 


--------------------------------------------------------------------------------
/src/parsingargs.h:
--------------------------------------------------------------------------------
 1 | #ifndef PARSINGARGS_H
 2 | #define PARSINGARGS_H
 3 | /* purpose @ 解析输入的参数，需先通过AddArgType将必须参数和可允许的参数key加入到判定列表中
 4 |  *          通过Parse中的result将结果返回，其中结果的key为合法的key，vecotr为参数列表
 5 |  *          参数列表支持去掉参数前后的引号和\对引号和\的转义
 6 |  *
 7 |  *          特殊合法字段：
 8 |  *          格式               实际存储值
 9 |  *          \\value\"            \value"
10 |  *          "\\\value\""         \value"
11 |  *
12 |  *          注意事项：
13 |  *              1、输入参数列表中参数分隔以空格区分
14 |  *              2、- 后跟单字符关键字，--后跟长字符串关键字
15 |  *              3、关键字不能重复出现，长短关键字不能同时出现在参数列表，否则会Parse函数会提示参数错误
16 |  *
17 |  *          用法：
18 |  *              ParsingArgs pa;
19 |  *              pa.AddArgType('l',"getlist", ParsingArgs::NO_VALUE); //NO_VALUE关键字后不能有参数
20 |  *              pa.AddArgType('p',"getuser", ParsingArgs::MAYBE_VALUE); //MAYBE_VALUE 关键字后可能有关键字
21 |  *              pa.AddArgType('o',"outFile", ParsingArgs::MUST_VALUE); // MUST_VALUE 关键字后必须有参数
22 |  *              std::map<std::string, std::vector<std::string> > result;
23 |  *              int iRet = pa.Parse(tmpPara,result); //result以输入关键字为key存储相关的值序列
24 |  *
25 |  * date    @ 2014.02.19
26 |  * author  @ haibin.wang
27 |  *
28 |  */
29 | 
30 | #include <map>
31 | #include <vector>
32 | #include <string>
33 | 
34 | class ParsingArgs
35 | {
36 | public:
37 |     ParsingArgs();
38 |     ~ParsingArgs();
39 |     enum KeyFlag{ INVALID_KEY=-1, NO_VALUE, MAYBE_VALUE, MUST_VALUE};
40 |     /* pur @ 添加解释参数，一个参数可以是长参数，也可以是缩写的段参数，短参数只能为单字符，longName和shortName至少要有一个
41 |      * para @ shortName 短参数名,0为不要短参数
42 |      * para @ longName 长参数名 ，NULL为不要长参数
43 |      * para @ flag 是否需要参数，0不需要，1必须要，2可要可不要
44 |      * return @ true 添加成功，false添加失败
45 |     */
46 |     bool AddArgType(const char shortName, const char * longName = NULL, KeyFlag flag=NO_VALUE);
47 | 
48 |     /* pur @ 根据参数类型解释传入的字符串
49 |      * para @ paras 需要解释的字符串
50 |      * para @ result 返回解析后的结果
51 |      * para @ errPos 当错误的时候返回出错的大概位置
52 |      * return @ 0 解释成功，负数 解释失败
53 |      *          -1 未知参数错误
54 |                 -2 不能有参数的选项有参数错误
55 |      *          -3 必有参数选项后没有跟参数
56 |      *          -4 关键字没有加入到AddArgType中
57 |      *          -5 关键字重复
58 |     */
59 |     int Parse(const std::string & paras, std::map<std::string, std::vector<std::string> > & result, std::string &errPos);
60 | 
61 | private:
62 |     /* pur @ 判定传入的参数是否是已经添加的参数类型,如果是则去掉-或--,并返回
63 |      * para @ key 要判定的参数
64 |      * return @ -1 不是合法参数类型 否则返回Option中的flag
65 |     */
66 |     KeyFlag GetKeyFlag(std::string &key);
67 | 
68 |     /* pur @ 删除关键字前的-或--
69 |     */
70 |     void RemoveKeyFlag(std::string & paras);
71 | 
72 |     /* pur @ 从Paras中获取一个单词，自动过滤掉单词前后引号，并实现\对空格和引号的转义
73 |      * para @ Paras 返回第一个单词后的所有内容
74 |      * para @ word 返回第一单词
75 |      * return @ 成功返回true，false失败
76 |      */
77 |     bool GetWord(std::string & Paras, std::string & word);
78 | 
79 |     /* pur @ 检查关键字是否重复
80 |      * para @ key 被检查的关键字
81 |      * para @  result已存储的关键字序列
82 |      * return @ true 是重复的，false不重复
83 |     */
84 |     bool IsDuplicateKey(const std::string &key, const std::map<std::string, std::vector<std::string> > & result);
85 | 
86 |     struct Option
87 |     {
88 |         std::string m_longName;
89 |         char m_shortName;
90 |         KeyFlag m_flag;
91 |     };
92 | 
93 |     std::vector<Option> m_args; //参数信息列表
94 | };
95 | 
96 | #endif
97 | 


--------------------------------------------------------------------------------
/src/HALC.h:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * SequencingProject.h
  3 |  *
  4 |  *  Created on: Dec 2, 2015
  5 |  *      Author: llx
  6 |  */
  7 | 
  8 | #ifndef SEQUENCINGPROJECT_H_
  9 | #define SEQUENCINGPROJECT_H_
 10 | #include<string.h>
 11 | #include<map>
 12 | #include<fstream>
 13 | #include<list>
 14 | #include<iostream>
 15 | #include<utility>
 16 | #include<vector>
 17 | #include<unordered_map>
 18 | #include<sstream>
 19 | #include<omp.h>
 20 | 
 21 | static omp_lock_t mylock;
 22 | 
 23 | class Ccutpoint
 24 | {
 25 | public:
 26 | 	std::string contigname;
 27 | 	int index; //start from 0
 28 | 	int longreadindex;
 29 | 	bool ishead;
 30 | 	bool changed = false;
 31 | 	std::fstream::pos_type fileindex;
 32 | 	//char strand;
 33 | };
 34 | 
 35 | struct Nspasenode
 36 | {
 37 | 	int longreadheadindex;
 38 | 	int longreadtailindex;
 39 | 	int contigheadindex;
 40 | 	int contigtailindex;
 41 | 	int rawlongreadheadindex;
 42 | 	int rawlongreadtailindex;
 43 | 	std::string longreadname;
 44 | 	Nspasenode() :
 45 | 			longreadheadindex(0), longreadtailindex(0), contigheadindex(0), contigtailindex(0)
 46 | 	{
 47 | 	}
 48 | };
 49 | 
 50 | class CSubcontig
 51 | {
 52 | public:
 53 | 	int headindex; //start from 0
 54 | 	int tailindex;
 55 | 	char strand;
 56 | 	int longreadheadindex;
 57 | 	int longreadtailindex;
 58 | 	unsigned int similarity;
 59 | 	std::string contigname;
 60 | 	int headoffset;
 61 | 	int tailoffset;
 62 | 	unsigned long indexofsubcontigs;
 63 | 	std::vector<Nspasenode>* Nspace;
 64 | 
 65 | 	bool operator==(const CSubcontig& obj) const;
 66 | 	CSubcontig() :
 67 | 			headoffset(0), tailoffset(0), Nspace(NULL)
 68 | 	{
 69 | 	}
 70 | };
 71 | 
 72 | struct Nnodeforsort
 73 | {
 74 | 	CSubcontig* me;
 75 | };
 76 | 
 77 | class Ccontig
 78 | {
 79 | public:
 80 | 	std::fstream::pos_type index; //start from 0
 81 | 	int length; //number of bases
 82 | };
 83 | 
 84 | class Clongread
 85 | {
 86 | public:
 87 | 	std::fstream::pos_type index; //start from 0
 88 | 	int length;
 89 | 	bool corrected;
 90 | 	Clongread() :
 91 | 			corrected(false), length(0), index(0)
 92 | 	{
 93 | 	}
 94 | };
 95 | 
 96 | inline bool ComIndex(Ccutpoint first, Ccutpoint second)
 97 | {
 98 | 	return first.index < second.index;
 99 | }
100 | 
101 | inline bool ComName(Ccutpoint first, Ccutpoint second)
102 | {
103 | 	return first.contigname < second.contigname;
104 | }
105 | 
106 | namespace __gnu_cxx
107 | {
108 | struct map_hash
109 | {
110 | 	size_t operator()(const std::pair<long, long>& m) const;
111 | };
112 | struct map_equal
113 | {
114 | 	bool operator()(const std::pair<long, long>& m1, const std::pair<long, long>& m2) const;
115 | };
116 | }
117 | 
118 | class Cfilebuffer
119 | {
120 | private:
121 | 	std::unordered_map<std::string, std::string> filebuffers;
122 | public:
123 | 	Cfilebuffer()
124 | 	{
125 | 	}
126 | 	;
127 | 	Cfilebuffer(std::string filename, long size = 0)
128 | 	{
129 | 		std::ifstream fin(filename.c_str());
130 | 		std::string fileContent;
131 | 		std::stringstream ss;
132 | #pragma omp critical
133 | {
134 | 		if (fin.is_open())
135 | 		{
136 | 			ss << fin.rdbuf();
137 | 			filebuffers[filename] = std::move(ss.str());
138 | 			fin.close();
139 | 		}
140 | }
141 | 	}
142 | 	void refreshbuffer(std::string filename, long size = 0)
143 | 	{
144 | 
145 | 		if (filebuffers.find(filename) == filebuffers.end())
146 | 		{
147 | 			omp_set_lock(&mylock);
148 | 			if(filebuffers.find(filename) != filebuffers.end()){
149 | 				return;
150 | 			}
151 | 			std::ifstream infile(filename.c_str());
152 | 			std::string fileContent;
153 | 			std::stringstream ss;
154 | 			if (infile.is_open())
155 | 			{
156 | 				ss << infile.rdbuf();
157 | 				filebuffers[filename] = std::move(ss.str());
158 | 				infile.close();
159 | 			}
160 | 			omp_unset_lock(&mylock);
161 | 		}
162 | 	}
163 | 	std::string Getstring(char *argv, std::fstream::pos_type begin, std::fstream::pos_type end);
164 | };
165 | 
166 | class CSubUndigraph
167 | {
168 | private:
169 | 	struct CSubcontigEx;
170 | 	static bool ComHeadIndex(CSubcontigEx first, CSubcontigEx second);
171 | public:
172 | 	static std::string lralignedseq;
173 | 	static std::string matchpattern;
174 | 	static std::string ctalignedseq;
175 | 	static std::vector<std::vector<CSubcontig> > contiglist; //contigs of a longread
176 | 	std::string longreadname;
177 | 	std::vector<CSubcontigEx> Subconitglist; //all subcontigs of a longread
178 | 	friend class CUndigraph;
179 | 	std::unordered_map<std::pair<unsigned long, unsigned long>, bool, __gnu_cxx ::map_hash, __gnu_cxx ::map_equal> edges[3];
180 | private:
181 | 	int ctoffset;
182 | 	int lroffset;
183 | //	int matchlength;
184 | 	int matchoffset;
185 | 	int numoflines(std::string& s, int begin, int end);
186 | 	int cutlongread(int length, CSubcontig& subcontig);
187 | 	int cutlongreadreverse(int length, CSubcontig& subcontig);
188 | 	//vector<CSubcontig> badcontigs;
189 | public:
190 | 	CSubUndigraph();
191 | 
192 | 	bool addToContigList(unsigned long head, unsigned long tail, char strand, int lrheadindex, int lrtailindex, int headoffset, int tailoffset, int ctheadindex, int cttailindex);
193 | 
194 | 	static bool clearcontiglist();
195 | 
196 | 	int getlrlength(int length, unsigned int& similarity);
197 | 
198 | 	int getlrlengthreverse(int length, unsigned int& similarity);
199 | 
200 | 	bool getAlignInf(int lrheadindex, int lrtailindex, int headoffset, int tailoffset, char strand);
201 | 
202 | 	bool getSubcontiglist();
203 | 
204 | 	bool
205 | 	drawLine(CSubcontigEx contig1, CSubcontigEx contig2);
206 | 
207 | 	bool getEdges();
208 | };
209 | 
210 | class CUndigraph
211 | {
212 | public:
213 | 	static std::unordered_map<std::pair<unsigned long, unsigned long>, unsigned short, __gnu_cxx ::map_hash, __gnu_cxx ::map_equal> graph[3];
214 | 	static void
215 | 	MakeUndigraph(std::ifstream& alignfile);
216 | 	static std::vector<CSubUndigraph> subundigraphs;
217 | 	static std::vector<Nnodeforsort> Nnodes;
218 | 	static double anveragesupport;
219 | 	static void replaceN();
220 | 	static void findanveragesupport();
221 | 	static inline bool comContigName(Nnodeforsort first, Nnodeforsort second);
222 | };
223 | 
224 | class CMyVectorInt: public std::vector<int>
225 | {
226 | private:
227 | 	unsigned int biggestindex = 0;
228 | public:
229 | 	int getnext();
230 | 	bool hasnext();
231 | };
232 | 
233 | class Ccorrector
234 | {
235 | private:
236 | 	CUndigraph undigraph;
237 | //	int* dist;
238 | //	int* path;
239 | //	std::vector<CMyVectorInt> pdist;
240 | //	std::vector<CMyVectorInt> ppath;
241 | 	char* lrfilename;
242 | 	char* ctfilename;
243 | 	bool fpathbysupport(int index, int *&dist);
244 | 	bool fpathbysimilarity(int index, int *&dist);
245 | 	bool nfpathbysimilarity(int index, int n, std::vector<CMyVectorInt> &pdist);
246 | 	int froutebysupport(int index, int *&dist, int *&path);
247 | 	int froutebysimilarity(int index, int *&dist, int *&path);
248 | 	void nfroutebysimilarity(int index, int* counter, int j, int pathposition, int jposition, int *&path, std::vector<CMyVectorInt> &pdist, std::vector<CMyVectorInt> &ppath);
249 | 	void bestnrouteofsimilarity(int index, int n, int *path, std::vector<CMyVectorInt> &pdist, std::vector<CMyVectorInt> &ppath);
250 | 	int leastcostofn(int index, std::vector<CMyVectorInt> &path, bool &hasrepeat);
251 | 	std::string& changetoreverse(std::string& s);
252 | public:
253 | 	Ccorrector(char* lrfile, char* ctfile);
254 | 	bool findBestRouteBySimilarity();
255 | 	bool findBestRouteBySupport();
256 | 	bool findBestNRoute(int n);
257 | 	void docorrect(int subundigraphindex, int ppathindex, std::ofstream &correctedfile, std::ofstream &repeatfile, std::vector<CMyVectorInt> &ppath, Cfilebuffer &longreadbuffer,
258 | 			Cfilebuffer &contigbuffer, bool & hasrepeat);
259 | };
260 | #endif /* SEQUENCINGPROJECT_H_ */
261 | 


--------------------------------------------------------------------------------
/thirdparty/Trimmer.cpp:
--------------------------------------------------------------------------------
  1 | // Copyright Leena Salmela and Eric Rivals 2014
  2 | //
  3 | // leena.salmela@cs.helsinki.fi
  4 | // rivals@lirmm.fr
  5 | //
  6 | // This software is a computer program whose purpose is to correct
  7 | // sequencing errors in PacBio reads using highly accurate short reads
  8 | // (e.g. Illumina).
  9 | //
 10 | // This software is governed by the CeCILL license under French law and
 11 | // abiding by the rules of distribution of free software. You can use,
 12 | // modify and/ or redistribute the software under the terms of the CeCILL
 13 | // license as circulated by CEA, CNRS and INRIA at the following URL
 14 | // "http://www.cecill.info".
 15 | //
 16 | // As a counterpart to the access to the source code and rights to copy,
 17 | // modify and redistribute granted by the license, users are provided only
 18 | // with a limited warranty and the software's author, the holder of the
 19 | // economic rights, and the successive licensors have only limited
 20 | // liability.
 21 | //
 22 | // In this respect, the user's attention is drawn to the risks associated
 23 | // with loading, using, modifying and/or developing or reproducing the
 24 | // software by the user in light of its specific status of free software,
 25 | // that may mean that it is complicated to manipulate, and that also
 26 | // therefore means that it is reserved for developers and experienced
 27 | // professionals having in-depth computer knowledge. Users are therefore
 28 | // encouraged to load and test the software's suitability as regards their
 29 | // requirements in conditions enabling the security of their systems and/or
 30 | // data to be ensured and, more generally, to use and operate it in the
 31 | // same conditions as regards security.
 32 | //
 33 | // The fact that you are presently reading this means that you have had
 34 | // knowledge of the CeCILL license and that you accept its terms.
 35 | #include <unistd.h>
 36 | #include <stdlib.h>
 37 | #include <fstream>  // for ifstream class
 38 | #include <iostream>
 39 | #define MAX_READ_LEN 500000
 40 | #define BASE_PER_LINE 70
 41 | 
 42 | std::string Realign(std::string source)
 43 | {
 44 | 	for (unsigned int i = BASE_PER_LINE; i < source.length(); i += BASE_PER_LINE + 1)
 45 | 	{
 46 | 		source.insert(i, 1, '\n');
 47 | 	}
 48 | 	return source;
 49 | }
 50 | void usage(char *prog)
 51 | {
 52 | 	std::cerr << "Usage: " << prog << " -i <FASTA-file> -o <output-file>" << std::endl;
 53 | }
 54 | 
 55 | std::string getword(char* str, long &pos, long size)
 56 | {
 57 | 	std::string temp;
 58 | 	while (pos < size && (str[pos] == '\t' || str[pos] == '\n' || str[pos] == ' '))
 59 | 	{
 60 | 		pos++;
 61 | 	}
 62 | 	while (pos < size && (str[pos] != '\t' && str[pos] != '\0' && str[pos] != '\n' && str[pos] != ' '))
 63 | 	{
 64 | 		temp += str[pos];
 65 | 		pos++;
 66 | 	}
 67 | 	return temp;
 68 | }
 69 | 
 70 | std::string getline(char* str, long &pos, long size)
 71 | {
 72 | 	std::string temp;
 73 | 	while (pos < size && str[pos] == '\n')
 74 | 	{
 75 | 		pos++;
 76 | 	}
 77 | 	while (pos < size && str[pos] != '\n' && str[pos] != '\0')
 78 | 	{
 79 | 		temp += str[pos];
 80 | 		pos++;
 81 | 	}
 82 | 	pos++;
 83 | 	return temp;
 84 | }
 85 | 
 86 | std::string getSequence(char* str, long &pos, long &total_len, long size)
 87 | {
 88 | 	std::string temp;
 89 | 	int read_len = 0;
 90 | 	while (pos < size && str[pos] != '>' && str[pos] != '\0')
 91 | 	{
 92 | 		if (str[pos] != '\n' && str[pos] != ' ')
 93 | 		{
 94 | 			read_len++;
 95 | 		}
 96 | 		else
 97 | 		{
 98 | 			temp.insert(temp.size(), str + pos - read_len, read_len);
 99 | 			total_len += read_len;
100 | 			read_len = 0;
101 | 		}
102 | 		pos++;
103 | 	}
104 | 	if (pos == size || str[pos] == '\0')
105 | 	{
106 | 		temp.insert(temp.size(), str + pos - read_len, read_len);
107 | 		total_len += read_len;
108 | 	}
109 | 	return temp;
110 | }
111 | 
112 | /********************************************************************************/
113 | /*                      Trim lower case letters from read ends                  */
114 | /********************************************************************************/
115 | int main(int argc, char* argv[])
116 | {
117 | 
118 | 	extern char *optarg;
119 | 	extern int optind;
120 | 	extern int opterr;
121 | 	opterr = 1;
122 | 
123 | 	std::string pacbioFile, outReadFile;
124 | 
125 | 	int c = 0, iFlag = 0, oFlag = 0;
126 | 
127 | 	// check nb of compulsory arguments (-i and -o followed by FASTA file URIs).
128 | 	if (argc < 5)
129 | 	{
130 | 		usage(argv[0]);
131 | 		return EXIT_FAILURE;
132 | 	}
133 | 
134 | 	while ((c = getopt(argc, argv, "i:o:")) != -1)
135 | 	{
136 | 		switch (c)
137 | 		{
138 | 		case 'i':
139 | 			if (iFlag)
140 | 			{
141 | 				std::cerr << "Option -i <long reads FASTA-file> should be used only once." << std::endl;
142 | 				usage(argv[0]);
143 | 				return EXIT_FAILURE;
144 | 			}
145 | 			iFlag = 1;
146 | 			pacbioFile = optarg;
147 | 			break;
148 | 
149 | 		case 'o':
150 | 			if (oFlag)
151 | 			{
152 | 				std::cerr << "Option -o <output-file> should be used only once." << std::endl;
153 | 				usage(argv[0]);
154 | 				return EXIT_FAILURE;
155 | 			}
156 | 			oFlag = 1;
157 | 			outReadFile = optarg;
158 | 			break;
159 | 
160 | 		case '?': 				// getopt_long already printed an error message.
161 | 			break;
162 | 		default:
163 | 			usage(argv[0]);
164 | 			return EXIT_FAILURE;
165 | 		}
166 | 	}
167 | 
168 | 	// Check for required arguments
169 | 	if (!iFlag || !oFlag)
170 | 	{
171 | 		usage(argv[0]);
172 | 		return EXIT_FAILURE;
173 | 	}
174 | 
175 | 	// check accessibility of pacbioFile
176 | 	if (-1 == access(pacbioFile.c_str(), R_OK))
177 | 	{
178 | 		std::cerr << "Cannot access the FASTA file for PacBio reads: " << pacbioFile << std::endl;
179 | 		return EXIT_FAILURE;
180 | 	}
181 | 
182 | 	std::ifstream infile(pacbioFile.c_str());
183 | 	char * inbuffer;
184 | 	std::filebuf *pbuf;
185 | 	pbuf = infile.rdbuf();
186 | 	long size = pbuf->pubseekoff(0, std::ios::end, std::ios::in);
187 | 	pbuf->pubseekpos(0, std::ios::in);
188 | 	inbuffer = new char[size];
189 | 	pbuf->sgetn(inbuffer, size);
190 | 	int len = 0;
191 | 	long pos = 0;
192 | 
193 | 	std::ofstream outfile(outReadFile.c_str());
194 | 	char *buffer = new char[MAX_READ_LEN];
195 | 	std::string title;
196 | 	std::string comment;
197 | 	long read_len;
198 | 	const char* read;
199 | 
200 | 	while (pos < size)
201 | 	{
202 | 		read_len = 0;
203 | 		title = getword(inbuffer, pos, size);
204 | 		comment.clear();
205 | 		if(inbuffer[pos] != '\n')
206 | 			comment = getline(inbuffer, pos, size);
207 | 		std::string tempstr = getSequence(inbuffer, pos, read_len, size);
208 | 		read = tempstr.c_str();
209 | 		int firstUC = -1, lastUC = -1; // first and last position of uppercase nuc in the input seq
210 | 		for (int i = 0; i < read_len; i++)
211 | 		{ // scan current input sequence
212 | 			if (isupper(read[i]))
213 | 			{ // current letter is uppercase
214 | 				firstUC = i;
215 | 				break;
216 | 
217 | 			}
218 | 		} // end scan of current sequence
219 | 		if (firstUC < 0) // no uppercase letter found: go to next input seq
220 | 			continue;
221 | 
222 | 		for (int i = read_len - 1; i >= 0; i--)
223 | 		{ // seek last uppercase letter from the end
224 | 			if (isupper(read[i]))
225 | 			{
226 | 				lastUC = i;
227 | 				break;
228 | 			}
229 | 		}
230 | 
231 | 		// copy desired region of the read to buffer
232 | 		// TODO: check the buffer limit !!!
233 | 		for (int i = firstUC; i <= lastUC; i++)
234 | 		{
235 | 			buffer[i - firstUC] = read[i];
236 | 		}
237 | 		buffer[lastUC - firstUC + 1] = '\0';
238 | 
239 | 		if(comment.empty())
240 | 			outfile << title << std::endl;
241 | 		else
242 | 			outfile << title + ' ' + comment << std::endl;
243 | 		outfile << Realign(buffer) << std::endl;
244 | 	}
245 | 	delete[] buffer;
246 | 
247 | 	return EXIT_SUCCESS;
248 | }
249 | 


--------------------------------------------------------------------------------
/src/parsingargs.cpp:
--------------------------------------------------------------------------------
  1 | #include "parsingargs.h"
  2 | #include <list>
  3 | 
  4 | ParsingArgs::ParsingArgs()
  5 | {
  6 | }
  7 | 
  8 | ParsingArgs::~ParsingArgs()
  9 | {
 10 | }
 11 | 
 12 | bool ParsingArgs::AddArgType(char shortName, const char * longName, KeyFlag flag)
 13 | {
 14 |     if(NULL == longName && 0 == shortName)
 15 |     {
 16 |         return false;
 17 |     }
 18 |     Option tmp;
 19 |     tmp.m_longName = longName;
 20 |     tmp.m_shortName = shortName;
 21 |     tmp.m_flag = flag;
 22 |     m_args.push_back(tmp);
 23 |     return true;
 24 | }
 25 | 
 26 | ParsingArgs::KeyFlag ParsingArgs::GetKeyFlag(std::string &key) //返回flag，
 27 | {
 28 |     for(int i=0; i<m_args.size(); ++i)
 29 |     {
 30 |         std::string shortName = "-";
 31 |         std::string longName = "--";
 32 |         shortName += m_args[i].m_shortName;
 33 |         longName += m_args[i].m_longName;
 34 |         if( 0 == key.compare(shortName) ||
 35 |                 (0==key.compare(longName))
 36 |             )
 37 |         {
 38 |             RemoveKeyFlag(key);
 39 |             return m_args[i].m_flag;
 40 |         }
 41 |     }
 42 |     return INVALID_KEY;
 43 | }
 44 | 
 45 | void ParsingArgs::RemoveKeyFlag(std::string & word)
 46 | {
 47 |     if(word.size()>=2)
 48 |     {
 49 |         if(word[1] == '-')
 50 |         {
 51 |             word.erase(1,1);
 52 |         }
 53 |         if(word[0] == '-')
 54 |         {
 55 |             word.erase(0,1);
 56 |         }
 57 |     }
 58 | }
 59 | /* pur @ 从Paras中获取一个单词，自动过滤掉单词前后引号，并实现\对空格和引号的转义
 60 |  * para @ Paras 返回第一个单词后的所有内容
 61 |  * para @ word 返回第一单词
 62 |  * return @ 成功返回true，false失败
 63 | */
 64 | bool ParsingArgs::GetWord(std::string & Paras, std::string & word)
 65 | {
 66 |     size_t iNotSpacePos = Paras.find_first_not_of(' ',0);//查找第一个非空格字符位置
 67 |     if(iNotSpacePos == std::string::npos)
 68 |     {
 69 |         Paras.clear();
 70 |         word.clear();
 71 |         return true;
 72 |     }
 73 |     int length = Paras.size();
 74 |     std::list<char> specialChar;
 75 |     int islashPos = -1;
 76 |     for(int i=iNotSpacePos; i<length; i++)
 77 |     {
 78 |         char cur=Paras[i];
 79 |         bool bOk = false;
 80 |         switch(cur)
 81 |         {
 82 |             case ' ':
 83 |                 if(specialChar.empty())
 84 |                 {
 85 |                     if(i!=(length-1))
 86 |                     {
 87 |                         Paras = std::string(Paras, i+1, length-i-1);
 88 |                     }
 89 |                     else
 90 |                     {//最后一个是空格
 91 |                         Paras.clear();
 92 |                     }
 93 |                     bOk = true;
 94 |                 }
 95 |                 else
 96 |                 {
 97 |                     if(specialChar.back() == '\\')
 98 |                     {
 99 |                         specialChar.pop_back();
100 |                     }
101 |                     word.append(1,cur);
102 |                 }
103 |                 break;
104 |             case '"':
105 |                 if(specialChar.empty())
106 |                 {
107 |                     specialChar.push_back(cur);
108 |                 }
109 |                 else if(specialChar.back() == cur)
110 |                 { //找到匹配的括号
111 |                     specialChar.pop_back();
112 |                 }
113 |                 else if(specialChar.back() == '\\')
114 |                 {
115 |                     word.append(1,cur);
116 |                     specialChar.pop_back();
117 |                 }
118 |                 else
119 |                 {
120 |                     word.clear();
121 |                     return false;
122 |                 }
123 |                 break;
124 |             case '\\':
125 |                 if(specialChar.empty())
126 |                 {
127 |                     specialChar.push_back(cur);
128 |                     islashPos = i;
129 |                 }
130 |                 else if(specialChar.back() == '"')
131 |                 {
132 |                     if(i<(length-1))
133 |                     {
134 |                         if('"'==Paras[i+1] || '\\'==Paras[i+1])
135 |                         {
136 |                             specialChar.push_back(cur);
137 |                         }
138 |                         else
139 |                         {
140 |                             word.append(1,cur);
141 |                         }
142 |                     }
143 |                     else
144 |                     {
145 |                         word.clear();
146 |                         return false;
147 |                     }
148 |                 }
149 |                 else if('\\' == specialChar.back())
150 |                 {
151 |                      word.append(1,cur);
152 |                      specialChar.pop_back();
153 |                 }
154 |                 else
155 |                 {
156 |                     word.clear();
157 |                     return false;
158 |                 }
159 |                 break;
160 |             default:
161 |                 word.append(1,Paras[i]);
162 |                 if(i==(length-1))
163 |                 {
164 |                     bOk = true;
165 |                     Paras.clear();
166 |                 }
167 |                 break;
168 |         }
169 |         if(bOk)
170 |         {
171 |             return true;
172 |         }
173 |     }//for end
174 |     if(specialChar.empty())
175 |     {
176 |         Paras.clear();
177 |         return true;
178 |     }
179 |     else
180 |     {
181 |         return false;
182 |     }
183 | }
184 | 
185 | bool ParsingArgs::IsDuplicateKey(const std::string &key, const std::map<std::string, std::vector<std::string> > & result)
186 | {
187 |     if(result.find(key) != result.end())
188 |     {
189 |         return true; //关键字重复
190 |     }
191 | 
192 |     for(int i=0; i<m_args.size(); ++i)
193 |     {
194 |         if( (key.compare(m_args[i].m_longName) == 0 && result.find(std::string(1, m_args[i].m_shortName)) != result.end())
195 |                 || (key.compare(std::string(1, m_args[i].m_shortName)) == 0 && result.find(m_args[i].m_longName) != result.end())
196 |           )
197 |         {
198 |             return true;
199 |         }
200 |     }
201 |     return false;
202 | }
203 | 
204 | int ParsingArgs::Parse(const std::string & Paras, std::map<std::string, std::vector<std::string> > & result, std::string &errPos)
205 | {
206 |     std::string tmpString = Paras;
207 |     KeyFlag keyFlag = INVALID_KEY; //参数标识
208 |     std::string sKey = ""; //key关键字
209 |     bool bFindValue = false; //是否已经有value
210 |     while(!tmpString.empty())
211 |     {
212 |         std::string word = "";
213 | 
214 |         bool bRet = GetWord(tmpString, word);
215 | 
216 |         if(bRet == false)
217 |         {
218 |             errPos = tmpString;
219 |             return -4;//参数解析错误
220 |         }
221 |         else
222 |         {
223 |             KeyFlag tmpFlag = GetKeyFlag(word);
224 |             if(IsDuplicateKey(word, result))
225 |             {
226 |                 errPos = tmpString;
227 |                 return -5;
228 |             }
229 |             if(tmpFlag != INVALID_KEY)
230 |             {
231 |                 if(tmpFlag == MUST_VALUE && keyFlag == MUST_VALUE && !bFindValue)
232 |                 {
233 |                     errPos = tmpString;
234 |                     return -3;
235 |                 }
236 |                 keyFlag = tmpFlag;
237 |                 std::vector<std::string> tmp;
238 |                 result[word] = tmp;
239 |                 sKey = word;
240 |                 bFindValue = false;
241 |             }
242 |             else
243 |             {
244 |                 switch(keyFlag)
245 |                 {
246 |                     case MAYBE_VALUE:
247 |                     case MUST_VALUE:
248 |                         {
249 |                             std::map<std::string, std::vector<std::string> >::iterator it = result.find(sKey);
250 |                             if(it != result.end())
251 |                             {
252 |                                 it->second.push_back(word);
253 |                                 bFindValue = true;
254 |                             }
255 |                             else
256 |                             {
257 |                                 errPos = tmpString;
258 |                                 return -1;// 没有发现相关的key
259 |                             }
260 |                         }
261 |                         break;
262 |                     case NO_VALUE:
263 |                         errPos = tmpString;
264 |                         return -2; //不能有参数的选项后有参数
265 |                     default:
266 |                         errPos = tmpString;
267 |                         return -1;//参数错误
268 |                 }//switch end
269 |             }
270 |         }
271 |     }//while end
272 |     return 0;
273 | }
274 | 
275 | 
276 | 
277 | 


--------------------------------------------------------------------------------
/thirdparty/Splitter.cpp:
--------------------------------------------------------------------------------
  1 | // Copyright Leena Salmela and Eric Rivals 2014
  2 | //
  3 | // leena.salmela@cs.helsinki.fi
  4 | // rivals@lirmm.fr
  5 | //
  6 | // This software is a computer program whose purpose is to correct
  7 | // sequencing errors in PacBio reads using highly accurate short reads
  8 | // (e.g. Illumina).
  9 | //
 10 | // This software is governed by the CeCILL license under French law and
 11 | // abiding by the rules of distribution of free software. You can use,
 12 | // modify and/ or redistribute the software under the terms of the CeCILL
 13 | // license as circulated by CEA, CNRS and INRIA at the following URL
 14 | // "http://www.cecill.info".
 15 | //
 16 | // As a counterpart to the access to the source code and rights to copy,
 17 | // modify and redistribute granted by the license, users are provided only
 18 | // with a limited warranty and the software's author, the holder of the
 19 | // economic rights, and the successive licensors have only limited
 20 | // liability.
 21 | //
 22 | // In this respect, the user's attention is drawn to the risks associated
 23 | // with loading, using, modifying and/or developing or reproducing the
 24 | // software by the user in light of its specific status of free software,
 25 | // that may mean that it is complicated to manipulate, and that also
 26 | // therefore means that it is reserved for developers and experienced
 27 | // professionals having in-depth computer knowledge. Users are therefore
 28 | // encouraged to load and test the software's suitability as regards their
 29 | // requirements in conditions enabling the security of their systems and/or
 30 | // data to be ensured and, more generally, to use and operate it in the
 31 | // same conditions as regards security.
 32 | //
 33 | // The fact that you are presently reading this means that you have had
 34 | // knowledge of the CeCILL license and that you accept its terms.
 35 | #include <unistd.h>
 36 | #include <stdlib.h>
 37 | #include <fstream>  // for ifstream class
 38 | #include <iostream>
 39 | #include <fstream>
 40 | #include <sstream>
 41 | #include <string.h>
 42 | #include<sys/types.h>
 43 | #include<sys/stat.h>
 44 | #include<fcntl.h>
 45 | #include <unistd.h>
 46 | #include <errno.h>
 47 | #include <omp.h>
 48 | //#include "lordec-gen.hpp"
 49 | #include <iostream>
 50 | 
 51 | #define DEF_MIN_REGION_LG 100
 52 | #define MAX_READ_LEN 500000
 53 | #define BASE_PER_LINE 70
 54 | 
 55 | std::string Realign(std::string source)
 56 | {
 57 | 	for (unsigned int i = BASE_PER_LINE; i < source.length(); i += BASE_PER_LINE + 1)
 58 | 	{
 59 | 		source.insert(i, 1, '\n');
 60 | 	}
 61 | 	return source;
 62 | }
 63 | 
 64 | void usage(char *prog)
 65 | {
 66 | 	std::cerr << "Usage: " << prog << " -i <FASTA-file> -o <output-file>" << std::endl;
 67 | 	std::cerr << "       scan a set of corrected long reads (in FASTA format) and output as sequence their regions that have indeed been corrected (which are in uppercase)." << std::endl;
 68 | }
 69 | 
 70 | std::string getword(char* str, long &pos, long size)
 71 | {
 72 | 	std::string temp;
 73 | 	while (pos < size && (str[pos] == '\t' || str[pos] == '\n' || str[pos] == ' '))
 74 | 	{
 75 | 		pos++;
 76 | 	}
 77 | 	while (pos < size && (str[pos] != '\t' && str[pos] != '\0' && str[pos] != '\n' && str[pos] != ' '))
 78 | 	{
 79 | 		temp += str[pos];
 80 | 		pos++;
 81 | 	}
 82 | 	return temp;
 83 | }
 84 | 
 85 | std::string getline(char* str, long &pos, long size)
 86 | {
 87 | 	std::string temp;
 88 | 	while (pos < size && str[pos] == '\n')
 89 | 	{
 90 | 		pos++;
 91 | 	}
 92 | 	while (pos < size && str[pos] != '\n' && str[pos] != '\0')
 93 | 	{
 94 | 		temp += str[pos];
 95 | 		pos++;
 96 | 	}
 97 | 	pos++;
 98 | 	return temp;
 99 | }
100 | 
101 | std::string getSequence(char* str, long &pos, long &total_len, long size)
102 | {
103 | 	std::string temp;
104 | 	int read_len = 0;
105 | 	while (pos < size && str[pos] != '>' && str[pos] != '\0')
106 | 	{
107 | 		if (str[pos] != '\n' && str[pos] != ' ')
108 | 		{
109 | 			read_len++;
110 | 		}
111 | 		else
112 | 		{
113 | 			temp.insert(temp.size(), str + pos - read_len, read_len);
114 | 			total_len += read_len;
115 | 			read_len = 0;
116 | 		}
117 | 		pos++;
118 | 	}
119 | 	if (pos == size || str[pos] == '\0')
120 | 	{
121 | 		temp.insert(temp.size(), str + pos - read_len, read_len);
122 | 		total_len += read_len;
123 | 	}
124 | 	return temp;
125 | }
126 | /**********************************************************************************/
127 | /* Trim lower case letters from reads and split them at lower/upper case boundary */
128 | /**********************************************************************************/
129 | int main(int argc, char* argv[])
130 | {
131 | 
132 | 	extern char *optarg;
133 | 	extern int optind;
134 | 	extern int opterr;
135 | 	opterr = 1;
136 | 
137 | 	std::string pacbioFile, outReadFile;
138 | 
139 | 	int c = 0, iFlag = 0, oFlag = 0;
140 | 
141 | 	int minRegionLg = DEF_MIN_REGION_LG;
142 | 
143 | 	// check nb of compulsory arguments (-i and -o followed by FASTA file URIs).
144 | 	if (argc < 5)
145 | 	{
146 | 		usage(argv[0]);
147 | 		return EXIT_FAILURE;
148 | 	}
149 | 
150 | 	while ((c = getopt(argc, argv, "i:o:")) != -1)
151 | 	{
152 | 		switch (c)
153 | 		{
154 | 		case 'i':
155 | 			if (iFlag)
156 | 			{
157 | 				std::cerr << "Option -i <FASTA-file> should be used only once." << std::endl;
158 | 				usage(argv[0]);
159 | 				return EXIT_FAILURE;
160 | 			}
161 | 			iFlag = 1;
162 | 			pacbioFile = optarg;
163 | 			break;
164 | 		case 'o':
165 | 			if (oFlag)
166 | 			{
167 | 				std::cerr << "Option -o <output-file> should be used only once." << std::endl;
168 | 				usage(argv[0]);
169 | 				return EXIT_FAILURE;
170 | 			}
171 | 			oFlag = 1;
172 | 			outReadFile = optarg;
173 | 			break;
174 | 		case '?': 				// getopt_long already printed an error message.
175 | 			break;
176 | 		default:
177 | 			usage(argv[0]);
178 | 			return EXIT_FAILURE;
179 | 		}
180 | 	}
181 | 
182 | 	// Check for required arguments
183 | 	if (!iFlag || !oFlag)
184 | 	{
185 | 		usage(argv[0]);
186 | 		return EXIT_FAILURE;
187 | 	}
188 | 
189 | 	// check accessibility of pacbioFile
190 | 	if (-1 == access(pacbioFile.c_str(), R_OK))
191 | 	{
192 | 		std::cerr << "Cannot access the FASTA file for PacBio reads: " << pacbioFile << std::endl;
193 | 		return EXIT_FAILURE;
194 | 	}
195 | 
196 | 	std::ifstream infile(pacbioFile.c_str());
197 | 	char * inbuffer;
198 | 	std::filebuf *pbuf;
199 | 	pbuf = infile.rdbuf();
200 | 	long size = pbuf->pubseekoff(0, std::ios::end, std::ios::in);
201 | 	pbuf->pubseekpos(0, std::ios::in);
202 | 	inbuffer = new char[size];
203 | 	pbuf->sgetn(inbuffer, size);
204 | 	int len = 0;
205 | 	long pos = 0;
206 | 
207 | 	std::ofstream outfile(outReadFile.c_str());
208 | 	char *buffer = new char[MAX_READ_LEN];
209 | 	std::string title;
210 | 	std::string comment;
211 | 	int part = 1;
212 | 	const char *ucstart = NULL;
213 | 	long read_len;
214 | 	const char* read;
215 | 
216 | 	while (pos < size)
217 | 	{
218 | 		part = 1;
219 | 		read_len = 0;
220 | 		title = getword(inbuffer, pos, size);
221 | 		comment.clear();
222 | 		if (inbuffer[pos] != '\n')
223 | 			comment = getline(inbuffer, pos, size);
224 | 		std::string tempstr = getSequence(inbuffer, pos, read_len, size);
225 | 		read = tempstr.c_str();
226 | 		for (int i = 0; i < read_len; i++)
227 | 		{ // scan current input sequence
228 | 			if (isupper(read[i]))
229 | 			{ // current letter is uppercase
230 | 				if (ucstart == NULL)
231 | 				{ // an uppercase region starts here
232 | 					ucstart = &read[i];
233 | 					len = 0;
234 | 				}
235 | 				len++; // increment the length of the current uppercase region
236 | 
237 | 			}
238 | 			else
239 | 			{  // current letter is lowercase
240 | 				if (ucstart != NULL)
241 | 				{ // an uppercase region ends here
242 | 					if (len >= minRegionLg)
243 | 					{ // if longer than minRegionLg: output
244 | 						strncpy(buffer, ucstart, len);  // copy into buffer
245 | 						buffer[len] = '\0';
246 | 
247 | 						char numbuf[22];
248 | 						sprintf(numbuf, "_%d", part);
249 | 						if (comment.empty())
250 | 							outfile << title + numbuf << std::endl;
251 | 						else
252 | 							outfile << title + numbuf + ' ' + comment << std::endl;
253 | 						outfile << Realign(buffer) << std::endl;
254 | 						part++;
255 | 					}
256 | 					ucstart = NULL; // reinitialise the start of an uppercase region
257 | 				}
258 | 			}
259 | 		} // end scan of current sequence
260 | 		if (ucstart != NULL)
261 | 		{ // an uppercase region ends with the input seq
262 | 			if (len >= minRegionLg)
263 | 			{ // see comments above
264 | 				strncpy(buffer, ucstart, len);
265 | 				buffer[len] = '\0';
266 | 
267 | 				char numbuf[22];
268 | 				sprintf(numbuf, "_%d", part);
269 | 				if (comment.empty())
270 | 					outfile << title + numbuf << std::endl;
271 | 				else
272 | 					outfile << title + numbuf + ' ' + comment << std::endl;
273 | 				outfile << Realign(buffer) << std::endl;
274 | 			}
275 | 			ucstart = NULL;
276 | 		}
277 | 	}
278 | 
279 | 	delete[] buffer;
280 | 
281 | 	return EXIT_SUCCESS;
282 | }
283 | 


--------------------------------------------------------------------------------
/src/BlasrAdapter.cpp:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * BlasrAdapter.cpp
  3 |  *
  4 |  *  Created on: Dec 2, 2015
  5 |  *      Author: llx
  6 |  */
  7 | #include "BlasrAdapter.h"
  8 | #include <algorithm>
  9 | 
 10 | using namespace __gnu_cxx;
 11 | using namespace std;
 12 | 
 13 | extern unordered_map<string, Ccontig> cthm;
 14 | extern unordered_map<string, Clongread> lrhm;
 15 | extern unordered_map<string, unsigned long> scthm;
 16 | extern vector<Ccutpoint> cutpoints;
 17 | extern void Sort();
 18 | extern string GetACut(char *argv, fstream::pos_type position, int begin, int end);
 19 | 
 20 | BlasrAdapter::BlasrAdapter(int threshold, char * contigfile, string outpath) :
 21 | 		mythreshold(threshold), contigfilename(contigfile),outputpath(outpath)
 22 | {
 23 | }
 24 | 
 25 | bool BlasrAdapter::ReSortCutPoints()
 26 | {
 27 | 	sort(cutpoints.begin(), cutpoints.end(), ComFileindex);
 28 | 	return true;
 29 | }
 30 | 
 31 | bool BlasrAdapter::RunAdapter(std::ifstream &blasrfile)
 32 | {
 33 | 	ChangeCutPoints();
 34 | 	ChangeCutPointsReverse();
 35 | 	ReSortCutPoints();
 36 | 	GetNewBlasrFile(blasrfile);
 37 | 	Sort();
 38 | 	return true;
 39 | }
 40 | 
 41 | bool BlasrAdapter::ChangeCutPoints()
 42 | {
 43 | 	Ccutpoint standard = cutpoints.front();
 44 | 	vector<Ccutpoint>::iterator it;
 45 | 	int temp;
 46 | 	for (it = cutpoints.begin() + 1; it != cutpoints.end(); it++)
 47 | 	{
 48 | 		if (it->contigname.compare(standard.contigname) != 0)
 49 | 		{
 50 | 			standard = (*it);
 51 | 		}
 52 | 		else
 53 | 		{
 54 | 			temp = it->index - standard.index;
 55 | 			if (temp != 0)
 56 | 			{
 57 | 				if (it->ishead == standard.ishead && temp <= mythreshold)
 58 | 				{
 59 | 					it->index = it->index - temp;
 60 | 					it->changed = true;
 61 | 				}
 62 | 				else if (it->ishead == standard.ishead) //it->index - standard.index > mythreshold
 63 | 				{
 64 | 					standard = (*it);
 65 | 				}
 66 | 			}
 67 | 		}
 68 | 	}
 69 | 	return true;
 70 | }
 71 | 
 72 | bool BlasrAdapter::ChangeCutPointsReverse()
 73 | {
 74 | 	Ccutpoint standard = cutpoints.back();
 75 | 	vector<Ccutpoint>::iterator it;
 76 | 	int temp;
 77 | 	for (it = cutpoints.end() - 2; it != cutpoints.begin() - 1; it--)
 78 | 	{
 79 | 		if (it->contigname.compare(standard.contigname) != 0)
 80 | 		{
 81 | 			standard = (*it);
 82 | 		}
 83 | 		else
 84 | 		{
 85 | 			temp = standard.index - it->index;
 86 | 			if (temp != 0)
 87 | 			{
 88 | 				if (it->ishead == standard.ishead && temp <= mythreshold)
 89 | 				{
 90 | 					it->index = it->index + temp;
 91 | 					it->changed = true;
 92 | 				}
 93 | 				else if (it->ishead == standard.ishead) //it->index - standard.index > mythreshold
 94 | 				{
 95 | 					standard = (*it);
 96 | 				}
 97 | 			}
 98 | 		}
 99 | 	}
100 | 	return true;
101 | }
102 | 
103 | bool BlasrAdapter::GetNewBlasrFile(std::ifstream &blasrfile)
104 | {
105 | 	ofstream newblasrfile;
106 | 	newblasrfile.open((outputpath).c_str());
107 | 	if (!newblasrfile.is_open())
108 | 	{
109 | 		cout << "Fail to create AdaptedBlasrResult.m5";
110 | 		return false;
111 | 	}
112 | 	blasrfile.clear();
113 | 	blasrfile.seekg(0, ios::beg);
114 | 
115 | 	char c;
116 | 	int counter = 0;
117 | 	int i = 0;
118 | 	int lrheadindex_old;
119 | 	int lrtailindex_old;
120 | 	int ctheadindex_old;
121 | 	int cttailindex_old;
122 | 	char contigstrand;
123 | 	bool headflag;
124 | 	bool tailflag;
125 | 	bool newheadflag;
126 | 	bool newtailflag;
127 | 	string longreadname;
128 | 	string lralinseq;
129 | 	string matchseq;
130 | 	string ctalinseq;
131 | 	string tempstr;
132 | 	blasrfile >> longreadname;
133 | 	newblasrfile << longreadname;
134 | 	if (lrhm.find(longreadname) == lrhm.end())
135 | 	{
136 | 		unsigned long p;
137 | 		if ((p = longreadname.rfind('/')) != string::npos)
138 | 		{
139 | 			longreadname = longreadname.substr(0, p);
140 | 		}
141 | 	}
142 | 	while (c = blasrfile.get(), blasrfile)
143 | 	{
144 | 		newblasrfile.put(c);
145 | 		if (c == ' ')
146 | 		{
147 | 			++counter;
148 | 			if (counter == 2)
149 | 			{
150 | 				blasrfile >> lrheadindex_old;
151 | 				if (lrheadindex_old == 0)
152 | 				{
153 | 					headflag = true;
154 | 				}
155 | 				newblasrfile << lrheadindex_old;
156 | 			}
157 | 			else if (counter == 3)
158 | 			{
159 | 				blasrfile >> lrtailindex_old;
160 | 				if (lrtailindex_old == lrhm[longreadname].length)
161 | 				{
162 | 					tailflag = true;
163 | 				}
164 | 				newblasrfile << lrtailindex_old;
165 | 			}
166 | 			else if (counter == 8)
167 | 			{
168 | 				blasrfile >> ctheadindex_old;
169 | 				if (cutpoints[i].changed)
170 | 				{
171 | 					newblasrfile << cutpoints[i].index;
172 | 
173 | 				}
174 | 				else
175 | 				{
176 | 					newblasrfile << ctheadindex_old;
177 | 				}
178 | 			}
179 | 			else if (counter == 9)
180 | 			{
181 | 				blasrfile >> cttailindex_old;
182 | 				if (cutpoints[i + 1].changed)
183 | 				{
184 | 					newblasrfile << cutpoints[i + 1].index + 1;
185 | 				}
186 | 				else
187 | 				{
188 | 					newblasrfile << cttailindex_old;
189 | 				}
190 | 			}
191 | 			else if (counter == 10)
192 | 			{
193 | 				blasrfile >> contigstrand;
194 | 				newblasrfile << contigstrand;
195 | 			}
196 | 			else if (counter == 17)
197 | 			{
198 | 				blasrfile >> lralinseq;
199 | 				if (contigstrand == '+')
200 | 				{
201 | 					if (cutpoints[i].changed)
202 | 					{
203 | 						for (int j = ctheadindex_old; j > cutpoints[i].index; j--)
204 | 						{
205 | 							newblasrfile << '-';
206 | 						}
207 | 					}
208 | 					newblasrfile << lralinseq;
209 | 					if (cutpoints[i + 1].changed)
210 | 					{
211 | 						for (int j = cttailindex_old - 1; j < cutpoints[i + 1].index; j++)
212 | 						{
213 | 							newblasrfile << '-';
214 | 						}
215 | 					}
216 | 				}
217 | 				else
218 | 				{
219 | 					if (cutpoints[i + 1].changed)
220 | 					{
221 | 						for (int j = cttailindex_old - 1; j < cutpoints[i + 1].index; j++)
222 | 						{
223 | 							newblasrfile << '-';
224 | 						}
225 | 					}
226 | 					newblasrfile << lralinseq;
227 | 					if (cutpoints[i].changed)
228 | 					{
229 | 						for (int j = ctheadindex_old; j > cutpoints[i].index; j--)
230 | 						{
231 | 							newblasrfile << '-';
232 | 						}
233 | 					}
234 | 				}
235 | 			}
236 | 			else if (counter == 18)
237 | 			{
238 | 				blasrfile >> matchseq;
239 | 				if (contigstrand == '+')
240 | 				{
241 | 					if (cutpoints[i].changed)
242 | 					{
243 | 						for (int j = ctheadindex_old; j > cutpoints[i].index; j--)
244 | 						{
245 | 							newblasrfile << '*';
246 | 						}
247 | 					}
248 | 					newblasrfile << matchseq;
249 | 					if (cutpoints[i + 1].changed)
250 | 					{
251 | 						for (int j = cttailindex_old - 1; j < cutpoints[i + 1].index; j++)
252 | 						{
253 | 							newblasrfile << '*';
254 | 						}
255 | 					}
256 | 				}
257 | 				else
258 | 				{
259 | 					if (cutpoints[i + 1].changed)
260 | 					{
261 | 						for (int j = cttailindex_old - 1; j < cutpoints[i + 1].index; j++)
262 | 						{
263 | 							newblasrfile << '*';
264 | 						}
265 | 					}
266 | 					newblasrfile << matchseq;
267 | 					if (cutpoints[i].changed)
268 | 					{
269 | 						for (int j = ctheadindex_old; j > cutpoints[i].index; j--)
270 | 						{
271 | 							newblasrfile << '*';
272 | 						}
273 | 					}
274 | 				}
275 | 			}
276 | 			else if (counter == 19)
277 | 			{
278 | 				blasrfile >> ctalinseq;
279 | 				if (contigstrand == '+')
280 | 				{
281 | 					if (cutpoints[i].changed)
282 | 					{
283 | 						newblasrfile << GetACut(contigfilename, cthm[cutpoints[i].contigname].index, cutpoints[i].index, ctheadindex_old - 1);
284 | 					}
285 | 					newblasrfile << ctalinseq;
286 | 					if (cutpoints[i + 1].changed)
287 | 					{
288 | 						newblasrfile << GetACut(contigfilename, cthm[cutpoints[i + 1].contigname].index, cttailindex_old, cutpoints[i + 1].index);
289 | 					}
290 | 				}
291 | 				else
292 | 				{
293 | 					if (cutpoints[i + 1].changed)
294 | 					{
295 | 						tempstr = GetACut(contigfilename, cthm[cutpoints[i + 1].contigname].index, cttailindex_old, cutpoints[i + 1].index);
296 | 						reverse(tempstr.begin(), tempstr.end());
297 | 						string::iterator it;
298 | 						for (it = tempstr.begin(); it != tempstr.end(); it++)
299 | 						{
300 | 							if ((*it) == 'A')
301 | 								(*it) = 'T';
302 | 							else if ((*it) == 'T')
303 | 								(*it) = 'A';
304 | 							else if ((*it) == 'G')
305 | 								(*it) = 'C';
306 | 							else if ((*it) == 'C')
307 | 								(*it) = 'G';
308 | 							else if ((*it) == 'a')
309 | 								(*it) = 't';
310 | 							else if ((*it) == 't')
311 | 								(*it) = 'a';
312 | 							else if ((*it) == 'g')
313 | 								(*it) = 'c';
314 | 							else if ((*it) == 'c')
315 | 								(*it) = 'g';
316 | 						}
317 | 						newblasrfile << tempstr;
318 | 					}
319 | 					newblasrfile << ctalinseq;
320 | 					if (cutpoints[i].changed)
321 | 					{
322 | 						tempstr = GetACut(contigfilename, cthm[cutpoints[i].contigname].index, cutpoints[i].index, ctheadindex_old - 1);
323 | 						reverse(tempstr.begin(), tempstr.end());
324 | 						string::iterator it;
325 | 						for (it = tempstr.begin(); it != tempstr.end(); it++)
326 | 						{
327 | 							if ((*it) == 'A')
328 | 								(*it) = 'T';
329 | 							else if ((*it) == 'T')
330 | 								(*it) = 'A';
331 | 							else if ((*it) == 'G')
332 | 								(*it) = 'C';
333 | 							else if ((*it) == 'C')
334 | 								(*it) = 'G';
335 | 							else if ((*it) == 'a')
336 | 								(*it) = 't';
337 | 							else if ((*it) == 't')
338 | 								(*it) = 'a';
339 | 							else if ((*it) == 'g')
340 | 								(*it) = 'c';
341 | 							else if ((*it) == 'c')
342 | 								(*it) = 'g';
343 | 						}
344 | 						newblasrfile << tempstr;
345 | 					}
346 | 				}
347 | 			}
348 | 		}
349 | 		else if (c == '\n')
350 | 		{
351 | 			blasrfile >> longreadname;
352 | 			if (!blasrfile)
353 | 				break;
354 | 			newblasrfile << longreadname;
355 | 			if (lrhm.find(longreadname) == lrhm.end())
356 | 			{
357 | 				unsigned long p;
358 | 				if ((p = longreadname.rfind('/')) != string::npos)
359 | 				{
360 | 					longreadname = longreadname.substr(0, p);
361 | 				}
362 | 			}
363 | 			counter = 0;
364 | 			i += 2;
365 | 			headflag = false;
366 | 			tailflag = false;
367 | 		}
368 | 	}
369 | 	newblasrfile.close();
370 | 	return true;
371 | }
372 | 


--------------------------------------------------------------------------------
/thirdparty/SeqChunker-sed:
--------------------------------------------------------------------------------
  1 | #!/bin/bash
  2 | 
  3 | # Author: Thomas Hackl, thomas.hackl@uni-wuerzburg.de
  4 | #         Simon Pfaff, simon.pfaff@stud-mail.uni-wuerzburg.de
  5 | # Last Modified: Simon Pfaff, 2014-04-08
  6 | # Version 0.12
  7 | 
  8 | ## TODO
  9 | ## CHUNK_STEP, e.g. 1~2 for only odds
 10 | ## split prefix multiple files
 11 | 
 12 | # Variable defaults
 13 | DEBUG=1;
 14 | CHUNK_NUM=0;
 15 | CHUNK_SIZE_BRUTTO=0;
 16 | CHUNK_SKIP=0;
 17 | CHUNK_COUNT=0;
 18 | CHUNK_RATIO=0;
 19 | SKIP=0;
 20 | OUT="";
 21 | OUT_SPLIT=0;
 22 | MAX_BLOCK_SIZE=1000000;
 23 | CHUNK_STEP=1;
 24 | CHUNK_STEP_NUM=1;
 25 | CHUNK_FIRST=1;
 26 | CHUNK_LAST=0;
 27 | CHUNK_END=0;
 28 | 
 29 | show_usage()
 30 | {
 31 |     echo "Usage: SeqChunker [OPTIONS ...] FILE1 FILE2 ..."
 32 | }
 33 | 
 34 | show_help()
 35 | {
 36 |     show_usage
 37 |     cat <<EOF
 38 | 
 39 | SeqChunker efficiently samples and outputs chunks from FASTA and FASTQ files, 
 40 |  e.g. to serve as on-the-fly input for other tools. 
 41 | 
 42 |   -n/--chunk-number       Number of chunks to be sampled [$CHUNK_NUM]
 43 |   -r/--chunk-ratio        Fraction of each chunk to  actually be outputted [$CHUNK_RATIO]
 44 |   -x/--chunk-step         Output a chunk every -x chunks [$CHUNK_STEP]
 45 |   -y/--chunks-per-step    Output -y chunks every -x chunks. Cannot be greater 
 46 |                           than -x [$CHUNK_STEP_NUM]
 47 |   -f/--first-chunk        Skip chunks before this chunk [$CHUNK_FIRST]
 48 |   -l/--last-chunk         Last after this chunk (including skipped ones) [$CHUNK_LAST]
 49 |   -q/--quiet              Suppress non-critical messages
 50 |   -d/--debug              Output more verbose messages
 51 |   -h/--help               Show this help screen
 52 | 
 53 | NOTE: This is just a hotfix alternative while the real SeqChunker undergoes maintenance.
 54 | 
 55 | NOTE: Do not use chunk sizes similar to or smaller than the expected record 
 56 |  sizes. Heuristics get messed up and cause unpredictable behaviour.
 57 | 
 58 | NOTE: Backfall SeqChunker (uses only sed) for old Bash versions, only use FASTQ! Can not handle --chunk-size --chunk-number. 
 59 | 
 60 | EOF
 61 | }
 62 | 
 63 | # Execute getopt
 64 | ARGS=`getopt --name "SeqChunker-sed" \
 65 |     --options "n:s:e:k:c:r:x:y:f:l:b:m:o:qdh" \
 66 |     --longoptions "chunk-number:,chunk-size:,chunk-end:,chunk-count:,chunk-skip:,chunk-ratio:,chunk-step:,chunks-per-step:,first-chunk:,last-chunk:,byte-skip:,out:,quiet,debug,help" \
 67 |     -- "$@"`
 68 | 
 69 | #Bad arguments
 70 | [ $? -ne 0 ] && exit 1;
 71 | 
 72 | # A little magic
 73 | eval set -- "$ARGS"
 74 | 
 75 | # Now go through all the options
 76 | while true; do
 77 |     case "$1" in
 78 |         -n|--chunk-number)
 79 |             [ ! -n "$2" ] && (echo "$1: value required" 1>&2 && exit 1);
 80 |             if [[ "$2" =~ [^0-9] ]]; then
 81 |                 echo "$1: '$2' not an INT" 1>&2;
 82 |                 exit 1;
 83 |             fi;
 84 |             CHUNK_NUM=$2;
 85 |             shift 2;;
 86 | 
 87 |         -s|--chunk-size)
 88 |             [ ! -n "$2" ] && (echo "$1: value required" 1>&2 && exit 1);
 89 |             if [[ "$2" =~ [^0-9] ]]; then
 90 |                 echo "$1: '$2' not an INT" 1>&2;
 91 |                 exit 1;
 92 |             fi;
 93 |             CHUNK_SIZE_BRUTTO=$2
 94 |             shift 2;;
 95 | 
 96 |         -e|--chunk-end)
 97 |             [ ! -n "$2" ] && (echo "$1: value required" 1>&2 && exit 1);
 98 |             if [[ "$2" =~ [^0-9] ]]; then
 99 |                 echo "$1: '$2' not an INT" 1>&2;
100 |                 exit 1;
101 |             fi;
102 |             CHUNK_END=$2;
103 |             shift 2;;
104 | 
105 |         -k|--chunk-skip)
106 |             [ ! -n "$2" ] && (echo "$1: value required" 1>&2 && exit 1);
107 |             if [[ "$2" =~ [^0-9~] ]] ; then
108 |                 echo "$1: '$2' not a valid input" 1>&2;
109 |                 exit 1;
110 |             fi;
111 |             if [[ "$2" =~ ~ ]]; then 
112 | 	        CHUNK_SKIP=${2%%~*};
113 | 	        CHUNK_STEP=${2##*~};
114 |             else
115 |                 CHUNK_SKIP=$2;
116 |                 CHUNK_STEP=1;
117 |             fi;
118 |             shift 2;;
119 | 
120 | 	-f|--first-chunk)
121 |             [ ! -n "$2" ] && (echo "$1: value required" 1>&2 && exit 1);
122 |             if [[ "$2" =~ [^0-9] ]]; then
123 |                 echo "$1: '$2' not an INT" 1>&2;
124 |                 exit 1;
125 |             fi;
126 |             CHUNK_FIRST=$2;
127 |             shift 2;;
128 | 
129 | 	-l|--last-chunk)
130 |             [ ! -n "$2" ] && (echo "$1: value required" 1>&2 && exit 1);
131 |             if [[ "$2" =~ [^0-9] ]]; then
132 |                 echo "$1: '$2' not an INT" 1>&2;
133 |                 exit 1;
134 |             fi;
135 |             CHUNK_LAST=$2
136 |             shift 2;;
137 | 
138 |         -c|--chunk-count)
139 |             [ ! -n "$2" ] && (echo "$1: value required" 1>&2 && exit 1);
140 |             if [[ "$2" =~ [^0-9] ]]; then
141 |                 echo "$1: '$2' not an INT" 1>&2;
142 |                 exit 1;
143 |             fi;
144 |             CHUNK_COUNT=$2
145 |             shift 2;;
146 | 
147 |         -r|--chunk-ratio)
148 |             [ ! -n "$2" ] && (echo "$1: value required" 1>&2 && exit 1);
149 |             if [[ "$2" =~ [^0-9.] ]]; then
150 |                 echo "$1: '$2' not a FLOAT" 1>&2;
151 |                 exit 1;
152 |             fi;
153 | 	    CHUNK_RATIO=$2
154 | 	    shift 2;;
155 | 
156 |         -x|--chunk-step)
157 |             [ ! -n "$2" ] && (echo "$1: value required" 1>&2 && exit 1);
158 |             if [[ "$2" =~ [^0-9] ]]; then
159 |                 echo "$1: '$2' not an INT" 1>&2;
160 |                 exit 1;
161 |             fi;
162 |             CHUNK_STEP=$2
163 |             shift 2;;
164 | 
165 | 	-y|--chunks-per-step)
166 |             [ ! -n "$2" ] && (echo "$1: value required" 1>&2 && exit 1);
167 |             if [[ "$2" =~ [^0-9] ]]; then
168 |                 echo "$1: '$2' not an INT" 1>&2;
169 |                 exit 1;
170 |             fi;
171 |             CHUNK_STEP_NUM=$2
172 |             shift 2;;
173 | 
174 |         -o|--out)
175 |             [ ! -n "$2" ] && (echo "$1: value required" 1>&2 && exit 1);
176 |             OUT=$2
177 |             shift 2;;
178 |         
179 |         -m|--max-block-size)
180 |             [ ! -n "$2" ] && (echo "$1: value required" 1>&2 && exit 1);
181 |             if [[ "$2" =~ [^0-9] ]]; then
182 |                 echo "$1: '$2' not an INT" 1>&2;
183 |                 exit 1;
184 |             fi;
185 |             MAX_BLOCK_SIZE=$2
186 |             shift 2;;
187 | 
188 |         -q|--quiet)
189 |             DEBUG=0
190 |             shift;;
191 | 
192 |         -d|--debug)
193 |             DEBUG=2
194 |             shift;;
195 |         
196 | 	-h|--help)
197 | 	    show_help && exit 0;;
198 | 
199 | 	-V|--version)
200 |             grep -m1 'Version' "$0" | sed 's/.*Version\s*//';
201 |             exit 0;;
202 | 
203 |         --)
204 |             shift
205 |             break;;
206 |         *)
207 |             echo "$1: Unknown option" 1>&2 && exit 1;;
208 |     esac
209 | done
210 | 
211 | if [ $# -eq 0 ]; then
212 |     echo "FILE required" 1>&2;
213 |     show_usage;
214 |     exit 1
215 | fi;
216 | 
217 | 
218 | # default output file descriptor
219 | if [ -z "$OUT" ] || [ "$OUT" == '-' ]; then
220 |     exec 3>&1;
221 | else
222 |     if [[ $OUT =~ % ]]; then
223 |         # dynamic descriptor for every chunk, no default ...
224 |         OUT_SPLIT=1;
225 |     else
226 |         eval "exec 3>$OUT";
227 |     fi;
228 | fi;
229 | 
230 | 
231 | 
232 | ##SeqChunker.sed-only can not handle CHUNK_SIZE & CHUNK_NUMBER - there are no Chunks, only Records
233 | if [ "$CHUNK_SIZE_BRUTTO" -gt 0 ]; then
234 |     echo "Backfall SeqChunker can not handle chunk size" 1>&2
235 |     echo "Set chunksize to 0" 1>&2
236 |     CHUNK_SIZE_BRUTTO=0
237 | fi;
238 | 
239 | 
240 | if [ "$CHUNK_NUM" -gt 0 ]; then
241 |     echo "Backfall SeqChunker can not handle chunk number" 1>&2
242 |     echo "Set chunk number to 0" 1>&2
243 |     CHUNK_NUM=0
244 | fi;
245 | 
246 | ## 1.03 need no chunk-ratio
247 | #if [ "$CHUNK_RATIO" -gt 0 ] &&  [[ "$CHUNK_STEP" -gt 0 || "$CHUNK_STEP_NUM" -gt 0  ]]; then
248 | #    echo "either --chunk-ratio or --chunk-step and --chunk-per-step, not both allowed" 1>&2
249 | #    exit 1
250 | #fi;
251 | 
252 | if [[ "$CHUNK_END" -gt 0 && "$CHUNK_LAST" -gt 0 ]]; then
253 |     echo "either --chunk-end or --last-chunk, not both" 1>&2
254 |     exit 1
255 | fi;
256 | 
257 | 
258 | for FILE in $@; do
259 | 
260 |     ##determine format (FASTA/FASTQ)
261 |     IS_FASTQ=$(sed -n '
262 | 1{
263 | s/^[^>@].*/!/
264 | t finished
265 | s/^>.*/0/
266 | t finished
267 | s/^@.*/1/
268 | :finished
269 | p;q
270 | }' "$FILE");
271 |     if [[ $IS_FASTQ == !* ]]; then
272 |         echo "FILE neither FASTA nor FASTQ" 1>&2
273 |         exit 1
274 |     fi;
275 | done;
276 | 
277 | ##Is FASTA
278 | if [ "$IS_FASTQ" -eq 0 ]; then
279 |     echo "Backfall SeqChunker needs FASTQ" 1>&2
280 |     exit 1
281 | 
282 | else
283 |     ##Is FASTQ
284 | 
285 |     RECORD_STEP=$(($CHUNK_STEP * 4));                #4 lines, cause of FASTQ
286 |     NUMBER_NS=$((($CHUNK_STEP_NUM * 4) - 1));        #additional 3 line -> 4 Lines
287 |     NS=$(printf 'N;%0.s' $(seq 1 $NUMBER_NS));       #prints 3 Ns for every Chunkstep
288 |     RECORD_FIRST=$(( 1 + ( 4 * ($CHUNK_FIRST - 1)) ));
289 |     LAST_RECORD=$(($CHUNK_LAST * 4 ));
290 | 
291 |     RECORD_NUMBER=$((($RECORD_FIRST+(($CHUNK_STEP*($CHUNK_END * 4)) / $CHUNK_STEP_NUM) - $RECORD_STEP + ($NUMBER_NS + 1)-1)));
292 |     if [ "$CHUNK_END" -gt 0 ]; then
293 |         SED=''$RECORD_FIRST'~'$RECORD_STEP'{'$NS'p;'$RECORD_NUMBER'Q}';    #e.g. SED = '1~12{N;N;N;p;21Q}'
294 |     elif [ "$LAST_RECORD" -gt 0 ]; then
295 |         SED=''$RECORD_FIRST'~'$RECORD_STEP'{'$NS'p;'$LAST_RECORD'Q}';
296 |     else 
297 |         SED=''$RECORD_FIRST'~'$RECORD_STEP'{'$NS'p}';
298 |     fi;
299 | 
300 | 
301 |     for FILE in $@; do
302 | 	sed -ne "$SED" < "$FILE"
303 |     done;
304 |     #echo $SED
305 | fi;
306 | 
307 | 
308 | 
309 | ## OLD CODE - only fix Ratios
310 | #if [ $CHUNK_RATIO == 0.2 ]; then
311 | #   SED='1~20{N;N;N;p}' # 4/20 == 0.2
312 | #   for FILE in $@; do
313 | #       sed -ne "$SED" < "$FILE"
314 | #   done;
315 | #elif [ $CHUNK_RATIO == 0.3 ]; then
316 | #   SED='1~40{N;N;N;N;N;N;N;N;N;N;N;p}' # 12/40 == 0.3
317 | #   for FILE in $@; do
318 | #       sed -ne "$SED" < "$FILE"
319 | #   done;
320 | #elif [ $CHUNK_RATIO == 0.5 ]; then
321 | #   SED='1~8{N;N;N;p}' # 4/8 == 0.5
322 | #   for FILE in $@; do
323 | #       sed -ne "$SED" < "$FILE"
324 | #   done;
325 | #else
326 | #   cat $@
327 | #fi;
328 | 


--------------------------------------------------------------------------------
/runHALC.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | 
  3 | import os
  4 | import argparse
  5 | import shutil
  6 | import datetime
  7 | 
  8 | parser = argparse.ArgumentParser(description='runHALC.py')
  9 | parser.add_argument('long_read_path', metavar='long_read.fa', help="The path to long_read.fa")
 10 | parser.add_argument('contig_path', metavar='contig.fa', help="The path to contig.fa")
 11 | parser.add_argument("-o", "--ordinary",help="Ordinary mode utilizing repeats to make correction. The error correction software LoRDEC and the initial short reads are required to refine the repeat corrected regions. It is exclusive with the -repeat-free option.(yes)")
 12 | parser.add_argument('-r', "--repeat-free",dest='repeatfree', help="Repeat-free mode without utilizing repeats to make correction. It is exclusive with the -ordinary option.(no)", action='store_true', default=False)
 13 | parser.add_argument('-b', '--boundary', type=int, help="Maximum boundary difference to split the subcontigs.(4)", default=4)
 14 | parser.add_argument('-a', '--accurate',  help="Accurate construction of the contig graph.(DEPRECATED)(yes)", action='store_true', default=True)
 15 | parser.add_argument('-c', '--coverage',  help="Expected long read coverage. If not specified, it can be automatically calculated(auto).", type=int)
 16 | parser.add_argument('-w', '--width', help="Maximum width of the dynamic programming table.(4)", type=int, default=4)
 17 | parser.add_argument('-k', '--kmer', help="Kmer length for LoRDEC refinement.(25)", default=25, type=int)
 18 | parser.add_argument('-t', '--threads', help="Number of threads for one process to create. It is automatically set to the number of computing cores.(auto)", type=int)
 19 | parser.add_argument('-l', '--log', help="System log to print.(no)", action='store_true', default=False)
 20 | args = parser.parse_args()
 21 | 
 22 | # Default Parameters#####################################
 23 | temp_dir = './temp'
 24 | output_dir = './output'
 25 | prefix = 'HALC'
 26 | repeat_free_mode = False
 27 | 
 28 | if os.path.exists(temp_dir + '/step1'):
 29 | 	if os.path.exists(temp_dir + '/step2'):
 30 | 		if os.path.exists(temp_dir + '/step3'):
 31 | 			if os.path.exists(temp_dir + '/step4'):
 32 | 				if os.path.exists(temp_dir + '/step5'):
 33 | 					print 'WARNING: ' + temp_dir + '/step5' + ' was found. Automatically started from step5'
 34 | 					shutil.rmtree(temp_dir + '/step5')
 35 | 					start_from_step = 5
 36 | 				else:
 37 | 					print 'WARNING: ' + temp_dir + '/step4' + ' was found. Automatically started from step4'
 38 | 					shutil.rmtree(temp_dir + '/step4')
 39 | 					start_from_step = 4
 40 | 			else:
 41 | 				print 'WARNING: ' + temp_dir + '/step3' + ' was found. Automatically started from step3'
 42 | 				shutil.rmtree(temp_dir + '/step3')
 43 | 				start_from_step = 3
 44 | 		else:
 45 | 			print 'WARNING: ' + temp_dir + '/step2' + ' was found. Automatically started from step2'
 46 | 			shutil.rmtree(temp_dir + '/step2')
 47 | 			start_from_step = 2
 48 | 	else:
 49 | 		print 'WARNING: ' + temp_dir + '/step1' + ' was found. Automatically started from step1'
 50 | 		shutil.rmtree(temp_dir + '/step1')
 51 | 		start_from_step = 1
 52 | else:
 53 | 	start_from_step = 1
 54 | # Parameters Analyzing###################################
 55 | long_read_path = args.long_read_path
 56 | contig_path = args.contig_path
 57 | 
 58 | if args.threads > 128 or args.threads < 1:
 59 | 	print 'ERROR: Argument -t/--threads  should be between 1 and 128'
 60 | 	exit(-1)
 61 | if args.kmer > 127 or args.kmer < 4:
 62 | 	print 'ERROR: Argument -k/--kmer  should be between 4 and 127'
 63 | 	exit(-1)
 64 | if args.width > 20 or args.width < 2:
 65 | 	print 'ERROR: Argument -w/--width  should be between 2 to 20'
 66 | 	exit(-1)
 67 | if args.coverage:
 68 | 	if args.coverage > 65535 or args.coverage < 1:
 69 | 		print 'ERROR: Argument -c/--coverage  should be between 1 and 65535'
 70 | 		exit(-1)
 71 | if args.boundary > 20 or args.boundary < 0:
 72 | 	print 'ERROR: Argument -b/--boundary  should be between 0 and 20'
 73 | 	exit(-1)
 74 | if args.ordinary and args.repeatfree:
 75 | 	print "ERROR: '-o' and '-r' cannot be set at the same time"
 76 | 	exit(-1)
 77 | if not (args.ordinary or args.repeatfree):
 78 | 	print "ERROR: Either '-o' or '-r' should be set"
 79 | 	exit(-1)
 80 | if args.ordinary:
 81 | 	short_read_path = args.ordinary
 82 | 	repeat_free_mode = False
 83 | elif args.repeatfree:
 84 | 	repeat_free_mode = True
 85 | 
 86 | start_time = datetime.datetime.now()
 87 | print start_time
 88 | 
 89 | # Step 1 chunk long reads#########################
 90 | if start_from_step <= 1:
 91 | 	print'''
 92 | /////STEP 1 STARTED//////////////////////////////////////////////////////////////////////////////////////////////////'''
 93 | 	if not os.path.exists(output_dir):
 94 | 		os.makedirs(output_dir)
 95 | 
 96 | 	if not os.path.exists(temp_dir):
 97 | 		os.makedirs(temp_dir)
 98 | 
 99 | 	if os.path.exists(temp_dir + '/step1'):
100 | 		print 'ERROR: ' + temp_dir + '/step1' + ' already exists, please delete it before running step 1'
101 | 		exit(-1)
102 | 	else:
103 | 		os.mkdir(temp_dir + '/step1')
104 | 
105 | 	Chunker_command = 'Chunker -s 200M -o ' + temp_dir + '/step1/pb-%03d.fa ' + long_read_path + ' 1>' + temp_dir + '/step1/SeqChunker.out ' + '2>' + temp_dir + '/step1/SeqChunker.err'
106 | 
107 | 	print 'Running command: ' + Chunker_command
108 | 	err = os.system(Chunker_command)
109 | 
110 | 	if err != 0:
111 | 		print 'ERROR: ' + 'Failed to run SeqChunker:' + os.strerror(err)
112 | 		exit(-1)
113 | 
114 | 	line_count = len(os.popen('ls -l ' + temp_dir + '/step1/ |grep pb-....fa').readlines())
115 | 	print str(line_count) + ' files created'
116 | 
117 | 	print '''
118 | /////STEP 1 DONE/////////////////////////////////////////////////////////////////////////////////////////////////////'''
119 | # Step 2 Run Blasr###############################
120 | if start_from_step <= 2:
121 | 	print '''
122 | /////STEP 2 STARTED//////////////////////////////////////////////////////////////////////////////////////////////////'''
123 | 	if not os.path.exists(output_dir):
124 | 		os.makedirs(output_dir)
125 | 
126 | 	if not os.path.exists(temp_dir):
127 | 		os.makedirs(temp_dir)
128 | 
129 | 	if os.path.exists(temp_dir + '/step2'):
130 | 		print 'ERROR: ' + temp_dir + '/step2' + ' already exists, please delete it before running step 2'
131 | 		exit(-1)
132 | 	else:
133 | 		os.mkdir(temp_dir + '/step2')
134 | 
135 | 	line_count = len(os.popen('ls -l ' + temp_dir + '/step1/ |grep pb-....fa').readlines())
136 | 	for i in range(1, line_count + 1):
137 | 		longread_name = 'pb-%03d.fa' % i
138 | 		outfile_name = 'blasrresult-%03d.m5' % i
139 | 		if repeat_free_mode:
140 | 			if args.threads:
141 | 				blasr_command = 'blasr ' + temp_dir + '/step1/' + longread_name + ' ' + contig_path + ' -m 5 --out ' + temp_dir + '/step2/' + outfile_name + ' --maxScore 2000 --minMatch 8 --minAlnLength 300 --nCandidates 30 --bestn 20 --nproc ' + str(args.threads)
142 | 			else:
143 | 				blasr_command = 'blasr ' + temp_dir + '/step1/' + longread_name + ' ' + contig_path + ' -m 5 --out ' + temp_dir + '/step2/' + outfile_name + ' --maxScore 2000 --minMatch 8 --minAlnLength 300 --nCandidates 30 --bestn 20 --nproc 4'
144 | 		else:
145 | 			if args.threads:
146 | 				blasr_command = 'blasr ' + temp_dir + '/step1/' + longread_name + ' ' + contig_path + ' -m 5 --out ' + temp_dir + '/step2/' + outfile_name + ' --maxScore 2000 --minMatch 8 --minAlnLength 15 --nCandidates 30 --bestn 20 --nproc ' + str(args.threads)
147 | 			else:
148 | 				blasr_command = 'blasr ' + temp_dir + '/step1/' + longread_name + ' ' + contig_path + ' -m 5 --out ' + temp_dir + '/step2/' + outfile_name + ' --maxScore 2000 --minMatch 8 --minAlnLength 15 --nCandidates 30 --bestn 20 --nproc 4'
149 | 		blasr_command += ' 1>' + temp_dir + '/step2/blasr_' + str(i) + '.out ' + '2>' + temp_dir + '/step2/blasr_' + str(i) + '.err'
150 | 
151 | 		print 'Running command: ' + blasr_command
152 | 		err = os.system(blasr_command)
153 | 		if err != 0:
154 | 			print 'ERROR: ' + 'Failed to run BLASR:' + os.strerror(err)
155 | 			exit(-1)
156 | 	print '''
157 | /////STEP 2 DONE/////////////////////////////////////////////////////////////////////////////////////////////////////'''
158 | # Step 3 Run HALC###############################
159 | if start_from_step <= 3:
160 | 	print '''
161 | /////STEP 3 STARTED//////////////////////////////////////////////////////////////////////////////////////////////////'''
162 | 	if not os.path.exists(output_dir):
163 | 		os.makedirs(output_dir)
164 | 
165 | 	if not os.path.exists(temp_dir):
166 | 		os.makedirs(temp_dir)
167 | 
168 | 	if os.path.exists(temp_dir + '/step3'):
169 | 		print 'ERROR: ' + temp_dir + '/step3' + ' already exists, please delete it before running step 3'
170 | 		exit(-1)
171 | 	else:
172 | 		os.mkdir(temp_dir + '/step3')
173 | 
174 | 	line_count = len(os.popen('ls -l ' + temp_dir + '/step1/ |grep pb-....fa').readlines())
175 | 	for i in range(1, line_count + 1):
176 | 		longread_name = 'pb-%03d' % i
177 | 		m5filename = 'blasrresult-%03d.m5' % i
178 | 		HALC_command = 'HALC ' + temp_dir + '/step2/' + m5filename + ' ' + contig_path + ' ' + temp_dir + '/step1/' + longread_name + '.fa -o ' + temp_dir + '/step3/' + ' -p ' + longread_name
179 | 		if args.threads:
180 | 			HALC_command += ' -t ' + str(args.threads)
181 | 		if args.coverage:
182 | 			HALC_command += ' -c ' + str(args.coverage)
183 | 		if args.repeatfree:
184 | 			HALC_command += ' -r'
185 | 		HALC_command += ' -b ' + str(args.boundary)
186 | 		HALC_command += ' -w ' + str(args.width)
187 | 		HALC_command += ' 1>' + temp_dir + '/step3/HALC_' + str(i) + '.out ' + '2>' + temp_dir + '/step3/HALC_' + str(i) + '.err'
188 | 		print 'Running command: ' + HALC_command
189 | 		err = os.system(HALC_command)
190 | 		if err != 0:
191 | 			print 'ERROR: ' + 'Failed to run HALC:' + os.strerror(err)
192 | 			exit(-1)
193 | 
194 | 	cat_command = 'cat ' + temp_dir + '/step3/' + '*.corrected.fa >' + temp_dir + '/step3/' + prefix + '.corrected.fa'
195 | 	err = os.system(cat_command)
196 | 	if err != 0:
197 | 		print 'ERROR: ' + 'Failed to combine corrected sequences:' + os.strerror(err)
198 | 		exit(-1)
199 | 
200 | 	if not repeat_free_mode:
201 | 		cat_command = 'cat ' + temp_dir + '/step3/' + '*.repeatused.fa >' + temp_dir + '/step3/' + prefix + '.repeatused.fa'
202 | 		err = os.system(cat_command)
203 | 		if err != 0:
204 | 			print 'ERROR: ' + 'Failed to combine corrected sequences:' + os.strerror(err)
205 | 			exit(-1)
206 | 
207 | 	if repeat_free_mode:
208 | 		cp_command = 'cp ' + temp_dir + '/step3/' + prefix + '.corrected.fa ' + output_dir
209 | 		err = os.system(cp_command)
210 | 		if err != 0:
211 | 			print 'ERROR: ' + 'Failed to copy corrected sequences to output directory' + os.strerror(err)
212 | 			exit(-1)
213 | 	print '''
214 | /////STEP 3 DONE/////////////////////////////////////////////////////////////////////////////////////////////////////'''
215 | 
216 | # Step 4 Run LoRDEC###############################
217 | if start_from_step <= 4 and not repeat_free_mode:
218 | 	print '''
219 | /////STEP 4 STARTED//////////////////////////////////////////////////////////////////////////////////////////////////'''
220 | 	if not os.path.exists(output_dir):
221 | 		os.makedirs(output_dir)
222 | 
223 | 	if not os.path.exists(temp_dir):
224 | 		os.makedirs(temp_dir)
225 | 
226 | 	if os.path.exists(temp_dir + '/step4'):
227 | 		print 'ERROR: ' + temp_dir + '/step4' + ' already exists, please delete it before running step 4'
228 | 		exit(-1)
229 | 	else:
230 | 		os.mkdir(temp_dir + '/step4')
231 | 
232 | 	LoRDEC_command = 'lordec-correct -2 ' + short_read_path + ' -s 3 -i ' + temp_dir + '/step3/' + prefix + '.repeatused.fa' + ' -o ' + temp_dir + '/step4/' + prefix + '.corrected.fa'
233 | 	LoRDEC_command += ' -k ' + str(args.kmer)
234 | 	LoRDEC_command += ' -T ' + str(args.threads)
235 | 	LoRDEC_command += ' 1>' + temp_dir + '/step4/LoRDEC.out ' + '2>' + temp_dir + '/step4/LoRDEC.err'
236 | 	print 'Running command: ' + LoRDEC_command
237 | 
238 | 	err = os.system(LoRDEC_command)
239 | 	if err != 0:
240 | 		print 'ERROR: ' + 'Failed to run LoRDEC:' + os.strerror(err)
241 | 		exit(-1)
242 | 
243 | 	cat_command = 'cat ' + temp_dir + '/step4/' + prefix + '.corrected.fa ' + temp_dir + '/step3/' + prefix + '.corrected.fa >' + output_dir + '/' + prefix + '.corrected.fa'
244 | 	err = os.system(cat_command)
245 | 	if err != 0:
246 | 		print 'ERROR: ' + 'Failed to combine corrected sequences:' + os.strerror(err)
247 | 		exit(-1)
248 | 
249 | 	print '''
250 | /////STEP 4 DONE/////////////////////////////////////////////////////////////////////////////////////////////////////'''
251 | 
252 | # Step 5 Trim+Split###############################
253 | if start_from_step <= 5:
254 | 	print '''
255 | /////STEP 5 STARTED//////////////////////////////////////////////////////////////////////////////////////////////////'''
256 | 	if not os.path.exists(output_dir):
257 | 		os.makedirs(output_dir)
258 | 
259 | 	if not os.path.exists(temp_dir):
260 | 		os.makedirs(temp_dir)
261 | 
262 | 	if os.path.exists(temp_dir + '/step5'):
263 | 		print 'ERROR: ' + temp_dir + '/step5' + ' already exists, please delete it before running step 5'
264 | 		exit(-1)
265 | 	else:
266 | 		os.mkdir(temp_dir + '/step5')
267 | 
268 | 	trim_command = 'Trimmer -i ' + output_dir + '/' + prefix + '.corrected.fa' + ' -o ' + output_dir + '/' + prefix + '.trim.fa'
269 | 	trim_command += ' 1>' + temp_dir + '/step5/trim.out ' + '2>' + temp_dir + '/step5/trim.err'
270 | 	print 'Running command: ' + trim_command
271 | 	err = os.system(trim_command)
272 | 	if err != 0:
273 | 		print 'ERROR: ' + 'Failed to trim corrected sequences:' + os.strerror(err)
274 | 		exit(-1)
275 | 
276 | 	split_command = 'Splitter -i ' + output_dir + '/' + prefix + '.corrected.fa' + ' -o ' + output_dir + '/' + prefix + '.split.fa'
277 | 	split_command += ' 1>' + temp_dir + '/step5/split.out ' + '2>' + temp_dir + '/step5/split.err'
278 | 	print 'Running command: ' + split_command
279 | 	err = os.system(split_command)
280 | 	if err != 0:
281 | 		print 'ERROR: ' + 'Failed to split corrected sequences:' + os.strerror(err)
282 | 		exit(-1)
283 | 	print '''
284 | /////Finished!!! Results are stored in output folder/////////////////////////////////////////////////////////////////'''
285 | 
286 | end_time = datetime.datetime.now()
287 | print end_time
288 | time_cost = end_time - start_time
289 | print 'Running time: ' + str(time_cost)
290 | 
291 | 


--------------------------------------------------------------------------------
/thirdparty/SeqChunker-dd:
--------------------------------------------------------------------------------
  1 | #!/bin/bash
  2 | 
  3 | ##----------------------------------------------------------------------------##
  4 | # Author: Thomas Hackl, thomas.hackl@uni-wuerzburg.de
  5 | # Last Modified: Thomas Hackl, 2013-08-20
  6 | # Version 0.14
  7 | 
  8 | 
  9 | ##----------------------------------------------------------------------------##
 10 | ## TODO
 11 | # BUGS
 12 | #  non known
 13 | 
 14 | # FEATURES
 15 | #  stdin - problem: cannot autocompute...
 16 | #  rand - if possible with seed to allow for consistent recomputation
 17 | 
 18 | ##----------------------------------------------------------------------------##
 19 | ## CHANGELOG
 20 | 
 21 | ## 0.14
 22 | # BugFix: stable chunk size which allow skipping by byte sizes only
 23 | #  ADJUST_CHUNK_SIZE=${#SED} + 1 : bash variables dont store trailing newlines, 
 24 | #  missing bytes
 25 | # BugFix: --version now also documented feature
 26 | # Feature: Fatal exception if CHUNK_SIZE_REAL < 0 => chunk size to small for 
 27 | #  given records
 28 | 
 29 | ## 0.13
 30 | # major refactoring to better support stepwise chunk maths and FASTA/FASTQ
 31 | # BugFix: empty newline at eof of last chunk
 32 | # Feature: --version
 33 | # added tests (test.sh)
 34 | 
 35 | ## 0.12
 36 | # removed chunk-ratio
 37 | # reverted to integer arithmetics
 38 | # autocompute chunk number
 39 | 
 40 | ## 0.11
 41 | # CHUNK_STEP: experimental
 42 | # LC_ALL instead of LC_NUMERIC
 43 | 
 44 | ## 0.10
 45 | # initial script
 46 | 
 47 | 
 48 | 
 49 | 
 50 | 
 51 | ##----------------------------------------------------------------------------##
 52 | # Variable defaults
 53 | DEBUG=1;
 54 | 
 55 | CHUNK_NUM=0;
 56 | 
 57 | CHUNK_SIZE=0;
 58 | CHUNK_FIRST=1
 59 | CHUNK_LAST=0
 60 | CHUNK_STEP=1
 61 | CHUNK_STEP_NUM=1
 62 | 
 63 | OUT="";
 64 | OUT_SPLIT=0;
 65 | MAX_BLOCK_SIZE=1000000;
 66 | 
 67 | SED_FASTA='/^>/{p;Q}'; # with -n, prints only matching line with, 
 68 | #  else prints all previous lines + matching line
 69 | 
 70 | SED_FIRST_FASTQ='/^@/ { # if ^@
 71 |                 N # read a second line
 72 |                 /\n@/ { s/^@.*\n// ; N} # remove first line and read another
 73 |                 N;N;p;q; # read two more lines, print and quit
 74 |         }'
 75 | 
 76 | SED_LAST_FASTQ='/^@/ { # if ^@
 77 |                 N # read a second line
 78 |                 /\n@/ { N } # read additonal line if second is head
 79 |                 N;N;q; # read two more lines, print and quit
 80 |         }'
 81 | 
 82 | PRINTF_DEBUG="  %-10s %10s : '%s'\n";
 83 | 
 84 | ##----------------------------------------------------------------------------##
 85 | # Subs
 86 | 
 87 | show_usage()
 88 | {
 89 |     echo "Usage: SeqChunker --chunk-number/--chunk-size INT [OPTIONS ...] FILE1 FILE2 ..."
 90 | }
 91 | 
 92 | show_help()
 93 | {
 94 |     show_usage
 95 |     cat <<EOF
 96 | 
 97 | SeqChunker efficiently samples and outputs chunks from FASTA and FASTQ files, 
 98 |  e.g. to serve as on-the-fly input for other tools. 
 99 | 
100 | Required: Either one of the following, but not both at the same time. The unset
101 |  parameter is computed respectively.
102 | 
103 |   -n/--chunk-number       Total number of chunks to be sampled (including 
104 |                           skipped ones).
105 |   -s/--chunk-size         Size of chunks to be sampled. Supports suffixes "k,M,G".
106 |   
107 | Optional:
108 |    
109 |   -f/--first-chunk        Skip chunks before this chunk [$CHUNK_FIRST]
110 |   -l/--last-chunk         Last after this chunk (including skipped ones) [$CHUNK_LAST]
111 |   -x/--chunk-step         Output a chunk every -x chunks [$CHUNK_STEP]
112 |   -y/--chunks-per-step    Output -y chunks every -x chunks. Cannot be greater 
113 |                           than -x [$CHUNK_PER_STEP] 
114 | 
115 |   -o/--out                Output filename. To split chunks into individual files
116 |                           provide a "printf" style pattern, e.g. "chunk%02d.fa",
117 |                           with a substitution for the chunk counter.
118 |   -m/--max-block-size     Maximum size of blocks in output stream [$MAX_BLOCK_SIZE]
119 |   -q/--quiet              Suppress non-critical messages
120 |   -V/--version            Version of the script.
121 |   -d/--debug              Output more verbose messages
122 |   -h/--help               Show this help screen
123 | 
124 | NOTE: Chunk sizes need to be at least twice as great as the longest record in 
125 |  the file. Otherwise results will be inconsistent.
126 | 
127 | NOTE: Chunk related computations are run each input file individually.
128 | 
129 |   
130 |   # output only even numbered chunks
131 |   SeqChunker --chunk-size 5M --chunk-first 2 --chunk-step 2 my.fa
132 |   
133 |   # Split file in 100 individual files of similar size
134 |   SeqChunker --chunk-number 100 --out "my.%03d.fa" my.fa
135 | 
136 | EOF
137 | }
138 | 
139 | # hr_to_byte <bytes with suffix>
140 | # convert suffixed byte number to full byte number, e.g. 1k to 1024 (byte)
141 | hr_to_byte()
142 | {
143 |     if [[ $1 =~ ^[0-9]+$ ]] ; then
144 |         echo -n $1;    
145 |     elif [[ $1 =~ ^[0-9]+[kMG]$ ]] ; then
146 |         echo  ${1:-1} ${1: -1} | awk '{printf("%d", $1 * 1024^index("kMG", $2))}'
147 |     else
148 |         echo "'$1' not an valid input" 1>&2;
149 |         exit 1;
150 |     fi;
151 | }
152 | 
153 | # out_split_descriptor
154 | # relay file_descriptor '3' to new chunkwise outfile
155 | out_split_descriptor()
156 | {
157 |     OUT_CHUNK=$(printf $OUT $I);
158 |     eval "exec 3>$OUT_CHUNK";
159 | }
160 | 
161 | # chunk_out_core
162 | # copy a chunks core (fixed size) using dd to output
163 | chunk_core_out()
164 | {
165 |     # chunk core
166 |     CHUNK_SIZE_REAL=$(( $CHUNK_SIZE - $CHUNK_SIZE_ADJUST ))
167 |     if [ $CHUNK_SIZE_REAL -lt 0 ]; then
168 |         echo "--chunk-size to small for records" 1>&2;
169 |         exit 1;
170 |     fi;
171 |     
172 |     if [ $DEBUG -gt 1 ]; then
173 |     	printf "$PRINTF_DEBUG" "CHUNK" "[$CHUNK_SIZE]" "$(($CHUNK_SIZE_REAL + $CHUNK_SIZE_ADJUST)) = $CHUNK_SIZE_REAL + $CHUNK_SIZE_ADJUST" 1>&2;
174 |     	printf "$PRINTF_DEBUG" "CHUNK_REAL" "[$CHUNK_SIZE_REAL]" "--content hidden--"  1>&2;
175 |     fi;
176 |     
177 |     # one ore multiple dd blocks
178 |     if [ $CHUNK_SIZE_REAL -gt $MAX_BLOCK_SIZE ]; then
179 |         BLOCK_COUNT=$(( $CHUNK_SIZE_REAL / $MAX_BLOCK_SIZE ))
180 |         BLOCK_REST=$(( $CHUNK_SIZE_REAL % $MAX_BLOCK_SIZE ))
181 |         [ $DEBUG -gt 1 ] && (
182 |             echo BLOCK_COUNT=$BLOCK_COUNT 1>&2;
183 |             echo BLOCK_REST=$BLOCK_REST 1>&2;
184 |         )
185 | 
186 | 	dd bs=$MAX_BLOCK_SIZE count=$BLOCK_COUNT 1>&3 2>/dev/null;
187 | 	dd bs=$BLOCK_REST count=1 1>&3 2>/dev/null;
188 |         
189 |     elif [ $CHUNK_SIZE_REAL -gt 0 ]; then
190 |         
191 |         dd bs=$CHUNK_SIZE_REAL count=1 1>&3 2>/dev/null;
192 |         
193 |     fi;
194 | }
195 | 
196 | # chunk_skip_core
197 | # skip a chunks core (fixed size) using dd
198 | chunk_core_skip()
199 | {
200 |     # chunk core
201 |     CHUNK_SIZE_REAL=$(( $CHUNK_SIZE - $CHUNK_SIZE_ADJUST ))
202 |     if [ $CHUNK_SIZE_REAL -lt 0 ]; then
203 |         echo "--chunk-size to small for records" 1>&2;
204 |         exit 1;
205 |     fi;
206 |     
207 |     if [ $DEBUG -gt 1 ]; then
208 |     	printf "$PRINTF_DEBUG" "CHUNK" "[$CHUNK_SIZE]" "$(($CHUNK_SIZE_REAL + $CHUNK_SIZE_ADJUST)) = $CHUNK_SIZE_REAL + $CHUNK_SIZE_ADJUST" 1>&2; 
209 |     	printf "$PRINTF_DEBUG" "CHUNK_REAL" "[$CHUNK_SIZE_REAL]" "--content hidden--"  1>&2;
210 |     fi;
211 |     # one ore multiple dd blocks
212 |     if [ $CHUNK_SIZE_REAL -gt $MAX_BLOCK_SIZE ]; then
213 |         BLOCK_COUNT=$(( $CHUNK_SIZE_REAL / $MAX_BLOCK_SIZE ))
214 |         BLOCK_REST=$(( $CHUNK_SIZE_REAL % $MAX_BLOCK_SIZE ))
215 |         [ $DEBUG -gt 1 ] && (
216 |             echo BLOCK_COUNT=$BLOCK_COUNT 1>&2;
217 |             echo BLOCK_REST=$BLOCK_REST 1>&2;
218 |         )
219 | 
220 |         dd bs=$MAX_BLOCK_SIZE skip=$BLOCK_COUNT count=0 2>/dev/null;
221 |         dd bs=$BLOCK_REST skip=1 count=0 2>/dev/null;
222 |         
223 |     elif [ $CHUNK_SIZE_REAL -gt 0 ]; then
224 |         
225 |         dd bs=$CHUNK_SIZE_REAL skip=1 count=0 2>/dev/null;
226 |         
227 |     fi;
228 | }
229 | 
230 | # chunk_out_fasta
231 | # process a chunk in fasta format
232 | chunk_fasta()
233 | {
234 |     [ $DEBUG -gt 1 ] && echo "#$I" 1>&2;
235 |     
236 |     # relay file descriptor if necessary
237 |     [ $SKIP -eq 0 ] && [ $OUT_SPLIT -eq 1 ] && out_split_descriptor;
238 | 
239 |     # check cache
240 |     if [ ! -z "$HEAD" ]; then
241 |         if [ $DEBUG -gt 1 ]; then
242 |             printf "$PRINTF_DEBUG" "HEAD (c)" "[${#HEAD}]" "$HEAD"  1>&2;
243 |         fi;
244 | 	[ $SKIP -eq 0 ] && echo "$HEAD" 1>&3;    
245 |         
246 |         # get first record
247 |     else
248 |         SED=$(sed --unbuffered -e "$SED_FASTA");
249 | 
250 |         TAIL="${SED%>*}";
251 |         HEAD="${SED:${#TAIL}}";
252 |         CHUNK_SIZE_ADJUST=$(( ${#SED} + 1 )); # number of bytes read, but not outputted
253 | 
254 |         # eof
255 | 	if [ -z "$HEAD" ]; then
256 | 	    # sometimes last chunk is empty -> remove
257 | 	    if [ -n "$OUT_CHUNK" ] && [ ! -s "$OUT_CHUNK" ]; then
258 | 		$(rm "$OUT_CHUNK");
259 | 	    fi;
260 | 	    [ $DEBUG -gt 1 ] && echo "EOF" 1>&2;
261 | 	    return 1; # return on eof
262 | 	fi;
263 | 	
264 |         
265 |         [ $SKIP -eq 0 ] && echo "$HEAD" 1>&3;    
266 |         
267 | 	if [ $DEBUG -gt 1 ]; then
268 | 	    printf "$PRINTF_DEBUG" "SED (f)" "[${#SED}]" "$SED"  1>&2;
269 | 	    printf "$PRINTF_DEBUG" "TAIL (f)" "[${#TAIL}]" "$TAIL"  1>&2;
270 | 	    printf "$PRINTF_DEBUG" "HEAD (f)" "[${#HEAD}]" "$HEAD"  1>&2;
271 |     	fi;
272 |     fi;
273 |     
274 |     
275 |     # output core
276 |     if [ $SKIP -eq 1 ]; then
277 |         chunk_core_skip
278 |     else
279 |         chunk_core_out
280 |     fi;
281 |     
282 |     # handle the final record of the chunk - needs to be complete
283 |     # last record -search for start of next record
284 |     SED=$(sed --unbuffered -e "$SED_FASTA");
285 |     # this many bytes have been read, but not yet outputted
286 |     CHUNK_SIZE_ADJUST=$(( ${#SED} + 1 ));
287 |     
288 |     # remove next FASTA record identifier, if present
289 |     if [ -z "$SED" ]; then # eof with dd
290 |         HEAD=''
291 |         TAIL=''
292 |     else
293 |         TAIL="${SED%>*}";
294 |         if [ ${#TAIL} -eq ${#SED} ]; then # eof in sed
295 | 	    HEAD=''
296 |             [ $SKIP -eq 0 ] && echo "$TAIL" 1>&3; # echo tail with '\n'
297 | 	else
298 | 	    HEAD="${SED:${#TAIL}}";
299 | 	    [ $SKIP -eq 0 ] && echo -n "$TAIL" 1>&3; # echo tail "as is"
300 | 	fi;
301 |     fi;
302 |     
303 |     if [ $DEBUG -gt 1 ]; then
304 |         printf "$PRINTF_DEBUG" "SED" "[${#SED}]" "$SED"  1>&2;
305 |         printf "$PRINTF_DEBUG" "TAIL" "[${#TAIL}]" "$TAIL"  1>&2;
306 |         printf "$PRINTF_DEBUG" "HEAD" "[${#HEAD}]" "$HEAD"  1>&2;
307 |     fi;
308 |     
309 |     return 0
310 | }
311 | 
312 | # chunk_fastq 
313 | # process a chunk in fastq format
314 | chunk_fastq()
315 | {
316 |     [ $DEBUG -gt 1 ] && echo "#$I" 1>&2;
317 |     
318 |     # relay file descriptor if necessary
319 |     [ $SKIP -eq 0 ] && [ $OUT_SPLIT -eq 1 ] && out_split_descriptor;
320 |     
321 |     # check cache
322 |     if [ ! -z "$HEAD" ]; then
323 |         if [ $DEBUG -gt 1 ]; then
324 |             printf "$PRINTF_DEBUG" "HEAD (c)" "[${#HEAD}]" "$HEAD"  1>&2;
325 |         fi;
326 |         [ $SKIP -eq 0 ] && echo "$HEAD" 1>&3;    
327 |         # get first record
328 |     else
329 |         SED=$(sed --unbuffered -n -e "$SED_FIRST_FASTQ");
330 | 
331 |         TAIL="${SED%@*$'\n'*$'\n'+$'\n'*}" # remove entire record
332 |         HEAD="${SED:${#TAIL}}";
333 |         CHUNK_SIZE_ADJUST=$(( ${#SED} + 1 ));
334 | 
335 |         # eof
336 |         if [ -z "$HEAD" ]; then
337 |             # sometimes last chunk is empty -> remove
338 |             if [ -n "$OUT_CHUNK" ] && [ ! -s "$OUT_CHUNK" ]; then
339 |                 $(rm "$OUT_CHUNK");
340 |             fi;
341 |             [ $DEBUG -gt 1 ] && echo "EOF" 1>&2;
342 |             return 1; # return on eof
343 |         fi;
344 |         
345 |         
346 |         [ $SKIP -eq 0 ] && echo "$HEAD" 1>&3;    
347 |         
348 |         if [ $DEBUG -gt 1 ]; then
349 |             printf "$PRINTF_DEBUG" "SED (f)" "[${#SED}]" "$SED"  1>&2;
350 |             printf "$PRINTF_DEBUG" "TAIL (f)" "[${#TAIL}]" "$TAIL"  1>&2;
351 |             printf "$PRINTF_DEBUG" "HEAD (f)" "[${#HEAD}]" "$HEAD"  1>&2;
352 |         fi;
353 |     fi;
354 |     
355 |     # output core
356 |     if [ $SKIP -eq 1 ]; then
357 |         chunk_core_skip
358 |     else
359 |         chunk_core_out
360 |     fi;    
361 |     
362 |     # handle the final record of the chunk - needs to be complete
363 |     # last record -search for start of next record
364 |     SED=$(sed --unbuffered -e "$SED_LAST_FASTQ");
365 |     # this many bytes have been read, but not yet outputted
366 |     CHUNK_SIZE_ADJUST=$(( ${#SED} + 1 ));
367 |     
368 |     if [ -z "$SED" ]; then # eof with dd
369 |         HEAD=''
370 |         TAIL=''
371 |     else
372 |         # remove next FASTQ record, if present
373 |         TAIL="${SED%@*$'\n'*$'\n'+$'\n'*}" # remove entire record
374 |         if [ ${#TAIL} -eq ${#SED} ]; then # eof in sed, no complete new record available
375 |             HEAD=''
376 |             [ $SKIP -eq 0 ] && echo "$TAIL" 1>&3; # echo tail with '\n'
377 |         else
378 |             HEAD="${SED:${#TAIL}}";
379 |             [ $SKIP -eq 0 ] && echo -n "$TAIL" 1>&3; # echo tail "as is"
380 |         fi;
381 |     fi;
382 |     
383 |     if [ $DEBUG -gt 1 ]; then
384 |         printf "$PRINTF_DEBUG" "SED" "[${#SED}]" "$SED"  1>&2;
385 |         printf "$PRINTF_DEBUG" "TAIL" "[${#TAIL}]" "$TAIL"  1>&2;
386 |         printf "$PRINTF_DEBUG" "HEAD" "[${#HEAD}]" "$HEAD"  1>&2;
387 |     fi;
388 |     
389 |     return 0
390 | }
391 | 
392 | 
393 | ##----------------------------------------------------------------------------##
394 | # Opts/Args
395 | 
396 | # Execute getopt
397 | ARGS=`getopt --name "SeqChunker" \
398 |     --options "n:s:f:l:x:y:o:m:qdhV" \
399 |     --longoptions "chunk-number:,chunk-size:,first-chunk:,last-chunk:,chunk-step:,chunks-per-step:,out:,max-block-size:,quiet,debug,help,version" \
400 |     -- "$@"`
401 | 
402 | #Bad arguments
403 | [ $? -ne 0 ] && exit 1;
404 | 
405 | # A little magic
406 | eval set -- "$ARGS"
407 | 
408 | # Now go through all the options
409 | while true; do
410 |     case "$1" in
411 |         -n|--chunk-number)
412 |             [ ! -n "$2" ] && (echo "$1: value required" 1>&2 && exit 1);
413 |             if [[ "$2" =~ [^0-9] ]]; then
414 |                 echo "$1: '$2' not an INT" 1>&2;
415 |                 exit 1;
416 |             fi;
417 |             CHUNK_NUM=$2;
418 |             shift 2;;
419 | 
420 |         -s|--chunk-size)
421 |             [ ! -n "$2" ] && (echo "$1: value required" 1>&2 && exit 1);
422 |             CHUNK_SIZE=$(hr_to_byte $2);
423 |             [ $? -gt 0 ] && exit 1;
424 |             shift 2;;
425 | 
426 |         -f|--first-chunk)
427 |             [ ! -n "$2" ] && (echo "$1: value required" 1>&2 && exit 1);
428 |             if [[ "$2" =~ [^0-9] ]]; then
429 |                 echo "$1: '$2' not an INT" 1>&2;
430 |                 exit 1;
431 |             fi;
432 |             CHUNK_FIRST=$2;
433 |             shift 2;;
434 | 
435 |         -l|--last-chunk)
436 |             [ ! -n "$2" ] && (echo "$1: value required" 1>&2 && exit 1);
437 |             if [[ "$2" =~ [^0-9] ]]; then
438 |                 echo "$1: '$2' not an INT" 1>&2;
439 |                 exit 1;
440 |             fi;
441 |             CHUNK_LAST=$2
442 |             shift 2;;
443 | 
444 |         -x|--chunk-step)
445 |             [ ! -n "$2" ] && (echo "$1: value required" 1>&2 && exit 1);
446 |             if [[ "$2" =~ [^0-9] ]]; then
447 |                 echo "$1: '$2' not an INT" 1>&2;
448 |                 exit 1;
449 |             fi;
450 |             CHUNK_STEP=$2
451 |             shift 2;;
452 | 
453 |         -y|--chunks-per-step)
454 |             [ ! -n "$2" ] && (echo "$1: value required" 1>&2 && exit 1);
455 |             if [[ "$2" =~ [^0-9] ]]; then
456 |                 echo "$1: '$2' not an INT" 1>&2;
457 |                 exit 1;
458 |             fi;
459 |             CHUNK_STEP_NUM=$2
460 |             shift 2;;
461 | 
462 |         -o|--out)
463 |             [ ! -n "$2" ] && (echo "$1: value required" 1>&2 && exit 1);
464 |             OUT=$2
465 |             shift 2;;
466 |         
467 |         -m|--max-block-size)
468 |             [ ! -n "$2" ] && (echo "$1: value required" 1>&2 && exit 1);
469 |             if [[ "$2" =~ [^0-9] ]]; then
470 |                 echo "$1: '$2' not an INT" 1>&2;
471 |                 exit 1;
472 |             fi;
473 |             MAX_BLOCK_SIZE=$2
474 |             shift 2;;
475 | 
476 |         -q|--quiet)
477 |             DEBUG=0
478 |             shift;;
479 | 
480 |         -d|--debug)
481 |             DEBUG=2
482 |             shift;;
483 |         
484 | 	-h|--help)
485 | 	    show_help && exit 0;;
486 | 
487 |         -V|--version)
488 |             grep -m1 'Version' "$0" | sed 's/.*Version\s*//';
489 |             exit 0;;
490 | 	
491 |         --)
492 |             shift
493 |             break;;
494 |         *)
495 |             echo "$1: Unknown option" 1>&2 && exit 1;;
496 |     esac
497 | done
498 | 
499 | # -b|--byte-skip)
500 | #    [ ! -n "$2" ] && (echo "$1: value required" 1>&2 && exit 1);
501 | #if [[ "$2" =~ [^0-9] ]]; then
502 | #    echo "$1: '$2' not an INT" 1>&2;
503 | #exit 1;
504 | #fi;
505 | #SKIP=$2
506 | #shift 2;;
507 | 
508 | # process some args and opts
509 | if [ $# -eq 0 ]; then
510 |     echo "FILE required" 1>&2 
511 |     show_usage
512 |     exit 1
513 | fi;
514 | 
515 | if [ $CHUNK_SIZE -gt 0 ] && [ $CHUNK_NUM -gt 0 ]; then
516 |     echo "Either --chunk-size or --chunk-number required, not both" 1>&2
517 |     show_usage
518 |     exit 1
519 | fi;
520 | 
521 | if [ $CHUNK_SIZE -eq 0 ] && [ $CHUNK_NUM -eq 0 ]; then
522 |     echo "Either --chunk-size or --chunk-number required, not both" 1>&2
523 |     show_usage
524 |     exit 1
525 | fi;
526 | 
527 | 
528 | # default output file descriptor
529 | if [ -z "$OUT" ] || [ "$OUT" == '-' ]; then
530 |     exec 3>&1;
531 | else
532 |     if [[ $OUT =~ % ]]; then
533 |         # dynamic descriptor for every chunk, no default ...
534 |         OUT_SPLIT=1;
535 |     else
536 | 	eval "exec 3>$OUT";
537 |     fi;
538 | fi;
539 | 
540 | ##----------------------------------------------------------------------------##
541 | # Main
542 | 
543 | # process files
544 | for FILE in $@; do
545 |     
546 |     # determine file size
547 |     FILE_SIZE=$( stat -L -c %s $FILE)
548 |     [ $? -ne 0 ] && exit 1;
549 |     [ $DEBUG -gt 1 ] && echo "FILE_SIZE=$FILE_SIZE" 1>&2;
550 | 
551 |     # determine format (FASTA/FASTQ)
552 |     IS_FASTQ=$(head -c 1 "$FILE" | sed 's/[^>@]/\!/;s/>/0/;s/@/1/;s/\n//'); 
553 |     if [ "$IS_FASTQ" = "!" ]; then
554 |         echo "FILE neither FASTA nor FASTQ" 1>&2 
555 |         exit 1
556 |     fi;
557 | 
558 |     # auto-compute chunk-size/chunk-number
559 |     # chunk-size
560 |     if [ $CHUNK_SIZE -eq 0 ]; then
561 | 	CHUNK_SIZE=$(( $FILE_SIZE/$CHUNK_NUM +1 ));
562 |     else
563 | 	CHUNK_NUM=$(( $FILE_SIZE/$CHUNK_SIZE +1 )); # better safe, lasts on eof anyway
564 |     fi;
565 |     
566 |     [ $CHUNK_LAST -eq 0 ] && CHUNK_LAST=$CHUNK_NUM;
567 |     
568 |     if [ $DEBUG -gt 1 ]; then
569 | 	echo "ARGS=$ARGS" 1>&2;
570 | 	echo "CHUNK_SIZE=$CHUNK_SIZE" 1>&2;
571 | 	echo "CHUNK_NUM=$CHUNK_NUM" 1>&2;
572 | 	echo "CHUNK_FIRST=$CHUNK_FIRST" 1>&2;
573 | 	echo "CHUNK_LAST=$CHUNK_LAST" 1>&2;
574 | 	echo "CHUNK_STEP=$CHUNK_STEP" 1>&2;
575 | 	echo "CHUNK_STEP_NUM=$CHUNK_STEP_NUM" 1>&2;
576 |     fi;
577 | 
578 |     # counter	
579 |     I=1; 
580 |     REC_CACHE="";
581 |     CHUNK_SIZE_ADJUST=0;
582 |     
583 |     (
584 |     	# FASTA
585 | 	if [ $IS_FASTQ -eq 0 ]; then
586 |             # skip until CHUNK_FIRST
587 | 	    SKIP=1;
588 | 	    while [ $I -lt $CHUNK_FIRST ]; do
589 | 		#chunk_fasta
590 | 		chunk_core_skip
591 | 		((I++));
592 | 	    done;
593 | 	    
594 |             # process until CHUNK_LAST
595 | 	    while [ $I -le $CHUNK_LAST ]; do
596 | 		if [ $(( ($I - $CHUNK_FIRST) % $CHUNK_STEP)) -lt $CHUNK_STEP_NUM ]; then
597 | 		    SKIP=0;
598 | 		else
599 | 		    SKIP=1;
600 | 		fi;
601 | 
602 | 		chunk_fasta
603 |                 [ $? -ne 0 ] && break; # eof
604 |     		((I++));
605 | 	    done;
606 | 	    
607 | 	    # FASTQ    
608 | 	else
609 | 	    # skip until CHUNK_FIRST
610 |             SKIP=1;
611 | 	    while [ $I -lt $CHUNK_FIRST ]; do
612 | 	        chunk_fastq
613 | 	        ((I++));
614 | 	    done;
615 | 	    
616 | 	    # process until CHUNK_LAST
617 | 	    while [ $I -le $CHUNK_LAST ]; do
618 | 	        if [ $(( ($I - $CHUNK_FIRST) % $CHUNK_STEP)) -lt $CHUNK_STEP_NUM ]; then
619 | 	            SKIP=0;
620 | 	        else
621 | 	            SKIP=1;
622 | 	        fi;
623 | 	        
624 |                 chunk_fastq
625 |                 [ $? -ne 0 ] && break; # eof
626 |                 ((I++));
627 |             done;
628 | 	fi;
629 | 	
630 |     ) < "$FILE";
631 |     
632 | done;
633 | 
634 | 
635 | 


--------------------------------------------------------------------------------
/thirdparty/SeqChunker-perl:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env perl
  2 | 
  3 | use strict;
  4 | use warnings;
  5 | 
  6 | use Getopt::Long;
  7 | use Pod::Usage;
  8 | 
  9 | use FindBin qw($RealBin $Script);
 10 | 
 11 | my $version = "0.22.2";
 12 | 
 13 | # Variable defaults
 14 | my $_debug=0;
 15 | my $help=0;
 16 | my $man=0;
 17 | my $version_requested=0;
 18 | 
 19 | my $CHUNK_NUM=0;
 20 | 
 21 | my $CHUNK_SIZE=0;
 22 | my $CHUNK_FIRST=1;
 23 | my $CHUNK_LAST=0;
 24 | my $CHUNK_STEP=1;
 25 | my $CHUNK_STEP_NUM=1;
 26 | 
 27 | my $OUT="";
 28 | my $OUT_SPLIT=0;
 29 | my $MAX_BLOCK_SIZE=1000000;
 30 | 
 31 | my $cache = "";
 32 | 
 33 | my $PATTERN;
 34 | my $interleaved;
 35 | my $paired;
 36 | my $percentage;
 37 | 
 38 | =pod
 39 | 
 40 | =head1 NAME
 41 | 
 42 | SeqChunker - split a fasta/fastq file into chunk of a specified size on the fly
 43 | 
 44 | =head1 SYNOPSIS
 45 | 
 46 |   SeqChunker --chunk-number|--chunk-size INT [OPTIONS...] FILE(S)";
 47 | 
 48 | =head1 DESCRIPTION
 49 | 
 50 | SeqChunker efficiently samples and outputs chunks from FASTA and FASTQ files,
 51 |  e.g. to serve as on-the-fly input for other tools.
 52 | 
 53 | =head1 ARGUMENTS
 54 | 
 55 | Required: Either one of the following, but not both at the same time. The unset
 56 |  parameter is computed respectively.
 57 | 
 58 |   -n/--chunk-number       Total number of chunks to be sampled (including
 59 |                           skipped ones).
 60 |   -s/--chunk-size         Size of chunks to be sampled. Supports suffixes "k,M,G".
 61 | 
 62 | =head2 NOTE
 63 | 
 64 | Chunk sizes need to be at least twice as great as the longest record
 65 | in the file. Otherwise results will be inconsistent.
 66 | 
 67 | Chunk related computations are run on each input file individually.
 68 | 
 69 | =head1 OPTIONS
 70 | 
 71 |   -f/--first-chunk        Skip chunks before this chunk
 72 |   -l/--last-chunk         Last after this chunk (including skipped ones)
 73 |   -x/--chunk-step         Output a chunk every -x chunks
 74 |   -y/--chunks-per-step    Output -y chunks every -x chunks. Cannot be greater
 75 |                           than -x
 76 |   -p/--percentage         Output this percentage of data, is internally transformed
 77 |                           into adequate -x/-y ratio. 0.1 % is the smallest unit.
 78 |   -o/--out                Output filename. To split chunks into individual files
 79 |                           provide a "printf" style pattern, e.g. "chunk%02d.fa",
 80 |                           with a substitution for the chunk counter.
 81 |   -i/--interleaved        Input data is interleaved FASTQ - keep pairs intact.
 82 |   -p/--paired             Input files are paired-end libraries in alternating order.
 83 |   -m/--max-block-size     Maximum size of blocks in output stream [default 1000000 bytes]
 84 |   -q/--quiet              Suppress non-critical messages
 85 |   -V/--version            Version of the script.
 86 |   -d/--debug              Output more verbose messages
 87 | 
 88 |   -h/--help               Show this help screen
 89 |   -m/--manual             Show the complete manual page
 90 | 
 91 | =head2 EXAMPLES
 92 | 
 93 |   # output only even numbered chunks
 94 |   SeqChunker --chunk-size 5M --chunk-first 2 --chunk-step 2 my.fa
 95 | 
 96 |   # Split file in 100 individual files of similar size
 97 |   SeqChunker --chunk-number 100 --out "my.%03d.fa" my.fa
 98 | 
 99 | =head1 TODO
100 | 
101 | =head2 BUGS
102 | 
103 | =over 8
104 | 
105 | =item B<Output differences to SeqChunker-dd>
106 | 
107 | Currently it seems that in small chunk sizes sometimes result in
108 | different output compared to the old SeqChunker-dd
109 | 
110 | It is still not clear was causes this behavior!
111 | 
112 | Solved in version 0.20.1
113 | 
114 | Bug was caused by the security margin. This can be removed in future versions.
115 | 
116 | =item B<Missing features>
117 | 
118 | =over
119 | 
120 | =item * Currently the option C<< -quiet >> is not implemented
121 | 
122 | =item * The usage of the parameter C<< max_blocksize >> is not implemented
123 | 
124 | =back
125 | 
126 | =back
127 | 
128 | =head2 PLANNED FEATURES
129 | 
130 | =over 8
131 | 
132 | =item rand
133 | 
134 | if possible with seed to allow for consistent recomputation
135 | 
136 | =back
137 | 
138 | =head1 HISTORY
139 | 
140 | =over 8
141 | 
142 | =item version 0.22.2
143 | 
144 | --percentage can be provided as a more intuitive sampling value. It
145 | will be transformed into an adequate -x/-y ratio.
146 | 
147 | =item version 0.22.1
148 | 
149 | --paired to support sampling for paired files. Uses interleaved
150 | intermediate to ensure proper pair handling. This however, largely
151 | reduces performance compared to regular mode.
152 | 
153 | =item version 0.21.1
154 | 
155 | --interleaved to support for interleaved files, respecting paired read
156 | record boundaries.
157 | 
158 | Support reading form STDIN.
159 | 
160 | =item version 0.20.3
161 | 
162 | Fixed another issue with the evaluation of the last-chunk
163 | parameter.
164 | 
165 | Added output for the version parameter.
166 | 
167 | Added a section for parameter manual to the documentation
168 | 
169 | =item version 0.20.2
170 | 
171 | Fixed an issue with the evaluation of the last-chunk parameter. Tests
172 | are now passed also with SeqChunker-perl
173 | 
174 | =item version 0.20.1
175 | 
176 | Solved the differences between old SeqChunker-dd and SeqChunker-perl
177 | under some parameter settings. New perl implementation should now
178 | return exact the same chunks.
179 | 
180 | =item version 0.20.0
181 | 
182 | First implementation of a perl based SeqChunker to circumvent some
183 | bash/sed/math issues in the script version. Is mainly based on
184 | SeqChunker-dd version 0.14 and give same result for larger chunk sizes.
185 | 
186 | =back
187 | 
188 | =head1 AUTHOR
189 | 
190 | Please report bugs to
191 | 
192 | Frank Foerster E<lt>frank.foerster@biozentrum.uni-wuerzburg.deE<gt>
193 | 
194 | Thomas Hackl E<lt>thomas.hackl@uni-wuerzburg.deE<gt>
195 | 
196 | Based on code B<SeqChunker.chunk> written by Thomas Hackl
197 | 
198 | =cut
199 | 
200 | 
201 | 
202 | GetOptions(
203 |     'chunk-number|n=i'    => \$CHUNK_NUM,
204 |     'chunk-size|s=s'      => \$CHUNK_SIZE,
205 |     'first-chunk|f=i'     => \$CHUNK_FIRST,
206 |     'last-chunk|l=i'      => \$CHUNK_LAST,
207 |     'chunk-step|x=i'      => \$CHUNK_STEP,
208 |     'chunks-per-step|y=i' => \$CHUNK_STEP_NUM,
209 |     'percentage|p=f'      => \$percentage,
210 |     'interleaved|i!'      => \$interleaved,
211 |     'paired|2!'           => \$paired,
212 |     'debug|d!'            => \$_debug,
213 |     'help|?|h'            => \$help,
214 |     'out|o=s'             => \$OUT,
215 |     'manual|m'            => \$man,
216 |     'version|V'           => \$version_requested
217 |     ) || pod2usage(2);
218 | 
219 | if ($version_requested)
220 | {
221 |     print "SeqChunker v",$version,"\n";
222 |     exit;
223 | }
224 | 
225 | pod2usage(1) if $help;
226 | pod2usage(-exitval => 0, -verbose => 2) if $man;
227 | 
228 | ## Expand the chunk size if required
229 | $CHUNK_SIZE=expand_byte_suffix($CHUNK_SIZE);
230 | 
231 | ## if out contains chunk number substitution set out_split
232 | if($paired){
233 |     unless($OUT){
234 |         print STDERR "--output required for --paired mode";
235 |         exit 2;
236 |     };
237 |     unless($OUT =~ tr/%//){
238 |         print STDERR "--output requires %d pattern for 1/2 in --paired mode\n";
239 |         exit 2;
240 |     };
241 | 
242 |     if($OUT =~ tr/%// == 2){
243 |         if($percentage){
244 |             print STDERR "in --percentage mode, --output must not have %d chunk-pattern as splitting isn't supported.\n";
245 |             exit 2;
246 |         }
247 |         
248 |         $OUT_SPLIT = 1;
249 |     }
250 | }else{
251 |     if($OUT =~ tr/%//){
252 |         if($percentage){
253 |             print STDERR "--output must not have %d chunk-pattern for splitting in --percentage mode.\n  --percentage doesn't support splitting.\n";
254 |             exit 2;
255 |         }
256 |         
257 |         $OUT_SPLIT=1;
258 |         
259 |     };
260 | }
261 | 
262 | ## the user needs to specify chunk size or chunk num
263 | unless ($percentage || ($CHUNK_SIZE > 0 && $CHUNK_NUM == 0) || ($CHUNK_SIZE == 0 && $CHUNK_NUM > 0))
264 | {
265 |     pod2usage(
266 |               -msg => "Either --chunk-size or --chunk-num required",
267 |               -exitval => 2
268 |              );
269 | }
270 | 
271 | if($percentage){
272 |     if($CHUNK_FIRST != 1 || $CHUNK_LAST ||  $CHUNK_STEP != 1 || $CHUNK_STEP_NUM != 1 ){
273 |         print STDERR "Ignoring --chunk-step/-step-num/-first/-last in --percentage mode\n";
274 |     }
275 | 
276 |     # percentage to -x/-y ratio
277 |     $CHUNK_STEP = 1000;
278 |     $CHUNK_STEP_NUM = int(($percentage*10)+.5);
279 |     $CHUNK_NUM = 10000 unless $CHUNK_NUM || $CHUNK_SIZE;
280 | }
281 | 
282 | ## if the user specified chunk last, this value has to be greater or equal to chunk first
283 | if ($CHUNK_LAST != 0 && $CHUNK_FIRST > $CHUNK_LAST)
284 | {
285 |     pod2usage(
286 |               -msg => "--last-chunk cannot be smaller then --first-chunk",
287 |               -exitval => 2
288 |              );
289 | }
290 | 
291 | ## if the user specified chunk step and chunk step num than this value has to be less or equal chunk_step
292 | unless ($CHUNK_STEP_NUM != 0 && $CHUNK_STEP != 0 && $CHUNK_STEP >= $CHUNK_STEP_NUM)
293 | {
294 |     pod2usage(
295 |               -msg => "--chunk-per-step cannot be greater than --chunk-step",
296 |               -exitval => 2
297 |              );
298 | }
299 | 
300 | if($CHUNK_NUM && $CHUNK_STEP > $CHUNK_NUM){
301 |     print STDERR "--chunk-step cannot be greater than --chunk-num \n";
302 |     exit 2;
303 | }
304 | ## if paired mode, files are read as interleaved
305 | $interleaved++ if $paired;
306 | 
307 | ##----------------------------------------------------------------------------##
308 | 
309 | 
310 | main_loop(@ARGV);
311 | 
312 | 
313 | ##----------------------------------------------------------------------------##
314 | 
315 | =head2 main_loop
316 | 
317 | =cut
318 | 
319 | sub main_loop
320 | {
321 |     my @files = @_;
322 | 
323 |     # generate the filehandle for the output
324 |     my $output_fh = undef;
325 |     my $output_filename = undef;
326 |     my $output_filename2 = undef;
327 | 
328 |     # if we are not splitting, we need a single output filehandle for
329 |     # everything
330 |     if ($OUT && ! $OUT_SPLIT) {
331 |         if ($paired) {
332 |             $output_filename = sprintf($OUT, 1);
333 |             $output_filename2 = sprintf($OUT, 2);
334 |                     
335 |             $_debug && printf STDERR "The next filenames for output are '%s', '%s'\n", $output_filename, $output_filename2;
336 |                     
337 |             open($output_fh, "| $RealBin/interleaved-split 1>$output_filename 2>$output_filename2")
338 |                 || die "Unable to open files '$output_filename', '$output_filename2' for writing! $!\n";
339 |         } else {
340 |                     
341 |             $output_filename = $OUT;
342 |             $_debug && printf STDERR "The next filename for output is '%s'\n", $output_filename;
343 |                     
344 |             open($output_fh, ">", $output_filename)
345 |                 || die "Unable to open file '$output_filename' for writing! $!\n";
346 |         }
347 |     } else {
348 |         if ($paired) {
349 |             open($output_fh, "| $RealBin/interleaved-split")
350 |                 || die "Unable to open files '$output_filename', '$output_filename2' for writing! $!\n";
351 | 
352 |         } else {
353 |             $output_fh = *STDOUT;
354 |         }
355 |     }
356 | 
357 | 
358 | 
359 |     for(my $i=0; $i<@files;$i++){
360 |         my $act_file = $files[$i];
361 |         my $fh;
362 |         my $filesize;
363 | 
364 |         if($paired){
365 |             $i++;
366 |             my $act_file2 = $files[$i];
367 | 
368 |             if ( $act_file eq '-' or -p $act_file or -t $act_file2
369 |                  or
370 |                  $act_file2 eq '-' or -p $act_file2 or -t $act_file2
371 |                ) {
372 |                 die "Reading STDIN is not supported in --paired mode";
373 |             } else {
374 |                 # check if the file exists
375 |                 unless (-e $act_file){
376 |                     die "The file '$act_file' does not exist!\n";
377 |                 }
378 |                 unless (-e $act_file2){
379 |                     die "The file '$act_file2' does not exist!\n";
380 |                 }
381 | 
382 |                 # get the filesize and open the file
383 |                 $filesize = (stat($act_file))[7] + (stat($act_file2))[7];
384 | 
385 |                 if ($_debug) {
386 |                     printf STDERR "Filesize for files '%s' and '%s' is %d bytes!\n", $act_file, $act_file2, $filesize;
387 |                 }
388 |             
389 |                 unless($filesize){
390 |                     print STDERR "Skipping empty files: $act_file, $act_file2\n";
391 |                     next;
392 |                 }
393 |                 print STDERR "Reading paired files $act_file, $act_file2\n";
394 |             }
395 | 
396 |             # create a filehandle to a intermediate interleaved pipe from the paired files
397 |             open($fh, "$RealBin/interleave $act_file $act_file2 |") or die "$!: $act_file, $act_file2";
398 |             
399 |         }else{
400 | 
401 |             # file or stdin
402 |             if ( $act_file eq '-' or -p $act_file or -t $act_file) {
403 |                 $filesize = 0;
404 |                 print STDERR "Reading STDIN\n";
405 |                 unless($CHUNK_SIZE){
406 |                     print STDERR "Cannot compute --chunk-size on STDIN, using 10000.\n  Decrease on small files or increase for long sequences.\n";
407 |                     $CHUNK_SIZE = 10000;
408 |                     $CHUNK_NUM = 0; # reset chunk_num, doesn't work on stdin
409 |                 }
410 |             } else {
411 |                 # check if the file exists
412 |                 unless (-e $act_file){
413 |                     die "The file '$act_file' does not exist!\n";
414 |                 }
415 |                 # get the filesize and open the file
416 |                 $filesize = (stat($act_file))[7];
417 | 
418 |                 if ($_debug) {
419 |                     printf STDERR "Filesize for file '%s' is %d bytes!\n", $act_file, $filesize;
420 |                 }
421 |             
422 |                 unless($filesize){
423 |                     print STDERR "Skipping empty file: $act_file\n";
424 |                     next;
425 |                 }
426 |                 print STDERR "Reading $act_file\n";
427 |             }
428 | 
429 |             if ($act_file ne '-') {
430 |                 open($fh, "<", $act_file) || die "Unable to open file '$act_file' for reading\n";
431 |             } else {
432 |                 open($fh, "<&=STDIN") || die "Unable to open file 'STDIN' for reading\n";
433 |             }
434 |         }
435 | 
436 | 
437 | 
438 |         my $fileformat=guess_file_format($fh) || die "The file '$act_file' seems to be neigher a FASTQ nor a FASTA file\n";
439 | 
440 |         # TODO: throw interleaved + fasta
441 | 
442 |         if($fileformat eq 'fasta' && $interleaved){
443 |             die "Interleaved FASTA isnt supported"
444 |         }
445 |         
446 |         if ($fileformat eq "fasta"){
447 |             $PATTERN = qr/^>|(?<=\n)>/;
448 |         } elsif ($fileformat eq "fastq"){
449 |             if( ! $interleaved){
450 |                 $PATTERN = qr/^@.+\n.+\n\+|(?<=\n)@.+\n.+\n\+/;
451 |             }else{
452 |                 $PATTERN = qr/(?:^|(?<=\n))                        # start at new line
453 |                               (@\S+?)(?:\/[12])?(?:[^\S\n].*)?\n   # seq_head: @+id + (picard) + (desc)
454 |                               .+\n                                 # seq
455 |                               \+.*\n                               # qual head
456 |                               .+\n                                 # qual
457 |                               (@\S+?)(?:\/[12])?(?:[^\S\n].*)?\n   # seq_head: @+id + (picard) + (desc)
458 |                               .+\n                                 # seq
459 |                               \+                                   # qual head
460 |                              /x;  
461 |             }
462 |         } else {
463 |             die "Unable to handle the file format '$fileformat'\n";
464 |         }
465 |         
466 |         
467 | 	# calculate the chunk size and the number of chunks expected
468 | 	if ($CHUNK_SIZE == 0)
469 | 	{
470 | 	    $CHUNK_SIZE=int($filesize/$CHUNK_NUM)+1;
471 | 	} else {
472 |             $CHUNK_NUM=int($filesize/$CHUNK_SIZE)+1;
473 | 	}
474 | 
475 | 	# set chunk last if not set
476 | 	if ($CHUNK_LAST == 0)
477 | 	{
478 | 	    $CHUNK_LAST=$CHUNK_NUM;
479 | 	}
480 | 
481 | 	# print debug information
482 | 	if ($_debug)
483 | 	{
484 | 	    printf STDERR "ARGV=%s\nCHUNK_SIZE=%d\nCHUNK_NUM=%d\nCHUNK_FIRST=%d\nCHUNK_LAST=%d\nCHUNK_STEP=%d\nCHUNK_STEP_NUM=%d\n", join(",", @_), $CHUNK_SIZE, $CHUNK_NUM, $CHUNK_FIRST, $CHUNK_LAST, $CHUNK_STEP, $CHUNK_STEP_NUM;
485 | 	}
486 | 
487 | 
488 |         # skip pre --first chunks immediately
489 |         if ($CHUNK_FIRST > 1){
490 |             my $first_skip = ($CHUNK_FIRST-1) * $CHUNK_SIZE;
491 |             if(-p $fh){ # cannot seek on pipe
492 |                 my $idontcareabout;
493 |                 read($fh, $idontcareabout, $first_skip-tell($fh)-1);
494 |             }else{
495 |                 seek($fh, $first_skip, 0);
496 |             }
497 |             $cache = ""; # remove guess format stuff
498 |         }
499 | 
500 |         
501 | 	# counter for the current chunk
502 | 	my $current_chunk=$CHUNK_FIRST;
503 | 
504 | 	# this variable determines if a block should be skipped (=1) or kept (=0)
505 | 	my $skip = 1;
506 | 
507 |         # process all chunks until LAST_CHUNK is reached
508 | 	while (!$filesize || $current_chunk<=$CHUNK_LAST)
509 | 	{
510 | 
511 | 	    # moreover if not every chunk should be returned, we have to determine if this chunk should be returned or not
512 | 	    if ((($current_chunk-$CHUNK_FIRST)%$CHUNK_STEP) < $CHUNK_STEP_NUM)
513 | 	    {
514 | 		$skip = 0;
515 | 	    } else {
516 | 		$skip = 1;
517 | 	    }
518 | 
519 | 	    if ($OUT_SPLIT && $skip == 0)
520 | 	    {
521 |                 if($paired){
522 |                     $output_filename = sprintf($OUT, $current_chunk, 1);
523 |                     $output_filename2 = sprintf($OUT, $current_chunk, 2);
524 |                     
525 |                     $_debug && printf STDERR "The next filenames for output are '%s', '%s'\n", $output_filename, $output_filename2;
526 |                     
527 |                     open($output_fh, "| $RealBin/interleaved-split 1>$output_filename 2>$output_filename2")
528 |                         || die "Unable to open files '$output_filename', '$output_filename2' for writing! $!\n";
529 |                 }else{
530 |                     
531 |                     $output_filename = sprintf($OUT, $current_chunk);
532 |                     $_debug && printf STDERR "The next filename for output is '%s'\n", $output_filename;
533 |                     
534 |                     open($output_fh, ">", $output_filename)
535 |                         || die "Unable to open file '$output_filename' for writing! $!\n";
536 |                 }
537 | 	    }
538 | 
539 | 	    get_chunk($fh, $current_chunk, $output_fh, $fileformat, $skip);
540 | 
541 | 	    # close the file if a new one was created
542 | 	    if ($OUT_SPLIT && $skip == 0)
543 | 	    {
544 | 		close($output_fh) || die "Unable to open file '$output_filename' after writing! $!\n";
545 | 	    }
546 | 
547 | 	    $current_chunk++;
548 | 
549 | 	    # check if the whole file was read
550 | 	    if (eof($fh))
551 | 	    {
552 |                 last;
553 | 	    }
554 | 	}
555 | 
556 |         # interleave returns with exits status != 0 because its output
557 |         # might not be read until the end. But in this case, $! is still unset.
558 | 	close($fh) || ($! && die "Unable to close file '$act_file' after reading\n");
559 |     }
560 | }
561 | 
562 | ##----------------------------------------------------------------------------##
563 | 
564 | sub expand_byte_suffix
565 | {
566 |     my ($value) = @_;
567 | 
568 |     if ($value =~ /^\d+$/)
569 |     {
570 | 	return $value+0;
571 |     } elsif  ($value =~ /^(\d+)\s*([kmg])$/i)
572 |     {
573 | 	my ($base, $suffix) = ($1, $2);
574 | 	return $base*1024**(index("kmg", lc($suffix))+1);
575 |     } else {
576 | 	die "The value '$value' does not represent a number or a suffixed number!\n";
577 |     }
578 | }
579 | 
580 | sub guess_file_format
581 | {
582 |     my ($fh) = @_;
583 | 
584 |     # read the first byte of the file
585 |     my $first_byte = "";
586 |     my $bytes_received=read $fh, $first_byte, 1;
587 |     if ($bytes_received != 1)
588 |     {
589 | 	die "Error on retriving first byte from file: $!\n";
590 |     }
591 | 
592 |     # check the first byte for indicating a fasta (>), fastq (@) or other file ([^>@])
593 | 
594 |     if ($first_byte eq "@")
595 |     {
596 | 	$cache = $first_byte;
597 | 	return "fastq";
598 |     } elsif ($first_byte eq ">")
599 |     {
600 | 	$cache = $first_byte;
601 | 	return "fasta";
602 |     } else {
603 | 	return undef;
604 |     }
605 | 
606 | }
607 | 
608 | sub strip_to_blockstart
609 | {
610 |     my ( $ref_string ) = @_;
611 | 
612 |     if ($$ref_string =~ $PATTERN)
613 |     {
614 |         my $pos;
615 |         unless($interleaved){
616 |             $pos = $-[0];
617 |         }else{
618 |             $_debug && print STDERR $1 eq $2 ? "pair:  $1 $2\n" : "split: $1 $2\n$&\n"; 
619 |             # compare ids, if identical, take both (/1,/2), otherwise take first which is /2
620 |             $pos = $1 eq $2 ? $-[1] : $-[2];
621 |         }
622 |         
623 | 	my $old_block = "";
624 | 	if ($pos > 0)
625 | 	{
626 | 	    $old_block = substr($$ref_string, 0, $pos, "");
627 | 	}
628 | 
629 | 	# print STDERR Dumper({
630 | 	#     pos => $pos,
631 | 	#     old_block => $old_block,
632 | 	#     buffer => $$ref_string,
633 | 	# 	     }); use Data::Dumper;
634 |     }
635 | 
636 | }
637 | 
638 | sub get_chunk
639 | {
640 |     my (
641 | 	$fh,
642 |         $current_chunk,
643 | 	$output_fh,
644 | 	$fileformat,
645 | 	$skip
646 | 	) = @_;
647 | 
648 |     my $starting_pos = tell($fh);
649 | 
650 |     # after some discussion with Thomas we changed the estimation of
651 |     # the chunksize2get each chunk begins at a multiple of the chunk
652 |     # size and the limit is therefore the next chunk start
653 |     my $next_chunk_pos = $current_chunk*$CHUNK_SIZE;
654 | 
655 |     if ($_debug)
656 |     {
657 | 	printf STDERR "Starting file position: current %d (expected %d), next chunk starts at %d\n", $starting_pos, $current_chunk*$CHUNK_SIZE, $next_chunk_pos;
658 |     }
659 | 
660 |     # we reduce the chunk size by 10 to find the right new block
661 |     my $security_margin = 10;
662 |     my $buffer = "";
663 | 
664 |     my $chunksize2get = $next_chunk_pos - $starting_pos - $security_margin;
665 | 
666 |     if($chunksize2get < 0){
667 |         die "--chunk-number to high/--chunk-size to small for record!"
668 |     }
669 |     
670 |     # read the number of chunksize2get bytes if skip is not set
671 |     if ($skip == 0)
672 |     {
673 | 	my $bytesread = read($fh, $buffer, $chunksize2get);
674 |     } else {
675 |         if(-p $fh){ # cannot seek on pipe
676 |             my $idontcareabout;
677 |             read($fh, $idontcareabout, $chunksize2get);
678 |         }else{
679 |             seek($fh, $next_chunk_pos, 0);
680 |         }
681 | 	$cache = "";
682 | 	return;
683 |     }
684 | 
685 |     # if the cache is set
686 |     if ($cache)
687 |     {
688 | 	$buffer = $cache.$buffer;
689 | 	$cache = "";
690 |     }else{
691 |         # this is only required after seeking, if cache exists, unless
692 |         # interleaved it always trims 0, which is just more work than
693 |         # necessary. On interleaved files it omits skip /2 records,
694 |         # which are than lost completely
695 |         strip_to_blockstart(\$buffer);
696 |     }
697 |     
698 |     # print the buffer to output_fh if skip is not set
699 |     if ($skip == 0 && $buffer)
700 |     {
701 |         print $output_fh $buffer;
702 |     }
703 | 
704 |     # find the next block
705 |     # read until the next line break
706 |     $buffer = <$fh>;
707 | 
708 |     # minimum length of buffer have to be 10
709 |     if ($buffer && length($buffer) < $security_margin)
710 |     {
711 | 	# print the buffer to output_fh if skip is not set
712 | 	if ($skip == 0  && $buffer)
713 | 	{
714 | 	    print $output_fh $buffer;
715 | 	}
716 | 	# and get the next line
717 | 	$buffer = <$fh>;
718 |     }
719 | 
720 |     # print the buffer to output_fh if skip is not set
721 |     if ($skip == 0 && $buffer)
722 |     {
723 | 	print $output_fh $buffer;
724 |     }
725 | 
726 |     # find next block
727 |     while (1)
728 |     {
729 | 	$buffer = <$fh>;
730 | 
731 | 	# check if we reached the eof and leave the loop in this case
732 | 	if (eof($fh))
733 | 	{
734 | 	    last;
735 | 	}
736 | 
737 | 	# check if buffer contains a new block
738 | 	if ($fileformat eq "fasta" && $buffer =~ /^>/)
739 | 	{
740 | 	    # store the new block inside the cache variable
741 | 	    $cache = $buffer;
742 | 	    # and leave the loop
743 | 	    last;
744 | 	} elsif  ($fileformat eq "fastq" && $buffer =~ /^@/)
745 | 	{
746 | 	    # for fastq the next line have to be the sequence and the next line have to start with a plus
747 | 	    my $seq = <$fh>;
748 | 	    my $qhead = <$fh>;
749 | 
750 | 	    if ($qhead =~ /^\+/)
751 | 	    {
752 |                 # okay... we reached a new block
753 |                 $buffer .= $seq.$qhead;
754 | 
755 |                 if($interleaved){
756 |                     # compare ids
757 |                     $buffer .= <$fh>; # qual to buffer
758 | 
759 |                     # read second record for comparison
760 |                     my $shead2 = <$fh>;
761 |                                         
762 |                     # handly possible eof
763 |                     unless(defined($shead2)){
764 |                         if ($skip == 0 && $buffer) {
765 |                             print $output_fh $buffer;
766 |                         }
767 |                         last;
768 |                     }
769 | 
770 |                     my $seq2 = <$fh>;
771 |                     my $qhead2 = <$fh>;
772 |                     my $qual2 = <$fh>;
773 | 
774 |                     my ($id1) = $buffer =~ /^@(\S+?)(?:\/[12])?\s/;
775 |                     my ($id2) = $shead2 =~ /^@(\S+?)(?:\/[12])?\s/;
776 | 
777 |                     if ($id1 ne $id2) {
778 |                         if ($skip == 0 && $buffer) {
779 |                             print $output_fh $buffer;
780 |                         }
781 |                         $buffer = $shead2.$seq2.$qhead2.$qual2;
782 |                     }else{
783 |                         if ($skip == 0 && $buffer) {
784 |                             print $output_fh $buffer, $shead2, $seq2, $qhead2, $qual2;
785 |                         }
786 |                         $buffer = "";
787 |                     }
788 |                 }
789 | 
790 |                 
791 | 	    } elsif ($seq =~ /^@/)
792 | 	    {
793 | 		# the block border was wrong and the start of a
794 | 		# quality line, therefore, the next line should be the
795 | 		# start of a new block write the buffer to the output
796 | 		# if required and fill the buffer with the right
797 | 		# content
798 | 		if ($skip == 0 && $buffer)
799 | 		{
800 |                     print $output_fh $buffer;
801 | 		}
802 | 
803 |                 $buffer = $seq.$qhead;
804 |                                 
805 |                 if($interleaved){
806 |                     # compare ids
807 |                     $buffer .= <$fh>.<$fh>; # qual to buffer
808 | 
809 |                     # read second record for comparison
810 |                     my $shead2 = <$fh>;
811 |                                         
812 |                     # handly possible eof
813 |                     unless(defined($shead2)){
814 |                         if ($skip == 0 && $buffer) {
815 |                             print $output_fh $buffer;
816 |                         }
817 |                         last;
818 |                     }
819 | 
820 |                     my $seq2 = <$fh>;
821 |                     my $qhead2 = <$fh>;
822 |                     my $qual2 = <$fh>;
823 | 
824 |                     my ($id1) = $buffer =~ /^@(\S+?)(?:\/[12])?\s/;
825 |                     my ($id2) = $shead2 =~ /^@(\S+?)(?:\/[12])?\s/;
826 | 
827 |                     if ($id1 ne $id2) {
828 |                         if ($skip == 0 && $buffer) {
829 |                             print $output_fh $buffer;
830 |                         }
831 |                         $buffer = $shead2.$seq2.$qhead2.$qual2;
832 |                     }else{
833 |                         if ($skip == 0 && $buffer) {
834 |                             print $output_fh $buffer, $shead2, $seq2, $qhead2, $qual2;
835 |                         }
836 |                         $buffer = "";
837 |                     }
838 |                 }
839 |                                 
840 | 	    } else {
841 | 		die "Unable to find a new block in a fastq file\n";
842 | 	    }
843 | 
844 |             
845 |             
846 | 	    $cache = $buffer;
847 | 	    # and leave the loop
848 | 	    last;
849 | 	} else {
850 | 	    # not a new block so print the buffer to output_fh if skip is not set
851 | 	    if ($skip == 0  && $buffer)
852 | 	    {
853 | 		print $output_fh $buffer;
854 | 	    }
855 | 	}
856 |     }
857 | }
858 | 
859 | 
860 | 
861 | 
862 | sub test_expand_byte_suffix
863 | {
864 |     foreach (qw(100 0001 15 10k 10K 10m 10M 10g 10G 5 9L4))
865 |     {
866 | 	print STDERR expand_byte_suffix($_), "\n";
867 |     }
868 | }
869 | 
870 | sub test_guess_file_format
871 | {
872 |     foreach (qw(test.fasta test.fastq test.other))
873 |     {
874 | 	open(my $fh, "<", $_) || die;
875 | 	my $fileformat=guess_file_format($fh) || die "The file seems to be neigher a FASTQ nor a FASTA file\n";
876 | 	print STDERR "$_ was detected as $fileformat\n";
877 | 	close($fh) || die;
878 |     }
879 | }
880 | 
881 | __END__
882 | 


--------------------------------------------------------------------------------