├── lib ├── x64 │ ├── pthreadVC2.lib │ └── pthreadVC2S.lib ├── x86 │ ├── pthreadVC2.lib │ └── pthreadVC2S.lib └── includes │ ├── semaphore.h │ └── sched.h ├── bin └── .gitignore ├── obj ├── .gitignore ├── getopt_pp │ └── .gitignore └── distanceCalculation │ └── .gitignore ├── test_data ├── strip_quotes.py ├── shorten_tree_names.py ├── make_phylip_trees.sh ├── run_tests.py ├── fasta2phylip.pl └── PF15224_full.fa ├── src ├── distanceCalculation │ ├── gpu │ │ ├── constants.hpp │ │ ├── cutil_inline.h │ │ ├── cutil_inline_bankchecker.h │ │ ├── cutil_inline_drvapi.h │ │ ├── distanceCalculatorProtein_gpu.cu │ │ └── kimuraDNA_gpu.cu │ ├── DistanceEstimate.cpp │ ├── dnaBitString.hpp │ ├── DistanceEstimate.hpp │ ├── simpleDistanceCalculator.hpp │ ├── hammingDistance.hpp │ ├── bitDistanceProtein.hpp │ ├── JCdistance.hpp │ ├── dataloaderPhylip.hpp │ ├── bitDistanceGap.hpp │ ├── KimuraDistance.hpp │ ├── dataloaderMemory.hpp │ ├── dnaBitString.cpp │ ├── bitStringUtils.hpp │ ├── hammingDistance.cpp │ ├── simpleDistanceCalculator.cpp │ ├── dataloader.hpp │ ├── sim_seq.cpp │ ├── dataloader.cpp │ ├── bitDistanceProtein.cpp │ ├── JCdistance.cpp │ ├── dataloaderFasta.hpp │ └── dataloaderStockholm.hpp ├── minFinder.h ├── cluster_pair.h ├── node.h ├── ProgressBar.hpp ├── threadedNJ.h ├── testNJ.h ├── simpleNJ.h ├── diskMatrix.h ├── getopt_pp │ ├── getopt_pp_standalone.h │ └── getopt_pp.cpp ├── ProgressBar.cpp ├── rapidNJ.h ├── rdDataInitialiser.h ├── distMatrixReader.hpp ├── stdinclude.h ├── rapidNJMem.hpp ├── rapidNJDisk.h ├── Bisection.h ├── polytree.h ├── node.cpp ├── diskMatrix.cpp ├── simpleNJ.cpp ├── threadedNJ.cpp ├── distMatrixReader.cpp ├── rdDataInitialiser.cpp └── testNJ.cpp ├── vcvars64.sh ├── vcvars32.sh ├── README ├── msvc2012.sln ├── createDnaDataPhylip.py ├── Makefile └── msvc2012.vcxproj.filters /lib/x64/pthreadVC2.lib: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/somme89/rapidNJ/HEAD/lib/x64/pthreadVC2.lib -------------------------------------------------------------------------------- /lib/x64/pthreadVC2S.lib: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/somme89/rapidNJ/HEAD/lib/x64/pthreadVC2S.lib -------------------------------------------------------------------------------- /lib/x86/pthreadVC2.lib: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/somme89/rapidNJ/HEAD/lib/x86/pthreadVC2.lib -------------------------------------------------------------------------------- /lib/x86/pthreadVC2S.lib: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/somme89/rapidNJ/HEAD/lib/x86/pthreadVC2S.lib -------------------------------------------------------------------------------- /bin/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Except this file 4 | !.gitignore 5 | 6 | -------------------------------------------------------------------------------- /obj/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Except this file 4 | !.gitignore 5 | 6 | -------------------------------------------------------------------------------- /obj/getopt_pp/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Except this file 4 | !.gitignore 5 | 6 | -------------------------------------------------------------------------------- /obj/distanceCalculation/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Except this file 4 | !.gitignore 5 | 6 | -------------------------------------------------------------------------------- /test_data/strip_quotes.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | for line in open(sys.argv[1]): 5 | line = line.replace("'","") 6 | print line, -------------------------------------------------------------------------------- /src/distanceCalculation/gpu/constants.hpp: -------------------------------------------------------------------------------- 1 | #ifndef CONSTANTS_HPP_ 2 | #define CONSTANTS_HPP_ 3 | 4 | #define threads_pr_block 128 5 | #define gpuMemory 512 6 | 7 | #endif 8 | -------------------------------------------------------------------------------- /test_data/shorten_tree_names.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | 4 | counter = 0 5 | for line in open(sys.argv[1]): 6 | if line.startswith(">"): 7 | print ">seq_" + str(counter) 8 | counter += 1 9 | else: 10 | print line, -------------------------------------------------------------------------------- /src/minFinder.h: -------------------------------------------------------------------------------- 1 | #ifndef MINFINDER_H_ 2 | #define MINFINDER_H_ 3 | #include "threadedNJ.h" 4 | 5 | class minFinder 6 | { 7 | 8 | public: 9 | minFinder(threadState* state); 10 | void run(); 11 | void findMin(); 12 | ~minFinder(); 13 | 14 | private: 15 | threadState* state; 16 | 17 | }; 18 | 19 | #endif /*MINFINDER_H_*/ 20 | -------------------------------------------------------------------------------- /src/cluster_pair.h: -------------------------------------------------------------------------------- 1 | #ifndef CLUSTER_PAIR_H 2 | #define CLUSTER_PAIR_H 3 | 4 | struct cluster_pair{ 5 | public: 6 | unsigned int id; /* index of the other cluster*/ 7 | distType distance; /* distance between clusters */ 8 | bool operator<(const cluster_pair& a) const{ 9 | return distance < a.distance; 10 | } 11 | }; 12 | 13 | #endif 14 | -------------------------------------------------------------------------------- /src/node.h: -------------------------------------------------------------------------------- 1 | #ifndef NODE_H 2 | #define NODE_H 3 | #include 4 | 5 | class node 6 | { 7 | public: 8 | node(); 9 | node(std::string name); 10 | void addEdge(node* n, distType distance); 11 | ~node(void); 12 | 13 | std::string name; 14 | distType* edgeDist; 15 | node** edges; 16 | std::string serializeTree(); 17 | std::string serializeNode(node* n); 18 | int edgeCount; 19 | 20 | private: 21 | 22 | }; 23 | 24 | #endif 25 | -------------------------------------------------------------------------------- /src/ProgressBar.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include "stdinclude.h" 5 | 6 | class ProgressBar 7 | { 8 | public: 9 | ProgressBar(void); 10 | void childProgress(double fraction); 11 | void setProgress(double progress); 12 | void finish(); 13 | ~ProgressBar(void); 14 | 15 | private: 16 | std::stack > intervals; 17 | double curStart; 18 | double curEnd; 19 | double currentProgress; 20 | }; 21 | 22 | -------------------------------------------------------------------------------- /src/distanceCalculation/DistanceEstimate.cpp: -------------------------------------------------------------------------------- 1 | #include "DistanceEstimate.hpp" 2 | #include "bitDistanceGap.hpp" 3 | #include "bitDistanceProtein.hpp" 4 | #include "dataloader.hpp" 5 | #include "simpleDistanceCalculator.hpp" 6 | 7 | DistanceEstimate::DistanceEstimate(dataloader* loader) { 8 | DistanceEstimate::type = loader->type; 9 | DistanceEstimate::fastDist = loader->fastdist; 10 | DistanceEstimate::loader = loader; 11 | } 12 | 13 | DistanceEstimate::~DistanceEstimate(void) { 14 | 15 | } 16 | -------------------------------------------------------------------------------- /src/distanceCalculation/dnaBitString.hpp: -------------------------------------------------------------------------------- 1 | #ifndef DNABITSTRING_H 2 | #define DNABITSTRING_H 3 | 4 | #include 5 | 6 | using namespace std; 7 | 8 | class dnaBitString { 9 | 10 | public: 11 | dnaBitString(); 12 | dnaBitString(char*); 13 | dnaBitString(v4ui initData); 14 | void printSequence(); 15 | 16 | v4ui_vector data; 17 | 18 | private: 19 | friend ostream& operator<<(ostream& out, const dnaBitString& s) ; 20 | }; 21 | 22 | ostream& operator<<(ostream& out, const dnaBitString& s) ; 23 | #endif 24 | -------------------------------------------------------------------------------- /src/distanceCalculation/DistanceEstimate.hpp: -------------------------------------------------------------------------------- 1 | #ifndef __DISTANCE_ESTIMATE_HPP__ 2 | #define __DISTANCE_ESTIMATE_HPP__ 3 | 4 | #include 5 | #include "dataloader.hpp" 6 | 7 | using namespace std; 8 | 9 | class DistanceEstimate { 10 | 11 | public: 12 | DistanceEstimate(dataloader* loader); 13 | virtual ~DistanceEstimate(); 14 | virtual void computeDistance(int i, int j, unsigned long long* data) = 0; 15 | 16 | protected: 17 | bool fastDist; 18 | dataloader* loader; 19 | InputType type; 20 | }; 21 | 22 | #endif 23 | -------------------------------------------------------------------------------- /src/threadedNJ.h: -------------------------------------------------------------------------------- 1 | #ifndef THREADEDNJ_H_ 2 | #define THREADEDNJ_H_ 3 | 4 | #include "node.h" 5 | #include "datareader.h" 6 | 7 | class threadedNJ 8 | { 9 | 10 | public: 11 | threadedNJ(datareader* reader); 12 | node* run(); 13 | ~threadedNJ(); 14 | static void* njThread( void* ptr ); 15 | int min1; 16 | int min2; 17 | 18 | 19 | private: 20 | void findMin(); 21 | 22 | void initialize(); 23 | void mergeMinNodes(); 24 | void updateMatrix(); 25 | }; 26 | 27 | struct threadState { 28 | int rowstart; 29 | int rowEnd; 30 | int min1; 31 | int min2; 32 | double min; 33 | bool run; 34 | }; 35 | 36 | void executefind(threadState* state); 37 | 38 | #endif /*THREADEDNJ_H_*/ 39 | -------------------------------------------------------------------------------- /src/distanceCalculation/simpleDistanceCalculator.hpp: -------------------------------------------------------------------------------- 1 | #ifndef SIMPLEDISTANCECALCULATOR_H 2 | #define SIMPLEDISTANCECALCULATOR_H 3 | 4 | #include "stdinclude.h" 5 | #include "dataloader.hpp" 6 | #include "DistanceEstimate.hpp" 7 | 8 | class simpleDistanceCalculator : public DistanceEstimate { 9 | 10 | public: 11 | simpleDistanceCalculator(dataloader* loader); 12 | ~simpleDistanceCalculator(); 13 | void computeDistance(int i, int j, unsigned long long* data); 14 | 15 | private: 16 | vector sequences; 17 | std::string* sequenceNames; 18 | int seqLength; 19 | void calculateTsTv(int i, int j, unsigned long long* data); 20 | void calculateHammingDistance(int i, int j, unsigned long long* data); 21 | }; 22 | 23 | #endif 24 | -------------------------------------------------------------------------------- /src/testNJ.h: -------------------------------------------------------------------------------- 1 | #ifndef TESTNJ_H 2 | #define TESTNJ_H 3 | 4 | #include "polytree.h" 5 | #include "distMatrixReader.hpp" 6 | 7 | class testNJ 8 | { 9 | public: 10 | testNJ(distMatrixReader* datareader, int matrixSize); 11 | ~testNJ(void); 12 | void step(int,int,distType); 13 | void stepNoVal(int idx1, int idx2); 14 | 15 | private: 16 | distType** matrix; 17 | polytree* mytree; 18 | int matrixSize; 19 | distType* separationsums; 20 | distType* separations; 21 | int clusterCount; 22 | int min1; 23 | int min2; 24 | double min; 25 | int* activeRows; 26 | int nextId; 27 | distMatrixReader* reader; 28 | 29 | void findMin(); 30 | void initialize(); 31 | void mergeMinNodes(); 32 | void updateMatrix(); 33 | }; 34 | #endif 35 | -------------------------------------------------------------------------------- /src/simpleNJ.h: -------------------------------------------------------------------------------- 1 | #ifndef SIMPLENJ_H 2 | #define SIMPLENJ_H 3 | 4 | #include "polytree.h" 5 | #include "distMatrixReader.hpp" 6 | 7 | class simpleNJ 8 | { 9 | public: 10 | simpleNJ(distMatrixReader* datareader, int matrixSize, bool negative_branches, ProgressBar* pb); 11 | ~simpleNJ(void); 12 | polytree* run(); 13 | 14 | private: 15 | distType** matrix; 16 | polytree* mytree; 17 | int matrixSize; 18 | ProgressBar* pb; 19 | distType* separationsums; 20 | distType* separations; 21 | int clusterCount; 22 | int min1; 23 | int min2; 24 | int* activeRows; 25 | int nextId; 26 | distMatrixReader* reader; 27 | bool negative_branches; 28 | 29 | void findMin(); 30 | void initialize(); 31 | void mergeMinNodes(); 32 | void updateMatrix(); 33 | }; 34 | #endif 35 | -------------------------------------------------------------------------------- /vcvars64.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export PATH="/cygdrive/c/Program Files (x86)/Microsoft Visual Studio 11.0/Common7/IDE/":"/cygdrive/c/Program Files (x86)/Microsoft Visual Studio 11.0/VC/bin/x86_amd64":"/cygdrive/c/Program Files (x86)/Microsoft Visual Studio 11.0/Common7/Tools":"/cygdrive/c/Program Files (x86)/Microsoft Visual Studio 11.0/VC/VCPackages":"/cygdrive/c/Program Files (x86)/Microsoft SDKs/Windows/v8.0A/bin/NETFX 4.0 Tools":"/cygdrive/c/Program Files (x86)/Microsoft SDKs/Windows/v8.0A/bin/x64":$PATH: 4 | 5 | export INCLUDE="C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\INCLUDE;C:\Program Files (x86)\Microsoft SDKs\Windows\v8.0A\include;C:\projects\rapidNJ\trunk\lib\includes" 6 | 7 | export LIB="C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\LIB\amd64;C:\Program Files (x86)\Windows Kits\8.0\Lib\win8\um\x64;C:\projects\rapidNJ\trunk\lib\x64;" -------------------------------------------------------------------------------- /vcvars32.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export PATH=$PATH:"/cygdrive/c/Program Files (x86)/Microsoft Visual Studio 11.0/VSTSDB/Deploy":"/cygdrive/c/Program Files (x86)/Microsoft Visual Studio 11.0/Common7/IDE/":"/cygdrive/c/Program Files (x86)/Microsoft Visual Studio 11.0/VC/BIN":"/cygdrive/c/Program Files (x86)/Microsoft Visual Studio 11.0/Common7/Tools":"/cygdrive/c/Windows/Microsoft.NET/Framework/v4.0.30319":"/cygdrive/c/Windows/Microsoft.NET/Framework/v3.5":"/cygdrive/c/Program Files (x86)/Microsoft Visual Studio 11.0/VC/VCPackages":"/cygdrive/c/Program Files (x86)/Microsoft SDKs/Windows/v8.0A/bin/NETFX 4.0 Tools":"/cygdrive/c/Program Files (x86)/Microsoft SDKs/Windows/v8.0A/bin": 4 | 5 | export INCLUDE="C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\INCLUDE;C:\Program Files (x86)\Microsoft SDKs\Windows\v8.0A\include;C:\projects\rapidNJ\trunk\lib\includes;" 6 | 7 | export LIB="C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\LIB;C:\pthreads-win32\Pre-built.2\lib;C:\Program Files (x86)\Windows Kits\8.0\Lib\win8\um\x86" -------------------------------------------------------------------------------- /test_data/make_phylip_trees.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | python shorten_tree_names.py LGT_70.fa > LGT_70_short_names.fa 3 | ./fasta2phylip.pl LGT_70_short_names.fa > LGT_70_short_names.phylip 4 | cp LGT_70_short_names.phylip infile 5 | rm outfile 6 | echo -e "D" > input 7 | echo -e "D" >> input 8 | echo -e "Y" >> input 9 | ../phylip_package/dnadist.exe < input 10 | mv outfile infile 11 | rm outtree 12 | echo -e "Y" > input 13 | ../phylip_package/neighbor.exe < input 14 | mv outtree LGT_70_short_names_phylip_tree.nwk 15 | 16 | python shorten_tree_names.py PF15224_full.fa > PF15224_full_short_names.fa 17 | ./fasta2phylip.pl PF15224_full_short_names.fa > PF15224_full_short_names.phylip 18 | cp PF15224_full_short_names.phylip infile 19 | rm outfile 20 | echo -e "P" > input 21 | echo -e "P" >> input 22 | echo -e "P" >> input 23 | echo -e "Y" >> input 24 | ../phylip_package/protdist.exe < input 25 | mv outfile infile 26 | rm outtree 27 | echo -e "Y" > input 28 | ../phylip_package/neighbor.exe < input 29 | mv outtree PF15224_full_short_names_phylip_tree.nwk -------------------------------------------------------------------------------- /src/distanceCalculation/hammingDistance.hpp: -------------------------------------------------------------------------------- 1 | #ifndef NAIVEHAMMING_H 2 | #define NAIVEHAMMING_H 3 | 4 | #include 5 | #include "stdinclude.h" 6 | #include "dataloader.hpp" 7 | 8 | using namespace std; 9 | 10 | class hammingDistance { 11 | 12 | public: 13 | hammingDistance(bool fastdist, dataloader* dataloader); 14 | void computeDistanceMatrix(); 15 | distType** getDistanceMatrix(); 16 | ~hammingDistance(); 17 | void printDistances(); 18 | 19 | private: 20 | void postProcessDistanceMatrix(); 21 | 22 | char** sequences; 23 | int seqLength; 24 | int seqCount; 25 | InputType type; 26 | int** distanceMatrixTS; 27 | int** distanceMatrixTV; 28 | distType** distanceMatrix; 29 | vector sequenceNames; 30 | static const int BUFFER_SIZE = 16384; 31 | void readHeader(); 32 | void parseName(char* input, int seq); 33 | int compareSequences(int i, int j); 34 | void createDataStructures(); 35 | void initializeData(string); 36 | dataloader* loader; 37 | DistanceEstimate* distEstimator; 38 | }; 39 | 40 | #endif 41 | -------------------------------------------------------------------------------- /src/diskMatrix.h: -------------------------------------------------------------------------------- 1 | #ifndef DISKMATRIX_H 2 | #define DISKMATRIX_H 3 | 4 | #include "stdinclude.h" 5 | #include 6 | #include 7 | 8 | class diskMatrix { 9 | 10 | public: 11 | diskMatrix(const std::string data_dir, int size); 12 | ~diskMatrix(void); 13 | void writeArray(distType *values, int row, int size); 14 | void writeEntry(distType value, int row, int col); 15 | string getTempFile(string directory); 16 | distType readEntry(int row, int col); 17 | void readArray(distType *values, int row, int size); 18 | void writeArrayNewSize(distType *values, int newRow, int newSize); 19 | void setNewSize(int newSize); 20 | void updateRowIndex(int newRow, int newSize); 21 | void flush(); 22 | void finishWrite(); 23 | 24 | private: 25 | int size; 26 | const static int MAX_FILE_SIZE = 1073741824; //bytes (1GB) 27 | std::string data_dir; 28 | int *rowToFileId; 29 | int *rowToFileIndex; 30 | std::fstream* fstreams; 31 | string* file_names; 32 | int rowsPerFile; 33 | int numberOfFiles; 34 | pthread_t thread; 35 | }; 36 | 37 | #endif 38 | 39 | -------------------------------------------------------------------------------- /src/getopt_pp/getopt_pp_standalone.h: -------------------------------------------------------------------------------- 1 | /* 2 | GetOpt_pp: Yet another C++ version of getopt. 3 | Copyright (C) 2007, 2008 Daniel Gutson, FuDePAN 4 | 5 | This file is part of GetOpt_pp. 6 | 7 | GetOpt_pp is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | board-games is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | */ 20 | 21 | #ifndef GETOPT_PP_STANDALONE_H 22 | #define GETOPT_PP_STANDALONE_H 23 | 24 | #define GETOPT_INLINE inline 25 | 26 | #include "getopt_pp.cpp" 27 | 28 | #undef GETOPT_INLINE 29 | 30 | #endif 31 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | Compilation: 2 | Dependencies: C++ (install on debian/ubuntu systems with the command "sudo apt install g++") 3 | 4 | On Linux and Mac run "make" from the root directory. Tip use "make -j" to comple on multiple cores 5 | 6 | Windows: Use Visual studio or cygwin to compile. The scripts vcvars32.sh or vcvars64.sh show an example on how to compile the binaries using cygwin commandline and a visual studio compiler 7 | 8 | Installation: 9 | Copy binaries from "./bin" to a suitable location. 10 | 11 | Version 2.3.2: 12 | -Fixed bug in bootstrap algorithm which caused the bootstrap values to be lower than the true value in most cases. 13 | -Fixed bug that caused the last two branches to have the wrong distance. 14 | 15 | For all publications and documentation see: https://birc.au.dk/software/rapidnj/ 16 | 17 | Pulication list: 18 | https://users-birc.au.dk/~cstorm/software/rapidnj/papers/SimonsenOthers2008_WABI.pdf 19 | https://users-birc.au.dk/~cstorm/software/rapidnj/papers/SimonsenOthers2011_CCIC.pdf 20 | https://users-birc.au.dk/~cstorm/software/rapidnj/papers/SimonsenPedersen2011_SAC.pdf 21 | 22 | Contact: 23 | Martin Simonsen - somme89@gmail.com 24 | -------------------------------------------------------------------------------- /src/distanceCalculation/gpu/cutil_inline.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 1993-2010 NVIDIA Corporation. All rights reserved. 3 | * 4 | * NVIDIA Corporation and its licensors retain all intellectual property and 5 | * proprietary rights in and to this software and related documentation. 6 | * Any use, reproduction, disclosure, or distribution of this software 7 | * and related documentation without an express license agreement from 8 | * NVIDIA Corporation is strictly prohibited. 9 | * 10 | */ 11 | 12 | #ifndef _CUTIL_INLINE_H_ 13 | #define _CUTIL_INLINE_H_ 14 | 15 | #include 16 | #include 17 | #include 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | inline void print_NVCC_min_spec(const char *sSDKsample, const char *sNVCCReq, const char *sDriverReq) 24 | { 25 | printf("CUDA %d.%02d Toolkit built this project.\n", CUDART_VERSION/1000, (CUDART_VERSION%100)); 26 | printf(" [ %s ] requirements:\n", sSDKsample); 27 | printf(" -> CUDA %s Toolkit\n" , sNVCCReq); 28 | printf(" -> %s NVIDIA Display Driver.\n", sDriverReq); 29 | } 30 | 31 | #define ALIGN_OFFSET(offset, alignment) offset = (offset + (alignment-1)) & ~((alignment-1)) 32 | 33 | 34 | #endif // _CUTIL_INLINE_H_ 35 | -------------------------------------------------------------------------------- /msvc2012.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio Express 2012 for Windows Desktop 4 | Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "msvc2012", "msvc2012.vcxproj", "{26CAEE12-46EF-4140-8D68-BF2068264352}" 5 | EndProject 6 | Global 7 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 8 | Debug|x64 = Debug|x64 9 | Debug|x86 = Debug|x86 10 | Release|x64 = Release|x64 11 | Release|x86 = Release|x86 12 | EndGlobalSection 13 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 14 | {26CAEE12-46EF-4140-8D68-BF2068264352}.Debug|x64.ActiveCfg = Debug|x64 15 | {26CAEE12-46EF-4140-8D68-BF2068264352}.Debug|x64.Build.0 = Debug|x64 16 | {26CAEE12-46EF-4140-8D68-BF2068264352}.Debug|x86.ActiveCfg = Debug|Win32 17 | {26CAEE12-46EF-4140-8D68-BF2068264352}.Debug|x86.Build.0 = Debug|Win32 18 | {26CAEE12-46EF-4140-8D68-BF2068264352}.Release|x64.ActiveCfg = Release|x64 19 | {26CAEE12-46EF-4140-8D68-BF2068264352}.Release|x64.Build.0 = Release|x64 20 | {26CAEE12-46EF-4140-8D68-BF2068264352}.Release|x86.ActiveCfg = Release|Win32 21 | {26CAEE12-46EF-4140-8D68-BF2068264352}.Release|x86.Build.0 = Release|Win32 22 | EndGlobalSection 23 | GlobalSection(SolutionProperties) = preSolution 24 | HideSolutionNode = FALSE 25 | EndGlobalSection 26 | EndGlobal 27 | -------------------------------------------------------------------------------- /src/distanceCalculation/bitDistanceProtein.hpp: -------------------------------------------------------------------------------- 1 | #ifndef bitDistanceProtein_HPP_INCLUDED 2 | #define bitDistanceProtein_HPP_INCLUDED 3 | 4 | #include "stdinclude.h" 5 | #include "dataloader.hpp" 6 | #include "DistanceEstimate.hpp" 7 | #include 8 | #include "bitStringUtils.hpp" 9 | 10 | class bitDistanceProtein : public DistanceEstimate{ 11 | 12 | public: 13 | bitDistanceProtein(dataloader* loader); 14 | ~bitDistanceProtein(); 15 | void computeDistance(int i, int j, unsigned long long* data); 16 | void printDistances(); 17 | long** distanceMatrixTS; 18 | long** distanceMatrixTV; 19 | 20 | private: 21 | v4ui** bitStrings; 22 | int seqCount; 23 | int seqLength; 24 | int bitStringsSize; 25 | int bitStringsCount; 26 | static v4ui mask1of8; 27 | static v4ui mask8; 28 | static v4ui mask16; 29 | static v4ui gapMask; 30 | std::vector sequenceNames; 31 | int seq1, seq2, idx; 32 | int numItr8Bit, numItr16Bit, numItr32Bit, numItrTop, numItr8Bit_cache, numItr16Bit_cache, numItr32Bit_cache, numItrTop_cache; 33 | int paddedSeqLength; 34 | 35 | v4ui increaseBlockSize16(v4ui); 36 | v4ui increaseBlockSize32(v4ui); 37 | void sum8_8BitVectors(v4ui&,v4ui&); 38 | void combineInto16BitVectors(v4ui&,v4ui&); 39 | void combineInto32BitVectors(v4ui&,v4ui&); 40 | void sum32Bit(v4ui&,v4ui&); 41 | }; 42 | 43 | #endif // bitDistanceProtein_HPP_INCLUDED 44 | -------------------------------------------------------------------------------- /src/distanceCalculation/JCdistance.hpp: -------------------------------------------------------------------------------- 1 | #ifndef JCDISTANCE_HPP_INCLUDED 2 | #define JCDISTANCE_HPP_INCLUDED 3 | 4 | #include "../stdinclude.h" 5 | #include "dataloader.hpp" 6 | #include "bitDistanceGap.hpp" 7 | #include "dnaBitString.hpp" 8 | #include "DistanceEstimate.hpp" 9 | #include "diskMatrix.h" 10 | 11 | class JCdistance{ 12 | 13 | public: 14 | JCdistance(bool verbose, bool fastdist, dataloader* loader, diskMatrix* dm); 15 | // void calculateDistances(); 16 | static void* distJCThread(void* ptr); 17 | distType** getDistanceMatrix(); 18 | void computeDistanceMatrix(int numThreads); 19 | 20 | private: 21 | DistanceEstimate* getDistanceEstimateInstance(dataloader* loader); 22 | void postProcessDistanceMatrix(); 23 | void computeDistanceMatrixMT(int); 24 | bool verbose; 25 | bool fastdist; 26 | int seqCount; 27 | int seqLength; 28 | dataloader* loader; 29 | vector sequenceNames; 30 | long** distanceMatrixTS; 31 | long** distanceMatrixTV; 32 | distType** jcDistMatrix; 33 | distType maxDistance; 34 | diskMatrix* dm; 35 | }; 36 | 37 | 38 | struct threadStateJC{ 39 | int seqIdx; 40 | unsigned int seqCount; 41 | unsigned int seqLength; 42 | dataloader* loader; 43 | distType** jcDistMatrix; 44 | bool verbose; 45 | int numThreads; 46 | distType maxDistance; 47 | diskMatrix* dm; 48 | volatile int availableBuffers; 49 | DistanceEstimate* estimator; 50 | pthread_mutex_t mutex; 51 | }; 52 | #endif 53 | -------------------------------------------------------------------------------- /src/ProgressBar.cpp: -------------------------------------------------------------------------------- 1 | #include "ProgressBar.hpp" 2 | #include 3 | 4 | ProgressBar::ProgressBar(void) 5 | { 6 | curStart = 0; 7 | curEnd = 100; 8 | currentProgress = 0; 9 | //intervals.push(make_pair(curStart, curEnd)); 10 | } 11 | 12 | void ProgressBar::childProgress(double fraction) 13 | { 14 | intervals.push(make_pair(curStart, curEnd)); 15 | curEnd = (curEnd - curStart) * fraction + currentProgress; 16 | curStart = currentProgress; 17 | if(curStart > 100) { 18 | curStart = 100; 19 | } 20 | if(curEnd > 100) { 21 | curEnd = 100; 22 | } 23 | if(curStart > curEnd) { 24 | curEnd = curStart; 25 | } 26 | } 27 | 28 | void ProgressBar::setProgress(double progress) { 29 | if(progress > 1 || progress < 0) { 30 | cerr << "INTERNAL ERROR: Progress has to be in [0;1]" << endl; 31 | exit(1); 32 | } 33 | double intervalProgress = (curEnd - curStart) * progress; 34 | currentProgress = curStart + intervalProgress; 35 | if(currentProgress > curEnd) { 36 | currentProgress = curEnd; 37 | } 38 | cerr << setiosflags(ios::fixed) << setprecision(2) << currentProgress << "% \r"; 39 | cerr.flush(); 40 | } 41 | 42 | void ProgressBar::finish() { 43 | setProgress(1); 44 | if(intervals.size() > 0) { 45 | pair temp = intervals.top(); 46 | curStart = temp.first; 47 | curEnd = temp.second; 48 | intervals.pop(); 49 | } 50 | } 51 | 52 | ProgressBar::~ProgressBar(void) 53 | { 54 | } 55 | -------------------------------------------------------------------------------- /src/rapidNJ.h: -------------------------------------------------------------------------------- 1 | #ifndef RAPIDNJ_H_ 2 | #define RAPIDNJ_H_ 3 | 4 | #include "polytree.h" 5 | #include "distMatrixReader.hpp" 6 | #include 7 | #include "cluster_pair.h" 8 | #include 9 | 10 | class rapidNJ { 11 | 12 | public: 13 | rapidNJ(distMatrixReader* datareader, int matrix_size, bool negative_branches, ProgressBar* pb); 14 | ~rapidNJ(void); 15 | polytree* run(); 16 | 17 | private: 18 | distType** matrix; 19 | polytree* mytree; 20 | int matrixSize; 21 | bool negative_branches; 22 | ProgressBar* pb; 23 | distType* separationsums; 24 | distType* separations; 25 | int clusterCount; 26 | int min1; 27 | int min2; 28 | cluster_pair **cluster_data; 29 | int currentId; 30 | int *idToIndex; 31 | int *indexToId; 32 | distType max_separation; 33 | distType global_min; 34 | int* garbage_flags; 35 | int min_row_cache1; 36 | int min_row_cache2; 37 | int *row_lengths; 38 | distMatrixReader* reader; 39 | std::list *redundantMap; 40 | int *redundantCount; 41 | int newRowIndex; 42 | int obsoleteRowIndex; 43 | int *redundantToMasterMap; 44 | distType* maxRowSeparations; 45 | 46 | void findMin(); 47 | int findRowMin(int i, int rowsize); 48 | void findRowMinGarbage(int i, int rowsize); 49 | void initialize(); 50 | void mergeMinNodes(); 51 | void updateData(); 52 | int next_id(); 53 | inline void handleRedundancy(int i, int j); 54 | inline void buildSortedRow(int i); 55 | void searchRedundantEntries(int i); 56 | }; 57 | 58 | #endif 59 | -------------------------------------------------------------------------------- /src/rdDataInitialiser.h: -------------------------------------------------------------------------------- 1 | #ifndef RDDATAINITIALISER_H 2 | #define RDDATAINITIALISER_H 3 | #include "polytree.h" 4 | #include 5 | #include "diskMatrix.h" 6 | #include "cluster_pair.h" 7 | 8 | class rdDataInitialiser { 9 | 10 | public: 11 | rdDataInitialiser(bool verbose, int sortedMatrixSize, string cacheDir, string filename); 12 | rdDataInitialiser(bool verbose, int sortedMatrixSize, string cacheDir, int matrixSize); 13 | ~rdDataInitialiser(void); 14 | bool read_data(); 15 | void initializeFromExistingMatrix(vector* sequenceNames, diskMatrix* dm); 16 | cluster_pair** getMatrix(); 17 | polytree* getTree(); 18 | int getSize(); 19 | int* getRowLengths(); 20 | int* getSortedRowLengths(); 21 | distType* getSeparationSums(); 22 | diskMatrix* getDiskMatrix(); 23 | int getDataStructureSize(); 24 | short* getPossibleRedundantRows(); 25 | std::string getFileName(); 26 | 27 | private: 28 | cluster_pair** matrix; 29 | polytree* mytree; 30 | distType* rowBuffer; 31 | int *row_lengths; 32 | int *sorted_row_lengths; 33 | distType *separation_sums; 34 | short *possibleRedundantRows; 35 | diskMatrix *dm; 36 | int matrixSize; 37 | int curRow; 38 | int curCol; 39 | bool verbose; 40 | distType curSeparationSum; 41 | int sortedMatrixSize; 42 | distType memSize; 43 | std::string cacheDir; 44 | std::string filename; 45 | 46 | void createDatastructures(); 47 | void parseData(char* input); 48 | void processRow(bool writeToDisk); 49 | }; 50 | 51 | #endif 52 | -------------------------------------------------------------------------------- /src/distanceCalculation/dataloaderPhylip.hpp: -------------------------------------------------------------------------------- 1 | #ifndef DATALOADER_PHILIP_H 2 | #define DATALOADER_PHILIP_H 3 | 4 | #include "../stdinclude.h" 5 | #include "dataloader.hpp" 6 | #include 7 | #include 8 | #include "dnaBitString.hpp" 9 | 10 | class dataloaderPhylip : public dataloader { 11 | 12 | public: 13 | dataloaderPhylip(); 14 | ~dataloaderPhylip(); 15 | 16 | void load(string filename); 17 | unsigned int** getBitStrings(); 18 | unsigned int** getGapFilters(); 19 | unsigned int getSequenceCount(); 20 | unsigned int getSequenceLength(); 21 | unsigned int getBitStringsCount(); 22 | vector* getSequenceNames(); 23 | vector* getSequences(); 24 | void setSequences(vector* val); 25 | 26 | vector* sequenceNames; 27 | int seqLength; 28 | int seqCount; 29 | unsigned int** bitStrings; 30 | unsigned int** gapFilters; 31 | char** buffers; 32 | int* bufferSizes; 33 | int bitStringsCount; 34 | vector sequences; 35 | int* sequencesIdx; 36 | static const int BUFFER_SIZE = 16384; 37 | 38 | private: 39 | int* dnaBitStringIdx; 40 | bool parsedNames; 41 | void readHeader(std::ifstream &is); 42 | void parseName(char* input, int seq); 43 | inline void parseBuffers(); 44 | inline void parseBuffersWithGaps(); 45 | inline void parseBuffersProtein(); 46 | void createBuffers(); 47 | void createBitStrings(); 48 | void createDataStructuresNaive(); 49 | void loadFastDist(string filename); 50 | void loadNormal(string filename); 51 | int counter; 52 | }; 53 | 54 | #endif 55 | -------------------------------------------------------------------------------- /src/distanceCalculation/bitDistanceGap.hpp: -------------------------------------------------------------------------------- 1 | #ifndef bitDistanceGap_HPP_INCLUDED 2 | #define bitDistanceGap_HPP_INCLUDED 3 | 4 | #include "stdinclude.h" 5 | #include "dataloader.hpp" 6 | #include "DistanceEstimate.hpp" 7 | 8 | 9 | class bitDistanceGap : public DistanceEstimate { 10 | 11 | public: 12 | bitDistanceGap(dataloader* loader); 13 | ~bitDistanceGap(); 14 | void computeDistance(int seq1, int seq2, unsigned long long* retVal); 15 | void printDistances(); 16 | long** distanceMatrixTS; 17 | long** distanceMatrixTV; 18 | 19 | private: 20 | v4ui** bitStrings; 21 | v4ui** gapFilters; 22 | int seqCount; 23 | int seqLength; 24 | int bitStringsSize; 25 | int bitStringsCount; 26 | static v4ui mask1; 27 | static v4ui negate_mask1; 28 | static v4ui mask2; 29 | static v4ui mask4; 30 | static v4ui mask8; 31 | static v4ui mask16; 32 | static v4ui mask32; 33 | static v4ui mask64; 34 | std::vector sequenceNames; 35 | int seq1, seq2, idx1, idx2; 36 | int numItr8Bit, numItr16Bit, numItr32Bit, numItrTop, numItr8Bit_cache, numItr16Bit_cache, numItr32Bit_cache, numItrTop_cache; 37 | 38 | v4ui increaseBlockSize4(v4ui); 39 | v4ui increaseBlockSize8(v4ui); 40 | v4ui increaseBlockSize16(v4ui); 41 | v4ui increaseBlockSize32(v4ui); 42 | v4ui increaseBlockSize64(v4ui); 43 | void initialiseDataStructures(); 44 | void combine2And4Bit(v4ui&, v4ui&, v4ui&); 45 | void combine8Bit(v4ui&, v4ui&, v4ui&); 46 | void combine16Bit(v4ui&, v4ui&, v4ui&); 47 | void combine32Bit(v4ui&, v4ui&, v4ui&); 48 | int printSums(v4ui, int); 49 | }; 50 | 51 | #endif // bitDistanceGap_HPP_INCLUDED 52 | -------------------------------------------------------------------------------- /src/distMatrixReader.hpp: -------------------------------------------------------------------------------- 1 | #ifndef DISTMATRIXREADER_H 2 | #define DISTMATRIXREADER_H 3 | #include "polytree.h" 4 | #include "stdinclude.h" 5 | #include 6 | #include 7 | #include "diskMatrix.h" 8 | 9 | using namespace std; 10 | 11 | class distMatrixReader { 12 | 13 | public: 14 | /*Constructer for reading a distance matrix from disk*/ 15 | distMatrixReader(bool verbose, std::string filename, int matrixSize, bool halfMatrix); 16 | 17 | /*Constructor for distance matrices computed from sequence data*/ 18 | distMatrixReader(bool verbose, int matrixSize, bool halfMatrix, vector* sequenceNames, distType** matrix); 19 | ~distMatrixReader(void); 20 | void read_data(diskMatrix* dm); 21 | distType** getMatrix(); 22 | vector* getSequenceNames(); 23 | string getFileName(); 24 | void initializeData(); 25 | 26 | void printMatrix(){ 27 | for(int i = 0; i < matrixSize; i++){ 28 | int rowSize = matrixSize; 29 | if(halfMatrix){ 30 | rowSize = i+1; 31 | } 32 | for(int j = 0; j < rowSize; j++){ 33 | cout << matrix[i][j] << "\t"; 34 | } 35 | cout << endl; 36 | } 37 | } 38 | 39 | 40 | private: 41 | distType** matrix; 42 | // node** nodes; 43 | int matrixSize; 44 | int curRow; 45 | int curCol; 46 | bool verbose; 47 | string filename; 48 | static const int bufsize = 65536; 49 | char* buf; 50 | bool halfMatrix; 51 | vector* sequenceNames; 52 | void createDatastructures(); 53 | inline void parseData(char* input); 54 | }; 55 | 56 | #endif 57 | -------------------------------------------------------------------------------- /src/distanceCalculation/KimuraDistance.hpp: -------------------------------------------------------------------------------- 1 | #ifndef KIMURADISTANCE_HPP 2 | #define KIMURADISTANCE_HPP 3 | 4 | #include "stdinclude.h" 5 | #include "dataloader.hpp" 6 | #include "DistanceEstimate.hpp" 7 | #include "diskMatrix.h" 8 | 9 | class KimuraDistance { 10 | 11 | public: 12 | KimuraDistance(bool verbose, bool fastdist, dataloader* loader, diskMatrix* dm); 13 | KimuraDistance(bool verbose, bool fastdist, dataloader* loader, distType** _distMatrix, diskMatrix* dm); 14 | ~KimuraDistance(void); 15 | static void* distThread(void* ptr); 16 | distType** getDistanceMatrix(); 17 | void computeDistances(int); 18 | void computeDistancesGPU(); 19 | 20 | private: 21 | DistanceEstimate* getDistanceEstimateInstance(dataloader* loader); 22 | void postProcessDistanceMatrix(); 23 | void computeDistanceMatrixMT(int); 24 | void computeDistancesDNAGPU(); 25 | void computeDistancesProteinGPU(); 26 | void computeDistancesDNAGPU2(); 27 | 28 | bool verbose; 29 | bool fastdist; 30 | bool popcnt; 31 | int seqCount; 32 | dataloader* loader; 33 | vector sequenceNames; 34 | long** distanceMatrixTS; 35 | long** distanceMatrixTV; 36 | distType** distMatrix; 37 | distType maxDistance; 38 | bool gpuInitialised; 39 | diskMatrix* dm; 40 | }; 41 | 42 | struct threadStateKimura{ 43 | int seqIdx; 44 | unsigned int seqCount; 45 | dataloader* loader; 46 | distType** distMatrix; 47 | bool verbose; 48 | int numThreads; 49 | distType maxDistance; 50 | DistanceEstimate* distanceCalculator; 51 | volatile int availableBuffers; 52 | diskMatrix* dm; 53 | pthread_mutex_t mutex; 54 | }; 55 | #endif 56 | -------------------------------------------------------------------------------- /src/distanceCalculation/gpu/cutil_inline_bankchecker.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 1993-2010 NVIDIA Corporation. All rights reserved. 3 | * 4 | * NVIDIA Corporation and its licensors retain all intellectual property and 5 | * proprietary rights in and to this software and related documentation. 6 | * Any use, reproduction, disclosure, or distribution of this software 7 | * and related documentation without an express license agreement from 8 | * NVIDIA Corporation is strictly prohibited. 9 | * 10 | */ 11 | 12 | #ifndef _CUTIL_INLINE_BANKCHECKER_H_ 13 | #define _CUTIL_INLINE_BANKCHECKER_H_ 14 | 15 | #ifdef _DEBUG 16 | #if __DEVICE_EMULATION__ 17 | #define cutilBankChecker(array, idx) (__cutilBankChecker (threadIdx.x, threadIdx.y, threadIdx.z, \ 18 | blockDim.x, blockDim.y, blockDim.z, \ 19 | #array, idx, __FILE__, __LINE__), \ 20 | array[idx]) 21 | 22 | #else 23 | #define cutilBankChecker(array, idx) array[idx] 24 | #endif 25 | #else 26 | #define cutilBankChecker(array, idx) array[idx] 27 | #endif 28 | 29 | // Interface for bank conflict checker 30 | inline void __cutilBankChecker(unsigned int tidx, unsigned int tidy, unsigned int tidz, 31 | unsigned int bdimx, unsigned int bdimy, unsigned int bdimz, 32 | char *aname, int index, char *file, int line) 33 | { 34 | cutCheckBankAccess( tidx, tidy, tidz, bdimx, bdimy, bdimz, file, line, aname, index); 35 | } 36 | 37 | #endif // _CUTIL_INLINE_BANKCHECKER_H_ 38 | -------------------------------------------------------------------------------- /src/stdinclude.h: -------------------------------------------------------------------------------- 1 | #ifndef STDINCLUDE_H 2 | #define STDINCLUDE_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "pthread.h" 12 | #include "ProgressBar.hpp" 13 | 14 | #if defined _WIN32 || defined _WIN64 15 | #else 16 | #include 17 | #endif 18 | 19 | using namespace std; 20 | 21 | #if defined _WIN32 || defined _WIN64 22 | #define __WINDOWS__ 1 23 | #endif 24 | 25 | enum {Abin=0, Cbin=3, Gbin=1, Tbin=2}; 26 | enum InputType {DNA, PROTEIN, UNKNOWN}; 27 | 28 | typedef __m128i v4ui; 29 | 30 | union v4ui_vector { 31 | v4ui v; 32 | unsigned int d[4]; 33 | char c[16]; 34 | unsigned long long ul[2]; 35 | }; 36 | 37 | typedef float distType; 38 | 39 | //RAPID_DISK CONSTANT: percentage of free space compared to the current memory usage before resizing and rebuilding the sorted matrix and cache 40 | const int DATA_STRUCTURE_SIZE_TRESSHOLD = 90; 41 | // MEMORY EFFICIENT RAPIDNJ CONSTANT: the minimum percentage of the sorted matrix which must fit in memory. 42 | const distType MIN_SORTED_MATRIX_SIZE = 0.05f; 43 | //Number of buffers used pr. thread for I/O efficient computation of distance matrices 44 | const static int THREAD_ROW_BUFFER_COUNT = 5; 45 | 46 | __inline void* malloc_aligned(size_t size, unsigned int alignment) { 47 | void *ptr; 48 | #ifdef __WINDOWS__ 49 | ptr = _aligned_malloc(size, alignment); 50 | #elif defined __APPLE__ 51 | //Mac OS X aligns memory to a 16 byte boundary per default 52 | ptr = malloc(size); 53 | #else 54 | int ret = posix_memalign(&ptr, alignment, size); 55 | if(ret != 0){ 56 | ptr = NULL; 57 | } 58 | #endif 59 | if (ptr == NULL) { 60 | cerr << "Could not allocate " << size << " bytes of aligned memory" << endl; 61 | exit(1); 62 | } 63 | return ptr; 64 | } 65 | 66 | #endif //STDINCLUDE_H 67 | -------------------------------------------------------------------------------- /src/rapidNJMem.hpp: -------------------------------------------------------------------------------- 1 | #ifndef RAPIDNJMEM_H_ 2 | #define RAPIDNJMEM_H_ 3 | 4 | #include "polytree.h" 5 | #include "distMatrixReader.hpp" 6 | #include 7 | #include "cluster_pair.h" 8 | #include 9 | 10 | class rapidNJMem { 11 | 12 | public: 13 | rapidNJMem(distMatrixReader* reader, int matrixSize, int sortedMatrixSize, bool verbose, bool negative_branches, ProgressBar* pb); 14 | ~rapidNJMem(void); 15 | polytree* run(); 16 | 17 | private: 18 | distType** distanceMatrix; 19 | polytree* mytree; 20 | int matrixSize; 21 | bool negative_branches; 22 | ProgressBar* pb; 23 | distType* separationsums; 24 | distType* separations; 25 | int clusterCount; 26 | int min1; 27 | int min2; 28 | cluster_pair **cluster_data; 29 | cluster_pair *cluster_buffer; 30 | cluster_pair min_pair; 31 | int currentId; 32 | int *idToIndex; 33 | int *indexToId; 34 | distType max_separation; 35 | double global_min; 36 | int* garbage_flags; 37 | int min_row_cache1; 38 | int min_row_cache2; 39 | int *row_lengths; 40 | distMatrixReader* reader; 41 | std::list *redundantMap; 42 | int *redundantCount; 43 | int newRowIndex; 44 | int obsoleteRowIndex; 45 | int *redundantToMasterMap; 46 | int sortedMatrixSize; 47 | int totalCount; 48 | distType* maxRowSeparations; 49 | bool verbose; 50 | 51 | void findMin(); 52 | int findRowMin(int i, int rowsize); 53 | void findRowMinGarbage(int i, int rowsize); 54 | void initialize(); 55 | void mergeMinNodes(); 56 | void updateData(); 57 | int next_id(); 58 | inline void handleRedundancy(int i, int j); 59 | inline void buildSortedRow(int i); 60 | void searchRedundantEntries(int i); 61 | 62 | inline distType getDist(int i, int j){ 63 | if(i >= j){ 64 | return distanceMatrix[i][j]; 65 | } else { 66 | return distanceMatrix[j][i]; 67 | } 68 | } 69 | 70 | inline void setDist(int i, int j, distType value){ 71 | if(i >= j){ 72 | distanceMatrix[i][j] = value; 73 | } else { 74 | distanceMatrix[j][i] = value; 75 | } 76 | } 77 | }; 78 | 79 | #endif 80 | -------------------------------------------------------------------------------- /src/distanceCalculation/dataloaderMemory.hpp: -------------------------------------------------------------------------------- 1 | #ifndef DATALOADER_MEMORY_H 2 | #define DATALOADER_MEMORY_H 3 | 4 | #include "stdinclude.h" 5 | #include "dataloader.hpp" 6 | 7 | /*Wrapper class used for bootstrapping where the dataset is kept in memory */ 8 | class dataloaderMemory : public dataloader{ 9 | 10 | public: 11 | 12 | dataloaderMemory(unsigned int sequenceLength, unsigned int sequenceCount, vector* sequenceNames, InputType type){ 13 | dataloaderMemory::sequenceLength = sequenceLength; 14 | dataloaderMemory::sequenceCount = sequenceCount; 15 | dataloaderMemory::sequenceNames = sequenceNames; 16 | dataloader::type = type; 17 | } 18 | 19 | void initialize(vector* sequences){ 20 | dataloaderMemory::sequences = sequences; 21 | } 22 | 23 | void initialize(unsigned int** bitStrings, unsigned int** gapFilters, unsigned int bitStringsCount, int paddingLength){ 24 | dataloaderMemory::bitStrings = bitStrings; 25 | dataloaderMemory::gapFilters = gapFilters; 26 | dataloaderMemory::bitStringsCount = bitStringsCount; 27 | dataloaderMemory::paddingLength = paddingLength; 28 | } 29 | 30 | unsigned int** getBitStrings(){ 31 | return bitStrings; 32 | } 33 | 34 | unsigned int** getGapFilters(){ 35 | return gapFilters; 36 | } 37 | 38 | unsigned int getSequenceCount(){ 39 | return sequenceCount; 40 | } 41 | 42 | unsigned int getSequenceLength(){ 43 | return sequenceLength; 44 | } 45 | 46 | unsigned int getBitStringsCount(){ 47 | return bitStringsCount; 48 | } 49 | 50 | vector* getSequenceNames(){ 51 | return sequenceNames; 52 | } 53 | 54 | vector* getSequences(){ 55 | return sequences; 56 | } 57 | 58 | void setSequences(vector* val){ 59 | sequences = val; 60 | } 61 | 62 | ~dataloaderMemory(){} 63 | 64 | private: 65 | unsigned int** bitStrings; 66 | unsigned int** gapFilters; 67 | vector* sequences; 68 | vector* sequenceNames; 69 | unsigned int sequenceLength; 70 | unsigned int sequenceCount; 71 | int bitStringsCount; 72 | int paddingLength; 73 | }; 74 | 75 | #endif 76 | 77 | -------------------------------------------------------------------------------- /src/rapidNJDisk.h: -------------------------------------------------------------------------------- 1 | #ifndef RAPIDNJDISK_H_ 2 | #define RAPIDNJDISK_H_ 3 | 4 | #include "polytree.h" 5 | #include "cluster_pair.h" 6 | #include "diskMatrix.h" 7 | #include "rdDataInitialiser.h" 8 | #include 9 | #include 10 | 11 | class rapidNJDisk { 12 | 13 | public: 14 | rapidNJDisk(rdDataInitialiser* datareader, bool verbose, bool negative_branches, ProgressBar* pb); 15 | ~rapidNJDisk(void); 16 | polytree* run(); 17 | 18 | private: 19 | diskMatrix* disk_matrix; 20 | polytree* mytree; 21 | bool negative_branches; 22 | ProgressBar* pb; 23 | int matrixSize; 24 | int matrixSizeCache; 25 | int newRowsStartId; 26 | distType* separationsums; 27 | distType* separations; 28 | int clusterCount; 29 | int min1; 30 | int min2; 31 | cluster_pair **cluster_data; 32 | cluster_pair min_pair; 33 | cluster_pair *cluster_buffer; 34 | int currentId; 35 | int *idToIndex; 36 | int *indexToId; 37 | distType max_separation; 38 | distType global_min; 39 | /*update_flags[i]==0 if the i'th row hasn't been cached and an id > 0 otherwise. The id is an increasing number starting from 1*/ 40 | int* update_flags; 41 | int updateCount; 42 | int min_row_cache1; 43 | int min_row_cache2; 44 | int *sorted_row_lengths; 45 | distType *rowBuffer1; 46 | distType *rowBuffer2; 47 | distType* maxRowSeparations; 48 | 49 | /*Contains all rows of the distance matrix which has been modified since the last cache flush.*/ 50 | distType **column_cache; 51 | int dataStructureSize; 52 | bool verbose; 53 | int *redundantToMasterMap; 54 | int newRowIndex; 55 | int obsoleteRowIndex; 56 | std::list *redundantMap; 57 | int *redundantCount; 58 | int totalCount; 59 | short *activeRowFlags; 60 | short *possibleRedundantRows; 61 | 62 | void findMin(); 63 | void findRowMin(int i); 64 | void initialize(); 65 | void mergeMinNodes(); 66 | void updateData(); 67 | int next_id(); 68 | void flushUpdateCache(); 69 | void handleRedundancy(int i, int j); 70 | void rebuildDataStructures(int newRowPos, int oldRowPos, int newDataStructureSize); 71 | void searchRedundantEntries(int i); 72 | }; 73 | 74 | #endif 75 | 76 | -------------------------------------------------------------------------------- /src/Bisection.h: -------------------------------------------------------------------------------- 1 | #ifndef BISECTION_H 2 | #define BISECTION_H 3 | #include "stdinclude.h" 4 | #include 5 | 6 | class bisection { 7 | public: 8 | vector* section1; 9 | vector* section2; 10 | int node1; 11 | int node2; 12 | int hashCached; 13 | int isHashCached; 14 | 15 | bisection(){ 16 | section1 = NULL; 17 | section2 = NULL; 18 | isHashCached = 0; 19 | } 20 | 21 | bisection(int n1, int n2){ 22 | node1 = n1; 23 | node2 = n2; 24 | isHashCached = 0; 25 | section1 = new vector(); 26 | section2 = new vector(); 27 | } 28 | 29 | int hash(){ 30 | if(isHashCached == 1) { 31 | return hashCached; 32 | } 33 | int hash = 0; 34 | if(section1->size() > section2->size()) { 35 | for(int i = 0; i < section1->size(); i++) { 36 | hash += section1->at(i); 37 | } 38 | } else if (section1->size() < section2->size()){ 39 | for(int i = 0; i < section2->size(); i++) { 40 | hash += section2->at(i); 41 | } 42 | } else { 43 | for(int i = 0; i < section1->size(); i++) { 44 | hash += section1->at(i); 45 | } 46 | for(int i = 0; i < section2->size(); i++) { 47 | hash += section2->at(i); 48 | } 49 | } 50 | return hash; 51 | } 52 | 53 | bool compareSections(vector* s1, vector* s2) { 54 | if(s1->size() != s2->size()) { 55 | return false; 56 | } 57 | sort(s1->begin(), s1->end()); 58 | sort(s2->begin(), s2->end()); 59 | for(int i = 0; i < s1->size(); i++) { 60 | if(s1->at(i) != s2->at(i)) { 61 | return false; 62 | } 63 | } 64 | return true; 65 | } 66 | 67 | bool equals(bisection* other){ 68 | //Only need to find one match as this implies that the other section will also match. 69 | if(compareSections(section1, other->section1)) { 70 | return true; 71 | } else if(compareSections(section1, other->section2)) { 72 | return true; 73 | } else { 74 | return false; 75 | } 76 | } 77 | 78 | ~bisection(){ 79 | if(section1 != NULL) { 80 | delete section1; 81 | } 82 | if(section2 != NULL) { 83 | delete section2; 84 | } 85 | } 86 | 87 | }; 88 | 89 | #endif 90 | -------------------------------------------------------------------------------- /src/polytree.h: -------------------------------------------------------------------------------- 1 | #ifndef POLYTREE_H 2 | #define POLYTREE_H 3 | #include "stdinclude.h" 4 | #include 5 | #include "Bisection.h" 6 | 7 | class polytree{ 8 | 9 | public: 10 | polytree(int size, vector* sequenceNames); 11 | ~polytree(void); 12 | 13 | void addLeaf(std::string name); 14 | void addInternalNode(double left_dist, double right_dist, int left_index, int right_index); 15 | int getParentIndex(int child_index); 16 | int getLeftChildIndex(int parentIndex); 17 | int getRightChildIndex(int parentIndex); 18 | int getSibling(int index); 19 | void setlastParentIndicies(); 20 | void compareTreeBootstrap(polytree* tree); 21 | void updateBSValues(unsigned int n1, unsigned int n2, map*>* hmap, 22 | map*>* hmapOther); 23 | void serialize_tree(ostream &out); 24 | void serialize_node(ostream &out, unsigned int left_index, unsigned int right_index, int index); 25 | void set_serialization_indices(int left, int right, distType dist); 26 | int* bootstrap_counts; 27 | bool test; 28 | 29 | protected: 30 | class edge { 31 | public: 32 | int n2; 33 | int bootstrapCount; 34 | 35 | edge(){ 36 | n2 = -1; 37 | bootstrapCount = 0; 38 | } 39 | 40 | }; 41 | unsigned int s_index_left; 42 | unsigned int s_index_right; 43 | unsigned int increaseLeafListsSize(vector** leafList, unsigned int newMaxSize); 44 | 45 | private: 46 | 47 | double *distances; //2n-1 48 | int *left_indexes;// n-1 49 | int *right_indexes;//n-1 50 | int *parent_indices; //2n-1 51 | std::string *leaf_names;//n 52 | unsigned int size; 53 | int current_index; 54 | char buffer[64]; 55 | int leaf_index; 56 | distType s_dist; 57 | map*>* hmap; 58 | void InitLeafLists(vector** list); 59 | int bootstrap_replicate_count; 60 | bisection* getBisectionsForNodePair(int n1, int n2); 61 | void getNextElements(stack* nextElements, int* nodeUseCount, stack* tempElements, vector* section); 62 | void findBisections(int i, int candidate, int* nodeUseCount, map*>* hmap, bool useIndexAsHash); 63 | map*>* findAllBisections(bool useIndexAsHash); 64 | }; 65 | 66 | #endif 67 | -------------------------------------------------------------------------------- /src/node.cpp: -------------------------------------------------------------------------------- 1 | #include "stdinclude.h" 2 | #include "node.h" 3 | using namespace std; 4 | 5 | /* This class represents a node in the final unrooted tree. 6 | * It has the nessasary logic to serialize the tree to a Newick formatted string representation. 7 | */ 8 | 9 | /* Creates a internal node*/ 10 | node::node() { 11 | edgeDist = new distType[3]; 12 | edges = new node*[3]; 13 | edgeCount = 0; 14 | } 15 | 16 | /* creates a named leaf node */ 17 | node::node(string name) { 18 | edgeDist = new distType[1]; 19 | edges = new node*[1]; 20 | node::name = name; 21 | edgeCount = 0; 22 | } 23 | 24 | /* Connect a node to this node */ 25 | void node::addEdge(node* n, distType distance){ 26 | if(n == this){ 27 | cout << "cannot add node to self \n"; 28 | exit(0); 29 | } 30 | edgeDist[edgeCount] = distance; 31 | 32 | edges[edgeCount] = n; 33 | edgeCount++; 34 | } 35 | 36 | /* serialize the three to newick format*/ 37 | string node::serializeTree(){ 38 | if(edgeCount == 3){ 39 | //internal node 40 | return serializeNode(NULL) + ";"; 41 | } else { 42 | if(edgeCount == 0){ 43 | // only one node, this one. 44 | return name + ";"; 45 | } else if(edgeCount == 1 && edges[0]->edgeCount == 1){ 46 | // only two nodes. 47 | return "(" + name + "," + edges[0]->name + ");"; 48 | } else { 49 | // leaf, use the connected internal node to start the recursion 50 | return edges[0]->serializeTree(); 51 | } 52 | } 53 | return "ERROR - could not serialize the tree"; 54 | } 55 | 56 | /* Recursive helper method for serializing the three*/ 57 | string node::serializeNode(node* n){ 58 | if(edgeCount == 2){ 59 | cout << "ERROR - only 2 edges found\n"; 60 | } 61 | char buffer [50]; 62 | if(edgeCount == 3){ 63 | // internal node 64 | string s = "("; 65 | for(int i = 0; i < edgeCount; i++){ 66 | node* edge = edges[i]; 67 | if(edge != n){ 68 | s += edge->serializeNode(this); 69 | s += ":"; 70 | int length = sprintf(buffer,"%g", edgeDist[i]); 71 | s.append(buffer,length); 72 | s += ","; 73 | } 74 | } 75 | s.replace(s.length()-1,1,")"); 76 | return s; 77 | } else { 78 | return "'" + name + "'"; 79 | } 80 | 81 | } 82 | 83 | node::~node(void) 84 | { 85 | } 86 | -------------------------------------------------------------------------------- /src/distanceCalculation/dnaBitString.cpp: -------------------------------------------------------------------------------- 1 | #include "../stdinclude.h" 2 | #include "dnaBitString.hpp" 3 | 4 | ostream& operator<<(ostream& out, const dnaBitString& s) { 5 | for (int i = 0;i < 4 ; i++) { 6 | cout << "["; 7 | for (int j = 0;j < 16 ; j++) { 8 | if(j%2 == 0 && j != 0){ 9 | cout << "-"; 10 | } 11 | out << ((s.data.d[i] >> (j*2)) & 1); 12 | out << ((s.data.d[i] >> (j*2+1)) & 1); 13 | } 14 | out << "] "; 15 | } 16 | return out; 17 | } 18 | 19 | dnaBitString::dnaBitString() {} 20 | 21 | /** 22 | * Builds a 128 bit sequence from an amino acid sequence. 23 | * [(part0),(part1),(part2),(part3)] 24 | * each part is 32 bit and represents 16 characters in reverse 25 | * ACGT is represented by the bit string 00000000 00000000 00000000 10011100 26 | */ 27 | dnaBitString::dnaBitString(char* seq) { 28 | for (int idx = 0; idx < 4; idx++) { 29 | data.d[idx] = 0; 30 | for (int i = 0; i < 16; i++) { 31 | switch (seq[i + (idx*16)]) { 32 | //A has value 00 so nothing is added 33 | case 'C': { 34 | data.d[idx] += (Cbin<<(i*2)); 35 | break; 36 | } 37 | case 'G': { 38 | data.d[idx] += (Gbin<<(i*2)); 39 | break; 40 | } 41 | case 'T': { 42 | data.d[idx] += (Tbin<<(i*2)); 43 | break; 44 | } 45 | default: 46 | break; 47 | } 48 | } 49 | } 50 | } 51 | 52 | dnaBitString::dnaBitString(v4ui initData){ 53 | data.v = initData; 54 | } 55 | 56 | 57 | void dnaBitString::printSequence() { 58 | for (int i = 0;i < 4 ; i++) { 59 | for (int j = 0;j < 16 ; j++) { 60 | switch ((data.d[i] >> (2*j) ) & 3) { 61 | case 0: { 62 | cout << 'A'; 63 | break; 64 | } 65 | case 1: { 66 | cout << 'G'; 67 | break; 68 | } 69 | case 2: { 70 | cout << 'T'; 71 | break; 72 | } 73 | case 3: { 74 | cout << 'C'; 75 | break; 76 | } 77 | default: { 78 | cout << '?'; 79 | } 80 | } 81 | } 82 | cout << " "; 83 | } 84 | } 85 | 86 | 87 | 88 | 89 | -------------------------------------------------------------------------------- /test_data/run_tests.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | from subprocess import PIPE, Popen, STDOUT 4 | 5 | cmd = './make_phylip_trees.sh' 6 | p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) 7 | output = p.stdout.read() 8 | 9 | cmd = '../x64/Release/bin/rapidnj_x64.exe LGT_70_short_names.fa -a jc -x LGT_70_short_names_rnj_tree.nwk' 10 | p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) 11 | output = p.stdout.read() 12 | 13 | cmd = 'python strip_quotes.py LGT_70_short_names_rnj_tree.nwk > LGT_70_short_names_rnj_tree_no_quotes.nwk ' 14 | p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) 15 | output = p.stdout.read() 16 | 17 | cmd = '../tqdist/quartet_dist.exe LGT_70_short_names_rnj_tree_no_quotes.nwk LGT_70_short_names_phylip_tree.nwk' 18 | p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) 19 | output = p.stdout.read() 20 | if int(output) != 0: 21 | print "ERROR: LG_70 tree distance was not 0 but", output 22 | sys.exit(1) 23 | else: 24 | print "LG_70 SUCCESS!" 25 | 26 | os.remove("LGT_70_short_names_rnj_tree.nwk") 27 | os.remove("LGT_70_short_names_rnj_tree_no_quotes.nwk") 28 | os.remove("LGT_70_short_names_phylip_tree.nwk") 29 | os.remove("LGT_70_short_names.fa") 30 | os.remove("LGT_70_short_names.phylip") 31 | 32 | cmd = '../x64/Release/bin/rapidnj_x64.exe PF15224_full_short_names.fa -x PF15224_full_short_names_rnj_tree.nwk' 33 | p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) 34 | output = p.stdout.read() 35 | 36 | cmd = 'python strip_quotes.py PF15224_full_short_names_rnj_tree.nwk > PF15224_full_short_names_rnj_tree_no_quotes.nwk ' 37 | p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) 38 | output = p.stdout.read() 39 | 40 | cmd = '../tqdist/quartet_dist.exe PF15224_full_short_names_rnj_tree_no_quotes.nwk PF15224_full_short_names_phylip_tree.nwk' 41 | p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) 42 | output = p.stdout.read() 43 | if int(output) != 112: 44 | print "ERROR: PF15224 tree distance was not 112 but", output 45 | sys.exit(1) 46 | else: 47 | print "PF15224 SUCCESS!" 48 | 49 | os.remove("PF15224_full_short_names_rnj_tree.nwk") 50 | os.remove("PF15224_full_short_names_rnj_tree_no_quotes.nwk") 51 | os.remove("PF15224_full_short_names_phylip_tree.nwk") 52 | os.remove("PF15224_full_short_names.fa") 53 | os.remove("PF15224_full_short_names.phylip") 54 | 55 | os.remove("outfile") 56 | os.remove("infile") 57 | os.remove("input") 58 | -------------------------------------------------------------------------------- /createDnaDataPhylip.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | import string 4 | import random 5 | import time 6 | 7 | if len(sys.argv) > 3 or len(sys.argv) < 2: 8 | print "USAGE: out_dir [-g]" 9 | sys.exit() 10 | out_dir = sys.argv[1] 11 | useGaps = 0 12 | if len(sys.argv) == 3: 13 | if sys.argv[2] == "-g": 14 | useGaps = 1 15 | else: 16 | print "USAGE: out_dir [-g]" 17 | sys.exit() 18 | 19 | seq_lengths = range(100000,3100000,100000) 20 | seq_lengths.append(10000) 21 | seq_lengths.append(50000) 22 | seq_count = 100 23 | 24 | for length in seq_lengths: 25 | print length 26 | if(useGaps): 27 | os.system("bin/sim_seq " + str(seq_count) + " " + str(length) + " d s " + " -g > " + out_dir + "/seq_" + str(seq_count) + "_" + str(length) + ".sth") 28 | else: 29 | os.system("bin/sim_seq " + str(seq_count) + " " + str(length) + " d s " + " > " + out_dir + "/seq_" + str(seq_count) + "_" + str(length) + ".sth") 30 | 31 | 32 | seq_lengths = range(10000,200000,10000) 33 | seq_lengths.append(100) 34 | seq_lengths.append(1000) 35 | seq_lengths.append(5000) 36 | seq_count = 1000 37 | 38 | for length in seq_lengths: 39 | print length 40 | if(useGaps): 41 | os.system("bin/sim_seq " + str(seq_count) + " " + str(length) + " d s " + " -g > " + out_dir + "/seq_" + str(seq_count) + "_" + str(length) + ".sth") 42 | else: 43 | os.system("bin/sim_seq " + str(seq_count) + " " + str(length) + " d s " + " > " + out_dir + "/seq_" + str(seq_count) + "_" + str(length) + ".sth") 44 | 45 | seq_lengths = range(5000,100000,5000) 46 | seq_lengths.append(50) 47 | seq_lengths.append(500) 48 | seq_lengths.append(2500) 49 | seq_count = 2000 50 | 51 | for length in seq_lengths: 52 | print length 53 | if(useGaps): 54 | os.system("bin/sim_seq " + str(seq_count) + " " + str(length) + " d s " + " -g > " + out_dir + "/seq_" + str(seq_count) + "_" + str(length) + ".sth") 55 | else: 56 | os.system("bin/sim_seq " + str(seq_count) + " " + str(length) + " d s " + " > " + out_dir + "/seq_" + str(seq_count) + "_" + str(length) + ".sth") 57 | 58 | seq_lengths = range(10000,200000,10000) 59 | seq_lengths.append(100) 60 | seq_lengths.append(1000) 61 | seq_lengths.append(5000) 62 | seq_lengths = [x/3 for x in seq_lengths] 63 | seq_count = 3000 64 | 65 | for length in seq_lengths: 66 | print length 67 | if(useGaps): 68 | os.system("bin/sim_seq " + str(seq_count) + " " + str(length) + " d s " + " -g > " + out_dir + "/seq_" + str(seq_count) + "_" + str(length) + ".sth") 69 | else: 70 | os.system("bin/sim_seq " + str(seq_count) + " " + str(length) + " d s " + " > " + out_dir + "/seq_" + str(seq_count) + "_" + str(length) + ".sth") 71 | -------------------------------------------------------------------------------- /src/distanceCalculation/bitStringUtils.hpp: -------------------------------------------------------------------------------- 1 | #ifndef BITSTRINGUTILS_HPP 2 | #define BITSTRINGUTILS_HPP 3 | #include "stdinclude.h" 4 | 5 | namespace { 6 | 7 | void printBitString(v4ui data){ 8 | v4ui_vector vector; 9 | vector.v = data; 10 | for (int i = 0;i < 4 ; i++) { 11 | cout << "["; 12 | for (int j = 0;j < 32 ; j++) { 13 | if(j%8 == 0 && j != 0){ 14 | cout << " "; 15 | } 16 | cout << ((vector.d[i] >> j) & 1); 17 | } 18 | cout << "] "; 19 | } 20 | cout << endl; 21 | } 22 | 23 | void printBitString(unsigned int data){ 24 | cout << "["; 25 | for (int j = 0;j < 32 ; j++) { 26 | if(j%8 == 0 && j != 0){ 27 | cout << " "; 28 | } 29 | cout << ((data >> j) & 1); 30 | } 31 | cout << "] "; 32 | cout << endl; 33 | } 34 | 35 | void printProteinSequence(v4ui data){ 36 | v4ui_vector vector; 37 | vector.v = data; 38 | for (int i = 0;i < 4 ; i++) { 39 | cout << "["; 40 | for (int j = 0;j < 4; j++) { 41 | if(j != 0){ 42 | cout << " "; 43 | } 44 | unsigned int c = ((vector.d[i] >> (j*8)) & 255); 45 | //cout << (char) c << " "; 46 | cout << c << " "; 47 | } 48 | cout << "] "; 49 | } 50 | cout << endl; 51 | } 52 | 53 | void printDNASequence(v4ui data){ 54 | v4ui_vector vector; 55 | vector.v = data; 56 | for (int i = 0;i < 4 ; i++) { 57 | cout << "["; 58 | for (int j = 0;j < 16; j++) { 59 | if(j % 4 == 0 && j != 0) { 60 | cout << " "; 61 | } 62 | unsigned int c = ((vector.d[i] >> (j*2)) & 3); 63 | switch (c) { 64 | case 0: { 65 | cout << "A "; 66 | break; 67 | } 68 | case 1: { 69 | cout << "G "; 70 | break; 71 | } 72 | case 2: { 73 | cout << "T "; 74 | break; 75 | } 76 | case 3: { 77 | cout << "C "; 78 | break; 79 | } 80 | default: 81 | break; 82 | } 83 | } 84 | cout << "] "; 85 | } 86 | cout << endl; 87 | } 88 | 89 | void printDNASequence(unsigned int data){ 90 | cout << "["; 91 | for (int j = 0;j < 16; j++) { 92 | if(j % 4 == 0 && j != 0) { 93 | cout << " "; 94 | } 95 | unsigned int c = ((data >> (j*2)) & 3); 96 | switch (c) { 97 | case 0: { 98 | cout << "A "; 99 | break; 100 | } 101 | case 1: { 102 | cout << "G "; 103 | break; 104 | } 105 | case 2: { 106 | cout << "T "; 107 | break; 108 | } 109 | case 3: { 110 | cout << "C "; 111 | break; 112 | } 113 | default: 114 | break; 115 | } 116 | } 117 | cout << "] "; 118 | cout << endl; 119 | } 120 | 121 | } 122 | 123 | #endif 124 | -------------------------------------------------------------------------------- /src/distanceCalculation/hammingDistance.cpp: -------------------------------------------------------------------------------- 1 | #include "simpleDistanceCalculator.hpp" 2 | #include "bitDistanceGap.hpp" 3 | #include "bitDistanceProtein.hpp" 4 | #include "DistanceEstimate.hpp" 5 | #include "hammingDistance.hpp" 6 | #include 7 | 8 | hammingDistance::hammingDistance(bool fastdist, dataloader* loader) { 9 | hammingDistance::seqCount = loader->getSequenceCount(); 10 | hammingDistance::seqLength = loader->getSequenceLength(); 11 | hammingDistance::sequenceNames = *loader->getSequenceNames(); 12 | hammingDistance::type = loader->type; 13 | hammingDistance::loader = loader; 14 | if(loader->fastdist) { 15 | if(type == DNA) { 16 | distEstimator = new bitDistanceGap(loader); 17 | } else if(type == PROTEIN) { 18 | distEstimator = new bitDistanceProtein(loader); 19 | } else { 20 | cerr << "ERROR: Unknown sequence type \"" << type << "\"" << endl; 21 | } 22 | } else { 23 | distEstimator = new simpleDistanceCalculator(loader); 24 | } 25 | } 26 | 27 | void hammingDistance::computeDistanceMatrix() { 28 | distanceMatrix = new distType*[seqCount]; 29 | unsigned long long data[3]; 30 | 31 | for(int i = 0; i < seqCount; i++){ 32 | distanceMatrix[i] = new distType[seqCount]; 33 | for(int j = i+1; j < seqCount; j++){ 34 | distEstimator->computeDistance(i,j, data); 35 | distanceMatrix[i][j] = (distType)(data[0]+data[1]) / (distType) data[2]; 36 | } 37 | } 38 | postProcessDistanceMatrix(); 39 | } 40 | 41 | void hammingDistance::postProcessDistanceMatrix(){ 42 | for (int i = 0; i < seqCount; i++) { 43 | for (int j = 0; j < seqCount; j++) { 44 | if(i > j){ 45 | distanceMatrix[i][j] = distanceMatrix[j][i]; 46 | } else if(j == i){ 47 | distanceMatrix[i][j] = 0; 48 | } 49 | } 50 | } 51 | } 52 | 53 | void hammingDistance::printDistances(){ 54 | for (int i = 0; i < seqCount; i++) { 55 | cout << sequenceNames[i] << "\t"; 56 | for (int j = 0; j < seqCount; j++) { 57 | cout << distanceMatrix[i][j] << "\t"; 58 | } 59 | cout << endl; 60 | } 61 | cout << endl; 62 | } 63 | 64 | // Test method 65 | /*void hammingDistance::getTsTv() { 66 | simpleDistanceCalculator* sc = new simpleDistanceCalculator(verbose, loader); 67 | unsigned int distances[2]; 68 | distanceMatrixTS = new int*[seqCount]; 69 | distanceMatrixTV = new int*[seqCount]; 70 | for(int i = 0; i < seqCount; i++){ 71 | distanceMatrixTS[i] = new int[seqCount]; 72 | distanceMatrixTV[i] = new int[seqCount]; 73 | for(int j = i+1; j < seqCount; j++){ 74 | sc->calculateDistanceDNA_TV_TS(i,j, distances); 75 | distanceMatrixTS[i][j] = distances[0]; 76 | distanceMatrixTV[i][j] = distances[1]; 77 | } 78 | } 79 | for (int i = 0; i < seqCount; i++) { 80 | cout << sequenceNames[i] << "\t"; 81 | for (int j = 0; j < seqCount; j++) { 82 | if(i <= j){ 83 | cout << "[" << distanceMatrixTS[i][j] << "," << distanceMatrixTV[i][j] << "]\t"; 84 | } else { 85 | cout << "[" << distanceMatrixTS[j][i] << "," << distanceMatrixTV[j][i] << "]\t"; 86 | } 87 | } 88 | cout << endl; 89 | } 90 | }*/ 91 | 92 | distType** hammingDistance::getDistanceMatrix(){ 93 | return distanceMatrix; 94 | } 95 | 96 | -------------------------------------------------------------------------------- /src/distanceCalculation/simpleDistanceCalculator.cpp: -------------------------------------------------------------------------------- 1 | #include "simpleDistanceCalculator.hpp" 2 | 3 | simpleDistanceCalculator::simpleDistanceCalculator(dataloader* loader) : DistanceEstimate(loader) { 4 | simpleDistanceCalculator::sequences = *loader->getSequences(); 5 | simpleDistanceCalculator::seqLength = loader->getSequenceLength(); 6 | } 7 | 8 | void simpleDistanceCalculator::computeDistance(int i, int j, unsigned long long* data) { 9 | if(type == DNA) { 10 | calculateTsTv(i, j, data); 11 | } else if(type == PROTEIN){ 12 | calculateHammingDistance(i, j, data); 13 | } else { 14 | cerr << "ERROR: simpleDistanceCalculator - unkown sequence type \"" << type << "\"" << endl; 15 | } 16 | } 17 | 18 | 19 | /* 20 | distType simpleDistanceCalculator::calculateDistance(int i, int j) { 21 | distType distance = 0; 22 | distType nucleotideCount = 0; 23 | for (int k = 0; k < seqLength; k++) { 24 | if(sequences[i][k] != '.' && sequences[i][k] != '-' && sequences[j][k] != '.' && sequences[j][k] != '-'){ 25 | // not a gap 26 | nucleotideCount++; 27 | if(sequences[i][k] != sequences[j][k]){ 28 | distance++; 29 | } 30 | } 31 | } 32 | if(nucleotideCount == 0){ 33 | return -1; 34 | } 35 | //cout << distance << "/" << nucleotideCount << endl; 36 | return distance / nucleotideCount; 37 | } 38 | */ 39 | 40 | void simpleDistanceCalculator::calculateHammingDistance(int i, int j, unsigned long long* data) { 41 | unsigned int distance = 0; 42 | unsigned int ungappedLength = 0; 43 | for (int k = 0; k < seqLength; k++) { 44 | if(sequences[i][k] != '.' && sequences[i][k] != '-' && sequences[j][k] != '.' && sequences[j][k] != '-'){ 45 | // not a gap 46 | if(sequences[i][k] != sequences[j][k]){ 47 | distance++; 48 | } 49 | ungappedLength++; 50 | } 51 | } 52 | data[0] = distance; 53 | data[1] = 0; 54 | data[2] = ungappedLength; 55 | } 56 | 57 | // Calculates the distance between two sequences. Transversion and Transitions are calculated 58 | void simpleDistanceCalculator::calculateTsTv(int i, int j, unsigned long long* data) { 59 | unsigned int ts = 0; 60 | unsigned int tv = 0; 61 | unsigned int length = 0; 62 | for (int k = 0; k < seqLength; k++) { 63 | if( sequences[i][k] == '-' || sequences[j][k] == '-'){ 64 | continue; 65 | } 66 | if(sequences[i][k] != sequences[j][k]){ 67 | if (sequences[i][k] == 'A') { 68 | if (sequences[j][k] == 'G') { 69 | ts++; 70 | } else { 71 | tv++; 72 | } 73 | } else if (sequences[i][k] == 'C') { 74 | if (sequences[j][k] == 'T') { 75 | ts++; 76 | } else { 77 | tv++; 78 | } 79 | } else if (sequences[i][k] == 'G') { 80 | if (sequences[j][k] == 'A') { 81 | ts++; 82 | } else { 83 | tv++; 84 | } 85 | } else if (sequences[i][k] == 'T') { 86 | if (sequences[j][k] == 'C') { 87 | ts++; 88 | } else { 89 | tv++; 90 | } 91 | } 92 | } 93 | length++; 94 | } 95 | data[0] = ts; 96 | data[1] = tv; 97 | data[2] = length; 98 | } 99 | 100 | simpleDistanceCalculator::~simpleDistanceCalculator(){} 101 | -------------------------------------------------------------------------------- /src/distanceCalculation/dataloader.hpp: -------------------------------------------------------------------------------- 1 | #ifndef DATALOADER_H 2 | #define DATALOADER_H 3 | 4 | #include 5 | #include 6 | #ifdef ENABLEGPU 7 | #include "cutil_inline.h" 8 | #include "gpu/constants.hpp" 9 | #endif 10 | 11 | using namespace std; 12 | 13 | class dataloader { 14 | 15 | public: 16 | dataloader(void); 17 | virtual ~dataloader(); 18 | void sample_sequences(); 19 | virtual void load(string filename){}; 20 | virtual unsigned int** getBitStrings() = 0; 21 | virtual unsigned int** getGapFilters() = 0; 22 | virtual unsigned int getSequenceCount() = 0; 23 | virtual unsigned int getSequenceLength() = 0; 24 | virtual unsigned int getBitStringsCount() = 0; 25 | virtual vector* getSequenceNames() = 0; 26 | virtual vector* getSequences() = 0; 27 | virtual void setSequences(vector*) = 0; 28 | unsigned int* bitStringsGPU; 29 | unsigned int* gapFiltersGPU; 30 | vector* sequences; 31 | vector* bitStrings; 32 | vector* gapFilters; 33 | /*Used to store the original data when performing bootstrapping*/ 34 | vector* sequences_original; 35 | vector* bitStrings_original; 36 | vector* gapFilters_original; 37 | 38 | void initGPUData(){ 39 | #ifdef ENABLEGPU 40 | unsigned int elmPrInt = 4; 41 | if(type == DNA){ 42 | elmPrInt = 16; 43 | } 44 | bsStride = getSequenceLength() / elmPrInt; 45 | if(getSequenceLength() % elmPrInt != 0) { 46 | bsStride++; 47 | } 48 | bsStride = (bsStride + threads_pr_block-1) & ~(threads_pr_block-1); 49 | unsigned int memSize = bsStride * getSequenceCount() * sizeof(unsigned int); 50 | cudaHostAlloc((void**)&bitStringsGPU,memSize,cudaHostAllocDefault); 51 | cudaHostAlloc((void**)&gapFiltersGPU,memSize,cudaHostAllocDefault); 52 | for(unsigned int i = 0; i < bsStride * getSequenceCount(); i++){ 53 | bitStringsGPU[i] = 0; 54 | gapFiltersGPU[i] = 0; 55 | } 56 | unsigned int** bitStrings = getBitStrings(); 57 | unsigned int** gapFilters = getGapFilters(); 58 | unsigned int dataSize = getBitStringsCount() * 4; 59 | dataSize = min(dataSize,bsStride); 60 | for(unsigned int i = 0; i < getSequenceCount(); i++) { 61 | memcpy(&bitStringsGPU[i*bsStride], bitStrings[i], dataSize*sizeof(unsigned int)); 62 | if(type == DNA){ 63 | memcpy(&gapFiltersGPU[i*bsStride], gapFilters[i], dataSize*sizeof(unsigned int)); 64 | } 65 | } 66 | #else 67 | cout << "GPU distance computation is not available" << endl; 68 | exit(1); 69 | #endif 70 | }; 71 | 72 | void readData(string filename, bool fastdist, bool verbose, InputType type, bool gpu) { 73 | dataloader::verbose = verbose; 74 | dataloader::fastdist = fastdist; 75 | 76 | dataloader::type = type; 77 | dataloader::initialised = false; 78 | load(filename); 79 | if(gpu){ 80 | initGPUData(); 81 | } 82 | }; 83 | 84 | InputType type; 85 | unsigned int bsStride; 86 | bool fastdist; 87 | 88 | protected: 89 | static const int BUFFER_SIZE = 16384; 90 | bool verbose; 91 | unsigned int sequenceLength; 92 | unsigned int sequenceCount; 93 | unsigned int bitStringsCount; 94 | int paddingLength; 95 | bool initialised; 96 | unsigned int BLOCK_SIZE; 97 | 98 | private: 99 | void copy_original_data(); 100 | void sample_fast_dist(); 101 | void sample_char(); 102 | }; 103 | #endif 104 | -------------------------------------------------------------------------------- /test_data/fasta2phylip.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | 3 | # Converts an aligned fasta (aa or dna) seq file to phylip format 4 | 5 | my $usage = "Usage: $0 [-h] [-v] [-c numChar] [infile]\n" . 6 | " -h: help\n" . 7 | " -c: long seq names are shortened to 10 char, default: 7 chars from the\n". 8 | " beggining is combined with the last 3 chars. You can change the\n". 9 | " behavior by this option. E.g., -c 10 will shorten the long name\n" . 10 | " by taking the first 10 characters of the name.\n". 11 | " -v: verbose (print name conversion in STDERR)\n" . 12 | " infile should be an aligned fasta, " . 13 | "STDIN is used if no infile is given\n"; 14 | 15 | use IO::File; 16 | use Getopt::Std; 17 | getopts('hvc:') || die "$usage\n"; 18 | 19 | die "$usage\n" if (defined ($opt_h)); 20 | 21 | my $totNumChar = 10; # number of characters allowed for name in phylip 22 | my $numFrontChar = 7; # When the name is too long, this amount of characters 23 | # are used from the beginning of the name, and the rest 24 | # are from the end of the name. 25 | if (defined ($opt_c)) { 26 | if ($opt_c <= $totNumChar && $opt_c >= 0) { 27 | $numFrontChar = $opt_c; 28 | } else { 29 | die "Error: give an integer between 0 and $totNumChar (ends inclusive)". 30 | " for -c.\n"; 31 | } 32 | } 33 | 34 | my $tmpFH = IO::File->new_tmpfile || die "Unable to make new temp file: $!"; 35 | 36 | my $firstLine = 1; 37 | my $maxLen = 0; 38 | my $numTaxa = 0; 39 | my $name; 40 | 41 | while(<>){ 42 | 43 | chomp; 44 | s/^\s+//; s/\s$//; 45 | next if (/^$/); 46 | 47 | if (s/^>\s*//) { 48 | 49 | if ($firstLine == 0) { 50 | if ($seqLen != $maxLen && $maxLen != 0) { 51 | warn "WARN: The $numTaxa-th species ($name) have ", 52 | "different seq length\n"; 53 | warn "Previous Seq Len: $maxLen, This Seq Len: $seqLen\n"; 54 | } 55 | print $tmpFH "\n"; # end of the previous sequence 56 | } else { 57 | $firstLine = 0; 58 | } 59 | 60 | $maxLen = $seqLen if ($seqLen > $maxLen); $seqLen = 0; 61 | $numTaxa ++; 62 | 63 | $name = $_; 64 | if (CharLen($_) <=10) { 65 | printf $tmpFH "%-10s", $_; 66 | } else { 67 | $shortName = TrimName($_); 68 | print STDERR "$_ => $shortName\n" if (defined ($opt_v)); 69 | printf $tmpFH "%10s", $shortName; 70 | } 71 | } else { 72 | $seqLen += CharLen ($_); 73 | print $tmpFH $_; 74 | } 75 | } 76 | 77 | print $tmpFH "\n"; 78 | 79 | ### print out to the STDOUT 80 | print "$numTaxa $maxLen\n"; 81 | 82 | seek ($tmpFH, 0, 0) || die "seek: $!"; 83 | my $line; 84 | while (defined ($line = $tmpFH->getline())) { 85 | chomp ($line); 86 | print "$line"; 87 | $missingBases = $maxLen - (CharLen($line) - $totNumChar); 88 | while ($missingBases > 0) { 89 | print "-"; 90 | $missingBases--; 91 | } 92 | print "\n"; 93 | } 94 | 95 | sub CharLen { # returns number of characters in a string 96 | my $string = shift; 97 | return scalar(split (//, $string)); 98 | } 99 | 100 | sub TrimName { # trim a long name 101 | my $name = shift; 102 | my @charArray = split (//, $name); 103 | 104 | if ($totNumChar == $numFrontChar) { 105 | return join ('', splice (@charArray, 0, $numFrontChar)); 106 | } else { 107 | return join ('', splice (@charArray, 0, $numFrontChar), 108 | splice (@charArray, - ($totNumChar - $numFrontChar))); 109 | } 110 | } 111 | 112 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Defining the compiler: 2 | ifeq ($(OS),Windows_NT) 3 | CC = cl 4 | LINK = link 5 | SWITCHES= -nologo -MT -D_CRT_SECURE_NO_DEPRECATE -EHsc -DPTW32_STATIC_LIB 6 | OPTIMIZATION_LEVEL=-O2 7 | DEBUG= #-WX #-pg 8 | OBJECTFLAG =-Fo 9 | COMPILEFLAG =-c 10 | LINK_OPTIONS =-nologo 11 | LINKER_COMMANDS =-out: 12 | LIBRARIES = pthreadVC2S.lib 13 | INCLUDES= -Isrc/distanceCalculation -Isrc/ -Ilib/includes 14 | else 15 | CC = g++ 16 | LINK = g++ 17 | OPTIMIZATION_LEVEL=-O3 -msse2 18 | DEBUG= #-Wall #-g #-pg 19 | OBJECTFLAG =-o 20 | COMPILEFLAG =-c 21 | LIBRARIES =-lpthread 22 | SWITCHES = 23 | LINK_OPTIONS = $(SWITCHES) 24 | LINKER_COMMANDS =-o 25 | INCLUDES= -Isrc/distanceCalculation -Isrc/ 26 | endif 27 | 28 | SRCPATH=src 29 | OBJPATH=obj 30 | LIBPATH=lib 31 | BINPATH=bin 32 | 33 | #use -pg for gprof profiling in both steps 34 | 35 | # Defining the object files: 36 | objects = $(OBJPATH)/main.o $(OBJPATH)/node.o $(OBJPATH)/distMatrixReader.o $(OBJPATH)/polytree.o $(OBJPATH)/diskMatrix.o $(OBJPATH)/rdDataInitialiser.o $(OBJPATH)/rapidNJ.o $(OBJPATH)/rapidNJDisk.o $(OBJPATH)/rapidNJMem.o $(OBJPATH)/simpleNJ.o $(OBJPATH)/testNJ.o $(OBJPATH)/distanceCalculation/dnaBitString.o $(OBJPATH)/distanceCalculation/dataloaderPhylip.o $(OBJPATH)/distanceCalculation/hammingDistance.o $(OBJPATH)/distanceCalculation/bitDistanceGap.o $(OBJPATH)/distanceCalculation/JCdistance.o $(OBJPATH)/distanceCalculation/simpleDistanceCalculator.o $(OBJPATH)/distanceCalculation/dataloader.o $(OBJPATH)/distanceCalculation/KimuraDistance.o $(OBJPATH)/distanceCalculation/bitDistanceProtein.o $(OBJPATH)/getopt_pp/getopt_pp.o $(OBJPATH)/distanceCalculation/DistanceEstimate.o $(OBJPATH)/ProgressBar.o 37 | 38 | all: rapidnj 39 | echo all: make complete 40 | 41 | 32 : SWITCHES=-m32 42 | 32 : all 43 | 44 | 64 : SWITCHES=-m64 45 | 64 : all 46 | 47 | rapidnj: $(objects) 48 | $(LINK) $(DEBUG) $(LINK_OPTIONS) $(LINKER_COMMANDS) $(BINPATH)/$@ $+ $(LIBRARIES) 49 | 50 | # compile to objectfiles 51 | $(OBJPATH)/%.o: $(SRCPATH)/%.cpp 52 | $(CC) $(DEBUG) $(SWITCHES) $(OPTIMIZATION_LEVEL) $(INCLUDES) $(COMPILEFLAG) $(OBJECTFLAG)$@ $< 53 | 54 | # clean target 55 | clean: 56 | -rm $(OBJPATH)/*.o 57 | -rm $(OBJPATH)/distanceCalculation/*.o 58 | -rm $(OBJPATH)/getopt_pp/*.o 59 | -rm $(BINPATH)/* 60 | echo clean: make complete 61 | 62 | release: 63 | mkdir rapidNJ 64 | cp -R src rapidNJ/ 65 | cp -R obj rapidNJ/ 66 | cp -R bin rapidNJ/ 67 | cp -R test_data rapidNJ/ 68 | cp -R tqdist rapidNJ/ 69 | cp Makefile rapidNJ/ 70 | find . -name "*.~" -exec rm {} \; 71 | zip rapidNJ.zip rapidNJ/README rapidNJ/Makefile rapidNJ/src/* rapidNJ/src/distanceCalculation/* rapidNJ/src/distanceCalculation/gpu/* rapidNJ/src/getopt_pp/* rapidNJ/obj rapidNJ/obj/distanceCalculation rapidNJ/obj/distanceCalculation/gpu rapidNJ/obj/getopt_pp rapidNJ/bin rapidNJ/test_data/* rapidNJ/tqdist/* 72 | rm -Rf rapidNJ/ 73 | 74 | util: $(SRCPATH)/distanceCalculation/sim_seq.cpp 75 | $(CC) $(DEBUG) $(SWITCHES) $(OPTIMIZATION_LEVEL) $(INCLUDES) $(COMPILEFLAG) $(OBJECTFLAG)$(OBJPATH)/distanceCalculation/sim_seq.o $< 76 | $(LINK) $(DEBUG) $(LINK_OPTIONS) $(LINKER_COMMANDS)$(BINPATH)/sim_seq $(OBJPATH)/distanceCalculation/sim_seq.o 77 | 78 | #Make sure that make rebuilds files if included headers change: 79 | $(objects): $(SRCPATH)/stdinclude.h $(SRCPATH)/polytree.h $(SRCPATH)/diskMatrix.h $(SRCPATH)/rdDataInitialiser.h $(SRCPATH)/distMatrixReader.hpp $(SRCPATH)/node.h $(SRCPATH)/rapidNJ.h $(SRCPATH)/rapidNJDisk.h $(SRCPATH)/rapidNJMem.hpp $(SRCPATH)/simpleNJ.h $(SRCPATH)/testNJ.h $(SRCPATH)/distanceCalculation/dataloader.hpp $(SRCPATH)/distanceCalculation/hammingDistance.hpp $(SRCPATH)/distanceCalculation/bitDistanceGap.hpp $(SRCPATH)/distanceCalculation/dnaBitString.hpp $(SRCPATH)/distanceCalculation/JCdistance.hpp $(SRCPATH)/distanceCalculation/simpleDistanceCalculator.hpp $(SRCPATH)/distanceCalculation/dataloaderStockholm.hpp $(SRCPATH)/distanceCalculation/dataloaderPhylip.hpp $(SRCPATH)/distanceCalculation/KimuraDistance.hpp $(SRCPATH)/distanceCalculation/bitStringUtils.hpp $(SRCPATH)/distanceCalculation/bitDistanceProtein.hpp $(SRCPATH)/distanceCalculation/dataloaderMemory.hpp $(SRCPATH)/distanceCalculation/dataloaderFasta.hpp $(SRCPATH)/distanceCalculation/gpu/constants.hpp $(SRCPATH)/getopt_pp/getopt_pp.h $(SRCPATH)/distanceCalculation/DistanceEstimate.hpp $(SRCPATH)/ProgressBar.hpp $(SRCPATH)/Bisection.h 80 | -------------------------------------------------------------------------------- /src/diskMatrix.cpp: -------------------------------------------------------------------------------- 1 | #include "stdinclude.h" 2 | #include "diskMatrix.h" 3 | #include 4 | #include 5 | 6 | using namespace std; 7 | 8 | /* A distance matrix stored in external memory. The matrix is stored in files of size 1 GB. 9 | */ 10 | 11 | diskMatrix::diskMatrix(std::string data_dir, int size) { 12 | diskMatrix::size = size; 13 | diskMatrix::data_dir = data_dir; 14 | rowToFileId = new int[size]; 15 | rowToFileIndex = new int[size]; 16 | rowsPerFile = MAX_FILE_SIZE / (size * ((int)sizeof(distType))); 17 | if (rowsPerFile == 0) rowsPerFile = 1; 18 | numberOfFiles = size / rowsPerFile; 19 | if(size % rowsPerFile != 0.0){ 20 | numberOfFiles++; 21 | } 22 | 23 | fstreams = new fstream[numberOfFiles]; 24 | file_names = new string[numberOfFiles]; 25 | int rowSize = size * sizeof(distType); 26 | int fileId = 0; 27 | int fileRowCount = 0; 28 | int totalRowCount = 0; 29 | 30 | //cout << "data_dir " << data_dir << endl; 31 | for(int i = 0; i < numberOfFiles; i++){ 32 | string fname = getTempFile(data_dir); 33 | fstreams[fileId].open(fname.data(),fstream::out | fstream::in | fstream::binary | fstream::trunc); 34 | file_names[fileId] = fname; 35 | if(!fstreams[fileId].is_open()){ 36 | cerr << "Could not open file " << fname << "\n"; 37 | cerr << "Check if the cache directory is accessible\n"; 38 | exit(1); 39 | } 40 | for(int j = 0; j < rowsPerFile; j++){ 41 | rowToFileIndex[totalRowCount] = fileRowCount * rowSize; 42 | rowToFileId[totalRowCount] = fileId; 43 | if(totalRowCount == size-1){ 44 | return; 45 | } 46 | fileRowCount++; 47 | totalRowCount++; 48 | } 49 | fileRowCount = 0; 50 | fileId++; 51 | } 52 | } 53 | 54 | string diskMatrix::getTempFile(string directory) { 55 | string fname; 56 | if(directory == "") { 57 | if(getenv("TMPDIR") != NULL) { 58 | fname = getenv("TMPDIR"); 59 | } else if (getenv("TMP") != NULL) { 60 | fname = getenv("TMP"); 61 | } else if (getenv("TEMP") != NULL) { 62 | fname = getenv("TEMP"); 63 | } else { 64 | fname = P_tmpdir; 65 | } 66 | } else { 67 | fname = directory; 68 | } 69 | char* buf = NULL; 70 | #ifdef __WINDOWS__ 71 | buf = new char[L_tmpnam]; 72 | tmpnam(buf); 73 | fname += buf; 74 | #else 75 | string s = fname + "/XXXXXX"; 76 | buf = new char [s.size()+1]; 77 | strcpy (buf, s.c_str()); 78 | int ret = mkstemp(buf); 79 | if(ret != -1) { 80 | close(ret); 81 | } 82 | fname = buf; 83 | #endif 84 | delete[] buf; 85 | return fname; 86 | } 87 | 88 | 89 | void diskMatrix::writeEntry(distType value, int row, int col){ 90 | int fileId = rowToFileId[row]; 91 | fstreams[fileId].seekp(rowToFileIndex[row] + col * sizeof(distType)); 92 | fstreams[fileId].write((char*)&value, sizeof(distType)); 93 | fstreams[fileId].flush(); 94 | } 95 | 96 | void diskMatrix::writeArray(distType *values, int row, int size){ 97 | int fileId = rowToFileId[row]; 98 | fstreams[fileId].seekp(rowToFileIndex[row]); 99 | fstreams[fileId].write((char*)values, sizeof(distType) * size); 100 | fstreams[fileId].flush(); 101 | } 102 | 103 | distType diskMatrix::readEntry(int row, int col){ 104 | char char_buf[sizeof(distType)]; 105 | distType distType_buf[1]; 106 | int fileId = rowToFileId[row]; 107 | fstreams[fileId].seekg(rowToFileIndex[row] + col * sizeof(distType)); 108 | fstreams[fileId].read(char_buf, sizeof(distType)); 109 | memcpy(distType_buf,char_buf,sizeof(distType)); 110 | return distType_buf[0]; 111 | } 112 | 113 | void diskMatrix::readArray(distType *values, int row, int size){ 114 | int fileId = rowToFileId[row]; 115 | fstreams[fileId].seekg(rowToFileIndex[row]); 116 | fstreams[fileId].read((char*)values, sizeof(distType) * size); 117 | } 118 | 119 | void diskMatrix::writeArrayNewSize(distType *values, int newRow, int newSize){ 120 | int fileId = rowToFileId[newRow]; 121 | int newIndex = (newRow % rowsPerFile) * newSize * sizeof(distType); 122 | fstreams[fileId].seekp(newIndex); 123 | rowToFileIndex[newRow] = newIndex; 124 | fstreams[fileId].write((char*)values, sizeof(distType) * newSize); 125 | fstreams[fileId].flush(); 126 | } 127 | 128 | void diskMatrix::setNewSize(int newSize){ 129 | size = newSize; 130 | } 131 | 132 | void diskMatrix::updateRowIndex(int newRow, int newSize){ 133 | rowToFileIndex[newRow] = (newRow % rowsPerFile) * newSize * sizeof(distType);; 134 | } 135 | 136 | void diskMatrix::flush(){ 137 | for(int i = 0; i < numberOfFiles; i++){ 138 | fstreams[i].flush(); 139 | } 140 | } 141 | 142 | diskMatrix::~diskMatrix(){ 143 | for(int i = 0; i < numberOfFiles; i++){ 144 | fstreams[i].close(); 145 | remove(file_names[i].c_str()); 146 | } 147 | delete[] file_names; 148 | delete[] fstreams; 149 | delete[] rowToFileId; 150 | delete[] rowToFileIndex; 151 | } 152 | -------------------------------------------------------------------------------- /src/simpleNJ.cpp: -------------------------------------------------------------------------------- 1 | /*A simple implementation of the neighbour-joining method*/ 2 | 3 | #include "stdinclude.h" 4 | #include "simpleNJ.h" 5 | #include "float.h" 6 | 7 | using namespace std; 8 | 9 | void printMatrix(distType** matrix, int size); 10 | void printArray(distType* a, int size); 11 | int countersim = 0; 12 | 13 | simpleNJ::simpleNJ(distMatrixReader* reader, int matrixSize, bool negative_branches, ProgressBar* pb) 14 | { 15 | simpleNJ::matrixSize = matrixSize; 16 | simpleNJ::reader = reader; 17 | simpleNJ::negative_branches = negative_branches; 18 | simpleNJ::pb = pb; 19 | matrix = reader->getMatrix(); 20 | separationsums = new distType[matrixSize]; 21 | separations = new distType[matrixSize]; 22 | clusterCount = matrixSize; 23 | activeRows = new int[matrixSize*2]; 24 | nextId = matrixSize; 25 | } 26 | 27 | polytree* simpleNJ::run(){ 28 | initialize(); 29 | while(clusterCount > 2){ 30 | findMin(); //O(n^2) 31 | mergeMinNodes(); 32 | updateMatrix(); 33 | } 34 | // finish by joining the two remaining clusters 35 | int index1 = -1; 36 | int index2 = -1; 37 | // find the last nodes 38 | for(int i = 0; i < matrixSize; i++){ 39 | if(activeRows[i] != -1){ 40 | if(index1 == -1){ 41 | index1 = i; 42 | } else { 43 | index2 = i; 44 | break; 45 | } 46 | } 47 | } 48 | double distance = matrix[index1][index2]; 49 | mytree->set_serialization_indices(activeRows[index1],activeRows[index2], distance / 2.0); 50 | pb->finish(); 51 | return mytree; 52 | } 53 | 54 | void simpleNJ::updateMatrix(){ 55 | distType newSeparationsum = 0; 56 | distType mutualDistance = matrix[min1][min2]; 57 | distType* row1 = matrix[min1]; 58 | distType* row2 = matrix[min2]; 59 | for(int i = 0; i < matrixSize; i++){ 60 | if(i == min1 || i == min2 || activeRows[i] == -1){ 61 | row1[i] = 0; 62 | } else { 63 | distType val1 = row1[i]; 64 | distType val2 = row2[i]; 65 | distType dist = (val1 + val2 - mutualDistance) / ((distType)2.0); 66 | if(simpleNJ::negative_branches && dist < 0) { 67 | dist = 0; 68 | } 69 | newSeparationsum += dist; 70 | // update the separationsum of cluster i. 71 | separationsums[i] += (dist - val1 - val2); 72 | separations[i] = separationsums[i] / (clusterCount -2); 73 | row1[i] = dist; 74 | matrix[i][min1] = dist; 75 | } 76 | } 77 | separationsums[min1] = newSeparationsum; 78 | separations[min1] = newSeparationsum / (clusterCount - 2); 79 | separationsums[min2] = 0; 80 | activeRows[min2] = -1; 81 | activeRows[min1] = nextId++ ; 82 | } 83 | 84 | void simpleNJ::mergeMinNodes() { 85 | // calculate distances 86 | double dist = matrix[min1][min2]; 87 | double sep1 = separations[min1]; 88 | double sep2 = separations[min2]; 89 | double dist1 = (0.5 * dist) + (0.5 * (sep1 - sep2)); 90 | double dist2 = (0.5 * dist) + (0.5 * (sep2 - sep1)); 91 | if(simpleNJ::negative_branches) { 92 | if(dist1 < 0.0) { 93 | dist2 += dist1; 94 | dist1 = 0; 95 | } 96 | if(dist2 < 0.0) { 97 | dist1 += dist2; 98 | if(dist1 < 0) { 99 | dist1 = 0; 100 | } 101 | dist2 = 0; 102 | } 103 | } 104 | // update tree 105 | mytree->addInternalNode(dist1, dist2, activeRows[min1], activeRows[min2]); 106 | clusterCount--; 107 | pb->setProgress((matrixSize - clusterCount) / (double) matrixSize); 108 | } 109 | 110 | void simpleNJ::initialize(){ 111 | //calculate initial seperation rows 112 | mytree = new polytree(matrixSize, reader->getSequenceNames()); 113 | for(int i = 0; i < matrixSize; i++){ 114 | distType sum = 0; 115 | for(int j = 0; j < matrixSize; j++){ 116 | sum += matrix[i][j]; 117 | } 118 | separationsums[i] = sum; 119 | separations[i] = sum / (clusterCount - 2); 120 | activeRows[i] = i; 121 | } 122 | } 123 | 124 | void simpleNJ::findMin() { 125 | min1 = -1; 126 | min2 = -1; 127 | double min = DBL_MAX; 128 | for (int i = 0; i < matrixSize; i++) { 129 | if(activeRows[i] != -1){ 130 | distType* row = matrix[i]; 131 | double sep1 = separations[i]; 132 | for(int j = 0; j < matrixSize; j++){ 133 | if(activeRows[j] != -1 && i != j){ 134 | double sep2 = separations[j]; 135 | double val = row[j] - sep1 - sep2; 136 | 137 | if(val < min){ 138 | // new minimum 139 | min1 = i; 140 | min2 = j; 141 | min = val; 142 | } 143 | } 144 | } 145 | } 146 | } 147 | //cout << "JOINING min1=" << min1 << " min2=" << min2 << " global_min=" << min << " clusterCount=" << clusterCount << "\n"; 148 | } 149 | 150 | simpleNJ::~simpleNJ(void) 151 | { 152 | } 153 | -------------------------------------------------------------------------------- /test_data/PF15224_full.fa: -------------------------------------------------------------------------------- 1 | >F6U190_ORNAN/130-206 2 | --PSARLSCYRKMLADHNCHSVPAGTTSLRRVDGGNLQDHFWEGKGCEIICYCNFRELLC 3 | CPKDIFFGPKISFVIPCN-V------ 4 | >G3WFF6_SARHA/20-95 5 | ST--SRLSCYRKILKDRNCHNIPEGVADLKQIDG-HLQDHFWDGKGCEMICYCNFSELLC 6 | CPKDIFFGPKISFVIPCNN------- 7 | >G1N4X3_MELGA/22-97 8 | --PSRRPKCYKRVLHDHSCHSIPEGMASLRQVDE-SLQDHFWEGEGCETICYCNFKELLC 9 | CPKEIFFGPKISFVIPCNS------- 10 | >G5B402_HETGA/21-96 11 | --PSNRLSCYRKILKDRNCHNLPEGVADLTQVDV-NVPDHFWEGKACEMICYCNFSELLC 12 | CPRDVFFGPKISFVIPCNN------- 13 | >H0UWX1_CAVPO/22-97 14 | --PSNRLSCYRKILKDRNCHSLPEGVAELTQVDV-NVPDHFWDGKECEMTCYCNFSELLC 15 | CPREIFFGPEVSFVIPCNN------- 16 | >M7BL74_CHEMY/22-81 17 | --PFSRLSCYKKVLKDRNCHSISEGVASLRRIDG-NLQDHFWEGQSCEMICYCNFSELLC 18 | CPK----------------------- 19 | >M3XS47_MUSPF/48-123 20 | --PANRLSCYRKTLKDRNCHNLPEGVTDLTKIDV-NVQDHFWDGKGCEMICYCNFSELLC 21 | CPKDIFFGPKISFVIPCNN------- 22 | >H0Z792_TAEGU/21-96 23 | --PSQRPSCYKRALQDHSCHTIPEGKENLRHVDD-GLQDHFWEGKGCEVICYCNLNELLC 24 | CPKDIFFGPKISFVIPCNS------- 25 | >L5L1P6_PTEAL/22-97 26 | --PTNRLSCYRKTVKDHNCHNLPEGVADLTKIDV-NVHEHFWKGKGCEMICYCNFRELFC 27 | CPKNVFFGPKISFVIPCS-N------ 28 | >F6PPY8_MACMU/22-97 29 | --PTNRLSCYRKILKDRNCHNLPEGVADLTEIDV-NVQDHFWDGKGCEMICYCNFSELLC 30 | CPKDVFFGPKISFVIPCN-I------ 31 | >U3I661_ANAPL/22-97 32 | --PSRHPSCYKRALKDHNCHGIHEGTENLRQIDG-SLKDHFWEGKGCETVCYCNFKELLC 33 | CPKEIFFGPKVSFVIPCNS------- 34 | >G1NY80_MYOLU/21-96 35 | --PANRLSCYRKILKDRNCHSLPEGVADLTKIDV-NVQDHFWDGKGCEMICYCNFSELLC 36 | CPKDVFFGPKISFVIPCNS------- 37 | >A6QPC2_BOVIN/22-97 38 | --PANRLSCYRKILKDRNCHSLPEGVADLTKIDV-NVQDHFWDGKGCEMVCYCNFSELLC 39 | CPKDVFFGPKISFVIPCNN------- 40 | >U3KAD0_FICAL/20-95 41 | --PSQRPACYKRALKEHSCHSIPGSTHSLRHVQG-GLQDHFWEGKGCEVICYCNLNELLC 42 | CPKDIFFGTKISFVIPCN-R------ 43 | >A0A0D9S254_CHLSB/22-97 44 | --PTNRLSCYRKILKDGNCHNLPEGVADLTEVDV-NVQDHFWDGKGCEMICYCNFSELLC 45 | CPKDVFFGPKISFVIPCN-I------ 46 | >G1SDP3_RABIT/22-97 47 | --PANRLSCYGKILKDRNCHNLPEGVADLTQIDA-NIQDHFWDGKGCEMICYCNFSELLC 48 | CPKDIFFGPKISFVIPCNN------- 49 | >F7H7Y2_CALJA/22-97 50 | --PANRLSCYRKILKDRNCHNLPEGVADLTQIDV-NVQDHFWDGKGCEMICYCNFSELLC 51 | CPKDIFFGPKISFVIPCNN------- 52 | >G3WFF5_SARHA/21-96 53 | ST--SRLSCYRKILKDRNCHNIPEGVADLKQIDG-HLQDHFWDGKGCEMICYCNFSELLC 54 | CPKDIFFGPKISFVIPCNN------- 55 | >H2REA2_PANTR/22-88 56 | --PANRLSCYRKILKDHNCHNLPEGVADLTQIDV-NVQDHFWDGKGCEMICYCNFSELLC 57 | CPK----------------CHHIFIL 58 | >G1KE53_ANOCA/24-99 59 | --PSVRLSCYRKILKDHNCHNIPAGVASLRPIDG-HLQDHFWEGKGCEMVCYCNFNELLC 60 | CPKDIFFGPKISFVIPCNS------- 61 | >I3LIJ2_PIG/23-98 62 | --PANRLSCYRKILKDRNCHNLPEGVADLTKIDV-NVQDHFWDGKGCEMICYCNFSELLC 63 | CPKDVFFGPKISFVIPCNN------- 64 | >SCRG1_HUMAN/22-97 65 | --PANRLSCYRKILKDHNCHNLPEGVADLTQIDV-NVQDHFWDGKGCEMICYCNFSELLC 66 | CPKDVFFGPKISFVIPCNN------- 67 | >G1R4I9_NOMLE/22-97 68 | --PANRLSCYRKILKDHNCHNLPEGVADLTQIDV-NVQDHFWDGKGCEMMCYCNFSELLC 69 | CPKDIFFGPKISFVIPCNN------- 70 | >M3W871_FELCA/23-98 71 | --PANRLSCYRKILKDRNCHNLPEGVADLTKMDV-NVQDHFWDGKGCEMICYCNFSELLC 72 | CPKDIFFGPKISFVIPCNN------- 73 | >H3AA38_LATCH/22-97 74 | --PSNRLSCSKKMLKDRDCHNIPEGVDHLRPIAR-NLQDHFWEGEGCEMVCYCNFSELLC 75 | CPKDVFFGPKISFVIPCNH------- 76 | >G3RVD3_GORGO/23-98 77 | --PANRLSCYRKILKDHNCHNLPEGVADLTQIDV-NVQDHFWDGKGCEMICYCNFSELLC 78 | CPKDVFFGPKISFVIPCNN------- 79 | >F6XL03_HORSE/22-97 80 | --PANRLSCYKKILKDRNCHNLPEGVADLTKMDV-NVQDHFWDGKGCEMICYCNFSELLC 81 | CPKDVFFGPKISFVIPCNN------- 82 | >F7GBP6_MONDO/22-97 83 | ST--SRLSCYRKILRDRNCHNIPEGVADLKQIDG-HLQDHFWDGKGCEMICYCNFSELLC 84 | CPKDIFFGPKISFVIPCNN------- 85 | >G3SR43_LOXAF/23-98 86 | --PVNRLSCYRKILRDGNCHNLPEGVADLTQMDV-NVQNHFWDGKGCEMICYCNFSELLC 87 | CPKDVFFGPKISFVIPCNN------- 88 | >G1M6N6_AILME/23-98 89 | --PANRLSCYRRTLRDRNCHDLPEGVTDLTKMDV-NVQDHFWDGKGCEMICYCNFSELLC 90 | CPKDIFFGPKISFVIPCNN------- 91 | >D6RD99_HUMAN/22-88 92 | --PANRLSCYRKILKDHNCHNLPEGVADLTQIDV-NVQDHFWDGKGCEMICYCNFSELLC 93 | CPK----------------CHHIFIL 94 | >SCRG1_RAT/22-97 95 | --PSSRLSCYRKLLKDRNCHNLPEGRADLKLIDE-NVQHHFWEGKGCEMICYCNFSELLC 96 | CPKDVFFGPKISFVIPCNS------- 97 | >W5Q192_SHEEP/23-98 98 | --PANRLSCYRKILKDRNCHSLPEGVADLTKIDV-NVQDHFWDGKGCEMICYCNFSELLC 99 | CPKDVFFGPRISFVIPCNN------- 100 | >G7P6K1_MACFA/22-97 101 | --PTNRLSCYRKILKDRNCHNLPEGVADLTEIDV-NVQDHFWDGKGCEMICYCNFSELLC 102 | CPKDVFFGPKISFVIPCN-I------ 103 | >A0A096N5P8_PAPAN/22-97 104 | --PTNRLSCYRKILKDRNCHNLPEGVADLTEIDV-NVQDHFWDGKGCEMICYCNFSELLC 105 | CPKDVFFGPKISFVIPCN-I------ 106 | >H2PES1_PONAB/22-97 107 | --PANRLSCYRKILKDHNCHNLPEGVADLTQIDV-NVQDHFWDGKGCEMICYCNFSELLC 108 | CPKDVFFGPKISFVIPCNN------- 109 | >SCRG1_MOUSE/22-97 110 | --PSSRLSCYRKLLKDRNCHNLPEGRADLKLIDA-NVQHHFWDGKGCEMICYCNFSELLC 111 | CPKDVFFGPKISFVIPCNN------- 112 | >I3MHH1_SPETR/21-96 113 | --PSNRLSCYRKILKDRNCHNLPEGVADLTQMDV-NVHDHFWDRNGCEMICYCNFSELLC 114 | CPRDIFFGPKISFVIPCNN------- 115 | >E2QXC1_CANFA/22-97 116 | --PASRLSCYRKTLKDRNCHNLPEGVADLTKMDV-NIQDHFWDGKGCEMICYCNFRELLC 117 | CPKDIFFGPKISFVIPCNN------- 118 | >F1NPU1_CHICK/22-97 119 | --PSRRPSCYKRILHNHSCHSIPEGMASLRRVDE-SLQDHFWEGEGCETICYCNFKELLC 120 | CPKEIFFGPKISFVIPCNS------- 121 | >H2QQF8_PANTR/22-97 122 | --PANRLSCYRKILKDHNCHNLPEGVADLTQIDV-NVQDHFWDGKGCEMICYCNFSELLC 123 | CPKDVFFGPKISFVIPCNN------- 124 | -------------------------------------------------------------------------------- /src/distanceCalculation/sim_seq.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | using namespace std; 9 | 10 | map nuc_map; 11 | 12 | void simPhylipSequences(int seq_num, int seq_length, bool useGaps){ 13 | srand(time(NULL)); 14 | int total_char = seq_length, remaining_char = seq_length; 15 | char* rnd_seq = new char[60]; 16 | for(int i= 0; i < 60; i++){ 17 | rnd_seq[i]= nuc_map[rand() % nuc_map.size()]; 18 | } 19 | cout << ' ' << seq_num << ' ' << seq_length << endl; 20 | char buffer[16]; 21 | while(remaining_char > 0){ 22 | for(int i = 0; i < seq_num; i++){ 23 | //change one random nucletide in each sequence 24 | int ranInt = rand() % 60; 25 | char original_nuc = rnd_seq[ranInt]; 26 | rnd_seq[ranInt] = nuc_map[rand() % nuc_map.size()]; 27 | sprintf(buffer,"seq_%06d ",i); 28 | cout << buffer << ' '; 29 | if(remaining_char <=60){ 30 | for(int j = 0; j < remaining_char; j++){ 31 | cout << rnd_seq[j]; 32 | } 33 | } else{ 34 | for(int j = 0; j < 60; j++){ 35 | cout << rnd_seq[j]; 36 | } 37 | } 38 | cout << endl; 39 | rnd_seq[ranInt] = original_nuc; 40 | } 41 | remaining_char -= 60; 42 | //generate new random sequence 43 | for(int i= 0; i < 60; i++){ 44 | rnd_seq[i]= nuc_map[rand() % nuc_map.size()]; 45 | } 46 | cout << endl; 47 | cerr << (1-remaining_char/ ((float)total_char))*100.0 << '\r'; 48 | } 49 | } 50 | 51 | void simStockholmSequences(int seq_num, int seq_length, bool useGaps){ 52 | double changeProb = 30; 53 | srand(time(NULL)); 54 | // build random sequence 55 | char* rnd_seq = new char[seq_length]; 56 | for(int i= 0; i < seq_length; i++){ 57 | rnd_seq[i]= nuc_map[rand() % nuc_map.size()]; 58 | } 59 | char buffer[16]; 60 | cout << "#STOCKHOLM 1.0" << endl; 61 | for(int i = 0; i < seq_num; i++){ 62 | sprintf(buffer,"seq_%06d ",i); 63 | cout << buffer << '\t'; 64 | for(int j = 0; j < seq_length; j++){ 65 | if(rand() % 100 > changeProb){ 66 | cout << rnd_seq[j]; 67 | } else { 68 | cout << nuc_map[rand() % nuc_map.size()]; 69 | } 70 | } 71 | cout << endl; 72 | } 73 | cout << "//" << endl; 74 | } 75 | 76 | void simFastaSequences(int seq_num, int seq_length, bool useGaps){ 77 | double changeProb = 30; 78 | srand(time(NULL)); 79 | // build random sequence 80 | char* rnd_seq = new char[seq_length]; 81 | for(int i= 0; i < seq_length; i++){ 82 | rnd_seq[i]= nuc_map[rand() % nuc_map.size()]; 83 | } 84 | char buffer[16]; 85 | for(int i = 0; i < seq_num; i++){ 86 | sprintf(buffer,">seq_%06d ",i); 87 | cout << buffer << endl; 88 | for(int j = 0; j < seq_length; j++){ 89 | if(j != 0 && j % 60 == 0){ 90 | cout << endl; 91 | } 92 | if(rand() % 100 > changeProb){ 93 | cout << rnd_seq[j]; 94 | } else { 95 | cout << nuc_map[rand() % nuc_map.size()]; 96 | } 97 | } 98 | cout << endl; 99 | } 100 | } 101 | 102 | int main(int argc, char* argv[]){ 103 | if(argc < 5 || argc > 6){ 104 | cout << "USAGE: sequence_count sequence_length type format [-g]" << endl; 105 | cout << "type: p=protein d=DNA" << endl; 106 | cout << "format: p=phylip, s=stockholm, f=fasta" << endl; 107 | exit(0); 108 | } 109 | int seq_num = atoi(argv[1]); 110 | int seq_length = atoi(argv[2]); 111 | string type = argv[3]; 112 | string format = argv[4]; 113 | bool useGaps = false; 114 | if(argc == 6){ 115 | if(string(argv[5]) == "-g"){ 116 | useGaps = true; 117 | } else { 118 | cout << "unrecognised argument " << argv[5] << endl; 119 | exit(0); 120 | } 121 | } 122 | if(type == "d"){ 123 | nuc_map[0] = 'A'; 124 | nuc_map[1] = 'C'; 125 | nuc_map[2] = 'G'; 126 | nuc_map[3] = 'T'; 127 | if(useGaps){ 128 | nuc_map[4] = '-'; 129 | } 130 | } else if(type == "p"){ 131 | nuc_map[0] = 'A'; 132 | nuc_map[1] = 'R'; 133 | nuc_map[2] = 'N'; 134 | nuc_map[3] = 'D'; 135 | nuc_map[4] = 'C'; 136 | nuc_map[5] = 'E'; 137 | nuc_map[6] = 'Q'; 138 | nuc_map[7] = 'G'; 139 | nuc_map[8] = 'H'; 140 | nuc_map[9] = 'I'; 141 | nuc_map[10] = 'L'; 142 | nuc_map[11] = 'K'; 143 | nuc_map[12] = 'M'; 144 | nuc_map[13] = 'F'; 145 | nuc_map[14] = 'P'; 146 | nuc_map[15] = 'S'; 147 | nuc_map[16] = 'T'; 148 | nuc_map[17] = 'W'; 149 | nuc_map[18] = 'Y'; 150 | nuc_map[19] = 'V'; 151 | if(useGaps){ 152 | nuc_map[20] = '-'; 153 | } 154 | } else { 155 | cout << "Unknown type " << type << endl; 156 | } 157 | if(format == "p"){ 158 | simPhylipSequences(seq_num, seq_length, useGaps); 159 | } else if(format == "s") { 160 | simStockholmSequences(seq_num, seq_length, useGaps); 161 | } else if(format == "f"){ 162 | simFastaSequences(seq_num, seq_length, useGaps); 163 | } else { 164 | cerr << "Unknown format" << endl; 165 | exit(1); 166 | } 167 | } 168 | -------------------------------------------------------------------------------- /src/distanceCalculation/dataloader.cpp: -------------------------------------------------------------------------------- 1 | #include "dataloader.hpp" 2 | #include "bitStringUtils.hpp" 3 | 4 | dataloader::dataloader(void){ 5 | type = UNKNOWN; 6 | bitStrings = NULL; 7 | gapFilters = NULL; 8 | sequences_original = NULL; 9 | bitStrings_original = NULL; 10 | gapFilters_original = NULL; 11 | BLOCK_SIZE = sizeof(unsigned int) * 8; 12 | //srand (time(NULL)); 13 | srand(0); 14 | } 15 | 16 | void dataloader::sample_fast_dist() { 17 | for(unsigned int i = 0; i < sequenceLength; i++){ 18 | //select random column 19 | unsigned int col = rand() % sequenceLength; 20 | // unsigned int col = i; //TEST 21 | unsigned int bits_per_site = 2; 22 | unsigned int mask = 3; 23 | if(type == PROTEIN) { 24 | bits_per_site = 8; 25 | mask = 255; 26 | } 27 | //cout << "Column: " << col << " -> " << i << endl; 28 | for(unsigned int row = 0; row < sequenceCount; row++){ 29 | unsigned int offset_target = i % (BLOCK_SIZE / bits_per_site); 30 | unsigned int offset_source = col % (BLOCK_SIZE / bits_per_site); 31 | unsigned int bitStringIdx_target = i / (BLOCK_SIZE / bits_per_site); 32 | unsigned int bitStringIdx_source = col / (BLOCK_SIZE / bits_per_site); 33 | unsigned int* bitString = bitStrings->at(row); 34 | unsigned int* gapFilter = NULL; 35 | if(gapFilters != NULL) { 36 | gapFilter = gapFilters->at(row); 37 | } 38 | //cout << "ROW: " << row << " bitstringindex: " << bitStringIdx_source << " " << bitStringIdx_target << endl; 39 | if(offset_target == 0) { 40 | bitString[bitStringIdx_target] = 0; 41 | if(gapFilter != NULL) { 42 | gapFilter[bitStringIdx_target] = 0; 43 | } 44 | } 45 | unsigned int temp; 46 | temp = bitStrings_original->at(row)[bitStringIdx_source] >> (offset_source * bits_per_site); 47 | temp &= mask; 48 | bitString[bitStringIdx_target] |= temp << (offset_target * bits_per_site); 49 | if(gapFilter != NULL) { 50 | temp = gapFilters_original->at(row)[bitStringIdx_source] >> (offset_source * bits_per_site); 51 | temp &= mask; 52 | gapFilter[bitStringIdx_target] |= temp << (offset_target * bits_per_site); 53 | } 54 | } 55 | } 56 | //printProteinSequence(((v4ui*)(bitStrings_original->at(0)))[0]); 57 | //printProteinSequence(((v4ui*)bitStrings->at(0))[0]); 58 | } 59 | 60 | void dataloader::sample_char() { 61 | for(unsigned int i = 0; i < sequenceLength; i++){ 62 | //select random column 63 | unsigned int col = rand() % sequenceLength; 64 | //unsigned int col = i; 65 | for(unsigned int row = 0; row < sequenceCount; row++){ 66 | sequences->at(row)[i] = sequences_original->at(row)[col]; 67 | } 68 | } 69 | } 70 | 71 | void dataloader::copy_original_data() { 72 | if(fastdist) { 73 | bitStrings_original = bitStrings; 74 | bitStrings = new vector; 75 | if(gapFilters != NULL) { 76 | gapFilters_original = gapFilters; 77 | gapFilters = new vector; 78 | } 79 | for(unsigned int i = 0; i < sequenceCount; i++) { 80 | unsigned int* bitString = (unsigned int*) _mm_malloc(bitStringsCount*4*sizeof(unsigned int), 16); 81 | for(unsigned int j = 0; j < bitStringsCount * 4; j++) { 82 | bitString[j] = bitStrings_original->at(i)[j]; 83 | } 84 | bitStrings->push_back(bitString); 85 | if(gapFilters != NULL) { 86 | unsigned int* gapFilter = (unsigned int*) _mm_malloc(bitStringsCount*4*sizeof(unsigned int), 16); 87 | for(unsigned int j = 0; j < bitStringsCount * 4; j++) { 88 | gapFilter[j] = gapFilters_original->at(i)[j]; 89 | } 90 | gapFilters->push_back(gapFilter); 91 | } 92 | } 93 | } else { 94 | sequences_original = sequences; 95 | sequences = new vector; 96 | for(unsigned int i = 0; i < sequenceCount; i++) { 97 | char* data = new char[sequenceLength]; 98 | sequences->push_back(data); 99 | } 100 | } 101 | } 102 | 103 | void dataloader::sample_sequences(){ 104 | if((fastdist && bitStrings_original == NULL) || (!fastdist && sequences_original == NULL)) { 105 | copy_original_data(); 106 | } 107 | if(fastdist) { 108 | sample_fast_dist(); 109 | } else { 110 | sample_char(); 111 | } 112 | } 113 | 114 | dataloader::~dataloader(){ 115 | if(sequences_original != NULL) { 116 | for(unsigned int i = 0; i < sequences_original->size(); i++) { 117 | delete[] sequences_original->at(i); 118 | } 119 | delete sequences_original; 120 | } 121 | if(bitStrings_original != NULL) { 122 | for(unsigned int i = 0; i < bitStrings_original->size(); i++) { 123 | _mm_free(bitStrings_original->at(i)); 124 | } 125 | delete bitStrings_original; 126 | } 127 | if(gapFilters_original != NULL) { 128 | for (unsigned int i = 0; i < gapFilters_original->size(); i++) { 129 | _mm_free(gapFilters_original->at(i)); 130 | } 131 | delete gapFilters_original; 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /src/distanceCalculation/bitDistanceProtein.cpp: -------------------------------------------------------------------------------- 1 | #include "bitDistanceProtein.hpp" 2 | #include 3 | 4 | v4ui bitDistanceProtein::mask1of8 = _mm_set_epi32(0x01010101,0x01010101,0x01010101,0x01010101); 5 | v4ui bitDistanceProtein::mask8 = _mm_set_epi32(0x00ff00ff,0x00ff00ff,0x00ff00ff,0x00ff00ff); 6 | v4ui bitDistanceProtein::mask16 = _mm_set_epi32(0x0000ffff,0x0000ffff,0x0000ffff,0x0000ffff); 7 | v4ui bitDistanceProtein::gapMask = _mm_set_epi32(0x2d2d2d2d,0x2d2d2d2d,0x2d2d2d2d,0x2d2d2d2d); 8 | 9 | bitDistanceProtein::bitDistanceProtein(dataloader* loader) : DistanceEstimate(loader) { 10 | bitDistanceProtein::bitStrings = (v4ui**) loader->getBitStrings(); 11 | bitDistanceProtein::seqCount = loader->getSequenceCount(); 12 | bitDistanceProtein::seqLength = loader->getSequenceLength(); 13 | bitDistanceProtein::bitStringsCount = loader->getBitStringsCount(); 14 | bitDistanceProtein::sequenceNames = *loader->getSequenceNames(); 15 | bitDistanceProtein::paddedSeqLength = (seqLength + 127) & ~127; 16 | numItr8Bit_cache = bitStringsCount / 8; // bitstrings are padded to a multiple of 8 17 | numItr16Bit_cache = numItr8Bit_cache / 31; 18 | if(numItr8Bit_cache % 31 != 0){ 19 | numItr16Bit_cache++; 20 | } 21 | numItr32Bit_cache = numItr16Bit_cache / 128; 22 | if(numItr16Bit_cache % 128 != 0){ 23 | numItr32Bit_cache++; 24 | } 25 | numItrTop_cache = numItr32Bit_cache / 32768; 26 | if(numItr32Bit_cache % 32768 != 0){ 27 | numItrTop_cache++; 28 | } 29 | } 30 | 31 | void bitDistanceProtein::computeDistance(int i, int j, unsigned long long* data) { 32 | numItr8Bit = numItr8Bit_cache; 33 | numItr16Bit = numItr16Bit_cache; 34 | numItr32Bit = numItr32Bit_cache; 35 | numItrTop = numItrTop_cache; 36 | long totalDist = 0; 37 | long totalGap = 0; 38 | bitDistanceProtein::seq1 = i; 39 | bitDistanceProtein::seq2 = j; 40 | idx = 0; 41 | for (int k = 0; k < numItrTop; k++) { 42 | v4ui_vector sumDist; 43 | v4ui_vector sumGap; 44 | sumDist.v = _mm_setzero_si128(); 45 | sumGap.v = _mm_setzero_si128(); 46 | 47 | sum32Bit(sumDist.v, sumGap.v); 48 | totalDist += sumDist.d[0]; 49 | totalDist += sumDist.d[1]; 50 | totalDist += sumDist.d[2]; 51 | totalDist += sumDist.d[3]; 52 | totalGap += sumGap.d[0]; 53 | totalGap += sumGap.d[1]; 54 | totalGap += sumGap.d[2]; 55 | totalGap += sumGap.d[3]; 56 | } 57 | data[0] = totalDist; 58 | data[1] = 0; 59 | data[2] = paddedSeqLength - totalGap; 60 | //cout << seq1 << " " << seq2 << ": " << totalDist << " " << paddedSeqLength - totalGap << endl; 61 | } 62 | 63 | /** Increases block size from 8 bits to 16 bits */ 64 | inline v4ui bitDistanceProtein::increaseBlockSize16(v4ui v) { 65 | return _mm_add_epi32(_mm_and_si128(v,mask8), _mm_and_si128(_mm_srli_epi32(v,8),mask8)); 66 | } 67 | 68 | /** Increases block size from 16 bits to 32 bits */ 69 | inline v4ui bitDistanceProtein::increaseBlockSize32(v4ui v) { 70 | return _mm_add_epi32(_mm_and_si128(v,mask16), _mm_and_si128(_mm_srli_epi32(v,16),mask16)); 71 | } 72 | 73 | inline void bitDistanceProtein::sum8_8BitVectors(v4ui& sumDist, v4ui& sumGap) { 74 | v4ui r, g; 75 | //unrolled by the compiler 76 | for(int i = 0; i < 8; i++){ 77 | r = _mm_cmpeq_epi8(bitStrings[seq1][idx],bitStrings[seq2][idx]); 78 | r = _mm_andnot_si128(r,mask1of8); 79 | g = _mm_and_si128(_mm_or_si128(_mm_cmpeq_epi8(bitStrings[seq1][idx],gapMask),_mm_cmpeq_epi8(bitStrings[seq2][idx],gapMask)),mask1of8); 80 | sumGap = _mm_add_epi32(g,sumGap); 81 | sumDist = _mm_add_epi32(_mm_andnot_si128(g,r),sumDist); 82 | idx++; 83 | } 84 | } 85 | 86 | /*sumDist is a 16 bit vector. This methods adds less than 512 to each element of the vector*/ 87 | void bitDistanceProtein::combineInto16BitVectors(v4ui& sumDist, v4ui& sumGap) { 88 | v4ui sum_dist_temp = _mm_setzero_si128(); 89 | v4ui sum_gap_temp = _mm_setzero_si128(); 90 | //TODO inline the sum function 91 | int iterations = min(numItr8Bit, 31); 92 | for (int i = 0; i < iterations; i++) { 93 | sum8_8BitVectors(sum_dist_temp, sum_gap_temp); 94 | } 95 | numItr8Bit -= iterations; 96 | sumDist = _mm_add_epi32(increaseBlockSize16(sum_dist_temp), sumDist); 97 | sumGap = _mm_add_epi32(increaseBlockSize16(sum_gap_temp), sumGap); 98 | } 99 | 100 | /* sumDist is a 32 bit vector. This methods adds less than 2*65536 to each element of the vector */ 101 | void bitDistanceProtein::combineInto32BitVectors(v4ui& sumDist, v4ui& sumGap) { 102 | int iterations = min(numItr16Bit, 128); 103 | v4ui sum_dist_temp = _mm_setzero_si128(); 104 | v4ui sum_gap_temp = _mm_setzero_si128(); 105 | for (int i = 0; i < iterations; i++) { 106 | combineInto16BitVectors(sum_dist_temp, sum_gap_temp); 107 | } 108 | sumDist = _mm_add_epi32(increaseBlockSize32(sum_dist_temp), sumDist); 109 | sumGap = _mm_add_epi32(increaseBlockSize32(sum_gap_temp), sumGap); 110 | numItr16Bit -= iterations; 111 | } 112 | 113 | void bitDistanceProtein::sum32Bit(v4ui& sumDist, v4ui& sumGap) { 114 | int iterations = min(numItr32Bit, 32768); 115 | for (int i = 0; i < iterations; i++) { 116 | combineInto32BitVectors(sumDist, sumGap); 117 | } 118 | numItr32Bit -= iterations; 119 | } 120 | 121 | bitDistanceProtein::~bitDistanceProtein(){ 122 | 123 | } 124 | 125 | -------------------------------------------------------------------------------- /src/threadedNJ.cpp: -------------------------------------------------------------------------------- 1 | /**The neighbour-joining method using two cores*/ 2 | 3 | #include "stdinclude.h" 4 | #include "threadedNJ.h" 5 | #include 6 | #include 7 | #include "minFinder.h" 8 | #include "float.h" 9 | 10 | using namespace std; 11 | 12 | distType** matrix; 13 | node** nodes; 14 | distType* separationsums; 15 | distType* separations; 16 | int matrixSize; 17 | int clusterCount; 18 | threadState * state1; 19 | threadState * state2; 20 | pthread_attr_t attrib1; 21 | sched_param s_param; 22 | 23 | threadedNJ::threadedNJ(datareader* reader) { 24 | matrixSize = reader->getSize(); 25 | matrix = reader->getMatrix(); 26 | nodes = reader->getNodes(); 27 | separationsums = new distType[matrixSize]; 28 | separations = new distType[matrixSize]; 29 | clusterCount = matrixSize; 30 | } 31 | 32 | void* threadedNJ::njThread( void* ptr ) { 33 | threadState* state = (threadState*) ptr; 34 | executefind(state); 35 | return 0; 36 | } 37 | 38 | node* threadedNJ::run() { 39 | initialize(); 40 | while(clusterCount > 2){ 41 | findMin(); 42 | mergeMinNodes(); 43 | updateMatrix(); 44 | } 45 | // finish by joining the two remaining clusters 46 | node* node1 = NULL; 47 | node* node2 = NULL; 48 | int index1 = -1; 49 | int index2 = -1; 50 | // find the last nodes 51 | for(int i = 0; i < matrixSize; i++){ 52 | node* node = nodes[i]; 53 | if(node != NULL){ 54 | if(index1 == -1){ 55 | node1 = node; 56 | index1 = i; 57 | } else { 58 | node2 = node; 59 | index2 = i; 60 | break; 61 | } 62 | } 63 | } 64 | double distance = matrix[index1][index2]; 65 | node1->addEdge(node2,distance); 66 | node2->addEdge(node1,distance); 67 | return node2; 68 | } 69 | 70 | void threadedNJ::initialize(){ 71 | //calculate initial seperation rows 72 | for(int i = 0; i < matrixSize; i++){ 73 | double sum = 0; 74 | for(int j = 0; j < matrixSize; j++){ 75 | sum += matrix[i][j]; 76 | } 77 | separationsums[i] = sum; 78 | separations[i] = sum / (clusterCount - 2); 79 | } 80 | // create threads 81 | state1 = new threadState(); 82 | state2 = new threadState(); 83 | state1->rowstart = 0; 84 | state1->rowEnd = (matrixSize / 2); 85 | state2->rowstart = matrixSize / 2; 86 | state2->rowEnd = matrixSize; 87 | } 88 | 89 | void threadedNJ::updateMatrix(){ 90 | double newSeparationsum = 0; 91 | double mutualDistance = matrix[min1][min2]; 92 | distType* row1 = matrix[min1]; 93 | distType* row2 = matrix[min2]; 94 | for(int i = 0; i < matrixSize; i++){ 95 | if(i == min1 || i == min2 || nodes[i] == NULL){ 96 | row1[i] = 0; 97 | } else { 98 | double val1 = row1[i]; 99 | double val2 = row2[i]; 100 | double dist = (val1 + val2 - mutualDistance) / 2.0; 101 | newSeparationsum += dist; 102 | // update the separationsum of cluster i. 103 | separationsums[i] += (dist - val1 - val2); 104 | separations[i] = separationsums[i] / (clusterCount -2); 105 | row1[i] = dist; 106 | matrix[i][min1] = dist; 107 | } 108 | } 109 | separationsums[min1] = newSeparationsum; 110 | separations[min1] = newSeparationsum / (clusterCount - 2); 111 | separationsums[min2] = 0; 112 | } 113 | 114 | void threadedNJ::mergeMinNodes(){ 115 | // calculate distances 116 | node* node1 = nodes[min1]; 117 | node* node2 = nodes[min2]; 118 | node* newNode = new node(); 119 | double dist = matrix[min1][min2]; 120 | double sep1 = separations[min1]; 121 | double sep2 = separations[min2]; 122 | double dist1 = (0.5 * dist) + (0.5 * (sep1 - sep2)); 123 | double dist2 = (0.5 * dist) + (0.5 * (sep2 - sep1)); 124 | // update tree 125 | newNode->addEdge(node1, dist1); 126 | node1->addEdge(newNode, dist1); 127 | newNode->addEdge(node2, dist2); 128 | node2->addEdge(newNode, dist2); 129 | // update data 130 | nodes[min1] = newNode; 131 | nodes[min2] = NULL; 132 | clusterCount--; 133 | } 134 | 135 | void threadedNJ::findMin() { 136 | pthread_t thread1, thread2; 137 | pthread_create( &thread1, NULL, threadedNJ::njThread, (void*) state1); 138 | pthread_create( &thread2, NULL, threadedNJ::njThread, (void*) state2); 139 | pthread_join(thread1, NULL); 140 | pthread_join(thread2, NULL); 141 | if(state1->min < state2->min){ 142 | min1 = state1->min1; 143 | min2 = state1->min2; 144 | } else { 145 | min1 = state2->min1; 146 | min2 = state2->min2; 147 | } 148 | } 149 | 150 | void executefind(threadState *state){ 151 | state->min1 = -1; 152 | state->min2 = -1; 153 | double min = DBL_MAX; 154 | for (int i = state->rowstart; i < state->rowEnd; i++) { 155 | if(nodes[i] != NULL){ 156 | distType* row = matrix[i]; 157 | double sep1 = separations[i]; 158 | for(int j = 0; j < matrixSize; j++){ 159 | if(nodes[j] != NULL && i != j){ 160 | double sep2 = separations[j]; 161 | double val = row[j] - sep1 - sep2; 162 | if(val < min){ 163 | // new minimum 164 | state->min1 = i; 165 | state->min2 = j; 166 | min = val; 167 | } 168 | } 169 | } 170 | } 171 | } 172 | state->min = min; 173 | } 174 | 175 | threadedNJ::~threadedNJ() 176 | { 177 | } 178 | -------------------------------------------------------------------------------- /lib/includes/semaphore.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Module: semaphore.h 3 | * 4 | * Purpose: 5 | * Semaphores aren't actually part of the PThreads standard. 6 | * They are defined by the POSIX Standard: 7 | * 8 | * POSIX 1003.1b-1993 (POSIX.1b) 9 | * 10 | * -------------------------------------------------------------------------- 11 | * 12 | * Pthreads-win32 - POSIX Threads Library for Win32 13 | * Copyright(C) 1998 John E. Bossom 14 | * Copyright(C) 1999,2005 Pthreads-win32 contributors 15 | * 16 | * Contact Email: rpj@callisto.canberra.edu.au 17 | * 18 | * The current list of contributors is contained 19 | * in the file CONTRIBUTORS included with the source 20 | * code distribution. The list can also be seen at the 21 | * following World Wide Web location: 22 | * http://sources.redhat.com/pthreads-win32/contributors.html 23 | * 24 | * This library is free software; you can redistribute it and/or 25 | * modify it under the terms of the GNU Lesser General Public 26 | * License as published by the Free Software Foundation; either 27 | * version 2 of the License, or (at your option) any later version. 28 | * 29 | * This library is distributed in the hope that it will be useful, 30 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 31 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 32 | * Lesser General Public License for more details. 33 | * 34 | * You should have received a copy of the GNU Lesser General Public 35 | * License along with this library in the file COPYING.LIB; 36 | * if not, write to the Free Software Foundation, Inc., 37 | * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA 38 | */ 39 | #if !defined( SEMAPHORE_H ) 40 | #define SEMAPHORE_H 41 | 42 | #undef PTW32_SEMAPHORE_LEVEL 43 | 44 | #if defined(_POSIX_SOURCE) 45 | #define PTW32_SEMAPHORE_LEVEL 0 46 | /* Early POSIX */ 47 | #endif 48 | 49 | #if defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 199309 50 | #undef PTW32_SEMAPHORE_LEVEL 51 | #define PTW32_SEMAPHORE_LEVEL 1 52 | /* Include 1b, 1c and 1d */ 53 | #endif 54 | 55 | #if defined(INCLUDE_NP) 56 | #undef PTW32_SEMAPHORE_LEVEL 57 | #define PTW32_SEMAPHORE_LEVEL 2 58 | /* Include Non-Portable extensions */ 59 | #endif 60 | 61 | #define PTW32_SEMAPHORE_LEVEL_MAX 3 62 | 63 | #if !defined(PTW32_SEMAPHORE_LEVEL) 64 | #define PTW32_SEMAPHORE_LEVEL PTW32_SEMAPHORE_LEVEL_MAX 65 | /* Include everything */ 66 | #endif 67 | 68 | #if defined(__GNUC__) && ! defined (__declspec) 69 | # error Please upgrade your GNU compiler to one that supports __declspec. 70 | #endif 71 | 72 | /* 73 | * When building the library, you should define PTW32_BUILD so that 74 | * the variables/functions are exported correctly. When using the library, 75 | * do NOT define PTW32_BUILD, and then the variables/functions will 76 | * be imported correctly. 77 | */ 78 | #if !defined(PTW32_STATIC_LIB) 79 | # if defined(PTW32_BUILD) 80 | # define PTW32_DLLPORT __declspec (dllexport) 81 | # else 82 | # define PTW32_DLLPORT __declspec (dllimport) 83 | # endif 84 | #else 85 | # define PTW32_DLLPORT 86 | #endif 87 | 88 | /* 89 | * This is a duplicate of what is in the autoconf config.h, 90 | * which is only used when building the pthread-win32 libraries. 91 | */ 92 | 93 | #if !defined(PTW32_CONFIG_H) 94 | # if defined(WINCE) 95 | # define NEED_ERRNO 96 | # define NEED_SEM 97 | # endif 98 | # if defined(__MINGW64__) 99 | # define HAVE_STRUCT_TIMESPEC 100 | # define HAVE_MODE_T 101 | # elif defined(_UWIN) || defined(__MINGW32__) 102 | # define HAVE_MODE_T 103 | # endif 104 | #endif 105 | 106 | /* 107 | * 108 | */ 109 | 110 | #if PTW32_SEMAPHORE_LEVEL >= PTW32_SEMAPHORE_LEVEL_MAX 111 | #if defined(NEED_ERRNO) 112 | #include "need_errno.h" 113 | #else 114 | #include 115 | #endif 116 | #endif /* PTW32_SEMAPHORE_LEVEL >= PTW32_SEMAPHORE_LEVEL_MAX */ 117 | 118 | #define _POSIX_SEMAPHORES 119 | 120 | #if defined(__cplusplus) 121 | extern "C" 122 | { 123 | #endif /* __cplusplus */ 124 | 125 | #if !defined(HAVE_MODE_T) 126 | typedef unsigned int mode_t; 127 | #endif 128 | 129 | 130 | typedef struct sem_t_ * sem_t; 131 | 132 | PTW32_DLLPORT int __cdecl sem_init (sem_t * sem, 133 | int pshared, 134 | unsigned int value); 135 | 136 | PTW32_DLLPORT int __cdecl sem_destroy (sem_t * sem); 137 | 138 | PTW32_DLLPORT int __cdecl sem_trywait (sem_t * sem); 139 | 140 | PTW32_DLLPORT int __cdecl sem_wait (sem_t * sem); 141 | 142 | PTW32_DLLPORT int __cdecl sem_timedwait (sem_t * sem, 143 | const struct timespec * abstime); 144 | 145 | PTW32_DLLPORT int __cdecl sem_post (sem_t * sem); 146 | 147 | PTW32_DLLPORT int __cdecl sem_post_multiple (sem_t * sem, 148 | int count); 149 | 150 | PTW32_DLLPORT int __cdecl sem_open (const char * name, 151 | int oflag, 152 | mode_t mode, 153 | unsigned int value); 154 | 155 | PTW32_DLLPORT int __cdecl sem_close (sem_t * sem); 156 | 157 | PTW32_DLLPORT int __cdecl sem_unlink (const char * name); 158 | 159 | PTW32_DLLPORT int __cdecl sem_getvalue (sem_t * sem, 160 | int * sval); 161 | 162 | #if defined(__cplusplus) 163 | } /* End of extern "C" */ 164 | #endif /* __cplusplus */ 165 | 166 | #undef PTW32_SEMAPHORE_LEVEL 167 | #undef PTW32_SEMAPHORE_LEVEL_MAX 168 | 169 | #endif /* !SEMAPHORE_H */ 170 | -------------------------------------------------------------------------------- /src/distMatrixReader.cpp: -------------------------------------------------------------------------------- 1 | #include "distMatrixReader.hpp" 2 | 3 | distMatrixReader::distMatrixReader(bool verbose, string filename, int matrixSize, bool halfMatrix) { 4 | distMatrixReader::matrixSize = matrixSize; 5 | curRow = 0; 6 | curCol = 0; 7 | distMatrixReader::verbose = verbose; 8 | distMatrixReader::filename = filename; 9 | buf = new char[bufsize]; 10 | distMatrixReader::halfMatrix = halfMatrix; 11 | distMatrixReader::sequenceNames = NULL; 12 | } 13 | 14 | distMatrixReader::distMatrixReader(bool verbose, int matrixSize, bool halfMatrix, vector* sequenceNames, distType** matrix) { 15 | distMatrixReader::matrixSize = matrixSize; 16 | curRow = 0; 17 | curCol = 0; 18 | distMatrixReader::verbose = verbose; 19 | buf = new char[bufsize]; 20 | distMatrixReader::halfMatrix = halfMatrix; 21 | distMatrixReader::matrix = matrix; 22 | distMatrixReader::sequenceNames = sequenceNames; 23 | } 24 | 25 | /*Used to initialise data structures when an alignment has been used as input */ 26 | void distMatrixReader::initializeData() { 27 | if(halfMatrix){ 28 | for(int k = 0; k < matrixSize; k++){ 29 | distType* newRow = new distType[k+1]; 30 | memcpy(newRow,matrix[k], sizeof(distType)*(k+1)); 31 | delete[] matrix[k]; 32 | matrix[k] = newRow; 33 | } 34 | } 35 | } 36 | 37 | /* Reads data from a file. The data must be formated as a phylip matrix.*/ 38 | void distMatrixReader::read_data(diskMatrix* dm){ 39 | char charInput[256]; 40 | ifstream is; 41 | // open the file 42 | is.open(filename.data(),ifstream::in); 43 | if(!is.is_open()){ 44 | cerr << "Could not read file: " << filename << endl; 45 | exit(1); 46 | } 47 | createDatastructures(); 48 | 49 | int j = 0; 50 | int i = 0; 51 | bool firstLine = true; 52 | 53 | while(is.good()) { 54 | is.read(buf,bufsize); 55 | i = 0; 56 | if(firstLine){ 57 | // scan over the first line containing the size 58 | while(firstLine && i < is.gcount()) { 59 | char c = buf[i]; 60 | i++; 61 | if(c == 10){ 62 | firstLine = false; 63 | break; 64 | } 65 | } 66 | } 67 | // fill matrix 68 | while(i < is.gcount()) { 69 | char c = buf[i]; 70 | if(c == 32 || c == 9){ 71 | if(j == 0){ 72 | // ignore. leading space 73 | } else { 74 | // parse the data found 75 | charInput[j] = '\0'; 76 | parseData(charInput); 77 | j = 0; 78 | curCol++; 79 | } 80 | } else if(c > 32 && c < 127){ 81 | // found data 82 | charInput[j] = c; 83 | j++; 84 | } else if(c == 10){ 85 | // new line 86 | charInput[j] = '\0'; 87 | if(j != 0){ 88 | // if j = 0 then we have read some trailing whitespaces. 89 | parseData(charInput); 90 | curCol++; 91 | } 92 | if(curCol < matrixSize+1) { 93 | cout << "Row " << curRow << " has fewer columns than the stated size of " << matrixSize << endl; 94 | exit(1); 95 | } 96 | j = 0; 97 | curCol = 0; 98 | curRow++; 99 | if(curRow > matrixSize){ 100 | // ignore the rest of the data, we have read enough 101 | is.close(); 102 | return; 103 | } 104 | } 105 | i++; 106 | } 107 | } 108 | if(j > 0){ 109 | // parse last piece of data, in case of missing newline on last line 110 | parseData(charInput); 111 | } 112 | is.close(); 113 | return; 114 | } 115 | 116 | /* Parse a piece of data as either text or a distType depending on the position of the data in the matrix */ 117 | inline void distMatrixReader::parseData(char* input){ 118 | if(curCol > matrixSize){ 119 | cout << "Row " << curRow << " has more columns than the stated size of " << matrixSize << endl; 120 | exit(1); 121 | } 122 | if(curCol > 0){ 123 | // parse as distType 124 | if(halfMatrix){ 125 | if(curRow >= curCol-1){ 126 | matrix[curRow][curCol-1] = atof(input); 127 | } 128 | } else { 129 | matrix[curRow][curCol-1] = atof(input); 130 | } 131 | } else { 132 | // parse as string 133 | sequenceNames->push_back(input); 134 | } 135 | } 136 | 137 | /* Create matrix datastructure and tree datastructure*/ 138 | void distMatrixReader::createDatastructures(){ 139 | if(halfMatrix){ 140 | matrix = new distType*[matrixSize]; 141 | for(int k = 0; k < matrixSize; k++){ 142 | matrix[k] = new distType[k+1]; 143 | } 144 | } else { 145 | matrix = new distType*[matrixSize]; 146 | for(int k = 0; k < matrixSize; k++){ 147 | matrix[k] = new distType[matrixSize]; 148 | } 149 | } 150 | if(sequenceNames == NULL) { 151 | sequenceNames = new vector(); 152 | } 153 | } 154 | 155 | distType** distMatrixReader::getMatrix(){ 156 | return matrix; 157 | } 158 | 159 | vector* distMatrixReader::getSequenceNames() { 160 | return sequenceNames; 161 | } 162 | 163 | string distMatrixReader::getFileName(){ 164 | return distMatrixReader::filename; 165 | } 166 | 167 | distMatrixReader::~distMatrixReader(void) 168 | { 169 | delete[] buf; 170 | } 171 | 172 | -------------------------------------------------------------------------------- /lib/includes/sched.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Module: sched.h 3 | * 4 | * Purpose: 5 | * Provides an implementation of POSIX realtime extensions 6 | * as defined in 7 | * 8 | * POSIX 1003.1b-1993 (POSIX.1b) 9 | * 10 | * -------------------------------------------------------------------------- 11 | * 12 | * Pthreads-win32 - POSIX Threads Library for Win32 13 | * Copyright(C) 1998 John E. Bossom 14 | * Copyright(C) 1999,2005 Pthreads-win32 contributors 15 | * 16 | * Contact Email: rpj@callisto.canberra.edu.au 17 | * 18 | * The current list of contributors is contained 19 | * in the file CONTRIBUTORS included with the source 20 | * code distribution. The list can also be seen at the 21 | * following World Wide Web location: 22 | * http://sources.redhat.com/pthreads-win32/contributors.html 23 | * 24 | * This library is free software; you can redistribute it and/or 25 | * modify it under the terms of the GNU Lesser General Public 26 | * License as published by the Free Software Foundation; either 27 | * version 2 of the License, or (at your option) any later version. 28 | * 29 | * This library is distributed in the hope that it will be useful, 30 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 31 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 32 | * Lesser General Public License for more details. 33 | * 34 | * You should have received a copy of the GNU Lesser General Public 35 | * License along with this library in the file COPYING.LIB; 36 | * if not, write to the Free Software Foundation, Inc., 37 | * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA 38 | */ 39 | #if !defined(_SCHED_H) 40 | #define _SCHED_H 41 | 42 | #undef PTW32_SCHED_LEVEL 43 | 44 | #if defined(_POSIX_SOURCE) 45 | #define PTW32_SCHED_LEVEL 0 46 | /* Early POSIX */ 47 | #endif 48 | 49 | #if defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 199309 50 | #undef PTW32_SCHED_LEVEL 51 | #define PTW32_SCHED_LEVEL 1 52 | /* Include 1b, 1c and 1d */ 53 | #endif 54 | 55 | #if defined(INCLUDE_NP) 56 | #undef PTW32_SCHED_LEVEL 57 | #define PTW32_SCHED_LEVEL 2 58 | /* Include Non-Portable extensions */ 59 | #endif 60 | 61 | #define PTW32_SCHED_LEVEL_MAX 3 62 | 63 | #if ( defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 200112 ) || !defined(PTW32_SCHED_LEVEL) 64 | #define PTW32_SCHED_LEVEL PTW32_SCHED_LEVEL_MAX 65 | /* Include everything */ 66 | #endif 67 | 68 | 69 | #if defined(__GNUC__) && !defined(__declspec) 70 | # error Please upgrade your GNU compiler to one that supports __declspec. 71 | #endif 72 | 73 | /* 74 | * When building the library, you should define PTW32_BUILD so that 75 | * the variables/functions are exported correctly. When using the library, 76 | * do NOT define PTW32_BUILD, and then the variables/functions will 77 | * be imported correctly. 78 | */ 79 | #if !defined(PTW32_STATIC_LIB) 80 | # if defined(PTW32_BUILD) 81 | # define PTW32_DLLPORT __declspec (dllexport) 82 | # else 83 | # define PTW32_DLLPORT __declspec (dllimport) 84 | # endif 85 | #else 86 | # define PTW32_DLLPORT 87 | #endif 88 | 89 | /* 90 | * This is a duplicate of what is in the autoconf config.h, 91 | * which is only used when building the pthread-win32 libraries. 92 | */ 93 | 94 | #if !defined(PTW32_CONFIG_H) 95 | # if defined(WINCE) 96 | # define NEED_ERRNO 97 | # define NEED_SEM 98 | # endif 99 | # if defined(__MINGW64__) 100 | # define HAVE_STRUCT_TIMESPEC 101 | # define HAVE_MODE_T 102 | # elif defined(_UWIN) || defined(__MINGW32__) 103 | # define HAVE_MODE_T 104 | # endif 105 | #endif 106 | 107 | /* 108 | * 109 | */ 110 | 111 | #if PTW32_SCHED_LEVEL >= PTW32_SCHED_LEVEL_MAX 112 | #if defined(NEED_ERRNO) 113 | #include "need_errno.h" 114 | #else 115 | #include 116 | #endif 117 | #endif /* PTW32_SCHED_LEVEL >= PTW32_SCHED_LEVEL_MAX */ 118 | 119 | #if (defined(__MINGW64__) || defined(__MINGW32__)) || defined(_UWIN) 120 | # if PTW32_SCHED_LEVEL >= PTW32_SCHED_LEVEL_MAX 121 | /* For pid_t */ 122 | # include 123 | /* Required by Unix 98 */ 124 | # include 125 | # else 126 | typedef int pid_t; 127 | # endif 128 | #else 129 | typedef int pid_t; 130 | #endif 131 | 132 | /* Thread scheduling policies */ 133 | 134 | enum { 135 | SCHED_OTHER = 0, 136 | SCHED_FIFO, 137 | SCHED_RR, 138 | SCHED_MIN = SCHED_OTHER, 139 | SCHED_MAX = SCHED_RR 140 | }; 141 | 142 | struct sched_param { 143 | int sched_priority; 144 | }; 145 | 146 | #if defined(__cplusplus) 147 | extern "C" 148 | { 149 | #endif /* __cplusplus */ 150 | 151 | PTW32_DLLPORT int __cdecl sched_yield (void); 152 | 153 | PTW32_DLLPORT int __cdecl sched_get_priority_min (int policy); 154 | 155 | PTW32_DLLPORT int __cdecl sched_get_priority_max (int policy); 156 | 157 | PTW32_DLLPORT int __cdecl sched_setscheduler (pid_t pid, int policy); 158 | 159 | PTW32_DLLPORT int __cdecl sched_getscheduler (pid_t pid); 160 | 161 | /* 162 | * Note that this macro returns ENOTSUP rather than 163 | * ENOSYS as might be expected. However, returning ENOSYS 164 | * should mean that sched_get_priority_{min,max} are 165 | * not implemented as well as sched_rr_get_interval. 166 | * This is not the case, since we just don't support 167 | * round-robin scheduling. Therefore I have chosen to 168 | * return the same value as sched_setscheduler when 169 | * SCHED_RR is passed to it. 170 | */ 171 | #define sched_rr_get_interval(_pid, _interval) \ 172 | ( errno = ENOTSUP, (int) -1 ) 173 | 174 | 175 | #if defined(__cplusplus) 176 | } /* End of extern "C" */ 177 | #endif /* __cplusplus */ 178 | 179 | #undef PTW32_SCHED_LEVEL 180 | #undef PTW32_SCHED_LEVEL_MAX 181 | 182 | #endif /* !_SCHED_H */ 183 | 184 | -------------------------------------------------------------------------------- /src/rdDataInitialiser.cpp: -------------------------------------------------------------------------------- 1 | #include "stdinclude.h" 2 | #include 3 | #include "rdDataInitialiser.h" 4 | #include 5 | #include "cluster_pair.h" 6 | 7 | using namespace std; 8 | 9 | rdDataInitialiser::rdDataInitialiser(bool verbose, int sortedMatrixSize, string cacheDir, string filename){ 10 | rdDataInitialiser::verbose = verbose; 11 | matrixSize = -1; 12 | curRow = 0; 13 | curCol = 0; 14 | curSeparationSum = 0; 15 | rdDataInitialiser::sortedMatrixSize = sortedMatrixSize; 16 | rdDataInitialiser::cacheDir = cacheDir; 17 | rdDataInitialiser::filename = filename; 18 | rdDataInitialiser::dm = NULL; 19 | } 20 | 21 | rdDataInitialiser::rdDataInitialiser(bool verbose, int sortedMatrixSize, string cacheDir, int matrixSize){ 22 | rdDataInitialiser::verbose = verbose; 23 | curRow = 0; 24 | curCol = 0; 25 | curSeparationSum = 0; 26 | rdDataInitialiser::sortedMatrixSize = sortedMatrixSize; 27 | rdDataInitialiser::cacheDir = cacheDir; 28 | rdDataInitialiser::filename = ""; 29 | rdDataInitialiser::dm = NULL; 30 | rdDataInitialiser::matrixSize = matrixSize; 31 | } 32 | 33 | void rdDataInitialiser::initializeFromExistingMatrix(vector* sequenceNames, diskMatrix* dm) { 34 | rdDataInitialiser::dm = dm; 35 | createDatastructures(); 36 | for(int i = 0; i < matrixSize; i++) { 37 | dm->readArray(rowBuffer, i, matrixSize); 38 | for(int j = 0; j < matrixSize; j++){ 39 | curSeparationSum += rowBuffer[j]; 40 | } 41 | processRow(false); 42 | curSeparationSum = 0; 43 | mytree->addLeaf(sequenceNames->at(i)); 44 | curRow++; 45 | } 46 | delete[] rowBuffer; 47 | } 48 | 49 | /* Reads data from a file. The data must be formated as a symmetrical phylip matix.*/ 50 | bool rdDataInitialiser::read_data(){ 51 | int bufsize = 65536; 52 | char * buf = new char[bufsize]; 53 | char charInput[256]; 54 | ifstream is; 55 | // open the file 56 | is.open(filename.data(),ifstream::in); 57 | if(!is.is_open()){ 58 | return false; 59 | } 60 | int j = 0; 61 | // main loop. Keep reading until no more data or enough data has been read 62 | while(is.good()){ 63 | is.read(buf,bufsize); 64 | int i = 0; 65 | // read the size of the matrix 66 | if(matrixSize == -1){ 67 | while(i < is.gcount()) { 68 | char c = buf[i]; 69 | if(c == 10){ 70 | charInput[j] = '\0'; 71 | i++; 72 | break; 73 | } else if(c >= 47 && c <= 57){ 74 | charInput[j] = c; 75 | j++; 76 | } 77 | i++; 78 | } 79 | matrixSize = atoi(charInput); 80 | j = 0; 81 | createDatastructures(); 82 | } 83 | // build matrix 84 | bool trailingSpace = false; 85 | while(i < is.gcount()) { 86 | char c = buf[i]; 87 | if(c == 32 || c == 9){ 88 | if(j == 0){ 89 | // ignore. leading space 90 | } else if(curCol == matrixSize){ 91 | trailingSpace = true; 92 | } else { 93 | // parse the data found 94 | charInput[j] = '\0'; 95 | parseData(charInput); 96 | j = 0; 97 | curCol++; 98 | if(curCol > matrixSize){ 99 | //cout << curCol << " " << matrixSize << endl; 100 | cerr << "Matrix doesn't match the stated size \n"; 101 | return false; 102 | } 103 | } 104 | } else if(c > 32 && c < 127){ 105 | // found data 106 | if(trailingSpace){ 107 | cerr << "Matrix doesn't match the stated size \n"; 108 | return false; 109 | } 110 | charInput[j] = c; 111 | j++; 112 | } else if(c == 10){ 113 | // new line 114 | if(curCol == matrixSize){ 115 | charInput[j] = '\0'; 116 | parseData(charInput); 117 | j = 0; 118 | trailingSpace = false; 119 | processRow(true); 120 | curSeparationSum = 0; 121 | curCol = 0; 122 | curRow++; 123 | } else { 124 | cerr << "Matrix doesn't match the stated size \n"; 125 | return false; 126 | } 127 | if(curRow == matrixSize){ 128 | // ignore the rest of the data if any. 129 | return true; 130 | } 131 | } 132 | i++; 133 | } 134 | } 135 | if(j > 0){ 136 | // parse last piece of data, in case of missing newline on last line 137 | parseData(charInput); 138 | processRow(true); 139 | } 140 | is.close(); 141 | return true; 142 | } 143 | 144 | /* Parse a piece of data as either text or a distType depending on the position of the data in the matrix */ 145 | inline void rdDataInitialiser::parseData(char* input){ 146 | if(curCol > 0){ 147 | // parse as distType 148 | rowBuffer[curCol-1] = (distType) atof(input); 149 | //cout << "row " << rowBuffer[curCol-1] << endl; 150 | curSeparationSum += rowBuffer[curCol-1]; 151 | } else { 152 | // parse as string 153 | mytree->addLeaf(input); 154 | } 155 | } 156 | 157 | /* Create matrix datastructure and tree datastructure*/ 158 | void rdDataInitialiser::createDatastructures(){ 159 | matrix = new cluster_pair*[matrixSize]; 160 | for(int k = 0; k < matrixSize; k++){ 161 | matrix[k] = new cluster_pair[sortedMatrixSize]; 162 | } 163 | mytree = new polytree(matrixSize, NULL); 164 | if(dm == NULL) { 165 | dm = new diskMatrix(cacheDir, matrixSize); 166 | } 167 | 168 | rowBuffer = new distType[matrixSize]; 169 | sorted_row_lengths = new int[matrixSize]; 170 | separation_sums = new distType[matrixSize]; 171 | possibleRedundantRows = new short[matrixSize]; 172 | } 173 | 174 | void rdDataInitialiser::processRow(bool writeToDisk){ 175 | // write the distance matrix row to the HDD 176 | if(writeToDisk) { 177 | dm->writeArray(rowBuffer,curRow,matrixSize); 178 | } 179 | //search for possible redundant rows 180 | possibleRedundantRows[curRow] = 0; 181 | for(int i = 0; i < matrixSize; i++){ 182 | if(rowBuffer[i] == 0 && i != curRow){ 183 | possibleRedundantRows[curRow] = 1; 184 | break; 185 | } 186 | } 187 | separation_sums[curRow] = curSeparationSum; 188 | } 189 | 190 | int rdDataInitialiser::getSize(){ 191 | return matrixSize; 192 | } 193 | 194 | cluster_pair** rdDataInitialiser::getMatrix(){ 195 | return matrix; 196 | } 197 | 198 | polytree* rdDataInitialiser::getTree(){ 199 | return mytree; 200 | } 201 | 202 | int* rdDataInitialiser::getSortedRowLengths(){ 203 | return sorted_row_lengths; 204 | } 205 | 206 | distType* rdDataInitialiser::getSeparationSums(){ 207 | return separation_sums; 208 | } 209 | 210 | diskMatrix* rdDataInitialiser::getDiskMatrix(){ 211 | return dm; 212 | } 213 | 214 | int rdDataInitialiser::getDataStructureSize(){ 215 | return sortedMatrixSize; 216 | } 217 | 218 | short* rdDataInitialiser::getPossibleRedundantRows(){ 219 | return possibleRedundantRows; 220 | } 221 | 222 | string rdDataInitialiser::getFileName(){ 223 | return filename; 224 | } 225 | 226 | rdDataInitialiser::~rdDataInitialiser(void) 227 | { 228 | for(int i = 0; i < matrixSize; i++){ 229 | delete[] matrix[i]; 230 | } 231 | delete[] matrix; 232 | delete[] possibleRedundantRows; 233 | delete[] separation_sums; 234 | delete[] sorted_row_lengths; 235 | } 236 | -------------------------------------------------------------------------------- /src/testNJ.cpp: -------------------------------------------------------------------------------- 1 | /*A simple implementation of the neighbour-joining method*/ 2 | 3 | #include "stdinclude.h" 4 | #include "testNJ.h" 5 | #include "float.h" 6 | 7 | using namespace std; 8 | 9 | void printMatrix(distType** matrix, int size); 10 | void printArray(distType* a, int size); 11 | 12 | testNJ::testNJ(distMatrixReader* reader, int matrixSize) 13 | { 14 | testNJ::matrixSize = matrixSize; 15 | testNJ::reader = reader; 16 | matrix = reader->getMatrix(); 17 | separationsums = new distType[matrixSize]; 18 | separations = new distType[matrixSize]; 19 | clusterCount = matrixSize; 20 | activeRows = new int[matrixSize*2]; 21 | nextId = matrixSize; 22 | initialize(); 23 | } 24 | 25 | void testNJ::step(int idx1, int idx2, distType value){ 26 | // cout << activeRows[38] << endl; 27 | if( clusterCount == 2){ 28 | // finish by joining the two remaining clusters 29 | min1 = -1; 30 | min2 = -1; 31 | // find the last nodes 32 | for(int i = 0; i < matrixSize; i++){ 33 | if(activeRows[i] != -1){ 34 | if(min1 == -1){ 35 | min1 = i; 36 | } else { 37 | min2 = i; 38 | break; 39 | } 40 | } 41 | } 42 | double distance = matrix[min1][min2]; 43 | if(idx1 == min1 && idx2 == min2){ 44 | if(distance != value){ 45 | cout << "ERROR: indexes match but value doesn't: " << value << "!=" << min << endl; 46 | exit(1); 47 | } 48 | } else if(value != distance){ 49 | cout << "ERROR: both value and indexes differ min1: " << min1 << "!=" << idx1 << ". min2: " << min2 << "!=" << idx2 << ". Value: " << min << "!=" << value << endl; 50 | exit(1); 51 | } 52 | //return mytree->serialize_tree(activeRows[index1],activeRows[index2], distance); 53 | } else { 54 | findMin(); 55 | // cout << "TEST: JOINING min1=" << min1 << " min2=" << min2 << " global_min=" << min << " clusterCount=" << clusterCount << "\n"; 56 | if(idx1 == min1 && idx2 == min2){ 57 | if((value - min) * (value - min) > 0.00001){ 58 | printf("%.20f %.20f \n",value,min); 59 | cout << "ERROR: indexes match but value doesn't: " << value << "!=" << min << endl; 60 | exit(1); 61 | } 62 | } else if((value - min) * (value - min) > 0.00001){ 63 | printf("%.20f %.20f \n",value,min); 64 | cout << "ERROR: both value and indexes differ min1: " << min1 << "!=" << idx1 << ". min2: " << min2 << "!=" << idx2 << ". Value: " << min << "!=" << value << endl; 65 | cout << matrix[970][943] << "-" << separations[970] << "-" << separations[943] << " = " << matrix[970][943] - separations[970] - separations[943] << endl; 66 | exit(1); 67 | } else { 68 | min1 = idx1; 69 | min2 = idx2; 70 | } 71 | } 72 | mergeMinNodes(); 73 | updateMatrix(); 74 | } 75 | 76 | void testNJ::stepNoVal(int idx1, int idx2){ 77 | // cout << idx2 << " " << idx1 << endl; 78 | if(clusterCount < 4){ 79 | return; 80 | } 81 | if( clusterCount == 2){ 82 | // finish by joining the two remaining clusters 83 | min1 = -1; 84 | min2 = -1; 85 | // find the last nodes 86 | for(int i = 0; i < matrixSize; i++){ 87 | if(activeRows[i] != -1){ 88 | if(min1 == -1){ 89 | min1 = i; 90 | } else { 91 | min2 = i; 92 | break; 93 | } 94 | } 95 | } 96 | // double distance = matrix[min1][min2]; 97 | if(!((idx1 == min1 && idx2 == min2) || (idx2 == min1 && idx1 == min2))){ 98 | double checkVal = matrix[idx1][idx2] - separations[idx1] - separations[idx2]; 99 | if(checkVal > min){ 100 | cout << "ERROR: bad join min1: " << min1 << "!=" << idx1 << ". min2: " << min2 << "!=" << idx2 << ". Value: " << min << "!=" << checkVal << endl << endl; 101 | } 102 | exit(1); 103 | } 104 | //return mytree->serialize_tree(activeRows[index1],activeRows[index2], distance); 105 | } else { 106 | findMin(); 107 | //cout << "TEST: JOINING min1=" << min1 << " min2=" << min2 << " global_min=" << min << " clusterCount=" << clusterCount << "\n"; 108 | if(!((idx1 == min1 && idx2 == min2) || (idx2 == min1 && idx1 == min2))){ 109 | //check if this join is ok 110 | double checkVal = matrix[idx1][idx2] - separations[idx1] - separations[idx2]; 111 | if((checkVal - min) * (checkVal - min) > 0.0001){ 112 | printf("%.20f %.20f \n",checkVal,min); 113 | cout << "ERROR: bad join min1: " << min1 << "!=" << idx1 << ". min2: " << min2 << "!=" << idx2 << ". Value: " << min << "!=" << checkVal << endl << endl; 114 | } 115 | exit(1); 116 | } else { 117 | min1 = idx1; 118 | min2 = idx2; 119 | } 120 | } 121 | mergeMinNodes(); 122 | updateMatrix(); 123 | } 124 | 125 | 126 | void testNJ::updateMatrix(){ 127 | distType newSeparationsum = 0; 128 | distType mutualDistance = matrix[min1][min2]; 129 | distType* row1 = matrix[min1]; 130 | distType* row2 = matrix[min2]; 131 | for(int i = 0; i < matrixSize; i++){ 132 | if(i == min1 || i == min2 || activeRows[i] == -1){ 133 | row1[i] = 0; 134 | } else { 135 | distType val1 = row1[i]; 136 | distType val2 = row2[i]; 137 | distType dist = (val1 + val2 - mutualDistance) / ((distType)2.0); 138 | newSeparationsum += dist; 139 | // update the separationsum of cluster i. 140 | separationsums[i] += (dist - val1 - val2); 141 | 142 | separations[i] = separationsums[i] / (clusterCount -2); 143 | row1[i] = dist; 144 | matrix[i][min1] = dist; 145 | } 146 | } 147 | separationsums[min1] = newSeparationsum; 148 | separations[min1] = newSeparationsum / (clusterCount - 2); 149 | separationsums[min2] = 0; 150 | activeRows[min2] = -1; 151 | activeRows[min1] = nextId++ ; 152 | } 153 | 154 | void testNJ::mergeMinNodes(){ 155 | // calculate distances 156 | double dist = matrix[min1][min2]; 157 | double sep1 = separations[min1]; 158 | double sep2 = separations[min2]; 159 | double dist1 = (0.5 * dist) + (0.5 * (sep1 - sep2)); 160 | double dist2 = (0.5 * dist) + (0.5 * (sep2 - sep1)); 161 | // update tree 162 | mytree->addInternalNode(dist1, dist2, activeRows[min1], activeRows[min2]); 163 | clusterCount--; 164 | } 165 | 166 | void testNJ::initialize(){ 167 | mytree = new polytree(matrixSize, reader->getSequenceNames()); 168 | //calculate initial seperation rows 169 | for(int i = 0; i < matrixSize; i++){ 170 | distType sum = 0; 171 | for(int j = 0; j < matrixSize; j++){ 172 | sum += matrix[i][j]; 173 | } 174 | separationsums[i] = sum; 175 | separations[i] = sum / (clusterCount - 2); 176 | activeRows[i] = i; 177 | } 178 | } 179 | 180 | void testNJ::findMin() { 181 | min1 = -1; 182 | min2 = -1; 183 | min = DBL_MAX; 184 | for (int i = 0; i < matrixSize; i++) { 185 | if(activeRows[i] != -1){ 186 | distType* row = matrix[i]; 187 | double sep1 = separations[i]; 188 | for(int j = 0; j < matrixSize; j++){ 189 | if(activeRows[j] != -1 && i != j){ 190 | double sep2 = separations[j]; 191 | double val = row[j] - sep1 - sep2; 192 | // cout << "TEST: [" << i << "," << j << "] " << row[j] << "-" << sep1 << "-" << sep2 << "=" << val << " min=" << min << endl; 193 | if(val < min){ 194 | // new minimum 195 | min1 = i; 196 | min2 = j; 197 | min = val; 198 | } 199 | } 200 | } 201 | } 202 | } 203 | // cout << "TEST: JOINING min1=" << min1 << " min2=" << min2 << " global_min=" << min << " clusterCount=" << clusterCount << "\n"; 204 | } 205 | 206 | testNJ::~testNJ(void) 207 | { 208 | } 209 | -------------------------------------------------------------------------------- /src/getopt_pp/getopt_pp.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | GetOpt_pp: Yet another C++ version of getopt. 3 | Copyright (C) 2007, 2008, 2009, 2010 Daniel Gutson, FuDePAN 4 | 5 | This file is part of GetOpt_pp. 6 | 7 | GetOpt_pp is free software: you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation, either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | GetOpt_pp is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License 18 | along with this program. If not, see . 19 | */ 20 | 21 | #if __APPLE__ 22 | extern char** environ; 23 | #elif _WIN32 24 | #include 25 | #define environ _environ 26 | #else 27 | #include 28 | #endif 29 | 30 | #include "getopt_pp.h" 31 | 32 | namespace GetOpt 33 | { 34 | 35 | GETOPT_INLINE Token* GetOpt_pp::_add_token(const std::string& value, Token::Type type) 36 | { 37 | Token* const ret = new Token(value, type); 38 | if (_first_token == NULL) 39 | _first_token = ret; 40 | else 41 | _last_token->link_to(ret); 42 | _last_token = ret; 43 | return ret; 44 | } 45 | 46 | GETOPT_INLINE void GetOpt_pp::_init_flags() 47 | { 48 | std::stringstream ss; 49 | _flags = ss.flags(); 50 | } 51 | 52 | GETOPT_INLINE void GetOpt_pp::_parse(int argc, char* argv[]) 53 | { 54 | _app_name = argv[0]; 55 | bool any_option_processed = false; 56 | 57 | // parse arguments by their '-' or '--': 58 | // (this will be a state machine soon) 59 | for (int i = 1; i < argc; i++) 60 | { 61 | const char current = argv[i][0]; 62 | const char next = argv[i][1]; 63 | 64 | if (current == '-' && (isalnum(next) || next == '-')) 65 | { 66 | // see what's next, differentiate whether it's short or long: 67 | if (next == '-' && argv[i][2] != 0) 68 | { 69 | // long option 70 | _longOps[&argv[i][2]].token = _add_token(&argv[i][2], Token::LongOption); 71 | any_option_processed = true; 72 | } 73 | else 74 | { 75 | // short option 76 | // iterate over all of them, keeping the last one in currentData 77 | // (so the intermediates will generate 'existent' arguments, as of '-abc') 78 | size_t j = 1; 79 | do 80 | { 81 | _shortOps[argv[i][j]].token = _add_token(std::string(&argv[i][j], 1), Token::ShortOption); 82 | j++; 83 | } 84 | while (argv[i][j] != 0); 85 | 86 | any_option_processed = true; 87 | } 88 | } 89 | else 90 | { 91 | _add_token(argv[i], any_option_processed ? Token::UnknownYet : Token::GlobalArgument); 92 | } 93 | } 94 | 95 | _last = _Option::OK; // TODO: IMPROVE!! 96 | } 97 | 98 | GETOPT_INLINE void GetOpt_pp::_parse_env() 99 | { 100 | // this will be optimized in version 3 101 | std::string var_name; 102 | std::string var_value; 103 | size_t var = 0; 104 | std::string::size_type pos; 105 | OptionData* data; 106 | 107 | while (environ[var] != NULL) 108 | { 109 | var_name = environ[var]; 110 | pos = var_name.find('='); 111 | 112 | if (pos != std::string::npos) 113 | { 114 | var_value = var_name.substr(pos + 1); 115 | var_name = var_name.substr(0, pos); 116 | 117 | if (_longOps.find(var_name) == _longOps.end()) 118 | { 119 | data = &_longOps[var_name]; 120 | data->token = _add_token(var_name, Token::LongOption); 121 | data->flags = OptionData::Envir; 122 | _add_token(var_value, Token::OptionArgument); 123 | } 124 | } 125 | else 126 | (data = &_longOps[var_name])->flags = OptionData::Envir; 127 | 128 | var++; 129 | } 130 | } 131 | 132 | GETOPT_INLINE GetOpt_pp::GetOpt_pp(int argc, char* argv[]) 133 | : _exc(std::ios_base::goodbit), _first_token(NULL), _last_token(NULL) 134 | { 135 | _init_flags(); 136 | _parse(argc, argv); 137 | } 138 | 139 | GETOPT_INLINE GetOpt_pp::GetOpt_pp(int argc, char* argv[], _EnvTag) 140 | : _first_token(NULL), _last_token(NULL) 141 | { 142 | _init_flags(); 143 | _parse(argc, argv); 144 | _parse_env(); 145 | } 146 | 147 | GETOPT_INLINE GetOpt_pp::~GetOpt_pp() 148 | { 149 | Token* next; 150 | Token* current(_first_token); 151 | while (current != NULL) 152 | { 153 | next = current->next; 154 | delete current; 155 | current = next; 156 | } 157 | } 158 | 159 | GETOPT_INLINE GetOpt_pp& GetOpt_pp::operator >> (const _Option& opt) 160 | { 161 | if (_last != _Option::ParsingError) 162 | { 163 | _last = opt(_shortOps, _longOps, _first_token, _flags); 164 | 165 | switch (_last) 166 | { 167 | case _Option::OK: 168 | break; 169 | 170 | case _Option::OptionNotFound: 171 | if (_exc & std::ios_base::eofbit) 172 | throw OptionNotFoundEx(); 173 | break; 174 | 175 | case _Option::BadType: 176 | if (_exc & std::ios_base::failbit) 177 | throw InvalidFormatEx(); 178 | break; 179 | 180 | case _Option::NoArgs: 181 | if (_exc & std::ios_base::eofbit) 182 | throw ArgumentNotFoundEx(); 183 | break; 184 | 185 | case _Option::TooManyArgs: 186 | if (_exc & std::ios_base::failbit) 187 | throw TooManyArgumentsEx(); 188 | break; 189 | 190 | case _Option::OptionNotFound_NoEx: 191 | break; // Ok, it will be read by casting to bool 192 | 193 | case _Option::ParsingError: 194 | break; // just to disable warning 195 | } 196 | } 197 | else if (_exc & std::ios_base::failbit) 198 | throw ParsingErrorEx(); 199 | 200 | return *this; 201 | } 202 | 203 | GETOPT_INLINE GetOpt_pp& GetOpt_pp::operator >> (std::ios_base& (*iomanip)(std::ios_base&)) 204 | { 205 | std::stringstream ss; 206 | ss.flags(_flags); 207 | _flags = (ss << iomanip).flags(); 208 | return *this; 209 | } 210 | 211 | GETOPT_INLINE bool GetOpt_pp::options_remain() const 212 | { 213 | bool remain = false; 214 | ShortOptions::const_iterator it = _shortOps.begin(); 215 | while (it != _shortOps.end() && !remain) 216 | { 217 | remain = (it->second.flags == OptionData::CmdLine_NotExtracted); 218 | ++it; 219 | } 220 | 221 | if (!remain) 222 | { 223 | LongOptions::const_iterator it = _longOps.begin(); 224 | while (it != _longOps.end() && !remain) 225 | { 226 | remain = (it->second.flags == OptionData::CmdLine_NotExtracted); 227 | ++it; 228 | } 229 | } 230 | 231 | if (!remain) 232 | { 233 | // check for the global arguments: 234 | Token* token = _first_token; 235 | while (!remain && token != NULL) 236 | { 237 | remain = ( token->type == Token::GlobalArgument || token->type == Token::UnknownYet ); 238 | token = token->next; 239 | } 240 | } 241 | 242 | return remain; 243 | } 244 | 245 | } 246 | -------------------------------------------------------------------------------- /src/distanceCalculation/JCdistance.cpp: -------------------------------------------------------------------------------- 1 | #include "JCdistance.hpp" 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include "simpleDistanceCalculator.hpp" 8 | #include "bitDistanceGap.hpp" 9 | #include "bitDistanceProtein.hpp" 10 | 11 | using namespace std; 12 | 13 | JCdistance::JCdistance(bool verbose, bool fastdist, dataloader* loader, diskMatrix* dm) { 14 | JCdistance::verbose = verbose; 15 | JCdistance::loader = loader; 16 | JCdistance::fastdist = fastdist; 17 | JCdistance::seqCount = loader->getSequenceCount(); 18 | JCdistance::seqLength = loader->getSequenceLength(); 19 | JCdistance::sequenceNames = *loader->getSequenceNames(); 20 | JCdistance::dm = dm; 21 | maxDistance = 0.0; 22 | } 23 | 24 | void JCdistance::computeDistanceMatrix(int numThreads) { 25 | maxDistance = 0.0; 26 | if(dm == NULL) { 27 | jcDistMatrix = new distType*[seqCount]; 28 | for (int i = 0; i < seqCount; i++) { 29 | jcDistMatrix[i] = new distType[seqCount]; 30 | } 31 | } else { 32 | jcDistMatrix = new distType*[numThreads* THREAD_ROW_BUFFER_COUNT]; 33 | for (int i = 0; i < numThreads* THREAD_ROW_BUFFER_COUNT; i++) { 34 | jcDistMatrix[i] = new distType[seqCount]; 35 | } 36 | } 37 | computeDistanceMatrixMT(numThreads); 38 | if(dm != NULL) { 39 | for (int i = 0; i < numThreads* THREAD_ROW_BUFFER_COUNT; i++) { 40 | delete[] jcDistMatrix[i]; 41 | } 42 | delete[] jcDistMatrix; 43 | } 44 | } 45 | 46 | 47 | /* 48 | pthread_attr_t thread_attr; 49 | struct sched_param thread_param; 50 | int thread_policy; 51 | int ret; 52 | 53 | // set thread priorities 54 | pthread_attr_init (&thread_attr); 55 | ret = pthread_attr_setschedpolicy (&thread_attr, SCHED_FIFO); 56 | if(ret != 0){ 57 | cout << "ERROR" << endl; 58 | exit(1); 59 | } 60 | ret= pthread_attr_setinheritsched (&thread_attr, PTHREAD_EXPLICIT_SCHED); 61 | if(ret != 0){ 62 | cout << "ERROR" << endl; 63 | exit(1); 64 | } 65 | int newPrio = 99; 66 | thread_param.sched_priority = newPrio; 67 | ret = pthread_attr_setschedparam (&thread_attr, &thread_param); 68 | if(ret != 0){ 69 | cout << "ERROR" << endl; 70 | exit(1); 71 | } 72 | pthread_setschedparam(pthread_self(),SCHED_FIFO, &thread_param); 73 | */ 74 | 75 | DistanceEstimate* JCdistance::getDistanceEstimateInstance(dataloader* loader) { 76 | DistanceEstimate* de = NULL; 77 | if(loader->fastdist) { 78 | if(loader->type == DNA) { 79 | de = new bitDistanceGap(loader); 80 | } else if(loader->type == PROTEIN) { 81 | de = new bitDistanceProtein(loader); 82 | } else { 83 | cerr << "ERROR: Unknown sequence type \"" << loader->type << "\"" << endl; 84 | exit(1); 85 | } 86 | } else { 87 | de = new simpleDistanceCalculator(loader); 88 | } 89 | return de; 90 | } 91 | 92 | void JCdistance::computeDistanceMatrixMT(int numThreads) { 93 | // Start threads 94 | pthread_t* threads = new pthread_t[numThreads]; 95 | threadStateJC** threadStates = new threadStateJC*[numThreads]; 96 | //start threads 97 | for (int i = 0; i < numThreads; i++) { 98 | threadStates[i] = new threadStateJC(); 99 | threadStates[i]->seqIdx = i; 100 | threadStates[i]->seqCount = seqCount; 101 | threadStates[i]->loader = loader; 102 | threadStates[i]->jcDistMatrix = jcDistMatrix; 103 | threadStates[i]->seqLength = seqLength; 104 | threadStates[i]->numThreads = numThreads; 105 | threadStates[i]->dm = dm; 106 | threadStates[i]->availableBuffers = THREAD_ROW_BUFFER_COUNT; 107 | threadStates[i]->estimator = getDistanceEstimateInstance(loader); 108 | if(pthread_mutex_init(&threadStates[i]->mutex, NULL)) { 109 | cerr << "Could not create mutex" << endl; 110 | exit(1); 111 | } 112 | //ret = pthread_create(&threads[i], &thread_attr, JCdistance::distJCThread, (void*)threadStates[i]); 113 | pthread_create(&threads[i], NULL, JCdistance::distJCThread, (void*)threadStates[i]); 114 | } 115 | if(dm != NULL) { 116 | for(int i = 0; i < seqCount; i++) { 117 | int threadIdx = i % numThreads; 118 | while(threadStates[threadIdx]->availableBuffers == THREAD_ROW_BUFFER_COUNT) { 119 | //busy wait 120 | } 121 | pthread_mutex_lock(&threadStates[threadIdx]->mutex); 122 | int bufIdx = i % (numThreads * THREAD_ROW_BUFFER_COUNT); 123 | //write data 124 | dm->writeArray(jcDistMatrix[bufIdx], i, seqCount); 125 | threadStates[threadIdx]->availableBuffers++; 126 | pthread_mutex_unlock(&threadStates[threadIdx]->mutex); 127 | } 128 | } 129 | for (int i = 0; i < numThreads; i++) { 130 | pthread_join(threads[i], NULL); 131 | if(maxDistance < threadStates[i]->maxDistance){ 132 | maxDistance = threadStates[i]->maxDistance; 133 | } 134 | } 135 | 136 | for (int i = 0; i < numThreads; i++) { 137 | pthread_mutex_destroy(&threadStates[i]->mutex); 138 | delete threadStates[i]->estimator; 139 | delete threadStates[i]; 140 | } 141 | delete[] threadStates; 142 | delete[] threads; 143 | 144 | postProcessDistanceMatrix(); 145 | } 146 | 147 | //thread entry point 148 | void* JCdistance::distJCThread(void* ptr) { 149 | distType maxDistance = 0.0; 150 | threadStateJC* state = (threadStateJC*) ptr; 151 | unsigned long long distances[3]; 152 | distType** jcDistMatrix = state->jcDistMatrix; 153 | int numThreads = state->numThreads; 154 | DistanceEstimate* estimator = state->estimator; 155 | double alpha = 4.0; 156 | if(state->loader->type == PROTEIN) { 157 | alpha = 20.0; 158 | } 159 | unsigned int offset = 0; 160 | for (unsigned int i = state->seqIdx; i < state->seqCount; i+=numThreads) { 161 | int bufIdx = i; 162 | if(state->dm != NULL) { 163 | while(state->availableBuffers == 0) { 164 | //Busy wait 165 | } 166 | bufIdx = i % (numThreads * THREAD_ROW_BUFFER_COUNT); 167 | } else { 168 | offset = i+1; 169 | } 170 | for (unsigned int j = offset; j < state->seqCount; j++) { 171 | estimator->computeDistance(i,j,distances); 172 | long total = distances[0] + distances[1]; 173 | distType distance; 174 | if(distances[2] == 0){ 175 | distance = 0; 176 | } else { 177 | distance = total / (distType) distances[2]; 178 | } 179 | if(distance / ((alpha-1)/alpha) >= 1.0) { 180 | distance = -1; 181 | } else { 182 | distance = -((alpha-1)/alpha) * log( 1.0 - distance / ((alpha-1)/alpha)); 183 | } 184 | if(maxDistance < distance){ 185 | maxDistance = distance; 186 | } 187 | jcDistMatrix[bufIdx][j] = distance; 188 | } 189 | if(state->dm != NULL) { 190 | pthread_mutex_lock(&state->mutex); 191 | state->availableBuffers--; 192 | pthread_mutex_unlock(&state->mutex); 193 | } 194 | } 195 | state->maxDistance = maxDistance; 196 | delete estimator; 197 | state->estimator = NULL; 198 | return NULL; 199 | } 200 | 201 | void JCdistance::postProcessDistanceMatrix(){ 202 | distType* buffer = NULL; 203 | if(dm != NULL) { 204 | buffer = new distType[seqCount]; 205 | } 206 | for (int i = 0; i < seqCount; i++) { 207 | if(dm == NULL) { 208 | buffer = jcDistMatrix[i]; 209 | } else { 210 | dm->readArray(buffer, i, seqCount); 211 | } 212 | for (int j = 0; j < seqCount; j++) { 213 | if (j < i && dm == NULL) { 214 | jcDistMatrix[i][j] = jcDistMatrix[j][i]; 215 | } 216 | if (i == j) { 217 | buffer[j] = 0.0; 218 | } 219 | if(buffer[j] == -1){ 220 | buffer[j] = maxDistance * 2; 221 | } 222 | } 223 | if(dm != NULL) { 224 | dm->writeArray(buffer, i, seqCount); 225 | } 226 | } 227 | if(dm != NULL) { 228 | delete[] buffer; 229 | } 230 | } 231 | 232 | distType** JCdistance::getDistanceMatrix(){ 233 | return jcDistMatrix; 234 | } 235 | -------------------------------------------------------------------------------- /src/distanceCalculation/gpu/cutil_inline_drvapi.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 1993-2010 NVIDIA Corporation. All rights reserved. 3 | * 4 | * NVIDIA Corporation and its licensors retain all intellectual property and 5 | * proprietary rights in and to this software and related documentation. 6 | * Any use, reproduction, disclosure, or distribution of this software 7 | * and related documentation without an express license agreement from 8 | * NVIDIA Corporation is strictly prohibited. 9 | * 10 | */ 11 | 12 | #ifndef _CUTIL_INLINE_FUNCTIONS_DRVAPI_H_ 13 | #define _CUTIL_INLINE_FUNCTIONS_DRVAPI_H_ 14 | 15 | #include 16 | #include 17 | #include 18 | 19 | 20 | // We define these calls here, so the user doesn't need to include __FILE__ and __LINE__ 21 | // The advantage is the developers gets to use the inline function so they can debug 22 | #define cutilDrvSafeCallNoSync(err) __cuSafeCallNoSync (err, __FILE__, __LINE__) 23 | #define cutilDrvSafeCall(err) __cuSafeCall (err, __FILE__, __LINE__) 24 | #define cutilDrvCtxSync() __cuCtxSync (__FILE__, __LINE__) 25 | #define cutilDrvCheckMsg(msg) __cuCheckMsg (msg, __FILE__, __LINE__) 26 | #define cutilDrvAlignOffset(offset, alignment) ( offset = (offset + (alignment-1)) & ~((alignment-1)) ) 27 | 28 | #define MIN(a,b) ((a < b) ? a : b) 29 | #define MAX(a,b) ((a > b) ? a : b) 30 | 31 | // This function returns the best GPU based on performance 32 | inline int cutilDrvGetMaxGflopsDeviceId() 33 | { 34 | CUdevice current_device = 0, max_perf_device = 0; 35 | int device_count = 0, sm_per_multiproc = 0; 36 | int max_compute_perf = 0, best_SM_arch = 0; 37 | int major = 0, minor = 0, multiProcessorCount, clockRate; 38 | int arch_cores_sm[3] = { 1, 8, 32 }; 39 | 40 | cuInit(0); 41 | CU_SAFE_CALL_NO_SYNC(cuDeviceGetCount(&device_count)); 42 | 43 | // Find the best major SM Architecture GPU device 44 | while ( current_device < device_count ) { 45 | CU_SAFE_CALL_NO_SYNC( cuDeviceComputeCapability(&major, &minor, current_device ) ); 46 | if (major > 0 && major < 9999) { 47 | best_SM_arch = MAX(best_SM_arch, major); 48 | } 49 | current_device++; 50 | } 51 | 52 | // Find the best CUDA capable GPU device 53 | current_device = 0; 54 | while( current_device < device_count ) { 55 | CU_SAFE_CALL_NO_SYNC( cuDeviceGetAttribute( &multiProcessorCount, 56 | CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT, 57 | current_device ) ); 58 | CU_SAFE_CALL_NO_SYNC( cuDeviceGetAttribute( &clockRate, 59 | CU_DEVICE_ATTRIBUTE_CLOCK_RATE, 60 | current_device ) ); 61 | 62 | if (major == 9999 && minor == 9999) { 63 | sm_per_multiproc = 1; 64 | } else if (major <= 2) { 65 | sm_per_multiproc = arch_cores_sm[major]; 66 | } else { 67 | sm_per_multiproc = arch_cores_sm[2]; 68 | } 69 | 70 | int compute_perf = multiProcessorCount * sm_per_multiproc * clockRate; 71 | if( compute_perf > max_compute_perf ) { 72 | // If we find GPU with SM major > 2, search only these 73 | if ( best_SM_arch > 2 ) { 74 | // If our device==dest_SM_arch, choose this, or else pass 75 | if (major == best_SM_arch) { 76 | max_compute_perf = compute_perf; 77 | max_perf_device = current_device; 78 | } 79 | } else { 80 | max_compute_perf = compute_perf; 81 | max_perf_device = current_device; 82 | } 83 | } 84 | ++current_device; 85 | } 86 | return max_perf_device; 87 | } 88 | 89 | // These are the inline versions for all of the CUTIL functions 90 | inline void __cuSafeCallNoSync( CUresult err, const char *file, const int line ) 91 | { 92 | if( CUDA_SUCCESS != err) { 93 | fprintf(stderr, "cuSafeCallNoSync() Driver API error = %04d from file <%s>, line %i.\n", 94 | err, file, line ); 95 | exit(-1); 96 | } 97 | } 98 | inline void __cuSafeCall( CUresult err, const char *file, const int line ) 99 | { 100 | __cuSafeCallNoSync( err, file, line ); 101 | } 102 | 103 | inline void __cuCtxSync(const char *file, const int line ) 104 | { 105 | CUresult err = cuCtxSynchronize(); 106 | if( CUDA_SUCCESS != err ) { 107 | fprintf(stderr, "cuCtxSynchronize() API error = %04d in file <%s>, line %i.\n", 108 | err, file, line ); 109 | exit(-1); 110 | } 111 | } 112 | 113 | inline void __cuCheckMsg( const char * msg, const char *file, const int line ) 114 | { 115 | CUresult err = cuCtxSynchronize(); 116 | if( CUDA_SUCCESS != err) { 117 | fprintf(stderr, "cutilDrvCheckMsg -> %s", msg); 118 | fprintf(stderr, "cutilDrvCheckMsg -> cuCtxSynchronize API error = %04d in file <%s>, line %i.\n", 119 | err, file, line ); 120 | exit(-1); 121 | } 122 | } 123 | 124 | 125 | #if __DEVICE_EMULATION__ 126 | inline void cutilDeviceInitDrv(int cuDevice, int ARGC, char **ARGV) { } 127 | #else 128 | inline void cutilDeviceInitDrv(int cuDevice, int ARGC, char ** ARGV) 129 | { 130 | cuDevice = 0; 131 | int deviceCount = 0; 132 | CUresult err = cuInit(0); 133 | if (CUDA_SUCCESS == err) 134 | cutilDrvSafeCallNoSync(cuDeviceGetCount(&deviceCount)); 135 | if (deviceCount == 0) { 136 | fprintf(stderr, "CUTIL DeviceInitDrv error: no devices supporting CUDA\n"); 137 | exit(-1); 138 | } 139 | int dev = 0; 140 | cutGetCmdLineArgumenti(ARGC, (const char **) ARGV, "device", &dev); 141 | if (dev < 0) dev = 0; 142 | if (dev > deviceCount-1) dev = deviceCount - 1; 143 | cutilDrvSafeCallNoSync(cuDeviceGet(&cuDevice, dev)); 144 | char name[100]; 145 | cuDeviceGetName(name, 100, cuDevice); 146 | if (cutCheckCmdLineFlag(ARGC, (const char **) ARGV, "quiet") == CUTFalse) 147 | fprintf(stderr, "Using CUDA device [%d]: %s\n", dev, name); 148 | } 149 | #endif 150 | 151 | //! Check for CUDA context lost 152 | inline void cutilDrvCudaCheckCtxLost(const char *errorMessage, const char *file, const int line ) 153 | { 154 | CUresult err = cuCtxSynchronize(); 155 | if( CUDA_ERROR_INVALID_CONTEXT != err) { 156 | fprintf(stderr, "Cuda error: %s in file '%s' in line %i\n", 157 | errorMessage, file, line ); 158 | exit(-1); 159 | } 160 | err = cuCtxSynchronize(); 161 | if( CUDA_SUCCESS != err) { 162 | fprintf(stderr, "Cuda error: %s in file '%s' in line %i\n", 163 | errorMessage, file, line ); 164 | exit(-1); 165 | } 166 | } 167 | 168 | // General check for CUDA GPU SM Capabilities for a specific device # 169 | inline bool cutilDrvCudaDevCapabilities(int major_version, int minor_version, int deviceNum) 170 | { 171 | int major, minor, dev; 172 | char device_name[256]; 173 | 174 | #ifdef __DEVICE_EMULATION__ 175 | printf("> Compute Device Emulation Mode \n"); 176 | #endif 177 | 178 | cutilDrvSafeCallNoSync( cuDeviceGet(&dev, deviceNum) ); 179 | cutilDrvSafeCallNoSync( cuDeviceComputeCapability(&major, &minor, dev)); 180 | cutilDrvSafeCallNoSync( cuDeviceGetName(device_name, 256, dev) ); 181 | 182 | if((major > major_version) || 183 | (major == major_version && minor >= minor_version)) 184 | { 185 | printf("> Compute SM %d.%d Device Detected\n", major, minor); 186 | printf("> Device %d: <%s>\n", dev, device_name); 187 | return true; 188 | } 189 | else 190 | { 191 | printf("There is no device supporting CUDA compute capability %d.%d.\n", major_version, minor_version); 192 | printf("PASSED\n"); 193 | return false; 194 | } 195 | } 196 | 197 | // General check for CUDA GPU SM Capabilities 198 | inline bool cutilDrvCudaCapabilities(int major_version, int minor_version) 199 | { 200 | return cutilDrvCudaDevCapabilities(major_version, minor_version, 0); 201 | } 202 | 203 | 204 | #endif // _CUTIL_INLINE_FUNCTIONS_DRVAPI_H_ 205 | -------------------------------------------------------------------------------- /msvc2012.vcxproj.filters: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | {4FC737F1-C7A5-4376-A066-2A32D752A2FF} 6 | cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx 7 | 8 | 9 | {93995380-89BD-4b04-88EB-625FBE52EBFB} 10 | h;hpp;hxx;hm;inl;inc;xsd 11 | 12 | 13 | {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} 14 | rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms 15 | 16 | 17 | 18 | 19 | Source Files 20 | 21 | 22 | Source Files 23 | 24 | 25 | Source Files 26 | 27 | 28 | Source Files 29 | 30 | 31 | Source Files 32 | 33 | 34 | Source Files 35 | 36 | 37 | Source Files 38 | 39 | 40 | Source Files 41 | 42 | 43 | Source Files 44 | 45 | 46 | Source Files 47 | 48 | 49 | Source Files 50 | 51 | 52 | Source Files 53 | 54 | 55 | Source Files 56 | 57 | 58 | Source Files 59 | 60 | 61 | Source Files 62 | 63 | 64 | Source Files 65 | 66 | 67 | Source Files 68 | 69 | 70 | Source Files 71 | 72 | 73 | Source Files 74 | 75 | 76 | Source Files 77 | 78 | 79 | Source Files 80 | 81 | 82 | Source Files 83 | 84 | 85 | Source Files 86 | 87 | 88 | 89 | 90 | Header Files 91 | 92 | 93 | Header Files 94 | 95 | 96 | Header Files 97 | 98 | 99 | Header Files 100 | 101 | 102 | Header Files 103 | 104 | 105 | Header Files 106 | 107 | 108 | Header Files 109 | 110 | 111 | Header Files 112 | 113 | 114 | Header Files 115 | 116 | 117 | Header Files 118 | 119 | 120 | Header Files 121 | 122 | 123 | Header Files 124 | 125 | 126 | Header Files 127 | 128 | 129 | Header Files 130 | 131 | 132 | Header Files 133 | 134 | 135 | Header Files 136 | 137 | 138 | Header Files 139 | 140 | 141 | Header Files 142 | 143 | 144 | Header Files 145 | 146 | 147 | Header Files 148 | 149 | 150 | Header Files 151 | 152 | 153 | Header Files 154 | 155 | 156 | Header Files 157 | 158 | 159 | Header Files 160 | 161 | 162 | Header Files 163 | 164 | 165 | Header Files 166 | 167 | 168 | Header Files 169 | 170 | 171 | Header Files 172 | 173 | 174 | Header Files 175 | 176 | 177 | Header Files 178 | 179 | 180 | Header Files 181 | 182 | 183 | Header Files 184 | 185 | 186 | Header Files 187 | 188 | 189 | Header Files 190 | 191 | 192 | Header Files 193 | 194 | 195 | Header Files 196 | 197 | 198 | Header Files 199 | 200 | 201 | Header Files 202 | 203 | 204 | Header Files 205 | 206 | 207 | -------------------------------------------------------------------------------- /src/distanceCalculation/gpu/distanceCalculatorProtein_gpu.cu: -------------------------------------------------------------------------------- 1 | #include 2 | #include "constants.hpp" 3 | #include 4 | #include 5 | 6 | __shared__ unsigned int shared_data_protein[threads_pr_block]; 7 | 8 | __device__ void parallelReductionProtein(unsigned int val){ 9 | //wait for all threads to finish and use shared memory to store results 10 | shared_data_protein[threadIdx.x] = val; 11 | __syncthreads(); 12 | 13 | //perform parallel reduction 14 | for(unsigned int s=blockDim.x/2; s>32; s>>=1){ 15 | if (threadIdx.x < s){ 16 | shared_data_protein[threadIdx.x ] += shared_data_protein[threadIdx.x + s]; 17 | } 18 | __syncthreads(); 19 | } 20 | if (threadIdx.x < 32) { 21 | shared_data_protein[threadIdx.x ] += shared_data_protein[threadIdx.x + 32 ]; 22 | } 23 | __syncthreads(); 24 | 25 | 26 | if (threadIdx.x < 16) { 27 | shared_data_protein[threadIdx.x ] += shared_data_protein[threadIdx.x + 16 ]; 28 | } 29 | __syncthreads(); 30 | if (threadIdx.x < 8) { 31 | shared_data_protein[threadIdx.x ] += shared_data_protein[threadIdx.x + 8 ]; 32 | } 33 | __syncthreads(); 34 | 35 | if (threadIdx.x < 4) { 36 | shared_data_protein[threadIdx.x ] += shared_data_protein[threadIdx.x + 4 ]; 37 | } 38 | __syncthreads(); 39 | 40 | if (threadIdx.x < 2) { 41 | shared_data_protein[threadIdx.x ] += shared_data_protein[threadIdx.x + 2 ]; 42 | } 43 | __syncthreads(); 44 | 45 | if(threadIdx.x == 0){ 46 | shared_data_protein[0 ] += shared_data_protein[1 ]; 47 | } 48 | __syncthreads(); 49 | } 50 | 51 | __global__ void computeDistanceProtein(unsigned int* gaps_gpu, unsigned int* dist_gpu, unsigned int* bitStrings_gpu, unsigned int bsStride, unsigned int rowOffset, unsigned int dataSize_protein){ 52 | 53 | int seq2 = blockIdx.y; 54 | if(seq2 < blockIdx.x+1+rowOffset){ 55 | return; 56 | } 57 | 58 | uchar4* charData = (uchar4*) bitStrings_gpu; 59 | unsigned int iterations = dataSize_protein / threads_pr_block; 60 | if(dataSize_protein % threads_pr_block != 0){ 61 | iterations++; 62 | } 63 | //printf(" %d \n",iterations); 64 | //for(int seq2 = blockIdx.x+1+rowOffset; seq2 < sequenceCount_gpu; seq2++) { 65 | unsigned int sumR = 0; 66 | unsigned int sumG = 0; 67 | for(int slice = 0; slice < iterations; slice++) { 68 | uchar4 r, g; 69 | // fetch next block of sequence data 70 | unsigned int idx0 = (rowOffset+blockIdx.x)*bsStride + slice*threads_pr_block + threadIdx.x; 71 | uchar4 base = charData[idx0]; 72 | unsigned int idx1 = seq2*bsStride + slice*threads_pr_block + threadIdx.x; 73 | uchar4 target = charData[idx1]; 74 | 75 | r.x = base.x != target.x; 76 | g.x = base.x < 64 || target.x < 64; 77 | r.x = r.x && !g.x; 78 | 79 | r.y = base.y != target.y; 80 | g.y = base.y < 64 || target.y < 64; 81 | r.y = r.y && !g.y; 82 | 83 | r.z = base.z != target.z; 84 | g.z = base.z < 64 || target.z < 64; 85 | r.z = r.z && !g.z; 86 | 87 | r.w = base.w != target.w; 88 | g.w = base.w < 64 || target.w < 64; 89 | r.w = r.w && !g.w; 90 | 91 | if(slice * threads_pr_block + threadIdx.x < dataSize_protein) { 92 | sumR += r.x; 93 | sumR += r.y; 94 | sumR += r.z; 95 | sumR += r.w; 96 | 97 | sumG += g.x; 98 | sumG += g.y; 99 | sumG += g.z; 100 | sumG += g.w; 101 | //printf("%d: %d %d: %d %d %d %d\n",bsStride,idx0,idx1, g.x, g.y, g.z, g.w); 102 | } 103 | } 104 | //sum over all threads 105 | parallelReductionProtein(sumR); 106 | sumR = shared_data_protein[0]; 107 | parallelReductionProtein(sumG); 108 | sumG = shared_data_protein[0]; 109 | 110 | //store result in global memory 111 | if(threadIdx.x == 0) { 112 | dist_gpu[blockIdx.x * gridDim.y + seq2] = sumR; 113 | gaps_gpu[blockIdx.x * gridDim.y + seq2] = sumG; 114 | } 115 | } 116 | 117 | //------------------------------------------------------------------------------------------------------------- 118 | 119 | unsigned int gridSize_protein; 120 | size_t resultMemSize_protein; 121 | size_t bitStringsMemSize_protein; 122 | unsigned int sequenceCount; 123 | //results 124 | unsigned int* dist_protein_gpu; 125 | unsigned int* gaps_protein_gpu; 126 | unsigned int bsStride_protein; 127 | //data 128 | unsigned int* bitStrings_protein_gpu; 129 | unsigned int numberOfKernelLaunches_protein; 130 | unsigned int rowsPrKernel_protein; 131 | unsigned int dataSize_protein; 132 | 133 | #ifdef TIMING 134 | extern float totalGpuComputation; 135 | extern timeval start,end; 136 | extern float totalTransfer; 137 | #endif 138 | 139 | extern "C" void computeDistancesProtein_gpu(unsigned int* bitStrings, unsigned int* dist, unsigned int* gaps) { 140 | // printf("%p %p %d \n",bitStrings_protein_gpu,bitStrings,bitStringsMemSize_protein); 141 | cutilSafeCall(cudaMemcpyAsync(bitStrings_protein_gpu, bitStrings, bitStringsMemSize_protein, cudaMemcpyHostToDevice,0)); 142 | for(int i = 0; i < numberOfKernelLaunches_protein; i++){ 143 | printf("kernel: %d -----------------------\n",i); 144 | // setup execution parameters 145 | unsigned int gridSize = 0; 146 | if(i != numberOfKernelLaunches_protein-1){ 147 | gridSize = rowsPrKernel_protein; 148 | } else { 149 | gridSize = sequenceCount - (numberOfKernelLaunches_protein-1)*rowsPrKernel_protein; 150 | } 151 | printf("gridSize: %d %d\n",gridSize,sequenceCount); 152 | dim3 grid(gridSize,sequenceCount); 153 | dim3 block(threads_pr_block, 1, 1); 154 | unsigned int transferSize = gridSize * sequenceCount * sizeof(unsigned int); 155 | computeDistanceProtein<<< grid, block >>>(gaps_protein_gpu, dist_protein_gpu, bitStrings_protein_gpu, bsStride_protein, i*rowsPrKernel_protein, dataSize_protein); 156 | 157 | #ifdef TIMING 158 | cudaThreadSynchronize(); 159 | gettimeofday(&end,NULL); 160 | totalGpuComputation += (end.tv_sec - start.tv_sec)*1000.0 + (end.tv_usec - start.tv_usec)/1000.0; 161 | gettimeofday(&start,NULL); 162 | #endif 163 | 164 | printf("Copying results...\n"); 165 | cutilSafeCall(cudaMemcpy(&dist[i * rowsPrKernel_protein * sequenceCount], dist_protein_gpu, transferSize, cudaMemcpyDeviceToHost)); 166 | cutilSafeCall(cudaMemcpy(&gaps[i * rowsPrKernel_protein * sequenceCount], gaps_protein_gpu, transferSize, cudaMemcpyDeviceToHost)); 167 | printf("Finished\n"); 168 | 169 | #ifdef TIMING 170 | cudaThreadSynchronize(); 171 | gettimeofday(&end,NULL); 172 | totalTransfer += (end.tv_sec - start.tv_sec)*1000.0 + (end.tv_usec - start.tv_usec)/1000.0; 173 | gettimeofday(&start,NULL); 174 | #endif 175 | } 176 | } 177 | 178 | extern "C" void getResultsProtein_gpu(unsigned int* dist, unsigned int* gaps) { 179 | /* // Wait for kernels to finish. 180 | cutilCheckMsg("Kernel execution failed"); 181 | 182 | // Copy results from device to host 183 | cutilSafeCall(cudaMemcpy(dist, dist_protein_gpu, resultMemSize_protein, cudaMemcpyDeviceToHost)); 184 | cutilSafeCall(cudaMemcpy(gaps, gaps_protein_gpu, resultMemSize_protein, cudaMemcpyDeviceToHost)); 185 | */ 186 | printf("getResult is not implemented for proteins \n"); 187 | exit(1); 188 | } 189 | 190 | extern "C" void initialiseProtein_gpu(unsigned int _sequenceCount, unsigned int _bsStride, unsigned int paddedLength) { 191 | // printf("initialising GPU... \n"); 192 | bsStride_protein = _bsStride; 193 | sequenceCount = _sequenceCount; 194 | dataSize_protein = paddedLength / 4; 195 | //printf("%d %d \n",bsStride_protein,paddedLength); 196 | //exit(0); 197 | 198 | resultMemSize_protein = sequenceCount * sequenceCount * sizeof(unsigned int); 199 | bitStringsMemSize_protein = bsStride_protein * sizeof(unsigned int) * sequenceCount; 200 | 201 | cudaDeviceProp deviceProp; 202 | cudaGetDeviceProperties(&deviceProp, 0); 203 | float freeGpuMem = deviceProp.totalGlobalMem - 250*1024*1024; //substract 100MB 204 | 205 | freeGpuMem -= bitStringsMemSize_protein; 206 | float temp = min(1.0,freeGpuMem / (resultMemSize_protein*2.0)); 207 | rowsPrKernel_protein = sequenceCount * temp; 208 | 209 | //rowsPrKernel_protein = 500; 210 | 211 | numberOfKernelLaunches_protein = sequenceCount/rowsPrKernel_protein; 212 | if(sequenceCount % rowsPrKernel_protein != 0){ 213 | numberOfKernelLaunches_protein++; 214 | } 215 | 216 | printf("Number of kernel launches needed: %d\n",numberOfKernelLaunches_protein); 217 | 218 | rowsPrKernel_protein = sequenceCount / numberOfKernelLaunches_protein; 219 | unsigned int exRows = sequenceCount - rowsPrKernel_protein*numberOfKernelLaunches_protein; 220 | rowsPrKernel_protein += exRows; 221 | 222 | printf("rows pr kernel: %d\n",rowsPrKernel_protein); 223 | resultMemSize_protein = rowsPrKernel_protein * sequenceCount * sizeof(unsigned int); 224 | 225 | // allocate device memory 226 | cutilSafeCall(cudaMalloc((void**) &bitStrings_protein_gpu, bitStringsMemSize_protein)); 227 | cutilSafeCall(cudaMalloc((void**) &dist_protein_gpu, resultMemSize_protein)); 228 | cutilSafeCall(cudaMalloc((void**) &gaps_protein_gpu, resultMemSize_protein)); 229 | } 230 | -------------------------------------------------------------------------------- /src/distanceCalculation/dataloaderFasta.hpp: -------------------------------------------------------------------------------- 1 | #ifndef DATALOADER_FASTA_H 2 | #define DATALOADER_FASTA_H 3 | 4 | #include "stdinclude.h" 5 | #include "dataloader.hpp" 6 | #include "bitStringUtils.hpp" 7 | 8 | class dataloaderFasta : public dataloader{ 9 | 10 | public: 11 | 12 | dataloaderFasta(){ 13 | sequences = NULL; 14 | bitStrings = NULL; 15 | gapFilters = NULL; 16 | buf = new char[BUFFER_SIZE]; 17 | firstSequence = true; 18 | sequenceLength = 0; 19 | sequenceCount = 0; 20 | is = new ifstream(); 21 | sequenceNames = new vector; 22 | } 23 | 24 | void load(string filename) { 25 | if(fastdist){ 26 | bitStrings = new vector; 27 | parseData(filename); 28 | } else { 29 | sequences = new vector; 30 | parseData(filename); 31 | } 32 | if(sequenceCount <= 1) { 33 | cerr << "Found " << sequenceCount << " sequences in the input file. At least 2 sequences are needed to build a tree." << endl; 34 | exit(1); 35 | } 36 | if(verbose) { 37 | cerr << "Input type determined as "; 38 | if(type == UNKNOWN){ 39 | cerr << "UNKNOWN. Please supply the type as parameter." << endl; 40 | exit(1); 41 | } else if(type == DNA){ 42 | cerr << "DNA." << endl; 43 | } else if(type == PROTEIN){ 44 | cerr << "PROTEIN." << endl; 45 | } else { 46 | cerr << "UNKNOWN. Please supply the type as parameter." << endl; 47 | exit(1); 48 | } 49 | cerr << "Number of sequences: " << sequenceCount << endl; 50 | cerr << "Sequence length: " << sequenceLength << endl; 51 | } 52 | } 53 | 54 | void discoverInputType(){ 55 | if(type != UNKNOWN) return; 56 | type = DNA; 57 | for(unsigned int i = 0; i < sequenceLength; i++){ 58 | char c = charBuffer.at(i); 59 | if(!(c < 65 || 60 | c == 'A' || 61 | c == 'a' || 62 | c == 'C' || 63 | c == 'c' || 64 | c == 'G' || 65 | c == 'g' || 66 | c == 'T' || 67 | c == 't' || 68 | c == 'U' || 69 | c == 'u' || 70 | c == 'N' || 71 | c == 'n')) { 72 | type = PROTEIN; 73 | return; 74 | } 75 | } 76 | } 77 | 78 | void parseData(string filename) { 79 | // open the file 80 | is->open(filename.data(),ifstream::in); 81 | if (!is->is_open()) { 82 | cerr << "ERROR: Could not read file: " << filename << "\n"; 83 | exit(1); 84 | } 85 | fillBuffer(); 86 | while(buf != NULL) { 87 | curChar = getNextChar(); 88 | if(curChar == ';') { 89 | parseComment(); 90 | } else if(curChar == '>') { 91 | storeSequence(); //Ignored if there is no sequence data 92 | curChar = getNextChar(); 93 | parseSequenceName(); 94 | } else { 95 | parseSequenceLine(); 96 | } 97 | } 98 | storeSequence(); 99 | } 100 | 101 | inline void fillBuffer(){ 102 | if(!is->good()){ 103 | delete[] buf; 104 | buf = NULL; 105 | is->close(); 106 | return; 107 | } 108 | is->read(buf,BUFFER_SIZE); 109 | if (is->gcount() == 0){ 110 | delete[] buf; 111 | buf = NULL; 112 | is->close(); 113 | return; 114 | } 115 | bufPos = 0; 116 | } 117 | 118 | inline char getNextChar() { 119 | if(bufPos == is->gcount()){ 120 | fillBuffer(); 121 | } 122 | if(buf != NULL) { 123 | return buf[bufPos++]; 124 | } 125 | return 10; 126 | } 127 | 128 | //Finds the next new line 129 | void parseComment(){ 130 | while (curChar != 10){ 131 | curChar = getNextChar(); 132 | } 133 | } 134 | 135 | inline void encodeProteinSequence(unsigned int* bitString, vector& data){ 136 | for(unsigned int i = 0; i < sequenceLength+paddingLength; i++) { 137 | int offset = i % 4; 138 | unsigned int bitStringIdx = i / 4; 139 | if(offset == 0){ 140 | bitString[bitStringIdx] = 0; 141 | } 142 | char c = resolveChar(data[i]); 143 | bitString[bitStringIdx] += (c << (offset*8)); 144 | } 145 | } 146 | 147 | inline void encodeDNASequence(unsigned int* bitString, unsigned int* gapFilter, vector& data){ 148 | for(unsigned int i = 0; i < sequenceLength+paddingLength; i++) { 149 | int offset = i % (BLOCK_SIZE / 2); 150 | unsigned int bitStringIdx = i / (BLOCK_SIZE / 2); 151 | if(offset == 0){ 152 | bitString[bitStringIdx] = 0; 153 | gapFilter[bitStringIdx] = 0; 154 | } 155 | char c = data[i]; 156 | switch (c) { 157 | //Nothing is done for gaps and ambigious nucleotides 158 | case 'A': 159 | case 'a': 160 | //A has value 00 so nothing is added to the bitStrings 161 | gapFilter[bitStringIdx] += (Gbin << (offset*2)); 162 | break; 163 | case 'C': 164 | case 'c': 165 | bitString[bitStringIdx] += (Cbin<<(offset*2)); 166 | gapFilter[bitStringIdx] += (Gbin << (offset*2)); 167 | break; 168 | case 'G': 169 | case 'g': 170 | bitString[bitStringIdx] += (Gbin<<(offset*2)); 171 | gapFilter[bitStringIdx] += (Gbin << (offset*2)); 172 | break; 173 | case 'T': 174 | case 't': 175 | bitString[bitStringIdx] += (Tbin<<(offset*2)); 176 | gapFilter[bitStringIdx] += (Gbin << (offset*2)); 177 | break; 178 | default: 179 | break; 180 | } 181 | } 182 | } 183 | 184 | inline char resolveChar(char c){ 185 | //ignore positions with ambigious/unknown nucleotides 186 | if(type == DNA){ 187 | if (!(c == 'a' || c == 'A' || c == 'c' || c == 'C' || c == 'g' || c == 'G' || c == 't' || c == 'T' || c == 'u' || c == 'U')){ 188 | return '-'; 189 | } else { 190 | return c; 191 | } 192 | } else { 193 | if(c == '-' || c == '.' || c == 'X' || c == 'x' || c == 'z' || c == 'Z' || c == 'b' || c == 'B' || c == 'J' || c == 'j' || c == '?') { 194 | return '-'; 195 | } else { 196 | return c; 197 | } 198 | } 199 | } 200 | 201 | void parseSequenceLine(){ 202 | while (curChar != 10){ 203 | if(curChar > 32){ 204 | charBuffer.push_back(curChar); 205 | } 206 | curChar = getNextChar(); 207 | } 208 | } 209 | 210 | void parseSequenceName(){ 211 | string name = ""; 212 | while(curChar > 31){ 213 | name.push_back(curChar); 214 | curChar = getNextChar(); 215 | } 216 | sequenceNames->push_back(name); 217 | } 218 | 219 | void storeSequenceFastDist(){ 220 | if(firstSequence){ 221 | if(type == DNA){ 222 | bitStringsCount = sequenceLength / 64 + 6; 223 | paddingLength = bitStringsCount*64 - sequenceLength; 224 | gapFilters = new vector; 225 | } else { 226 | bitStringsCount = sequenceLength / 16 + 8; 227 | paddingLength = bitStringsCount*16 - sequenceLength; 228 | } 229 | } 230 | // insert paddings 231 | for(int i = 0; i < paddingLength; i++){ 232 | charBuffer.push_back('-'); 233 | } 234 | // Encode the data in char vectors. 235 | // unsigned int* bitString = new unsigned int[bitStringsCount*4+3]; 236 | // bitString = (unsigned int*) ((((unsigned int) bitString) + 12) & ~15); 237 | unsigned int* bitString = (unsigned int*) _mm_malloc(bitStringsCount*4*sizeof(unsigned int), 16); 238 | if(type == DNA){ 239 | unsigned int* gapFilter; 240 | gapFilter = (unsigned int*) _mm_malloc(bitStringsCount*4*sizeof(unsigned int), 16); 241 | encodeDNASequence(bitString, gapFilter, charBuffer); 242 | gapFilters->push_back(gapFilter); 243 | } else { 244 | encodeProteinSequence(bitString, charBuffer); 245 | } 246 | 247 | bitStrings->push_back(bitString); 248 | firstSequence = false; 249 | sequenceCount++; 250 | charBuffer.clear(); 251 | } 252 | 253 | void storeSequence(){ 254 | if(charBuffer.size() != 0) { 255 | 256 | if(firstSequence) { 257 | sequenceLength = charBuffer.size(); 258 | discoverInputType(); 259 | } else if(charBuffer.size() != sequenceLength){ 260 | cerr << "ERROR: Sequence " << sequenceCount << " has length " << charBuffer.size() << " while the preceding sequences in the alignment which have length " << sequenceLength << "." << endl; 261 | exit(1); 262 | } 263 | if(fastdist) { 264 | storeSequenceFastDist(); 265 | return; 266 | } 267 | firstSequence = false; 268 | char* data = new char[sequenceLength]; 269 | for(unsigned int i = 0; i < sequenceLength; i++){ 270 | data[i] = resolveChar(charBuffer[i]); 271 | } 272 | sequences->push_back(data); 273 | charBuffer.clear(); 274 | sequenceCount++; 275 | } 276 | } 277 | 278 | unsigned int** getBitStrings(){ 279 | return &(*bitStrings)[0]; 280 | } 281 | 282 | unsigned int** getGapFilters(){ 283 | return &(*gapFilters)[0]; 284 | } 285 | 286 | unsigned int getSequenceCount(){ 287 | return sequenceCount; 288 | } 289 | 290 | unsigned int getSequenceLength(){ 291 | return sequenceLength; 292 | } 293 | 294 | unsigned int getBitStringsCount(){ 295 | return bitStringsCount; 296 | } 297 | 298 | vector* getSequenceNames(){ 299 | return sequenceNames; 300 | } 301 | 302 | vector* getSequences(){ 303 | return sequences; 304 | } 305 | 306 | void setSequences(vector* val){ 307 | sequences = val; 308 | } 309 | 310 | ~dataloaderFasta(){ 311 | if(sequenceNames != NULL) { 312 | delete sequenceNames; 313 | } 314 | delete is; 315 | if(sequences != NULL) { 316 | for(unsigned int i = 0; i < sequences->size(); i++) { 317 | delete[] sequences->at(i); 318 | } 319 | delete sequences; 320 | } 321 | if(bitStrings != NULL) { 322 | for(unsigned int i = 0; i < bitStrings->size(); i++) { 323 | _mm_free(bitStrings->at(i)); 324 | } 325 | delete bitStrings; 326 | } 327 | if(gapFilters != NULL) { 328 | for (unsigned int i = 0; i < gapFilters->size(); i++) { 329 | _mm_free(gapFilters->at(i)); 330 | } 331 | delete gapFilters; 332 | } 333 | } 334 | 335 | private: 336 | int bufPos; 337 | char* buf; 338 | vector* sequenceNames; 339 | bool firstSequence; 340 | char curChar; 341 | ifstream* is; 342 | vector charBuffer; 343 | }; 344 | 345 | #endif 346 | -------------------------------------------------------------------------------- /src/distanceCalculation/gpu/kimuraDNA_gpu.cu: -------------------------------------------------------------------------------- 1 | /* 2 | __device__ void printBitString(unsigned int data){ 3 | for (int j = 0;j < 32 ; j++) { 4 | if(j % 4 == 0 && j != 0){ 5 | printf(" "); 6 | } 7 | printf("%d",((data >> j) & 1)); 8 | } 9 | printf("\n"); 10 | } 11 | 12 | 13 | __device__ void printDNASequence(unsigned int data){ 14 | for (int j = 0;j < 16; j++) { 15 | if(j % 4 == 0 && j != 0){ 16 | printf(" "); 17 | } 18 | unsigned int c = ((data >> (j*2)) & 3); 19 | switch (c) { 20 | case 0: { 21 | printf("A "); 22 | break; 23 | } 24 | case 1: { 25 | printf("G "); 26 | break; 27 | } 28 | case 2: { 29 | printf("T "); 30 | break; 31 | } 32 | case 3: { 33 | printf("C "); 34 | break; 35 | } 36 | default: 37 | break; 38 | } 39 | } 40 | printf("\n"); 41 | } 42 | */ 43 | 44 | #include "cutil_inline.h" 45 | #include "constants.hpp" 46 | #include 47 | 48 | __shared__ unsigned int shared_data_dna[threads_pr_block*3]; 49 | 50 | __device__ void parallelReductionDNA2(){ 51 | //wait for all threads to finish and reuse shared memory to store results 52 | __syncthreads(); 53 | 54 | //perform parallel reduction 55 | for(unsigned int s=blockDim.x/2; s>32; s>>=1){ 56 | if (threadIdx.x < s){ 57 | shared_data_dna[threadIdx.x] += shared_data_dna[threadIdx.x + s]; 58 | shared_data_dna[threadIdx.x + threads_pr_block] += shared_data_dna[threadIdx.x + s + threads_pr_block]; 59 | shared_data_dna[threadIdx.x + threads_pr_block*2] += shared_data_dna[threadIdx.x + s + threads_pr_block*2]; 60 | } 61 | __syncthreads(); 62 | } 63 | 64 | if (threadIdx.x < 32) { 65 | shared_data_dna[threadIdx.x] += shared_data_dna[threadIdx.x + 32]; 66 | shared_data_dna[threadIdx.x + threads_pr_block] += shared_data_dna[threadIdx.x + 32 + threads_pr_block]; 67 | shared_data_dna[threadIdx.x + threads_pr_block*2] += shared_data_dna[threadIdx.x + 32 + threads_pr_block*2]; 68 | } 69 | __syncthreads(); 70 | if (threadIdx.x < 16) { 71 | shared_data_dna[threadIdx.x] += shared_data_dna[threadIdx.x + 16]; 72 | shared_data_dna[threadIdx.x + threads_pr_block] += shared_data_dna[threadIdx.x + 16 + threads_pr_block]; 73 | shared_data_dna[threadIdx.x + threads_pr_block*2] += shared_data_dna[threadIdx.x + 16 + threads_pr_block*2]; 74 | } 75 | __syncthreads(); 76 | if (threadIdx.x < 8) { 77 | shared_data_dna[threadIdx.x] += shared_data_dna[threadIdx.x + 8]; 78 | shared_data_dna[threadIdx.x + threads_pr_block] += shared_data_dna[threadIdx.x + 8 + threads_pr_block]; 79 | shared_data_dna[threadIdx.x + threads_pr_block*2] += shared_data_dna[threadIdx.x + 8 + threads_pr_block*2]; 80 | } 81 | __syncthreads(); 82 | if (threadIdx.x < 4) { 83 | shared_data_dna[threadIdx.x] += shared_data_dna[threadIdx.x + 4]; 84 | shared_data_dna[threadIdx.x + threads_pr_block] += shared_data_dna[threadIdx.x + 4 + threads_pr_block]; 85 | shared_data_dna[threadIdx.x + threads_pr_block*2] += shared_data_dna[threadIdx.x + 4 + threads_pr_block*2]; 86 | } 87 | __syncthreads(); 88 | 89 | if (threadIdx.x < 2) { 90 | shared_data_dna[threadIdx.x] += shared_data_dna[threadIdx.x + 2]; 91 | shared_data_dna[threadIdx.x + threads_pr_block] += shared_data_dna[threadIdx.x + 2 + threads_pr_block]; 92 | shared_data_dna[threadIdx.x + threads_pr_block*2] += shared_data_dna[threadIdx.x + 2 + threads_pr_block*2]; 93 | } 94 | __syncthreads(); 95 | 96 | if(threadIdx.x == 0){ 97 | shared_data_dna[0] += shared_data_dna[1]; 98 | shared_data_dna[0 + threads_pr_block] += shared_data_dna[1 + threads_pr_block]; 99 | shared_data_dna[0 + threads_pr_block*2] += shared_data_dna[1 + threads_pr_block*2]; 100 | } 101 | __syncthreads(); 102 | } 103 | 104 | /*each block computes all distances between a sequence and all other sequences with higher indexes. In this way no summation is needed but shared memory is not utillised*/ 105 | __global__ void computeSliceDNA(float* results, unsigned int* bitStrings_gpu, unsigned int* gapFilters_gpu, unsigned int bsStride, unsigned int dataSize, unsigned int rowOffset){ 106 | 107 | int i = blockIdx.y; 108 | if( i < blockIdx.x+1+rowOffset){ 109 | return; 110 | } 111 | unsigned int iterations = dataSize / threads_pr_block; 112 | if(dataSize % threads_pr_block != 0){ 113 | iterations++; 114 | } 115 | //for(int i = blockIdx.x+1+rowOffset; i < sequenceCount_GPU; i++) { 116 | //set counters in shared memory 117 | shared_data_dna[threadIdx.x] = 0; 118 | shared_data_dna[threadIdx.x + threads_pr_block] = 0; 119 | shared_data_dna[threadIdx.x + threads_pr_block*2] = 0; 120 | for(int slice = 0; slice < iterations; slice++) { 121 | // fetch next block of sequence data 122 | unsigned int idx0 = (rowOffset + blockIdx.x)*bsStride + slice*threads_pr_block + threadIdx.x; 123 | unsigned int baseSeq = bitStrings_gpu[idx0]; 124 | unsigned int baseGap = gapFilters_gpu[idx0]; 125 | unsigned int idx1 = i*bsStride + slice*threads_pr_block + threadIdx.x; 126 | //printf("%d*%d+%d*%d+%d=%d \n",i,bsStride,slice,threads_pr_block,threadIdx.x,idx1); 127 | unsigned int r = bitStrings_gpu[idx1]; 128 | unsigned int gf = gapFilters_gpu[idx1]; 129 | 130 | // compute distances and gaps 131 | r = r ^ baseSeq; 132 | gf = gf & baseGap; 133 | unsigned int tv = (r >> 1) & 0x55555555; 134 | unsigned int ts = (~tv) & (r & 0x55555555); 135 | 136 | // handle gaps 137 | tv = tv & gf; 138 | ts = ts & gf; 139 | // sum distances in this thread 140 | tv = __popc(tv); 141 | ts = __popc(ts); 142 | gf = __popc(gf); 143 | 144 | //Remove invalid results 145 | if(slice * threads_pr_block + threadIdx.x >= dataSize) { 146 | tv = 0; 147 | ts = 0; 148 | gf = 0; 149 | } 150 | 151 | //save intermediate result 152 | shared_data_dna[threadIdx.x] += tv; 153 | shared_data_dna[threadIdx.x + threads_pr_block] += ts; 154 | shared_data_dna[threadIdx.x + threads_pr_block*2] += gf; 155 | } 156 | 157 | //sum over all threads 158 | parallelReductionDNA2(); 159 | 160 | //store data in global memory 161 | if(threadIdx.x == 0){ 162 | //printf("%d \n", blockIdx.x * sequenceCount_GPU + i); 163 | //tv_gpu[blockIdx.x * gridDim.y + i] = shared_data_dna[0]; 164 | //ts_gpu[blockIdx.x * gridDim.y + i] = shared_data_dna[threads_pr_block]; 165 | unsigned int lengthWithoutGaps = shared_data_dna[threads_pr_block*2]; 166 | float ts = float(shared_data_dna[threads_pr_block]) / lengthWithoutGaps; 167 | float tv = float(shared_data_dna[0]) / lengthWithoutGaps; 168 | float temp1 = 1.0f-2.0f*ts-tv; 169 | float temp2 = 1.0f-2.0f*tv; 170 | float distance = -1.0f; 171 | if(!(temp1 <= 0 || temp2 <= 0)){ 172 | distance = -0.5f*log(1.0f-2.0f*ts-tv)-0.25f*log(1.0f-2.0f*tv); 173 | } 174 | results[blockIdx.x * gridDim.y + i] = distance; 175 | } 176 | } 177 | 178 | unsigned int sequenceCount2_dna; 179 | unsigned int dataSize2_dna; 180 | size_t resultMemSize2_dna; 181 | size_t bitStringsMemSize2_dna; 182 | 183 | //results 184 | float* results_dna_gpu; 185 | unsigned int bsStride2_dna; 186 | 187 | //data 188 | unsigned int* gapFilters2_dna_gpu; 189 | unsigned int* bitStrings2_dna_gpu; 190 | unsigned int rowsPrKernel2; 191 | unsigned int numberOfKernelLaunches2; 192 | 193 | #ifdef TIMING 194 | extern float totalGpuComputation; 195 | extern timeval start,end; 196 | extern float totalTransfer; 197 | #endif 198 | 199 | extern "C" void computeDistancesDNA2_gpu(float* results) { 200 | 201 | // execute the kernel 202 | // unsigned int startIdx = i * threads_pr_block; 203 | for(int i = 0; i < numberOfKernelLaunches2; i++) { 204 | printf("kernel: %d -----------------------\n",i); 205 | // setup execution parameters 206 | unsigned int gridSize_dna = 0; 207 | if(i != numberOfKernelLaunches2-1){ 208 | gridSize_dna = rowsPrKernel2; 209 | } else { 210 | gridSize_dna = sequenceCount2_dna - (numberOfKernelLaunches2-1)*rowsPrKernel2; 211 | } 212 | dim3 grid(gridSize_dna,sequenceCount2_dna); 213 | dim3 block(threads_pr_block, 1, 1); 214 | unsigned int transferSize = gridSize_dna * sequenceCount2_dna * sizeof(unsigned int); 215 | computeSliceDNA<<< grid, block >>>(results_dna_gpu, bitStrings2_dna_gpu, gapFilters2_dna_gpu, bsStride2_dna, dataSize2_dna, i*rowsPrKernel2); 216 | 217 | #ifdef TIMING 218 | cudaThreadSynchronize(); 219 | gettimeofday(&end,NULL); 220 | totalGpuComputation += (end.tv_sec - start.tv_sec)*1000.0 + (end.tv_usec - start.tv_usec)/1000.0; 221 | gettimeofday(&start,NULL); 222 | #endif 223 | 224 | cutilSafeCall(cudaMemcpy(&results[i * rowsPrKernel2 * sequenceCount2_dna], results_dna_gpu, transferSize, cudaMemcpyDeviceToHost)); 225 | 226 | #ifdef TIMING 227 | cudaThreadSynchronize(); 228 | gettimeofday(&end,NULL); 229 | totalTransfer += (end.tv_sec - start.tv_sec)*1000.0 + (end.tv_usec - start.tv_usec)/1000.0; 230 | gettimeofday(&start,NULL); 231 | #endif 232 | } 233 | } 234 | 235 | extern "C" void storeDataDNA2_gpu(unsigned int* bitStrings, unsigned int* gapFilters) { 236 | // Copy results from device to host 237 | cutilSafeCall(cudaMemcpyAsync(bitStrings2_dna_gpu, bitStrings, bitStringsMemSize2_dna, cudaMemcpyHostToDevice,0)); 238 | cutilSafeCall(cudaMemcpyAsync(gapFilters2_dna_gpu, gapFilters, bitStringsMemSize2_dna, cudaMemcpyHostToDevice,0)); 239 | } 240 | 241 | extern "C" void initialiseDNA2_gpu(unsigned int sequenceCount, unsigned int bitStringCount, unsigned int _bsStride) { 242 | printf("initialising GPU... \n"); 243 | sequenceCount2_dna = sequenceCount; 244 | bsStride2_dna = _bsStride; 245 | dataSize2_dna = min(bsStride2_dna,bitStringCount * 4); 246 | 247 | resultMemSize2_dna = sequenceCount * sequenceCount * sizeof(float); 248 | bitStringsMemSize2_dna = bsStride2_dna * sizeof(unsigned int) * sequenceCount; 249 | 250 | cudaDeviceProp deviceProp; 251 | cudaGetDeviceProperties(&deviceProp, 0); 252 | float gpuMemSize = deviceProp.totalGlobalMem * 0.8f; 253 | 254 | printf("Gpu memsize %fMB\n",gpuMemSize/1024.0f/1024.0f); 255 | 256 | //printf("TODO REMOVE THIS \n"); 257 | //float gpuMemSize = 50.0 * 1024 * 1024; 258 | 259 | 260 | rowsPrKernel2 = sequenceCount * min(1.0f,(gpuMemSize - bitStringsMemSize2_dna*2) / resultMemSize2_dna); 261 | numberOfKernelLaunches2 = sequenceCount/rowsPrKernel2; 262 | if(sequenceCount % rowsPrKernel2 != 0){ 263 | numberOfKernelLaunches2++; 264 | } 265 | rowsPrKernel2 = sequenceCount / numberOfKernelLaunches2; 266 | while(sequenceCount > rowsPrKernel2*numberOfKernelLaunches2){ 267 | rowsPrKernel2++; 268 | } 269 | printf("rows pr kernel: %d\n",rowsPrKernel2); 270 | resultMemSize2_dna = rowsPrKernel2 * sequenceCount * sizeof(unsigned int); 271 | 272 | // allocate device memory 273 | cutilSafeCall(cudaMalloc((void**) &bitStrings2_dna_gpu, bitStringsMemSize2_dna)); 274 | cutilSafeCall(cudaMalloc((void**) &gapFilters2_dna_gpu, bitStringsMemSize2_dna)); 275 | cutilSafeCall(cudaMalloc((void**) &results_dna_gpu, resultMemSize2_dna)); 276 | } 277 | -------------------------------------------------------------------------------- /src/distanceCalculation/dataloaderStockholm.hpp: -------------------------------------------------------------------------------- 1 | #ifndef DATALOADER_STOCKHOLM_H 2 | #define DATALOADER_STOCKHOLM_H 3 | 4 | #include "stdinclude.h" 5 | #include "dataloader.hpp" 6 | #include "bitStringUtils.hpp" 7 | 8 | class dataloaderStockholm : public dataloader{ 9 | 10 | public: 11 | 12 | dataloaderStockholm() { 13 | buf = new char[BUFFER_SIZE]; 14 | firstSequence = true; 15 | sequenceLength = 0; 16 | sequenceCount = 0; 17 | tempBuffer = ""; 18 | is = new ifstream(); 19 | sequenceNames = new vector; 20 | } 21 | 22 | void load(string filename) { 23 | if(fastdist){ 24 | bitStrings = new vector; 25 | parseData(filename); 26 | } else { 27 | sequences = new vector; 28 | parseData(filename); 29 | } 30 | if(sequenceCount <= 1) { 31 | cerr << "Found " << sequenceCount << " sequences in the input file. At least 2 sequences are needed to build a tree." << endl; 32 | exit(1); 33 | } 34 | if(verbose){ 35 | cerr << "Input type determined as "; 36 | if(type == UNKNOWN){ 37 | cerr << "UNKNOWN. Please supply the type as parameter." << endl; 38 | exit(1); 39 | } else if(type == DNA){ 40 | cerr << "DNA." << endl; 41 | } else if(type == PROTEIN){ 42 | cerr << "PROTEIN." << endl; 43 | } else { 44 | cerr << "UNKNOWN. Please supply the type as parameter." << endl; 45 | exit(1); 46 | } 47 | cerr << "Number of sequences: " << sequenceCount << endl; 48 | cerr << "Sequence length: " << sequenceLength << endl; 49 | } 50 | } 51 | 52 | void discoverInputType(char* data){ 53 | if(type != UNKNOWN) return; 54 | type = DNA; 55 | for(unsigned int i = 0; i < sequenceLength; i++){ 56 | if(!(data[i] < 65 || 57 | data[i] == 'A' || 58 | data[i] == 'a' || 59 | data[i] == 'C' || 60 | data[i] == 'c' || 61 | data[i] == 'G' || 62 | data[i] == 'g' || 63 | data[i] == 'T' || 64 | data[i] == 't' || 65 | data[i] == 'U' || 66 | data[i] == 'u' || 67 | data[i] == 'N' || 68 | data[i] == 'n')) 69 | { 70 | type = PROTEIN; 71 | return; 72 | } 73 | } 74 | } 75 | 76 | void parseData(string filename) { 77 | // open the file 78 | is->open(filename.data(),ifstream::in); 79 | if (!is->is_open()) { 80 | cerr << "ERROR: Could not read file: " << filename << "\n"; 81 | exit(1); 82 | } 83 | fillBuffer(); 84 | curChar = getNextChar(); 85 | 86 | while(true) { 87 | while(curChar < 32){ 88 | curChar = getNextChar(); 89 | } 90 | if(curChar == 35) { 91 | // # markup line. 92 | parseMarkupLine(); 93 | } else if(curChar == 47) { 94 | // '/' might be the end of the alignment. 95 | curChar = getNextChar(); 96 | if(curChar == 47){ 97 | //finished 98 | is->close(); 99 | return; 100 | } else { 101 | tempBuffer = "/"; 102 | parseSequenceLine(); 103 | } 104 | } else { 105 | parseSequenceLine(); 106 | } 107 | curChar = getNextChar(); 108 | } 109 | } 110 | 111 | inline void fillBuffer(){ 112 | if(is->good()){ 113 | is->read(buf,BUFFER_SIZE); 114 | bufPos = 0; 115 | } else { 116 | is->close(); 117 | cerr << "Stream ended unexpectedly. Did not find '//' marking the end of the alignment" << endl; 118 | exit(1); 119 | } 120 | } 121 | 122 | inline char getNextChar() { 123 | if(bufPos == is->gcount()){ 124 | fillBuffer(); 125 | } 126 | return buf[bufPos++]; 127 | } 128 | 129 | //Finds the next new line 130 | void parseMarkupLine(){ 131 | while (curChar != 10){ 132 | curChar = getNextChar(); 133 | } 134 | } 135 | 136 | inline void encodeProteinSequence(unsigned int* bitString, vector& data){ 137 | for(unsigned int i = 0; i < sequenceLength+paddingLength; i++) { 138 | int offset = i % 4; 139 | unsigned int bitStringIdx = i / 4; 140 | if(offset == 0){ 141 | bitString[bitStringIdx] = 0; 142 | } 143 | char c = resolveChar(data[i]); 144 | bitString[bitStringIdx] += (c << (offset*8)); 145 | } 146 | } 147 | 148 | inline void encodeDNASequence(unsigned int* bitString, unsigned int* gapFilter, vector& data){ 149 | for(unsigned int i = 0; i < sequenceLength+paddingLength; i++) { 150 | int offset = i % 16; 151 | unsigned int bitStringIdx = i / 16; 152 | if(offset == 0){ 153 | bitString[bitStringIdx] = 0; 154 | gapFilter[bitStringIdx] = 0; 155 | } 156 | char c = data[i]; 157 | switch (c) { 158 | //Nothing is done for gaps and ambigious nucleotides 159 | case 'A': 160 | case 'a': 161 | //A has value 00 so nothing is added to the bitStrings 162 | gapFilter[bitStringIdx] += (Gbin << (offset*2)); 163 | break; 164 | case 'C': 165 | case 'c': 166 | bitString[bitStringIdx] += (Cbin<<(offset*2)); 167 | gapFilter[bitStringIdx] += (Gbin << (offset*2)); 168 | break; 169 | case 'G': 170 | case 'g': 171 | bitString[bitStringIdx] += (Gbin<<(offset*2)); 172 | gapFilter[bitStringIdx] += (Gbin << (offset*2)); 173 | break; 174 | case 'T': 175 | case 't': 176 | bitString[bitStringIdx] += (Tbin<<(offset*2)); 177 | gapFilter[bitStringIdx] += (Gbin << (offset*2)); 178 | break; 179 | default: 180 | break; 181 | } 182 | } 183 | } 184 | 185 | void parseSequenceLineFastDist(){ 186 | // find the length and type of the sequence 187 | parseSequenceName(); 188 | while (curChar != 10){ 189 | if(curChar > 32){ 190 | charBuffer.push_back(curChar); 191 | } 192 | curChar = getNextChar(); 193 | } 194 | if(firstSequence){ 195 | sequenceLength = charBuffer.size(); 196 | discoverInputType(&charBuffer[0]); 197 | if(type == DNA){ 198 | bitStringsCount = sequenceLength / 64 + 6; 199 | paddingLength = bitStringsCount*64 - sequenceLength; 200 | gapFilters = new vector; 201 | } else { 202 | bitStringsCount = sequenceLength / 16 + 8; 203 | paddingLength = bitStringsCount*16 - sequenceLength; 204 | } 205 | } 206 | if(charBuffer.size() != sequenceLength) { 207 | cerr << "ERROR: Sequence " << sequenceCount << " has length " << charBuffer.size() << " while the preceding sequences in the alignment has length " << sequenceLength << "." << endl; 208 | exit(1); 209 | } 210 | // insert paddings 211 | for(int i = 0; i < paddingLength; i++){ 212 | charBuffer.push_back('-'); 213 | } 214 | 215 | // Encode the data in char vectors. 216 | // unsigned int* bitString = new unsigned int[bitStringsCount*4+3]; 217 | // bitString = (unsigned int*) ((((unsigned int) bitString) + 12) & ~15); 218 | unsigned int* bitString; 219 | bitString = (unsigned int*) _mm_malloc(bitStringsCount*4*sizeof(unsigned int), 16); 220 | if(type == DNA){ 221 | unsigned int* gapFilter; 222 | gapFilter = (unsigned int*) _mm_malloc(bitStringsCount*4*sizeof(unsigned int), 16); 223 | encodeDNASequence(bitString, gapFilter, charBuffer); 224 | gapFilters->push_back(gapFilter); 225 | } else { 226 | encodeProteinSequence(bitString, charBuffer); 227 | } 228 | bitStrings->push_back(bitString); 229 | firstSequence = false; 230 | sequenceCount++; 231 | charBuffer.clear(); 232 | } 233 | 234 | inline char resolveChar(char c){ 235 | //ignore positions with ambigious/unknown nucleotides 236 | if(type == DNA){ 237 | if (!(c == 'a' || c == 'A' || c == 'c' || c == 'C' || c == 'g' || c == 'G' || c == 't' || c == 'T' || c == 'u' || c == 'U')){ 238 | return '-'; 239 | } else { 240 | return c; 241 | } 242 | } else { 243 | if(c == '-' || c == '.' || c == 'X' || c == 'x' || c == 'z' || c == 'Z' || c == 'b' || c == 'B' || c == 'J' || c == 'j' || c == '?') { 244 | return '-'; 245 | } else { 246 | return c; 247 | } 248 | } 249 | } 250 | 251 | void parseSequenceLine(){ 252 | if(fastdist){ 253 | parseSequenceLineFastDist(); 254 | return; 255 | } 256 | if(firstSequence){ 257 | // find the length of the sequence 258 | parseSequenceName(); 259 | while (curChar != 10){ 260 | if(curChar > 32) { 261 | charBuffer.push_back(curChar); 262 | } 263 | curChar = getNextChar(); 264 | } 265 | // We have the length. Use char arrays to store the char data in. 266 | sequenceLength = charBuffer.size(); 267 | char* data = new char[sequenceLength]; 268 | sequences->push_back(data); 269 | 270 | for(unsigned int i = 0; i < sequenceLength; i++){ 271 | data[i] = resolveChar(charBuffer[i]); 272 | } 273 | charBuffer.clear(); 274 | discoverInputType(sequences->at(0)); 275 | firstSequence = false; 276 | } else { 277 | parseSequenceName(); 278 | unsigned int counter = 0; 279 | char* data = new char[sequenceLength]; 280 | sequences->push_back(data); 281 | while (curChar != 10){ 282 | if(curChar > 32){ 283 | data[counter] = resolveChar(curChar); 284 | counter++; 285 | } 286 | curChar = getNextChar(); 287 | } 288 | if(counter != sequenceLength){ 289 | cerr << "ERROR: Sequence " << sequenceCount << " has length " << counter << " while the preceding sequences in the alignment has length " << sequenceLength << "." << endl; 290 | exit(1); 291 | } 292 | } 293 | sequenceCount++; 294 | } 295 | 296 | void parseSequenceName(){ 297 | string name = ""; 298 | if(tempBuffer != ""){ 299 | name.append(tempBuffer); 300 | tempBuffer = ""; 301 | } 302 | while(curChar > 32){ 303 | name.push_back(curChar); 304 | curChar = getNextChar(); 305 | } 306 | sequenceNames->push_back(name); 307 | while(curChar < 32){ 308 | curChar = getNextChar(); 309 | if(curChar == 10) { 310 | cerr << "ERROR: alignment with name \"" << sequenceNames->back() << "\" has zero length" << endl; 311 | exit(1); 312 | } 313 | } 314 | } 315 | 316 | unsigned int** getBitStrings(){ 317 | return &(*bitStrings)[0]; 318 | } 319 | 320 | unsigned int** getGapFilters(){ 321 | return &(*gapFilters)[0]; 322 | } 323 | 324 | unsigned int getSequenceCount(){ 325 | return sequenceCount; 326 | } 327 | 328 | unsigned int getSequenceLength(){ 329 | return sequenceLength; 330 | } 331 | 332 | unsigned int getBitStringsCount(){ 333 | return bitStringsCount; 334 | } 335 | 336 | vector* getSequenceNames(){ 337 | return sequenceNames; 338 | } 339 | 340 | vector* getSequences(){ 341 | return sequences; 342 | } 343 | 344 | void setSequences(vector* val){ 345 | sequences = val; 346 | } 347 | 348 | ~dataloaderStockholm(){ 349 | delete sequenceNames; 350 | } 351 | 352 | private: 353 | int bufPos; 354 | char* buf; 355 | vector* sequenceNames; 356 | bool firstSequence; 357 | char curChar; 358 | string tempBuffer; 359 | ifstream* is; 360 | vector charBuffer; 361 | 362 | }; 363 | 364 | #endif 365 | --------------------------------------------------------------------------------