├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── data ├── query_file └── test.fastq ├── include ├── CLI │ ├── App.hpp │ ├── CLI.hpp │ ├── Error.hpp │ ├── Ini.hpp │ ├── Option.hpp │ ├── Split.hpp │ ├── StringTools.hpp │ ├── Timer.hpp │ ├── TypeTools.hpp │ └── Validators.hpp ├── ProgOpts.h ├── SqueakrFS.h ├── chunk.h ├── clipp.h ├── gqf │ ├── gqf.h │ ├── gqf_file.h │ ├── gqf_int.h │ ├── hashutil.h │ └── partitioned_counter.h ├── gqf_cpp.h ├── kmer.h ├── reader.h ├── spdlog │ ├── async_logger.h │ ├── common.h │ ├── details │ │ ├── async_log_helper.h │ │ ├── async_logger_impl.h │ │ ├── file_helper.h │ │ ├── log_msg.h │ │ ├── logger_impl.h │ │ ├── mpmc_bounded_q.h │ │ ├── null_mutex.h │ │ ├── os.h │ │ ├── pattern_formatter_impl.h │ │ ├── registry.h │ │ └── spdlog_impl.h │ ├── fmt │ │ ├── bundled │ │ │ ├── format.cc │ │ │ ├── format.h │ │ │ ├── ostream.cc │ │ │ ├── ostream.h │ │ │ ├── posix.cc │ │ │ ├── posix.h │ │ │ └── time.h │ │ ├── fmt.h │ │ └── ostr.h │ ├── formatter.h │ ├── logger.h │ ├── sinks │ │ ├── android_sink.h │ │ ├── ansicolor_sink.h │ │ ├── base_sink.h │ │ ├── dist_sink.h │ │ ├── file_sinks.h │ │ ├── msvc_sink.h │ │ ├── null_sink.h │ │ ├── ostream_sink.h │ │ ├── sink.h │ │ ├── stdout_sinks.h │ │ ├── syslog_sink.h │ │ └── wincolor_sink.h │ ├── spdlog.h │ └── tweakme.h ├── squeakrconfig.h └── util.h ├── scripts ├── lognumslots.sh ├── merge_into_develop.sh └── merge_into_master.sh └── src ├── SqueakrFS.cc ├── count.cc ├── gqf ├── gqf.c ├── gqf_file.c ├── hashutil.c └── partitioned_counter.c ├── info.cc ├── innerprod.cc ├── kmer.cc ├── list.cc ├── query.cc ├── squeakr.cc └── util.cc /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files 2 | *.slo 3 | *.lo 4 | *.o 5 | *.obj 6 | 7 | # Precompiled Headers 8 | *.gch 9 | *.pch 10 | 11 | # Compiled Dynamic libraries 12 | *.so 13 | *.dylib 14 | *.dll 15 | 16 | # Fortran module files 17 | *.mod 18 | *.smod 19 | 20 | # Compiled Static libraries 21 | *.lai 22 | *.la 23 | *.a 24 | *.lib 25 | 26 | # Executables 27 | *.exe 28 | *.out 29 | *.app 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2017, 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | TARGETS= squeakr 2 | 3 | ifdef D 4 | DEBUG=-g 5 | OPT= 6 | else 7 | DEBUG= 8 | OPT=-Ofast 9 | endif 10 | 11 | ifdef NH 12 | ARCH= 13 | else 14 | ARCH=-msse4.2 -D__SSE4_2_ 15 | endif 16 | 17 | ifdef P 18 | PROFILE=-pg -no-pie # for bug in gprof. 19 | endif 20 | 21 | CXX = g++ -std=c++11 22 | CC = gcc -std=gnu11 23 | LD= g++ -std=c++11 24 | 25 | LOC_INCLUDE=include 26 | LOC_SRC=src 27 | OBJDIR=obj 28 | 29 | CXXFLAGS += -Wall $(DEBUG) $(PROFILE) $(OPT) $(ARCH) -m64 -I. -I$(LOC_INCLUDE) 30 | 31 | CFLAGS += -Wall $(DEBUG) $(PROFILE) $(OPT) $(ARCH) -m64 -I. -I$(LOC_INCLUDE) 32 | 33 | LDFLAGS += $(DEBUG) $(PROFILE) $(OPT) -lpthread -lboost_system \ 34 | -lboost_thread -lm -lbz2 -lz -lrt 35 | 36 | # 37 | # declaration of dependencies 38 | # 39 | 40 | all: $(TARGETS) 41 | 42 | # dependencies between programs and .o files 43 | squeakr: $(OBJDIR)/kmer.o $(OBJDIR)/hashutil.o $(OBJDIR)/util.o \ 44 | $(OBJDIR)/gqf.o $(OBJDIR)/gqf_file.o \ 45 | $(OBJDIR)/partitioned_counter.o $(OBJDIR)/SqueakrFS.o \ 46 | $(OBJDIR)/count.o $(OBJDIR)/query.o $(OBJDIR)/innerprod.o \ 47 | $(OBJDIR)/list.o $(OBJDIR)/info.o $(OBJDIR)/squeakr.o 48 | 49 | # dependencies between .o files and .h files 50 | 51 | $(OBJDIR)/squeakr.o: $(LOC_SRC)/squeakr.cc 52 | $(OBJDIR)/count.o: $(LOC_INCLUDE)/gqf_cpp.h $(LOC_INCLUDE)/chunk.h \ 53 | $(LOC_INCLUDE)/kmer.h \ 54 | $(LOC_INCLUDE)/reader.h $(LOC_INCLUDE)/util.h \ 55 | $(LOC_INCLUDE)/SqueakrFS.h 56 | $(OBJDIR)/query.o: $(LOC_INCLUDE)/gqf_cpp.h $(LOC_INCLUDE)/kmer.h \ 57 | $(LOC_INCLUDE)/util.h 58 | $(OBJDIR)/innerprod.o: $(LOC_INCLUDE)/gqf_cpp.h 59 | $(OBJDIR)/list.o: $(LOC_INCLUDE)/gqf_cpp.h $(LOC_INCLUDE)/kmer.h \ 60 | $(LOC_INCLUDE)/util.h 61 | $(OBJDIR)/info.o: $(LOC_INCLUDE)/gqf_cpp.h $(LOC_INCLUDE)/kmer.h \ 62 | $(LOC_INCLUDE)/util.h 63 | $(OBJDIR)/kmer.o: $(LOC_SRC)/kmer.cc $(LOC_INCLUDE)/kmer.h 64 | $(OBJDIR)/util.o: $(LOC_SRC)/util.cc $(LOC_INCLUDE)/util.h 65 | 66 | # dependencies between .o files and .cc (or .c) files 67 | $(OBJDIR)/gqf.o: $(LOC_SRC)/gqf/gqf.c $(LOC_INCLUDE)/gqf/gqf.h 68 | $(OBJDIR)/gqf_file.o: $(LOC_SRC)/gqf/gqf_file.c $(LOC_INCLUDE)/gqf/gqf_file.h 69 | $(OBJDIR)/hashutil.o: $(LOC_INCLUDE)/gqf/hashutil.h 70 | $(OBJDIR)/partitioned_counter.o: $(LOC_INCLUDE)/gqf/partitioned_counter.h 71 | 72 | # 73 | # generic build rules 74 | # 75 | 76 | $(TARGETS): 77 | $(LD) $^ $(LDFLAGS) -o $@ 78 | 79 | $(OBJDIR)/%.o: $(LOC_SRC)/%.cc | $(OBJDIR) 80 | $(CXX) $(CXXFLAGS) $(INCLUDE) -c -o $@ $< 81 | 82 | $(OBJDIR)/%.o: $(LOC_SRC)/%.c | $(OBJDIR) 83 | $(CXX) $(CFLAGS) $(INCLUDE) -c -o $@ $< 84 | 85 | $(OBJDIR)/%.o: $(LOC_SRC)/gqf/%.c | $(OBJDIR) 86 | $(CXX) $(CFLAGS) $(INCLUDE) -c -o $@ $< 87 | 88 | $(OBJDIR): 89 | @mkdir -p $(OBJDIR) 90 | 91 | clean: 92 | rm -rf $(OBJDIR) core $(TARGETS) 93 | 94 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # squeakr 2 | Squeakr: An Exact and Approximate k -mer Counting System 3 | 4 | This work was published in Bioinformatics. If you use this software please cite us: 5 | ``` 6 | @article{doi:10.1093/bioinformatics/btx636, 7 | author = {Pandey, Prashant and Bender, Michael A and Johnson, Rob and Patro, Rob}, 8 | title = {Squeakr: An Exact and Approximate k-mer Counting System}, 9 | journal = {Bioinformatics}, 10 | volume = {}, 11 | number = {}, 12 | pages = {btx636}, 13 | year = {2017}, 14 | doi = {10.1093/bioinformatics/btx636}, 15 | URL = { + http://dx.doi.org/10.1093/bioinformatics/btx636}, 16 | eprint = {/oup/backfile/content_public/journal/bioinformatics/pap/10.1093_bioinformatics_btx636/1/btx636.pdf} 17 | } 18 | ``` 19 | 20 | Overview 21 | -------- 22 | 23 | Squeakr is a k-mer-counting and multiset-representation system using the 24 | recently-introduced counting quotient filter (CQF) Pandey et al. (2017), a 25 | feature-rich approximate membership query (AMQ) data structure. 26 | 27 | Squeakr is memory-efficient, consuming 1.5X–4.3X less memory than the 28 | state-of-the-art. It offers competitive counting performance, in fact, it is 29 | faster for larger k-mers, and answers queries about a particular k-mer over an 30 | order-of- magnitude faster than other systems. The Squeakr representation of the 31 | k-mer multiset turns out to be immediately useful for downstream processing 32 | (e.g., De Bruijn graph traversal) because it supports fast queries and dynamic 33 | k-mer insertion, deletion, and modification. 34 | 35 | k-mer counts can be validated by hooking into the C++ level query API. An 36 | example query program is also available in "kmer_query.cc". 37 | 38 | Release notes 39 | -------- 40 | 41 | Squeakr now has a new k-mer representation (version 2) based on the new version of the CQF and some few Squeakr specific changes. The new version of Squeakr is not compatible with the old version. We have added some new features to the 'squeakr count' command and a couple of new commands. 42 | 43 | 44 | * Squeakr count command now supports auto-resizing. However, auto-resizing only works when the count command is run with a single thread. 45 | * Squeakr count command can now filter out k-mers in the final representation below a certain count value. 46 | * Squeakr count command can now exclude counts in the final representation and only keep k-mers. 47 | 48 | * Squeakr list: to list k-mers present in a Squeakr representation. This command only works when the representation is exact. 49 | 50 | * Squeakr info: to get the infomation about the Squeakr representation. For example, version, k-mer size, number of k-mers, CQF specific info, etc. 51 | 52 | API 53 | -------- 54 | * 'squeakr count': count k-mers in a read dataset. 55 | * 'squeakr query': query k-mers in the Squeakr representation. 56 | * 'squeakr inner-prod': compute inner products of two Squeakr representations. 57 | * 'squeakr list': list k-mers in the Squeakr representation. Only in exact 58 | representation. 59 | * 'squeakr info': get information about the Squeakr representation. 60 | 61 | Build 62 | ------- 63 | 64 | Library dependencies (given version or higher): 65 | - libboost-dev 1.58.0.1ubuntu1 66 | - libssl-dev 1.0.2g-1ubuntu4.6 67 | - zlib1g-dev 1:1.2.8.dfsg-2ubuntu4 68 | - bzip2 1.0.6-8 69 | 70 | Squeakr currently only supports fastq files. If any other file formats are 71 | passed as input then it will throw a segmentation fault. 72 | 73 | The CQF code uses two new instructions to implement select on machine words 74 | introduced in intel's Haswell line of CPUs. However, there is also an alternate 75 | implementation of select on machine words to work on CPUs older than Haswell. 76 | To build on an older hardware (older than Haswell) use "NH=1" as a make argument. 77 | 78 | ```bash 79 | $ make squeakr 80 | $ ./squeakr count -e -k 28 -s 20 -t 1 -o data/tmp.squeakr data/test.fastq 81 | ``` 82 | 83 | The usage of `./squeakr count` is as follows: 84 | 85 | ```bash 86 | SYNOPSIS 87 | squeakr count [-e] -k [-c ] [-n] [-s ] [-t ] -o ... 88 | 89 | OPTIONS 90 | -e, --exact squeakr-exact (default is Squeakr approximate) 91 | length of k-mers to count 92 | only output k-mers with count greater than or equal to cutoff (default = 1) 93 | 94 | -n, --no-counts 95 | only output k-mers and no counts (default = false) 96 | 97 | log of number of slots in the CQF. (Size argument is only optional when numthreads is exactly 1.) 98 | 99 | 100 | number of threads to use to count (default = number of hardware threads) 101 | 102 | file in which output should be written 103 | ... list of files to be counted (supported files: fastq and compressed gzip or bzip2 fastq files) 104 | ``` 105 | 106 | squeakr-count creates a file which is the k-mer representation. 107 | 108 | `lognumslots.sh` script can be used to estimate the `log of number of slots in the CQF` argument. The script takes as input the path to the output file of 'ntCard' (https://github.com/bcgsc/ntCard). It then calculates log of the number of slots needed by Squeakr to count k-mers. 109 | 110 | ```bash 111 | $ ./squeakr query -f data/tmp.squeakr -q data/query_file -o data/query.output 112 | ``` 113 | The usage of `./squeakr query` is as follows: 114 | 115 | ```bash 116 | SYNOPSIS 117 | squeakr query -f -q -o 118 | OPTIONS 119 | 120 | input squeakr file 121 | 122 | 123 | input query file 124 | 125 | 126 | output file 127 | ``` 128 | 129 | ```bash 130 | $ make squeakr-inner-prod 131 | $ ./squeakr inner_prod data/tmp.squeakr data/tmp.squeakr 132 | ``` 133 | The usage of `./squeakr inner_prod` is as follows: 134 | 135 | ```bash 136 | SYNOPSIS 137 | squeakr inner_prod 138 | 139 | OPTIONS 140 | 141 | first input squeakr file 142 | 143 | 144 | second input squeakr file 145 | 146 | ``` 147 | 148 | Contributing 149 | ------------ 150 | Contributions via GitHub pull requests are welcome. 151 | 152 | 153 | Authors 154 | ------- 155 | - Prashant Pandey 156 | - Rob Patro 157 | - Rob Johnson 158 | -------------------------------------------------------------------------------- /include/CLI/CLI.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | // Distributed under the 3-Clause BSD License. See accompanying 4 | // file LICENSE or https://github.com/CLIUtils/CLI11 for details. 5 | 6 | // CLI Library includes 7 | // Order is important for combiner script 8 | 9 | #include "CLI/Error.hpp" 10 | #include "CLI/TypeTools.hpp" 11 | #include "CLI/StringTools.hpp" 12 | #include "CLI/Split.hpp" 13 | #include "CLI/Ini.hpp" 14 | #include "CLI/Validators.hpp" 15 | #include "CLI/Option.hpp" 16 | #include "CLI/App.hpp" 17 | -------------------------------------------------------------------------------- /include/CLI/Error.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | // Distributed under the 3-Clause BSD License. See accompanying 4 | // file LICENSE or https://github.com/CLIUtils/CLI11 for details. 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | namespace CLI { 11 | 12 | /// These codes are part of every error in CLI. They can be obtained from e using e.exit_code or as a quick shortcut, 13 | /// int values from e.get_error_code(). 14 | enum class ExitCodes { 15 | Success = 0, 16 | IncorrectConstruction = 100, 17 | BadNameString, 18 | OptionAlreadyAdded, 19 | File, 20 | Conversion, 21 | Validation, 22 | Required, 23 | Requires, 24 | Excludes, 25 | Extras, 26 | ExtrasINI, 27 | Invalid, 28 | Horrible, 29 | OptionNotFound, 30 | BaseClass = 255 31 | }; 32 | 33 | // Error definitions 34 | 35 | /// @defgroup error_group Errors 36 | /// @brief Errors thrown by CLI11 37 | /// 38 | /// These are the errors that can be thrown. Some of them, like CLI::Success, are not really errors. 39 | /// @{ 40 | 41 | /// All errors derive from this one 42 | struct Error : public std::runtime_error { 43 | int exit_code; 44 | bool print_help; 45 | int get_exit_code() const { return exit_code; } 46 | Error(std::string parent, std::string name, ExitCodes exit_code = ExitCodes::BaseClass, bool print_help = true) 47 | : runtime_error(parent + ": " + name), exit_code(static_cast(exit_code)), print_help(print_help) {} 48 | Error(std::string parent, 49 | std::string name, 50 | int exit_code = static_cast(ExitCodes::BaseClass), 51 | bool print_help = true) 52 | : runtime_error(parent + ": " + name), exit_code(exit_code), print_help(print_help) {} 53 | }; 54 | 55 | /// Construction errors (not in parsing) 56 | struct ConstructionError : public Error { 57 | // Using Error::Error constructors seem to not work on GCC 4.7 58 | ConstructionError(std::string parent, 59 | std::string name, 60 | ExitCodes exit_code = ExitCodes::BaseClass, 61 | bool print_help = true) 62 | : Error(parent, name, exit_code, print_help) {} 63 | }; 64 | 65 | /// Thrown when an option is set to conflicting values (non-vector and multi args, for example) 66 | struct IncorrectConstruction : public ConstructionError { 67 | IncorrectConstruction(std::string name) 68 | : ConstructionError("IncorrectConstruction", name, ExitCodes::IncorrectConstruction) {} 69 | }; 70 | 71 | /// Thrown on construction of a bad name 72 | struct BadNameString : public ConstructionError { 73 | BadNameString(std::string name) : ConstructionError("BadNameString", name, ExitCodes::BadNameString) {} 74 | }; 75 | 76 | /// Thrown when an option already exists 77 | struct OptionAlreadyAdded : public ConstructionError { 78 | OptionAlreadyAdded(std::string name) 79 | : ConstructionError("OptionAlreadyAdded", name, ExitCodes::OptionAlreadyAdded) {} 80 | }; 81 | 82 | // Parsing errors 83 | 84 | /// Anything that can error in Parse 85 | struct ParseError : public Error { 86 | ParseError(std::string parent, std::string name, ExitCodes exit_code = ExitCodes::BaseClass, bool print_help = true) 87 | : Error(parent, name, exit_code, print_help) {} 88 | }; 89 | 90 | // Not really "errors" 91 | 92 | /// This is a successful completion on parsing, supposed to exit 93 | struct Success : public ParseError { 94 | Success() : ParseError("Success", "Successfully completed, should be caught and quit", ExitCodes::Success, false) {} 95 | }; 96 | 97 | /// -h or --help on command line 98 | struct CallForHelp : public ParseError { 99 | CallForHelp() 100 | : ParseError("CallForHelp", "This should be caught in your main function, see examples", ExitCodes::Success) {} 101 | }; 102 | 103 | /// Thrown when parsing an INI file and it is missing 104 | struct FileError : public ParseError { 105 | FileError(std::string name) : ParseError("FileError", name, ExitCodes::File) {} 106 | }; 107 | 108 | /// Thrown when conversion call back fails, such as when an int fails to coerse to a string 109 | struct ConversionError : public ParseError { 110 | ConversionError(std::string name) : ParseError("ConversionError", name, ExitCodes::Conversion) {} 111 | }; 112 | 113 | /// Thrown when validation of results fails 114 | struct ValidationError : public ParseError { 115 | ValidationError(std::string name) : ParseError("ValidationError", name, ExitCodes::Validation) {} 116 | }; 117 | 118 | /// Thrown when a required option is missing 119 | struct RequiredError : public ParseError { 120 | RequiredError(std::string name) : ParseError("RequiredError", name, ExitCodes::Required) {} 121 | }; 122 | 123 | /// Thrown when a requires option is missing 124 | struct RequiresError : public ParseError { 125 | RequiresError(std::string name, std::string subname) 126 | : ParseError("RequiresError", name + " requires " + subname, ExitCodes::Requires) {} 127 | }; 128 | 129 | /// Thrown when a exludes option is present 130 | struct ExcludesError : public ParseError { 131 | ExcludesError(std::string name, std::string subname) 132 | : ParseError("ExcludesError", name + " excludes " + subname, ExitCodes::Excludes) {} 133 | }; 134 | 135 | /// Thrown when too many positionals or options are found 136 | struct ExtrasError : public ParseError { 137 | ExtrasError(std::string name) : ParseError("ExtrasError", name, ExitCodes::Extras) {} 138 | }; 139 | 140 | /// Thrown when extra values are found in an INI file 141 | struct ExtrasINIError : public ParseError { 142 | ExtrasINIError(std::string name) : ParseError("ExtrasINIError", name, ExitCodes::ExtrasINI) {} 143 | }; 144 | 145 | /// Thrown when validation fails before parsing 146 | struct InvalidError : public ParseError { 147 | InvalidError(std::string name) : ParseError("InvalidError", name, ExitCodes::Invalid) {} 148 | }; 149 | 150 | /// This is just a safety check to verify selection and parsing match 151 | struct HorribleError : public ParseError { 152 | HorribleError(std::string name) 153 | : ParseError("HorribleError", "(You should never see this error) " + name, ExitCodes::Horrible) {} 154 | }; 155 | 156 | // After parsing 157 | 158 | /// Thrown when counting a non-existent option 159 | struct OptionNotFound : public Error { 160 | OptionNotFound(std::string name) : Error("OptionNotFound", name, ExitCodes::OptionNotFound) {} 161 | }; 162 | 163 | /// @} 164 | 165 | } // namespace CLI 166 | -------------------------------------------------------------------------------- /include/CLI/Ini.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | // Distributed under the 3-Clause BSD License. See accompanying 4 | // file LICENSE or https://github.com/CLIUtils/CLI11 for details. 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "CLI/StringTools.hpp" 12 | 13 | namespace CLI { 14 | namespace detail { 15 | 16 | inline std::string inijoin(std::vector args) { 17 | std::ostringstream s; 18 | size_t start = 0; 19 | for(const auto &arg : args) { 20 | if(start++ > 0) 21 | s << " "; 22 | 23 | auto it = std::find_if(arg.begin(), arg.end(), [](char ch) { return std::isspace(ch, std::locale()); }); 24 | if(it == arg.end()) 25 | s << arg; 26 | else if(arg.find(R"(")") == std::string::npos) 27 | s << R"(")" << arg << R"(")"; 28 | else 29 | s << R"(')" << arg << R"(')"; 30 | } 31 | 32 | return s.str(); 33 | } 34 | 35 | struct ini_ret_t { 36 | /// This is the full name with dots 37 | std::string fullname; 38 | 39 | /// Listing of inputs 40 | std::vector inputs; 41 | 42 | /// Current parent level 43 | size_t level = 0; 44 | 45 | /// Return parent or empty string, based on level 46 | /// 47 | /// Level 0, a.b.c would return a 48 | /// Level 1, a.b.c could return b 49 | std::string parent() const { 50 | std::vector plist = detail::split(fullname, '.'); 51 | if(plist.size() > (level + 1)) 52 | return plist[level]; 53 | else 54 | return ""; 55 | } 56 | 57 | /// Return name 58 | std::string name() const { 59 | std::vector plist = detail::split(fullname, '.'); 60 | return plist.at(plist.size() - 1); 61 | } 62 | }; 63 | 64 | /// Internal parsing function 65 | inline std::vector parse_ini(std::istream &input) { 66 | std::string name, line; 67 | std::string section = "default"; 68 | 69 | std::vector output; 70 | 71 | while(getline(input, line)) { 72 | std::vector items; 73 | 74 | detail::trim(line); 75 | size_t len = line.length(); 76 | if(len > 1 && line[0] == '[' && line[len - 1] == ']') { 77 | section = line.substr(1, len - 2); 78 | } else if(len > 0 && line[0] != ';') { 79 | output.emplace_back(); 80 | ini_ret_t &out = output.back(); 81 | 82 | // Find = in string, split and recombine 83 | auto pos = line.find("="); 84 | if(pos != std::string::npos) { 85 | name = detail::trim_copy(line.substr(0, pos)); 86 | std::string item = detail::trim_copy(line.substr(pos + 1)); 87 | items = detail::split_up(item); 88 | } else { 89 | name = detail::trim_copy(line); 90 | items = {"ON"}; 91 | } 92 | 93 | if(detail::to_lower(section) == "default") 94 | out.fullname = name; 95 | else 96 | out.fullname = section + "." + name; 97 | 98 | out.inputs.insert(std::end(out.inputs), std::begin(items), std::end(items)); 99 | } 100 | } 101 | return output; 102 | } 103 | 104 | /// Parse an INI file, throw an error (ParseError:INIParseError or FileError) on failure 105 | inline std::vector parse_ini(const std::string &name) { 106 | 107 | std::ifstream input{name}; 108 | if(!input.good()) 109 | throw FileError(name); 110 | 111 | return parse_ini(input); 112 | } 113 | 114 | } // namespace detail 115 | } // namespace CLI 116 | -------------------------------------------------------------------------------- /include/CLI/Split.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | // Distributed under the 3-Clause BSD License. See accompanying 4 | // file LICENSE or https://github.com/CLIUtils/CLI11 for details. 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | #include "CLI/Error.hpp" 11 | #include "CLI/StringTools.hpp" 12 | 13 | namespace CLI { 14 | namespace detail { 15 | 16 | // Returns false if not a short option. Otherwise, sets opt name and rest and returns true 17 | inline bool split_short(const std::string ¤t, std::string &name, std::string &rest) { 18 | if(current.size() > 1 && current[0] == '-' && valid_first_char(current[1])) { 19 | name = current.substr(1, 1); 20 | rest = current.substr(2); 21 | return true; 22 | } else 23 | return false; 24 | } 25 | 26 | // Returns false if not a long option. Otherwise, sets opt name and other side of = and returns true 27 | inline bool split_long(const std::string ¤t, std::string &name, std::string &value) { 28 | if(current.size() > 2 && current.substr(0, 2) == "--" && valid_first_char(current[2])) { 29 | auto loc = current.find("="); 30 | if(loc != std::string::npos) { 31 | name = current.substr(2, loc - 2); 32 | value = current.substr(loc + 1); 33 | } else { 34 | name = current.substr(2); 35 | value = ""; 36 | } 37 | return true; 38 | } else 39 | return false; 40 | } 41 | 42 | // Splits a string into multiple long and short names 43 | inline std::vector split_names(std::string current) { 44 | std::vector output; 45 | size_t val; 46 | while((val = current.find(",")) != std::string::npos) { 47 | output.push_back(trim_copy(current.substr(0, val))); 48 | current = current.substr(val + 1); 49 | } 50 | output.push_back(trim_copy(current)); 51 | return output; 52 | } 53 | 54 | /// Get a vector of short names, one of long names, and a single name 55 | inline std::tuple, std::vector, std::string> 56 | get_names(const std::vector &input) { 57 | 58 | std::vector short_names; 59 | std::vector long_names; 60 | std::string pos_name; 61 | 62 | for(std::string name : input) { 63 | if(name.length() == 0) 64 | continue; 65 | else if(name.length() > 1 && name[0] == '-' && name[1] != '-') { 66 | if(name.length() == 2 && valid_first_char(name[1])) 67 | short_names.emplace_back(1, name[1]); 68 | else 69 | throw BadNameString("Invalid one char name: " + name); 70 | } else if(name.length() > 2 && name.substr(0, 2) == "--") { 71 | name = name.substr(2); 72 | if(valid_name_string(name)) 73 | long_names.push_back(name); 74 | else 75 | throw BadNameString("Bad long name: " + name); 76 | } else if(name == "-" || name == "--") { 77 | throw BadNameString("Must have a name, not just dashes"); 78 | } else { 79 | if(pos_name.length() > 0) 80 | throw BadNameString("Only one positional name allowed, remove: " + name); 81 | pos_name = name; 82 | } 83 | } 84 | 85 | return std::tuple, std::vector, std::string>( 86 | short_names, long_names, pos_name); 87 | } 88 | 89 | } // namespace detail 90 | } // namespace CLI 91 | -------------------------------------------------------------------------------- /include/CLI/StringTools.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | // Distributed under the 3-Clause BSD License. See accompanying 4 | // file LICENSE or https://github.com/CLIUtils/CLI11 for details. 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | namespace CLI { 14 | namespace detail { 15 | 16 | // Based on http://stackoverflow.com/questions/236129/split-a-string-in-c 17 | /// Split a string by a delim 18 | inline std::vector split(const std::string &s, char delim) { 19 | std::vector elems; 20 | // Check to see if emtpy string, give consistent result 21 | if(s.empty()) 22 | elems.emplace_back(""); 23 | else { 24 | std::stringstream ss; 25 | ss.str(s); 26 | std::string item; 27 | while(std::getline(ss, item, delim)) { 28 | elems.push_back(item); 29 | } 30 | } 31 | return elems; 32 | } 33 | 34 | /// Simple function to join a string 35 | template std::string join(const T &v, std::string delim = ",") { 36 | std::ostringstream s; 37 | size_t start = 0; 38 | for(const auto &i : v) { 39 | if(start++ > 0) 40 | s << delim; 41 | s << i; 42 | } 43 | return s.str(); 44 | } 45 | 46 | /// Join a string in reverse order 47 | template std::string rjoin(const T &v, std::string delim = ",") { 48 | std::ostringstream s; 49 | for(size_t start = 0; start < v.size(); start++) { 50 | if(start > 0) 51 | s << delim; 52 | s << v[v.size() - start - 1]; 53 | } 54 | return s.str(); 55 | } 56 | 57 | // Based roughly on http://stackoverflow.com/questions/25829143/c-trim-whitespace-from-a-string 58 | 59 | /// Trim whitespace from left of string 60 | inline std::string <rim(std::string &str) { 61 | auto it = std::find_if(str.begin(), str.end(), [](char ch) { return !std::isspace(ch, std::locale()); }); 62 | str.erase(str.begin(), it); 63 | return str; 64 | } 65 | 66 | /// Trim anything from left of string 67 | inline std::string <rim(std::string &str, const std::string &filter) { 68 | auto it = std::find_if(str.begin(), str.end(), [&filter](char ch) { return filter.find(ch) == std::string::npos; }); 69 | str.erase(str.begin(), it); 70 | return str; 71 | } 72 | 73 | /// Trim whitespace from right of string 74 | inline std::string &rtrim(std::string &str) { 75 | auto it = std::find_if(str.rbegin(), str.rend(), [](char ch) { return !std::isspace(ch, std::locale()); }); 76 | str.erase(it.base(), str.end()); 77 | return str; 78 | } 79 | 80 | /// Trim anything from right of string 81 | inline std::string &rtrim(std::string &str, const std::string &filter) { 82 | auto it = 83 | std::find_if(str.rbegin(), str.rend(), [&filter](char ch) { return filter.find(ch) == std::string::npos; }); 84 | str.erase(it.base(), str.end()); 85 | return str; 86 | } 87 | 88 | /// Trim whitespace from string 89 | inline std::string &trim(std::string &str) { return ltrim(rtrim(str)); } 90 | 91 | /// Trim anything from string 92 | inline std::string &trim(std::string &str, const std::string filter) { return ltrim(rtrim(str, filter), filter); } 93 | 94 | /// Make a copy of the string and then trim it 95 | inline std::string trim_copy(const std::string &str) { 96 | std::string s = str; 97 | return trim(s); 98 | } 99 | 100 | /// Make a copy of the string and then trim it, any filter string can be used (any char in string is filtered) 101 | inline std::string trim_copy(const std::string &str, const std::string &filter) { 102 | std::string s = str; 103 | return trim(s, filter); 104 | } 105 | /// Print a two part "help" string 106 | inline void format_help(std::stringstream &out, std::string name, std::string description, size_t wid) { 107 | name = " " + name; 108 | out << std::setw(static_cast(wid)) << std::left << name; 109 | if(!description.empty()) { 110 | if(name.length() >= wid) 111 | out << std::endl << std::setw(static_cast(wid)) << ""; 112 | out << description; 113 | } 114 | out << std::endl; 115 | } 116 | 117 | /// Verify the first character of an option 118 | template bool valid_first_char(T c) { return std::isalpha(c, std::locale()) || c == '_'; } 119 | 120 | /// Verify following characters of an option 121 | template bool valid_later_char(T c) { 122 | return std::isalnum(c, std::locale()) || c == '_' || c == '.' || c == '-'; 123 | } 124 | 125 | /// Verify an option name 126 | inline bool valid_name_string(const std::string &str) { 127 | if(str.empty() || !valid_first_char(str[0])) 128 | return false; 129 | for(auto c : str.substr(1)) 130 | if(!valid_later_char(c)) 131 | return false; 132 | return true; 133 | } 134 | 135 | /// Return a lower case version of a string 136 | inline std::string to_lower(std::string str) { 137 | std::transform(std::begin(str), std::end(str), std::begin(str), [](const std::string::value_type &x) { 138 | return std::tolower(x, std::locale()); 139 | }); 140 | return str; 141 | } 142 | 143 | /// Split a string '"one two" "three"' into 'one two', 'three' 144 | inline std::vector split_up(std::string str) { 145 | 146 | std::vector delims = {'\'', '\"'}; 147 | auto find_ws = [](char ch) { return std::isspace(ch, std::locale()); }; 148 | trim(str); 149 | 150 | std::vector output; 151 | 152 | while(!str.empty()) { 153 | if(str[0] == '\'') { 154 | auto end = str.find('\'', 1); 155 | if(end != std::string::npos) { 156 | output.push_back(str.substr(1, end - 1)); 157 | str = str.substr(end + 1); 158 | } else { 159 | output.push_back(str.substr(1)); 160 | str = ""; 161 | } 162 | } else if(str[0] == '\"') { 163 | auto end = str.find('\"', 1); 164 | if(end != std::string::npos) { 165 | output.push_back(str.substr(1, end - 1)); 166 | str = str.substr(end + 1); 167 | } else { 168 | output.push_back(str.substr(1)); 169 | str = ""; 170 | } 171 | 172 | } else { 173 | auto it = std::find_if(std::begin(str), std::end(str), find_ws); 174 | if(it != std::end(str)) { 175 | std::string value = std::string(str.begin(), it); 176 | output.push_back(value); 177 | str = std::string(it, str.end()); 178 | } else { 179 | output.push_back(str); 180 | str = ""; 181 | } 182 | } 183 | trim(str); 184 | } 185 | 186 | return output; 187 | } 188 | 189 | } // namespace detail 190 | } // namespace CLI 191 | -------------------------------------------------------------------------------- /include/CLI/Timer.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | // Distributed under the 3-Clause BSD License. See accompanying 4 | // file LICENSE or https://github.com/CLIUtils/CLI11 for details. 5 | 6 | // On GCC < 4.8, the following define is often missing. Due to the 7 | // fact that this library only uses sleep_for, this should be safe 8 | #if defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 5 && __GNUC_MINOR__ < 8 9 | #define _GLIBCXX_USE_NANOSLEEP 10 | #endif 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | namespace CLI { 19 | 20 | class Timer { 21 | protected: 22 | /// This is a typedef to make clocks easier to use 23 | using clock = std::chrono::steady_clock; 24 | 25 | /// This typedef is for points in time 26 | using time_point = std::chrono::time_point; 27 | 28 | /// This is the type of a printing function, you can make your own 29 | using time_print_t = std::function; 30 | 31 | /// This is the title of the timer 32 | std::string title_; 33 | 34 | /// This is the function that is used to format most of the timing message 35 | time_print_t time_print_; 36 | 37 | /// This is the starting point (when the timer was created) 38 | time_point start_; 39 | 40 | /// This is the number of times cycles (print divides by this number) 41 | size_t cycles{1}; 42 | 43 | public: 44 | /// Standard print function, this one is set by default 45 | static std::string Simple(std::string title, std::string time) { return title + ": " + time; } 46 | 47 | /// This is a fancy print function with --- headers 48 | static std::string Big(std::string title, std::string time) { 49 | return std::string("-----------------------------------------\n") + "| " + title + " | Time = " + time + "\n" + 50 | "-----------------------------------------"; 51 | } 52 | 53 | public: 54 | /// Standard constructor, can set title and print function 55 | Timer(std::string title = "Timer", time_print_t time_print = Simple) 56 | : title_(std::move(title)), time_print_(std::move(time_print)), start_(clock::now()) {} 57 | 58 | /// Time a function by running it multiple times. Target time is the len to target. 59 | std::string time_it(std::function f, double target_time = 1) { 60 | time_point start = start_; 61 | double total_time; 62 | 63 | start_ = clock::now(); 64 | size_t n = 0; 65 | do { 66 | f(); 67 | std::chrono::duration elapsed = clock::now() - start_; 68 | total_time = elapsed.count(); 69 | } while(n++ < 100 && total_time < target_time); 70 | 71 | std::string out = make_time_str(total_time / n) + " for " + std::to_string(n) + " tries"; 72 | start_ = start; 73 | return out; 74 | } 75 | 76 | /// This formats the numerical value for the time string 77 | std::string make_time_str() const { 78 | time_point stop = clock::now(); 79 | std::chrono::duration elapsed = stop - start_; 80 | double time = elapsed.count() / cycles; 81 | return make_time_str(time); 82 | } 83 | 84 | // LCOV_EXCL_START 85 | std::string make_time_str(double time) const { 86 | auto print_it = [](double x, std::string unit) { 87 | char buffer[50]; 88 | std::snprintf(buffer, 50, "%.5g", x); 89 | return buffer + std::string(" ") + unit; 90 | }; 91 | 92 | if(time < .000001) 93 | return print_it(time * 1000000000, "ns"); 94 | else if(time < .001) 95 | return print_it(time * 1000000, "us"); 96 | else if(time < 1) 97 | return print_it(time * 1000, "ms"); 98 | else 99 | return print_it(time, "s"); 100 | } 101 | // LCOV_EXCL_END 102 | 103 | /// This is the main function, it creates a string 104 | std::string to_string() const { return time_print_(title_, make_time_str()); } 105 | 106 | /// Division sets the number of cycles to divide by (no graphical change) 107 | Timer &operator/(size_t val) { 108 | cycles = val; 109 | return *this; 110 | } 111 | }; 112 | 113 | /// This class prints out the time upon destruction 114 | class AutoTimer : public Timer { 115 | public: 116 | /// Reimplementing the constructor is required in GCC 4.7 117 | AutoTimer(std::string title = "Timer", time_print_t time_print = Simple) : Timer(title, time_print) {} 118 | // GCC 4.7 does not support using inheriting constructors. 119 | 120 | /// This desctructor prints the string 121 | ~AutoTimer() { std::cout << to_string() << std::endl; } 122 | }; 123 | 124 | } // namespace CLI 125 | 126 | /// This prints out the time if shifted into a std::cout like stream. 127 | inline std::ostream &operator<<(std::ostream &in, const CLI::Timer &timer) { return in << timer.to_string(); } 128 | -------------------------------------------------------------------------------- /include/CLI/TypeTools.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | // Distributed under the 3-Clause BSD License. See accompanying 4 | // file LICENSE or https://github.com/CLIUtils/CLI11 for details. 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | namespace CLI { 12 | 13 | // Type tools 14 | 15 | // Copied from C++14 16 | #if __cplusplus < 201402L 17 | template using enable_if_t = typename std::enable_if::type; 18 | #else 19 | // If your compiler supports C++14, you can use that definition instead 20 | using std::enable_if_t; 21 | #endif 22 | 23 | template struct is_vector { static const bool value = false; }; 24 | 25 | template struct is_vector> { static bool const value = true; }; 26 | 27 | template struct is_bool { static const bool value = false; }; 28 | 29 | template <> struct is_bool { static bool const value = true; }; 30 | 31 | namespace detail { 32 | // Based generally on https://rmf.io/cxx11/almost-static-if 33 | /// Simple empty scoped class 34 | enum class enabler {}; 35 | 36 | /// An instance to use in EnableIf 37 | constexpr enabler dummy = {}; 38 | 39 | // Type name print 40 | 41 | /// Was going to be based on 42 | /// http://stackoverflow.com/questions/1055452/c-get-name-of-type-in-template 43 | /// But this is cleaner and works better in this case 44 | 45 | template ::value && std::is_signed::value, detail::enabler> = detail::dummy> 47 | constexpr const char *type_name() { 48 | return "INT"; 49 | } 50 | 51 | template ::value && std::is_unsigned::value, detail::enabler> = detail::dummy> 53 | constexpr const char *type_name() { 54 | return "UINT"; 55 | } 56 | 57 | template ::value, detail::enabler> = detail::dummy> 58 | constexpr const char *type_name() { 59 | return "FLOAT"; 60 | } 61 | 62 | /// This one should not be used, since vector types print the internal type 63 | template ::value, detail::enabler> = detail::dummy> 64 | constexpr const char *type_name() { 65 | return "VECTOR"; 66 | } 67 | 68 | template ::value && !std::is_integral::value && !is_vector::value, 70 | detail::enabler> = detail::dummy> 71 | constexpr const char *type_name() { 72 | return "TEXT"; 73 | } 74 | 75 | // Lexical cast 76 | 77 | /// Integers / enums 78 | template ::value || std::is_enum::value, detail::enabler> = detail::dummy> 80 | bool lexical_cast(std::string input, T &output) { 81 | try { 82 | output = static_cast(std::stoll(input)); 83 | return true; 84 | } catch(const std::invalid_argument &) { 85 | return false; 86 | } catch(const std::out_of_range &) { 87 | return false; 88 | } 89 | } 90 | 91 | /// Floats 92 | template ::value, detail::enabler> = detail::dummy> 93 | bool lexical_cast(std::string input, T &output) { 94 | try { 95 | output = static_cast(std::stold(input)); 96 | return true; 97 | } catch(const std::invalid_argument &) { 98 | return false; 99 | } catch(const std::out_of_range &) { 100 | return false; 101 | } 102 | } 103 | 104 | /// String and similar 105 | template ::value && !std::is_integral::value && !std::is_enum::value, 107 | detail::enabler> = detail::dummy> 108 | bool lexical_cast(std::string input, T &output) { 109 | output = input; 110 | return true; 111 | } 112 | 113 | } // namespace detail 114 | } // namespace CLI 115 | -------------------------------------------------------------------------------- /include/CLI/Validators.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | // Distributed under the 3-Clause BSD License. See accompanying 4 | // file LICENSE or https://github.com/CLIUtils/CLI11 for details. 5 | 6 | #include "CLI/TypeTools.hpp" 7 | #include 8 | #include 9 | #include 10 | 11 | // C standard library 12 | // Only needed for existence checking 13 | // Could be swapped for filesystem in C++17 14 | #include 15 | #include 16 | 17 | namespace CLI { 18 | 19 | /// @defgroup validator_group Validators 20 | /// @brief Some validators that are provided 21 | /// 22 | /// These are simple `bool(std::string)` validators that are useful. 23 | /// @{ 24 | 25 | /// Check for an existing file 26 | inline bool ExistingFile(std::string filename) { 27 | struct stat buffer; 28 | bool exist = stat(filename.c_str(), &buffer) == 0; 29 | bool is_dir = (buffer.st_mode & S_IFDIR) != 0; 30 | if(!exist) { 31 | std::cerr << "File does not exist: " << filename << std::endl; 32 | return false; 33 | } else if(is_dir) { 34 | std::cerr << "File is actually a directory: " << filename << std::endl; 35 | return false; 36 | } else { 37 | return true; 38 | } 39 | } 40 | 41 | /// Check for an existing directory 42 | inline bool ExistingDirectory(std::string filename) { 43 | struct stat buffer; 44 | bool exist = stat(filename.c_str(), &buffer) == 0; 45 | bool is_dir = (buffer.st_mode & S_IFDIR) != 0; 46 | if(!exist) { 47 | std::cerr << "Directory does not exist: " << filename << std::endl; 48 | return false; 49 | } else if(is_dir) { 50 | return true; 51 | } else { 52 | std::cerr << "Directory is actually a file: " << filename << std::endl; 53 | return false; 54 | } 55 | } 56 | 57 | /// Check for a non-existing path 58 | inline bool NonexistentPath(std::string filename) { 59 | struct stat buffer; 60 | bool exist = stat(filename.c_str(), &buffer) == 0; 61 | if(!exist) { 62 | return true; 63 | } else { 64 | std::cerr << "Path exists: " << filename << std::endl; 65 | return false; 66 | } 67 | } 68 | 69 | /// Produce a range validator function 70 | template std::function Range(T min, T max) { 71 | return [min, max](std::string input) { 72 | T val; 73 | detail::lexical_cast(input, val); 74 | return val >= min && val <= max; 75 | }; 76 | } 77 | 78 | /// Range of one value is 0 to value 79 | template std::function Range(T max) { return Range(static_cast(0), max); } 80 | 81 | /// @} 82 | 83 | } // namespace CLI 84 | -------------------------------------------------------------------------------- /include/ProgOpts.h: -------------------------------------------------------------------------------- 1 | /* 2 | * ============================================================================ 3 | * 4 | * Authors: Prashant Pandey 5 | * Rob Johnson 6 | * Rob Patro (rob.patro@cs.stonybrook.edu) 7 | * 8 | * ============================================================================ 9 | */ 10 | 11 | #ifndef __PROG_OPTS__ 12 | #define __PROG_OPTS__ 13 | 14 | #include 15 | #include "spdlog/spdlog.h" 16 | 17 | class CountOpts { 18 | public: 19 | int exact {0}; 20 | int ksize; 21 | int cutoff {1}; 22 | int contains_counts{1}; 23 | int qbits {28}; 24 | bool setqbits{false}; 25 | int numthreads{0}; 26 | std::string output_file; 27 | std::vector filenames; 28 | std::shared_ptr console{nullptr}; 29 | }; 30 | 31 | class QueryOpts { 32 | public: 33 | std::string squeakr_file; 34 | std::string queryfile; 35 | std::string output_file; 36 | std::shared_ptr console{nullptr}; 37 | }; 38 | 39 | class InnerProdOpts { 40 | public: 41 | std::string squeakr_filea; 42 | std::string squeakr_fileb; 43 | std::shared_ptr console{nullptr}; 44 | }; 45 | 46 | class ListOpts { 47 | public: 48 | std::string squeakr_file; 49 | std::string output_file; 50 | std::shared_ptr console{nullptr}; 51 | }; 52 | 53 | class InfoOpts { 54 | public: 55 | std::string squeakr_file; 56 | std::shared_ptr console{nullptr}; 57 | }; 58 | 59 | #endif //__MANTIS_PROG_OPTS__ 60 | -------------------------------------------------------------------------------- /include/SqueakrFS.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (C) 2017 Prahsant Pandey, Michael Bender, 3 | // Rob Johnson, Rob Patro 4 | // 5 | // This file is part of Squeakr. 6 | // 7 | 8 | #ifndef __SQUEAKR_FILESYSTEM_HPP__ 9 | #define __SQUEAKR_FILESYSTEM_HPP__ 10 | 11 | #include 12 | #include 13 | 14 | namespace squeakr { 15 | namespace fs { 16 | std::string GetDir(std::string str); 17 | // Taken from 18 | // http://stackoverflow.com/questions/12774207/fastest-way-to-check-if-a-file-exist-using-standard-c-c11-c 19 | bool FileExists(const char* path); 20 | // Taken from 21 | // http://stackoverflow.com/questions/12774207/fastest-way-to-check-if-a-file-exist-using-standard-c-c11-c 22 | bool DirExists(const char* path); 23 | void MakeDir(const char* path); 24 | // Taken from 25 | // https://stackoverflow.com/questions/19189014/how-do-i-find-files-with-a-specific-extension-in-a-directory-that-is-provided-by 26 | std::vector GetFilesExt(const char *dir, const char *ext); 27 | } 28 | } 29 | 30 | #endif //__SQUEAKR_FILESYSTEM_HPP__ 31 | -------------------------------------------------------------------------------- /include/chunk.h: -------------------------------------------------------------------------------- 1 | /* 2 | * ============================================================================ 3 | * 4 | * Authors: Prashant Pandey 5 | * Rob Johnson 6 | * Rob Patro (rob.patro@cs.stonybrook.edu) 7 | * 8 | * ============================================================================ 9 | */ 10 | 11 | #ifndef _CHUNK_H_ 12 | #define _CHUNK_H_ 13 | 14 | #include 15 | 16 | class chunk { 17 | public: 18 | chunk(); 19 | chunk(char *reads, uint32_t size); 20 | inline char *get_reads(); 21 | inline uint32_t get_size(); 22 | 23 | private: 24 | char *_reads; 25 | uint32_t _size; 26 | }; 27 | 28 | chunk::chunk() 29 | { 30 | _reads = NULL; 31 | _size = 0; 32 | } 33 | 34 | chunk::chunk(char *reads, uint32_t size) 35 | { 36 | _reads = reads; 37 | _size = size; 38 | } 39 | 40 | inline char *chunk::get_reads() 41 | { 42 | return _reads; 43 | } 44 | 45 | inline uint32_t chunk::get_size() 46 | { 47 | return _size; 48 | } 49 | 50 | #endif 51 | -------------------------------------------------------------------------------- /include/gqf/gqf_file.h: -------------------------------------------------------------------------------- 1 | /* 2 | * ============================================================================ 3 | * 4 | * Authors: Prashant Pandey 5 | * Rob Johnson 6 | * 7 | * ============================================================================ 8 | */ 9 | 10 | #ifndef _GQF_FILE_H_ 11 | #define _GQF_FILE_H_ 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | #include "gqf.h" 18 | 19 | #ifdef __cplusplus 20 | extern "C" { 21 | #endif 22 | 23 | /* Initialize a file-backed (i.e. mmapped) CQF at "filename". */ 24 | bool qf_initfile(QF *qf, uint64_t nslots, uint64_t key_bits, uint64_t 25 | value_bits, enum qf_hashmode hash, uint32_t seed, const char* 26 | filename); 27 | 28 | #define QF_USEFILE_READ_ONLY (0x01) 29 | #define QF_USEFILE_READ_WRITE (0x02) 30 | 31 | /* mmap existing cqf in "filename" into "qf". */ 32 | uint64_t qf_usefile(QF* qf, const char* filename, int flag); 33 | 34 | /* Resize the QF to the specified number of slots. Uses mmap to 35 | * initialize the new file, and calls munmap() on the old memory. 36 | * Return value: 37 | * >= 0: number of keys copied during resizing. 38 | * */ 39 | int64_t qf_resize_file(QF *qf, uint64_t nslots); 40 | 41 | bool qf_closefile(QF* qf); 42 | 43 | bool qf_deletefile(QF* qf); 44 | 45 | /* write data structure of to the disk */ 46 | uint64_t qf_serialize(const QF *qf, const char *filename); 47 | 48 | /* read data structure off the disk */ 49 | uint64_t qf_deserialize(QF *qf, const char *filename); 50 | 51 | /* This wraps qfi_next, using madvise(DONTNEED) to reduce our RSS. 52 | Only valid on mmapped QFs, i.e. cqfs from qf_initfile and 53 | qf_usefile. */ 54 | int qfi_next_madvise(QFi *qfi); 55 | 56 | /* Furthermore, you can call this immediately after constructing the 57 | qfi to call madvise(DONTNEED) on the portion of the cqf up to the 58 | first element visited by the qfi. */ 59 | int qfi_initial_madvise(QFi *qfi); 60 | 61 | #ifdef __cplusplus 62 | } 63 | #endif 64 | 65 | #endif // _GQF_FILE_H_ 66 | -------------------------------------------------------------------------------- /include/gqf/gqf_int.h: -------------------------------------------------------------------------------- 1 | /* 2 | * ============================================================================ 3 | * 4 | * Authors: Prashant Pandey 5 | * Rob Johnson 6 | * 7 | * ============================================================================ 8 | */ 9 | 10 | #ifndef _GQF_INT_H_ 11 | #define _GQF_INT_H_ 12 | 13 | #include 14 | #include 15 | 16 | #include "gqf.h" 17 | #include "partitioned_counter.h" 18 | 19 | #ifdef __cplusplus 20 | extern "C" { 21 | #endif 22 | 23 | #define MAGIC_NUMBER 1018874902021329732 24 | 25 | /* Can be 26 | 0 (choose size at run-time), 27 | 8, 16, 32, or 64 (for optimized versions), 28 | or other integer <= 56 (for compile-time-optimized bit-shifting-based versions) 29 | */ 30 | #define QF_BITS_PER_SLOT 0 31 | 32 | /* Must be >= 6. 6 seems fastest. */ 33 | #define QF_BLOCK_OFFSET_BITS (6) 34 | 35 | #define QF_SLOTS_PER_BLOCK (1ULL << QF_BLOCK_OFFSET_BITS) 36 | #define QF_METADATA_WORDS_PER_BLOCK ((QF_SLOTS_PER_BLOCK + 63) / 64) 37 | 38 | typedef struct __attribute__ ((__packed__)) qfblock { 39 | /* Code works with uint16_t, uint32_t, etc, but uint8_t seems just as fast as 40 | * anything else */ 41 | uint8_t offset; 42 | uint64_t occupieds[QF_METADATA_WORDS_PER_BLOCK]; 43 | uint64_t runends[QF_METADATA_WORDS_PER_BLOCK]; 44 | 45 | #if QF_BITS_PER_SLOT == 8 46 | uint8_t slots[QF_SLOTS_PER_BLOCK]; 47 | #elif QF_BITS_PER_SLOT == 16 48 | uint16_t slots[QF_SLOTS_PER_BLOCK]; 49 | #elif QF_BITS_PER_SLOT == 32 50 | uint32_t slots[QF_SLOTS_PER_BLOCK]; 51 | #elif QF_BITS_PER_SLOT == 64 52 | uint64_t slots[QF_SLOTS_PER_BLOCK]; 53 | #elif QF_BITS_PER_SLOT != 0 54 | uint8_t slots[QF_SLOTS_PER_BLOCK * QF_BITS_PER_SLOT / 8]; 55 | #else 56 | uint8_t slots[]; 57 | #endif 58 | } qfblock; 59 | 60 | //struct __attribute__ ((__packed__)) qfblock; 61 | //typedef struct qfblock qfblock; 62 | 63 | typedef struct file_info { 64 | int fd; 65 | char *filepath; 66 | } file_info; 67 | 68 | // The below struct is used to instrument the code. 69 | // It is not used in normal operations of the CQF. 70 | typedef struct { 71 | uint64_t total_time_single; 72 | uint64_t total_time_spinning; 73 | uint64_t locks_taken; 74 | uint64_t locks_acquired_single_attempt; 75 | } wait_time_data; 76 | 77 | typedef struct quotient_filter_runtime_data { 78 | file_info f_info; 79 | uint32_t auto_resize; 80 | int64_t (*container_resize)(QF *qf, uint64_t nslots); 81 | pc_t pc_nelts; 82 | pc_t pc_ndistinct_elts; 83 | pc_t pc_noccupied_slots; 84 | uint64_t num_locks; 85 | volatile int metadata_lock; 86 | volatile int *locks; 87 | wait_time_data *wait_times; 88 | } quotient_filter_runtime_data; 89 | 90 | typedef quotient_filter_runtime_data qfruntime; 91 | 92 | typedef struct quotient_filter_metadata { 93 | uint64_t magic_endian_number; 94 | enum qf_hashmode hash_mode; 95 | uint32_t reserved; 96 | uint64_t total_size_in_bytes; 97 | uint32_t seed; 98 | uint64_t nslots; 99 | uint64_t xnslots; 100 | uint64_t key_bits; 101 | uint64_t value_bits; 102 | uint64_t key_remainder_bits; 103 | uint64_t bits_per_slot; 104 | __uint128_t range; 105 | uint64_t nblocks; 106 | uint64_t nelts; 107 | uint64_t ndistinct_elts; 108 | uint64_t noccupied_slots; 109 | } quotient_filter_metadata; 110 | 111 | typedef quotient_filter_metadata qfmetadata; 112 | 113 | typedef struct quotient_filter { 114 | qfruntime *runtimedata; 115 | qfmetadata *metadata; 116 | qfblock *blocks; 117 | } quotient_filter; 118 | 119 | typedef quotient_filter QF; 120 | 121 | #if QF_BITS_PER_SLOT > 0 122 | static inline qfblock * get_block(const QF *qf, uint64_t block_index) 123 | { 124 | return &qf->blocks[block_index]; 125 | } 126 | #else 127 | static inline qfblock * get_block(const QF *qf, uint64_t block_index) 128 | { 129 | return (qfblock *)(((char *)qf->blocks) 130 | + block_index * (sizeof(qfblock) + QF_SLOTS_PER_BLOCK * 131 | qf->metadata->bits_per_slot / 8)); 132 | } 133 | #endif 134 | 135 | // The below struct is used to instrument the code. 136 | // It is not used in normal operations of the CQF. 137 | typedef struct { 138 | uint64_t start_index; 139 | uint16_t length; 140 | } cluster_data; 141 | 142 | typedef struct quotient_filter_iterator { 143 | const QF *qf; 144 | uint64_t run; 145 | uint64_t current; 146 | uint64_t cur_start_index; 147 | uint16_t cur_length; 148 | uint32_t num_clusters; 149 | cluster_data *c_info; 150 | } quotient_filter_iterator; 151 | 152 | #ifdef __cplusplus 153 | } 154 | #endif 155 | 156 | #endif /* _GQF_INT_H_ */ 157 | -------------------------------------------------------------------------------- /include/gqf/hashutil.h: -------------------------------------------------------------------------------- 1 | /* 2 | * ============================================================================ 3 | * 4 | * Authors: Prashant Pandey 5 | * Rob Johnson 6 | * 7 | * ============================================================================ 8 | */ 9 | 10 | #ifndef _HASHUTIL_H_ 11 | #define _HASHUTIL_H_ 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | uint64_t MurmurHash64B ( const void * key, int len, unsigned int seed ); 18 | uint64_t MurmurHash64A ( const void * key, int len, unsigned int seed ); 19 | 20 | uint64_t hash_64(uint64_t key, uint64_t mask); 21 | uint64_t hash_64i(uint64_t key, uint64_t mask); 22 | 23 | #endif // #ifndef _HASHUTIL_H_ 24 | 25 | 26 | -------------------------------------------------------------------------------- /include/gqf/partitioned_counter.h: -------------------------------------------------------------------------------- 1 | /* 2 | * ============================================================================ 3 | * 4 | * Author: Prashant Pandey (), ppandey@cs.stonybrook.edu 5 | * Organization: Stony Brook University 6 | * 7 | * ============================================================================ 8 | */ 9 | 10 | #ifndef _PARTITIONED_COUNTER_H_ 11 | #define _PARTITIONED_COUNTER_H_ 12 | 13 | #include 14 | #include 15 | 16 | #ifdef __cplusplus 17 | extern "C" { 18 | #endif 19 | 20 | typedef struct local_counter { 21 | int64_t counter; 22 | int64_t padding[7]; 23 | } local_counter; 24 | 25 | typedef struct local_counter lctr_t; 26 | 27 | typedef struct partitioned_counter { 28 | lctr_t *local_counters; 29 | int64_t *global_counter; 30 | uint32_t num_counters; 31 | int32_t threshold; 32 | } partitioned_counter; 33 | 34 | typedef struct partitioned_counter pc_t; 35 | 36 | #define PC_ERROR -1 37 | 38 | /* on success returns 0. 39 | * If allocation fails returns PC_ERROR 40 | */ 41 | int pc_init(pc_t *pc, int64_t *global_counter, uint32_t num_counters, 42 | int32_t threshold); 43 | 44 | void pc_destructor(pc_t *pc); 45 | 46 | void pc_add(pc_t *pc, int64_t count); 47 | 48 | void pc_sync(pc_t *pc); 49 | 50 | #ifdef __cplusplus 51 | } 52 | #endif 53 | 54 | #endif /* _PARTITIONED_COUNTER_H_ */ 55 | -------------------------------------------------------------------------------- /include/gqf_cpp.h: -------------------------------------------------------------------------------- 1 | /* 2 | * ============================================================================ 3 | * 4 | * Authors: Prashant Pandey 5 | * Rob Johnson 6 | * Rob Patro (rob.patro@cs.stonybrook.edu) 7 | * 8 | * ============================================================================ 9 | */ 10 | 11 | #ifndef _CQF_H_ 12 | #define _CQF_H_ 13 | 14 | #include 15 | #include 16 | #include 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | #include "gqf/gqf.h" 24 | #include "gqf/gqf_int.h" 25 | #include "gqf/gqf_file.h" 26 | #include "util.h" 27 | 28 | #define NUM_HASH_BITS 24 29 | #define NUM_Q_BITS 16 30 | #define SEED 2038074761 31 | 32 | enum readmode { 33 | MMAP, 34 | FREAD 35 | }; 36 | 37 | template 38 | class CQF { 39 | public: 40 | CQF(); 41 | CQF(uint64_t q_bits, uint64_t key_bits, enum qf_hashmode hash, uint32_t seed); 42 | CQF(std::string& filename, enum readmode flag); 43 | CQF(const CQF& copy_cqf); 44 | 45 | int insert(const key_obj& k, uint8_t flags); 46 | 47 | /* Will return the count. */ 48 | uint64_t query(const key_obj& k, uint8_t flags); 49 | 50 | uint64_t inner_prod(const CQF& in_cqf); 51 | 52 | void serialize(std::string filename) { 53 | qf_serialize(&cqf, filename.c_str()); 54 | } 55 | 56 | void set_auto_resize(void) { qf_set_auto_resize(&cqf, true); } 57 | int64_t get_unique_index(const key_obj& k, uint8_t flags) const { 58 | return qf_get_unique_index(&cqf, k.key, k.value, flags); 59 | } 60 | 61 | bool is_exact(void); 62 | 63 | const QF* get_cqf(void) const { return &cqf; } 64 | __uint128_t range(void) const { return cqf.metadata->range; } 65 | uint32_t seed(void) const { return cqf.metadata->seed; } 66 | uint64_t numslots(void) const { return cqf.metadata->nslots; } 67 | uint32_t keybits(void) const { return cqf.metadata->key_bits; } 68 | uint64_t total_elts(void) const { return cqf.metadata->nelts; } 69 | uint64_t dist_elts(void) const { return cqf.metadata->ndistinct_elts; } 70 | //uint64_t set_size(void) const { return set.size(); } 71 | void reset(void) { qf_reset(&cqf); } 72 | 73 | void dump_metadata(void) const { qf_dump_metadata(&cqf); } 74 | 75 | void drop_pages(uint64_t cur); 76 | 77 | class Iterator { 78 | public: 79 | Iterator(QFi it); 80 | key_obj operator*(void) const; 81 | void operator++(void); 82 | bool done(void) const; 83 | 84 | key_obj get_cur_hash(void) const; 85 | 86 | QFi iter; 87 | private: 88 | uint64_t end_hash; 89 | }; 90 | 91 | Iterator begin(void) const; 92 | Iterator end(void) const; 93 | 94 | private: 95 | QF cqf; 96 | //std::unordered_set set; 97 | }; 98 | 99 | class KeyObject { 100 | public: 101 | KeyObject() : key(0), value(0), count(0) {}; 102 | 103 | KeyObject(uint64_t k, uint64_t v, uint64_t c) : key(k), 104 | value(v), count(c) {}; 105 | 106 | KeyObject(const KeyObject& k) : key(k.key), value(k.value), count(k.count) {}; 107 | 108 | bool operator==(KeyObject k) { return key == k.key; } 109 | 110 | uint64_t key; 111 | uint64_t value; 112 | uint64_t count; 113 | }; 114 | 115 | template 116 | CQF::CQF() { 117 | if (!qf_malloc(&cqf, 1ULL << NUM_Q_BITS, NUM_HASH_BITS, 0, QF_HASH_DEFAULT, 118 | SEED)) { 119 | ERROR("Can't allocate the CQF"); 120 | exit(EXIT_FAILURE); 121 | } 122 | } 123 | 124 | template 125 | CQF::CQF(uint64_t q_bits, uint64_t key_bits, enum qf_hashmode hash, 126 | uint32_t seed) { 127 | if (!qf_malloc(&cqf, 1ULL << q_bits, key_bits, 0, hash, SEED)) { 128 | ERROR("Can't allocate the CQF"); 129 | exit(EXIT_FAILURE); 130 | } 131 | } 132 | 133 | template 134 | CQF::CQF(std::string& filename, enum readmode flag) { 135 | uint64_t size = 0; 136 | if (flag == MMAP) 137 | size = qf_usefile(&cqf, filename.c_str(), PROT_READ); 138 | else 139 | size = qf_deserialize(&cqf, filename.c_str()); 140 | 141 | if (size == 0) { 142 | ERROR("Can't read/deserialize the CQF"); 143 | exit(EXIT_FAILURE); 144 | } 145 | } 146 | 147 | template CQF::CQF(const CQF& copy_cqf) { 148 | memcpy(reinterpret_cast(&cqf), 149 | reinterpret_cast(const_cast(©_cqf.cqf)), sizeof(QF)); 150 | } 151 | 152 | template 153 | int CQF::insert(const key_obj& k, uint8_t flags) { 154 | return qf_insert(&cqf, k.key, k.value, k.count, flags); 155 | // To validate the CQF 156 | //set.insert(k.key); 157 | } 158 | 159 | template 160 | uint64_t CQF::query(const key_obj& k, uint8_t flags) { 161 | return qf_count_key_value(&cqf, k.key, k.value, flags); 162 | } 163 | 164 | template 165 | uint64_t CQF::inner_prod(const CQF& in_cqf) { 166 | return qf_inner_product(&cqf, in_cqf.get_cqf()); 167 | } 168 | 169 | template 170 | bool CQF::is_exact(void) { 171 | if (cqf.metadata->hash_mode == QF_HASH_INVERTIBLE) 172 | return true; 173 | return false; 174 | } 175 | 176 | template 177 | CQF::Iterator::Iterator(QFi it) 178 | : iter(it) {}; 179 | 180 | template 181 | key_obj CQF::Iterator::operator*(void) const { 182 | uint64_t key = 0, value = 0, count = 0; 183 | qfi_get_key(&iter, &key, &value, &count); 184 | return key_obj(key, value, count); 185 | } 186 | 187 | template 188 | key_obj CQF::Iterator::get_cur_hash(void) const { 189 | uint64_t key = 0, value = 0, count = 0; 190 | qfi_get_hash(&iter, &key, &value, &count); 191 | return key_obj(key, value, count); 192 | } 193 | 194 | template 195 | void CQF::Iterator::operator++(void) { 196 | qfi_next(&iter); 197 | } 198 | 199 | /* Currently, the iterator only traverses forward. So, we only need to check 200 | * the right side limit. 201 | */ 202 | template 203 | bool CQF::Iterator::done(void) const { 204 | return qfi_end(&iter); 205 | } 206 | 207 | template 208 | typename CQF::Iterator CQF::begin(void) const { 209 | QFi qfi; 210 | qf_iterator_from_position(&this->cqf, &qfi, 0); 211 | return Iterator(qfi); 212 | } 213 | 214 | template 215 | typename CQF::Iterator CQF::end(void) const { 216 | QFi qfi; 217 | qf_iterator_from_position(&this->cqf, &qfi, 0xffffffffffffffff); 218 | return Iterator(qfi, UINT64_MAX); 219 | } 220 | 221 | #endif 222 | -------------------------------------------------------------------------------- /include/kmer.h: -------------------------------------------------------------------------------- 1 | /* 2 | * ============================================================================ 3 | * 4 | * Authors: Prashant Pandey 5 | * Rob Johnson 6 | * Rob Patro (rob.patro@cs.stonybrook.edu) 7 | * 8 | * ============================================================================ 9 | */ 10 | 11 | #ifndef _KMER_H_ 12 | #define _KMER_H_ 13 | 14 | #include 15 | #include 16 | #include 17 | 18 | #define BITMASK(nbits) ((nbits) == 64 ? 0xffffffffffffffff : (1ULL << (nbits)) \ 19 | - 1ULL) 20 | enum DNA_MAP {C, A, T, G}; // A=1, C=0, T=2, G=3 21 | 22 | class Kmer { 23 | public: 24 | static char map_int(uint8_t base); 25 | static uint8_t map_base(char base); 26 | static __int128_t str_to_int(std::string str); 27 | static std::string int_to_str(__int128_t kmer, uint64_t kmer_size); 28 | static int reverse_complement_base(int x); 29 | static __int128_t reverse_complement(__int128_t kmer, uint64_t kmer_size); 30 | static bool compare_kmers(__int128_t kmer, __int128_t kmer_rev); 31 | 32 | static void parse_kmers(const char *filename, uint64_t ksize, 33 | std::unordered_set& kmerset); 34 | 35 | private: 36 | Kmer(); 37 | }; 38 | 39 | #endif 40 | -------------------------------------------------------------------------------- /include/reader.h: -------------------------------------------------------------------------------- 1 | /* 2 | * ============================================================================ 3 | * 4 | * Authors: Prashant Pandey 5 | * Rob Johnson 6 | * Rob Patro (rob.patro@cs.stonybrook.edu) 7 | * 8 | * ============================================================================ 9 | */ 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | #include 16 | #include 17 | 18 | #ifndef _READER_H_ 19 | #define _READER_H_ 20 | 21 | struct file_pointer; 22 | 23 | class reader { 24 | public: 25 | reader(); 26 | reader(FILE *in, gzFile in_gzip, BZFILE *in_bzip2, int bzerror); 27 | 28 | bool is_eof(int mode); 29 | 30 | static bool skip_next_eol(char *part, int64_t &pos, int64_t max_pos); 31 | static bool fastq_read_parts(int mode, file_pointer *fp); 32 | static bool getFileReader(int mode, const char* fastq_file, reader* 33 | file_reader); 34 | 35 | FILE *in = nullptr; 36 | gzFile in_gzip = nullptr; 37 | BZFILE *in_bzip2 = nullptr; 38 | int bzerror; 39 | }; 40 | 41 | struct file_pointer { 42 | std::unique_ptr freader{nullptr}; 43 | char* part{nullptr}; 44 | char* part_buffer{nullptr}; 45 | int mode{0}; 46 | uint64_t size{0}; 47 | uint64_t part_filled{0}; 48 | }; 49 | 50 | reader::reader() 51 | { 52 | in = nullptr; 53 | in_gzip = nullptr; 54 | in_bzip2 = nullptr; 55 | bzerror = 0; 56 | } 57 | 58 | reader::reader(FILE *_in, gzFile _in_gzip, BZFILE *_in_bzip2, int _bzerror) 59 | { 60 | in = _in; 61 | in_gzip = _in_gzip; 62 | in_bzip2 = _in_bzip2; 63 | bzerror = _bzerror; 64 | } 65 | 66 | /* check if it's the end of the file. */ 67 | bool reader::is_eof(int mode) { 68 | if (mode == 0) 69 | return feof(in) != 0; 70 | else if (mode == 1) 71 | return gzeof(in_gzip) != 0; 72 | else if (mode == 2) 73 | return bzerror == BZ_STREAM_END; 74 | 75 | return true; 76 | } 77 | 78 | /* move the pointer to the end of the next newline. */ 79 | bool reader::skip_next_eol(char *part, int64_t &pos, int64_t max_pos) { 80 | int64_t i; 81 | for(i = pos; i < max_pos-2; ++i) 82 | if((part[i] == '\n' || part[i] == '\r') && !(part[i+1] == '\n' || 83 | part[i+1] == '\r')) 84 | break; 85 | 86 | if(i >= max_pos-2) 87 | return false; 88 | pos = i+1; 89 | 90 | return true; 91 | } 92 | 93 | /* read a part of the fastq file. */ 94 | bool reader::fastq_read_parts(int mode, file_pointer *fp) { 95 | char *& _part = (fp->part); 96 | uint64_t& _size = fp->size; 97 | char*& part_buffer = (fp->part_buffer); 98 | uint64_t& part_filled = fp->part_filled; 99 | reader& file_reader = *(fp->freader.get()); 100 | 101 | uint32_t OVERHEAD_SIZE = 65535; 102 | uint64_t part_size = 1ULL << 25; 103 | char *part = (char *)malloc((part_size + OVERHEAD_SIZE)*sizeof(char)); 104 | memcpy(part, part_buffer, part_filled); 105 | 106 | if(file_reader.is_eof(mode)) 107 | return false; 108 | 109 | uint64_t readed = 0; 110 | 111 | if (mode == 0) 112 | readed = fread(part+part_filled, 1, part_size, file_reader.in); 113 | else if (mode == 1) 114 | readed = gzread(file_reader.in_gzip, part+part_filled, (int) part_size); 115 | else if (mode == 2) 116 | readed = BZ2_bzRead(&file_reader.bzerror, file_reader.in_bzip2, 117 | part+part_filled, (int) part_size); 118 | else 119 | readed = 0; 120 | 121 | int64_t total_filled = part_filled + readed; 122 | int64_t i; 123 | if(part_filled >= OVERHEAD_SIZE) 124 | { 125 | std::cout << "Error: Wrong input file!" << std::endl; 126 | exit(EXIT_FAILURE); 127 | } 128 | if(file_reader.is_eof(mode)) 129 | { 130 | _part = part; 131 | _size = total_filled; 132 | part = NULL; 133 | return true; 134 | } 135 | // Looking for a FASTQ record at the end of the area 136 | { 137 | int64_t line_start[9]; 138 | int32_t j; 139 | i = total_filled - OVERHEAD_SIZE / 2; 140 | for(j = 0; j < 9; ++j) 141 | { 142 | if(!skip_next_eol(part, i, total_filled)) 143 | break; 144 | line_start[j] = i; 145 | } 146 | _part = part; 147 | if(j < 9) 148 | _size = 0; 149 | else 150 | { 151 | int k; 152 | for(k = 0; k < 4; ++k) 153 | { 154 | if(part[line_start[k]+0] == '@' && part[line_start[k+2]+0] == '+') 155 | { 156 | if(part[line_start[k+2]+1] == '\n' || part[line_start[k+2]+1] == '\r') 157 | break; 158 | if(line_start[k+1]-line_start[k] == line_start[k+3]-line_start[k+2] && 159 | memcmp(part+line_start[k]+1, part+line_start[k+2]+1, 160 | line_start[k+3]-line_start[k+2]-1) == 0) 161 | break; 162 | } 163 | } 164 | if(k == 4) 165 | _size = 0; 166 | else 167 | _size = line_start[k]; 168 | } 169 | } 170 | 171 | std::copy(_part+_size, _part+total_filled, part_buffer); 172 | part_filled = total_filled - _size; 173 | 174 | return true; 175 | } 176 | 177 | bool reader::getFileReader(int mode, const char* fastq_file, reader* 178 | file_reader) { 179 | uint64_t gzip_buffer_size = 1ULL << 26; 180 | uint64_t bzip2_buffer_size = 1ULL << 26; 181 | 182 | if (mode == 0) { 183 | if ((file_reader->in = fopen(fastq_file, "rb")) == NULL) 184 | return false; 185 | } else if (mode == 1) { 186 | if ((file_reader->in_gzip = gzopen(fastq_file, "rb")) == NULL) 187 | return false; 188 | gzbuffer(file_reader->in_gzip, gzip_buffer_size); 189 | } else if (mode == 2) { 190 | file_reader->in = fopen(fastq_file, "rb"); 191 | if (!file_reader->in) 192 | return false; 193 | setvbuf(file_reader->in, NULL, _IOFBF, bzip2_buffer_size); 194 | if ((file_reader->in_bzip2 = BZ2_bzReadOpen(&file_reader->bzerror, 195 | file_reader->in, 0, 0, NULL, 196 | 0)) == NULL) { 197 | fclose(file_reader->in); 198 | return false; 199 | } 200 | } 201 | return true; 202 | } 203 | 204 | 205 | #endif 206 | -------------------------------------------------------------------------------- /include/spdlog/async_logger.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2015 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | // Very fast asynchronous logger (millions of logs per second on an average desktop) 9 | // Uses pre allocated lockfree queue for maximum throughput even under large number of threads. 10 | // Creates a single back thread to pop messages from the queue and log them. 11 | // 12 | // Upon each log write the logger: 13 | // 1. Checks if its log level is enough to log the message 14 | // 2. Push a new copy of the message to a queue (or block the caller until space is available in the queue) 15 | // 3. will throw spdlog_ex upon log exceptions 16 | // Upon destruction, logs all remaining messages in the queue before destructing.. 17 | 18 | #include "spdlog/common.h" 19 | #include "spdlog/logger.h" 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | namespace spdlog 27 | { 28 | 29 | namespace details 30 | { 31 | class async_log_helper; 32 | } 33 | 34 | class async_logger SPDLOG_FINAL :public logger 35 | { 36 | public: 37 | template 38 | async_logger(const std::string& name, 39 | const It& begin, 40 | const It& end, 41 | size_t queue_size, 42 | const async_overflow_policy overflow_policy = async_overflow_policy::block_retry, 43 | const std::function& worker_warmup_cb = nullptr, 44 | const std::chrono::milliseconds& flush_interval_ms = std::chrono::milliseconds::zero(), 45 | const std::function& worker_teardown_cb = nullptr); 46 | 47 | async_logger(const std::string& logger_name, 48 | sinks_init_list sinks, 49 | size_t queue_size, 50 | const async_overflow_policy overflow_policy = async_overflow_policy::block_retry, 51 | const std::function& worker_warmup_cb = nullptr, 52 | const std::chrono::milliseconds& flush_interval_ms = std::chrono::milliseconds::zero(), 53 | const std::function& worker_teardown_cb = nullptr); 54 | 55 | async_logger(const std::string& logger_name, 56 | sink_ptr single_sink, 57 | size_t queue_size, 58 | const async_overflow_policy overflow_policy = async_overflow_policy::block_retry, 59 | const std::function& worker_warmup_cb = nullptr, 60 | const std::chrono::milliseconds& flush_interval_ms = std::chrono::milliseconds::zero(), 61 | const std::function& worker_teardown_cb = nullptr); 62 | 63 | //Wait for the queue to be empty, and flush synchronously 64 | //Warning: this can potentially last forever as we wait it to complete 65 | void flush() override; 66 | 67 | // Error handler 68 | virtual void set_error_handler(log_err_handler) override; 69 | virtual log_err_handler error_handler() override; 70 | 71 | protected: 72 | void _sink_it(details::log_msg& msg) override; 73 | void _set_formatter(spdlog::formatter_ptr msg_formatter) override; 74 | void _set_pattern(const std::string& pattern, pattern_time_type pattern_time) override; 75 | 76 | private: 77 | std::unique_ptr _async_log_helper; 78 | }; 79 | } 80 | 81 | 82 | #include "spdlog/details/async_logger_impl.h" 83 | -------------------------------------------------------------------------------- /include/spdlog/common.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2015 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #if defined(_WIN32) && defined(SPDLOG_WCHAR_FILENAMES) 17 | #include 18 | #include 19 | #endif 20 | 21 | #include "spdlog/details/null_mutex.h" 22 | 23 | //visual studio upto 2013 does not support noexcept nor constexpr 24 | #if defined(_MSC_VER) && (_MSC_VER < 1900) 25 | #define SPDLOG_NOEXCEPT throw() 26 | #define SPDLOG_CONSTEXPR 27 | #else 28 | #define SPDLOG_NOEXCEPT noexcept 29 | #define SPDLOG_CONSTEXPR constexpr 30 | #endif 31 | 32 | // See tweakme.h 33 | #if !defined(SPDLOG_FINAL) 34 | #define SPDLOG_FINAL 35 | #endif 36 | 37 | #if defined(__GNUC__) || defined(__clang__) 38 | #define SPDLOG_DEPRECATED __attribute__((deprecated)) 39 | #elif defined(_MSC_VER) 40 | #define SPDLOG_DEPRECATED __declspec(deprecated) 41 | #else 42 | #define SPDLOG_DEPRECATED 43 | #endif 44 | 45 | 46 | #include "spdlog/fmt/fmt.h" 47 | 48 | namespace spdlog 49 | { 50 | 51 | class formatter; 52 | 53 | namespace sinks 54 | { 55 | class sink; 56 | } 57 | 58 | using log_clock = std::chrono::system_clock; 59 | using sink_ptr = std::shared_ptr < sinks::sink >; 60 | using sinks_init_list = std::initializer_list < sink_ptr >; 61 | using formatter_ptr = std::shared_ptr; 62 | #if defined(SPDLOG_NO_ATOMIC_LEVELS) 63 | using level_t = details::null_atomic_int; 64 | #else 65 | using level_t = std::atomic; 66 | #endif 67 | 68 | using log_err_handler = std::function; 69 | 70 | //Log level enum 71 | namespace level 72 | { 73 | typedef enum 74 | { 75 | trace = 0, 76 | debug = 1, 77 | info = 2, 78 | warn = 3, 79 | err = 4, 80 | critical = 5, 81 | off = 6 82 | } level_enum; 83 | 84 | #if !defined(SPDLOG_LEVEL_NAMES) 85 | #define SPDLOG_LEVEL_NAMES { "trace", "debug", "info", "warning", "error", "critical", "off" }; 86 | #endif 87 | static const char* level_names[] SPDLOG_LEVEL_NAMES 88 | 89 | static const char* short_level_names[] { "T", "D", "I", "W", "E", "C", "O" }; 90 | 91 | inline const char* to_str(spdlog::level::level_enum l) 92 | { 93 | return level_names[l]; 94 | } 95 | 96 | inline const char* to_short_str(spdlog::level::level_enum l) 97 | { 98 | return short_level_names[l]; 99 | } 100 | } //level 101 | 102 | 103 | // 104 | // Async overflow policy - block by default. 105 | // 106 | enum class async_overflow_policy 107 | { 108 | block_retry, // Block / yield / sleep until message can be enqueued 109 | discard_log_msg // Discard the message it enqueue fails 110 | }; 111 | 112 | // 113 | // Pattern time - specific time getting to use for pattern_formatter. 114 | // local time by default 115 | // 116 | enum class pattern_time_type 117 | { 118 | local, // log localtime 119 | utc // log utc 120 | }; 121 | 122 | // 123 | // Log exception 124 | // 125 | namespace details 126 | { 127 | namespace os 128 | { 129 | std::string errno_str(int err_num); 130 | } 131 | } 132 | class spdlog_ex: public std::exception 133 | { 134 | public: 135 | spdlog_ex(const std::string& msg):_msg(msg) 136 | {} 137 | spdlog_ex(const std::string& msg, int last_errno) 138 | { 139 | _msg = msg + ": " + details::os::errno_str(last_errno); 140 | } 141 | const char* what() const SPDLOG_NOEXCEPT override 142 | { 143 | return _msg.c_str(); 144 | } 145 | private: 146 | std::string _msg; 147 | 148 | }; 149 | 150 | // 151 | // wchar support for windows file names (SPDLOG_WCHAR_FILENAMES must be defined) 152 | // 153 | #if defined(_WIN32) && defined(SPDLOG_WCHAR_FILENAMES) 154 | using filename_t = std::wstring; 155 | #else 156 | using filename_t = std::string; 157 | #endif 158 | 159 | 160 | } //spdlog 161 | -------------------------------------------------------------------------------- /include/spdlog/details/async_logger_impl.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2015 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | // Async Logger implementation 9 | // Use an async_sink (queue per logger) to perform the logging in a worker thread 10 | 11 | #include "spdlog/details/async_log_helper.h" 12 | #include "spdlog/async_logger.h" 13 | 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | template 20 | inline spdlog::async_logger::async_logger(const std::string& logger_name, 21 | const It& begin, 22 | const It& end, 23 | size_t queue_size, 24 | const async_overflow_policy overflow_policy, 25 | const std::function& worker_warmup_cb, 26 | const std::chrono::milliseconds& flush_interval_ms, 27 | const std::function& worker_teardown_cb) : 28 | logger(logger_name, begin, end), 29 | _async_log_helper(new details::async_log_helper(_formatter, _sinks, queue_size, _err_handler, overflow_policy, worker_warmup_cb, flush_interval_ms, worker_teardown_cb)) 30 | { 31 | } 32 | 33 | inline spdlog::async_logger::async_logger(const std::string& logger_name, 34 | sinks_init_list sinks_list, 35 | size_t queue_size, 36 | const async_overflow_policy overflow_policy, 37 | const std::function& worker_warmup_cb, 38 | const std::chrono::milliseconds& flush_interval_ms, 39 | const std::function& worker_teardown_cb) : 40 | async_logger(logger_name, sinks_list.begin(), sinks_list.end(), queue_size, overflow_policy, worker_warmup_cb, flush_interval_ms, worker_teardown_cb) {} 41 | 42 | inline spdlog::async_logger::async_logger(const std::string& logger_name, 43 | sink_ptr single_sink, 44 | size_t queue_size, 45 | const async_overflow_policy overflow_policy, 46 | const std::function& worker_warmup_cb, 47 | const std::chrono::milliseconds& flush_interval_ms, 48 | const std::function& worker_teardown_cb) : 49 | async_logger(logger_name, 50 | { 51 | single_sink 52 | }, queue_size, overflow_policy, worker_warmup_cb, flush_interval_ms, worker_teardown_cb) {} 53 | 54 | 55 | inline void spdlog::async_logger::flush() 56 | { 57 | _async_log_helper->flush(true); 58 | } 59 | 60 | // Error handler 61 | inline void spdlog::async_logger::set_error_handler(spdlog::log_err_handler err_handler) 62 | { 63 | _err_handler = err_handler; 64 | _async_log_helper->set_error_handler(err_handler); 65 | 66 | } 67 | inline spdlog::log_err_handler spdlog::async_logger::error_handler() 68 | { 69 | return _err_handler; 70 | } 71 | 72 | 73 | inline void spdlog::async_logger::_set_formatter(spdlog::formatter_ptr msg_formatter) 74 | { 75 | _formatter = msg_formatter; 76 | _async_log_helper->set_formatter(_formatter); 77 | } 78 | 79 | inline void spdlog::async_logger::_set_pattern(const std::string& pattern, pattern_time_type pattern_time) 80 | { 81 | _formatter = std::make_shared(pattern, pattern_time); 82 | _async_log_helper->set_formatter(_formatter); 83 | } 84 | 85 | 86 | inline void spdlog::async_logger::_sink_it(details::log_msg& msg) 87 | { 88 | try 89 | { 90 | #if defined(SPDLOG_ENABLE_MESSAGE_COUNTER) 91 | msg.msg_id = _msg_counter.fetch_add(1, std::memory_order_relaxed); 92 | #endif 93 | _async_log_helper->log(msg); 94 | if (_should_flush_on(msg)) 95 | _async_log_helper->flush(false); // do async flush 96 | } 97 | catch (const std::exception &ex) 98 | { 99 | _err_handler(ex.what()); 100 | } 101 | catch (...) 102 | { 103 | _err_handler("Unknown exception"); 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /include/spdlog/details/file_helper.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2015 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | // Helper class for file sink 9 | // When failing to open a file, retry several times(5) with small delay between the tries(10 ms) 10 | // Throw spdlog_ex exception on errors 11 | 12 | #include "spdlog/details/os.h" 13 | #include "spdlog/details/log_msg.h" 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | namespace spdlog 22 | { 23 | namespace details 24 | { 25 | 26 | class file_helper 27 | { 28 | 29 | public: 30 | const int open_tries = 5; 31 | const int open_interval = 10; 32 | 33 | explicit file_helper() : 34 | _fd(nullptr) 35 | {} 36 | 37 | file_helper(const file_helper&) = delete; 38 | file_helper& operator=(const file_helper&) = delete; 39 | 40 | ~file_helper() 41 | { 42 | close(); 43 | } 44 | 45 | 46 | void open(const filename_t& fname, bool truncate = false) 47 | { 48 | 49 | close(); 50 | auto *mode = truncate ? SPDLOG_FILENAME_T("wb") : SPDLOG_FILENAME_T("ab"); 51 | _filename = fname; 52 | for (int tries = 0; tries < open_tries; ++tries) 53 | { 54 | if (!os::fopen_s(&_fd, fname, mode)) 55 | return; 56 | 57 | std::this_thread::sleep_for(std::chrono::milliseconds(open_interval)); 58 | } 59 | 60 | throw spdlog_ex("Failed opening file " + os::filename_to_str(_filename) + " for writing", errno); 61 | } 62 | 63 | void reopen(bool truncate) 64 | { 65 | if (_filename.empty()) 66 | throw spdlog_ex("Failed re opening file - was not opened before"); 67 | open(_filename, truncate); 68 | 69 | } 70 | 71 | void flush() 72 | { 73 | std::fflush(_fd); 74 | } 75 | 76 | void close() 77 | { 78 | if (_fd) 79 | { 80 | std::fclose(_fd); 81 | _fd = nullptr; 82 | } 83 | } 84 | 85 | void write(const log_msg& msg) 86 | { 87 | 88 | size_t msg_size = msg.formatted.size(); 89 | auto data = msg.formatted.data(); 90 | if (std::fwrite(data, 1, msg_size, _fd) != msg_size) 91 | throw spdlog_ex("Failed writing to file " + os::filename_to_str(_filename), errno); 92 | } 93 | 94 | size_t size() 95 | { 96 | if (!_fd) 97 | throw spdlog_ex("Cannot use size() on closed file " + os::filename_to_str(_filename)); 98 | return os::filesize(_fd); 99 | } 100 | 101 | const filename_t& filename() const 102 | { 103 | return _filename; 104 | } 105 | 106 | static bool file_exists(const filename_t& name) 107 | { 108 | 109 | return os::file_exists(name); 110 | } 111 | 112 | private: 113 | FILE* _fd; 114 | filename_t _filename; 115 | }; 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /include/spdlog/details/log_msg.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2015 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | #include "spdlog/common.h" 9 | #include "spdlog/details/os.h" 10 | 11 | 12 | #include 13 | #include 14 | 15 | namespace spdlog 16 | { 17 | namespace details 18 | { 19 | struct log_msg 20 | { 21 | log_msg() = default; 22 | log_msg(const std::string *loggers_name, level::level_enum lvl) : 23 | logger_name(loggers_name), 24 | level(lvl), 25 | msg_id(0) 26 | { 27 | #ifndef SPDLOG_NO_DATETIME 28 | time = os::now(); 29 | #endif 30 | 31 | #ifndef SPDLOG_NO_THREAD_ID 32 | thread_id = os::thread_id(); 33 | #endif 34 | } 35 | 36 | log_msg(const log_msg& other) = delete; 37 | log_msg& operator=(log_msg&& other) = delete; 38 | log_msg(log_msg&& other) = delete; 39 | 40 | 41 | const std::string *logger_name; 42 | level::level_enum level; 43 | log_clock::time_point time; 44 | size_t thread_id; 45 | fmt::MemoryWriter raw; 46 | fmt::MemoryWriter formatted; 47 | size_t msg_id; 48 | }; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /include/spdlog/details/mpmc_bounded_q.h: -------------------------------------------------------------------------------- 1 | /* 2 | A modified version of Bounded MPMC queue by Dmitry Vyukov. 3 | 4 | Original code from: 5 | http://www.1024cores.net/home/lock-free-algorithms/queues/bounded-mpmc-queue 6 | 7 | licensed by Dmitry Vyukov under the terms below: 8 | 9 | Simplified BSD license 10 | 11 | Copyright (c) 2010-2011 Dmitry Vyukov. All rights reserved. 12 | Redistribution and use in source and binary forms, with or without modification, 13 | are permitted provided that the following conditions are met: 14 | 1. Redistributions of source code must retain the above copyright notice, this list of 15 | conditions and the following disclaimer. 16 | 17 | 2. Redistributions in binary form must reproduce the above copyright notice, this list 18 | of conditions and the following disclaimer in the documentation and/or other materials 19 | provided with the distribution. 20 | 21 | THIS SOFTWARE IS PROVIDED BY DMITRY VYUKOV "AS IS" AND ANY EXPRESS OR IMPLIED 22 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 23 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT 24 | SHALL DMITRY VYUKOV OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 25 | INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, 27 | OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 28 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 29 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 30 | ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | The views and conclusions contained in the software and documentation are those of the authors and 33 | should not be interpreted as representing official policies, either expressed or implied, of Dmitry Vyukov. 34 | */ 35 | 36 | /* 37 | The code in its current form adds the license below: 38 | 39 | Copyright(c) 2015 Gabi Melman. 40 | Distributed under the MIT License (http://opensource.org/licenses/MIT) 41 | 42 | */ 43 | 44 | #pragma once 45 | 46 | #include "spdlog/common.h" 47 | 48 | #include 49 | #include 50 | 51 | namespace spdlog 52 | { 53 | namespace details 54 | { 55 | 56 | template 57 | class mpmc_bounded_queue 58 | { 59 | public: 60 | 61 | using item_type = T; 62 | mpmc_bounded_queue(size_t buffer_size) 63 | :max_size_(buffer_size), 64 | buffer_(new cell_t [buffer_size]), 65 | buffer_mask_(buffer_size - 1) 66 | { 67 | //queue size must be power of two 68 | if(!((buffer_size >= 2) && ((buffer_size & (buffer_size - 1)) == 0))) 69 | throw spdlog_ex("async logger queue size must be power of two"); 70 | 71 | for (size_t i = 0; i != buffer_size; i += 1) 72 | buffer_[i].sequence_.store(i, std::memory_order_relaxed); 73 | enqueue_pos_.store(0, std::memory_order_relaxed); 74 | dequeue_pos_.store(0, std::memory_order_relaxed); 75 | } 76 | 77 | ~mpmc_bounded_queue() 78 | { 79 | delete [] buffer_; 80 | } 81 | 82 | 83 | bool enqueue(T&& data) 84 | { 85 | cell_t* cell; 86 | size_t pos = enqueue_pos_.load(std::memory_order_relaxed); 87 | for (;;) 88 | { 89 | cell = &buffer_[pos & buffer_mask_]; 90 | size_t seq = cell->sequence_.load(std::memory_order_acquire); 91 | intptr_t dif = (intptr_t)seq - (intptr_t)pos; 92 | if (dif == 0) 93 | { 94 | if (enqueue_pos_.compare_exchange_weak(pos, pos + 1, std::memory_order_relaxed)) 95 | break; 96 | } 97 | else if (dif < 0) 98 | { 99 | return false; 100 | } 101 | else 102 | { 103 | pos = enqueue_pos_.load(std::memory_order_relaxed); 104 | } 105 | } 106 | cell->data_ = std::move(data); 107 | cell->sequence_.store(pos + 1, std::memory_order_release); 108 | return true; 109 | } 110 | 111 | bool dequeue(T& data) 112 | { 113 | cell_t* cell; 114 | size_t pos = dequeue_pos_.load(std::memory_order_relaxed); 115 | for (;;) 116 | { 117 | cell = &buffer_[pos & buffer_mask_]; 118 | size_t seq = 119 | cell->sequence_.load(std::memory_order_acquire); 120 | intptr_t dif = (intptr_t)seq - (intptr_t)(pos + 1); 121 | if (dif == 0) 122 | { 123 | if (dequeue_pos_.compare_exchange_weak(pos, pos + 1, std::memory_order_relaxed)) 124 | break; 125 | } 126 | else if (dif < 0) 127 | return false; 128 | else 129 | pos = dequeue_pos_.load(std::memory_order_relaxed); 130 | } 131 | data = std::move(cell->data_); 132 | cell->sequence_.store(pos + buffer_mask_ + 1, std::memory_order_release); 133 | return true; 134 | } 135 | 136 | size_t approx_size() 137 | { 138 | size_t first_pos = dequeue_pos_.load(std::memory_order_relaxed); 139 | size_t last_pos = enqueue_pos_.load(std::memory_order_relaxed); 140 | if (last_pos <= first_pos) 141 | return 0; 142 | auto size = last_pos - first_pos; 143 | return size < max_size_ ? size : max_size_; 144 | } 145 | 146 | private: 147 | struct cell_t 148 | { 149 | std::atomic sequence_; 150 | T data_; 151 | }; 152 | 153 | size_t const max_size_; 154 | 155 | static size_t const cacheline_size = 64; 156 | typedef char cacheline_pad_t [cacheline_size]; 157 | 158 | cacheline_pad_t pad0_; 159 | cell_t* const buffer_; 160 | size_t const buffer_mask_; 161 | cacheline_pad_t pad1_; 162 | std::atomic enqueue_pos_; 163 | cacheline_pad_t pad2_; 164 | std::atomic dequeue_pos_; 165 | cacheline_pad_t pad3_; 166 | 167 | mpmc_bounded_queue(mpmc_bounded_queue const&) = delete; 168 | void operator= (mpmc_bounded_queue const&) = delete; 169 | }; 170 | 171 | } // ns details 172 | } // ns spdlog 173 | -------------------------------------------------------------------------------- /include/spdlog/details/null_mutex.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2015 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | #include 9 | // null, no cost dummy "mutex" and dummy "atomic" int 10 | 11 | namespace spdlog 12 | { 13 | namespace details 14 | { 15 | struct null_mutex 16 | { 17 | void lock() {} 18 | void unlock() {} 19 | bool try_lock() 20 | { 21 | return true; 22 | } 23 | }; 24 | 25 | struct null_atomic_int 26 | { 27 | int value; 28 | null_atomic_int() = default; 29 | 30 | null_atomic_int(int val):value(val) 31 | {} 32 | 33 | int load(std::memory_order) const 34 | { 35 | return value; 36 | } 37 | 38 | void store(int val) 39 | { 40 | value = val; 41 | } 42 | }; 43 | 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /include/spdlog/details/registry.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2015 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | // Loggers registy of unique name->logger pointer 9 | // An attempt to create a logger with an already existing name will be ignored 10 | // If user requests a non existing logger, nullptr will be returned 11 | // This class is thread safe 12 | 13 | #include "spdlog/details/null_mutex.h" 14 | #include "spdlog/logger.h" 15 | #include "spdlog/async_logger.h" 16 | #include "spdlog/common.h" 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | namespace spdlog 26 | { 27 | namespace details 28 | { 29 | template class registry_t 30 | { 31 | public: 32 | 33 | void register_logger(std::shared_ptr logger) 34 | { 35 | std::lock_guard lock(_mutex); 36 | auto logger_name = logger->name(); 37 | throw_if_exists(logger_name); 38 | _loggers[logger_name] = logger; 39 | } 40 | 41 | 42 | std::shared_ptr get(const std::string& logger_name) 43 | { 44 | std::lock_guard lock(_mutex); 45 | auto found = _loggers.find(logger_name); 46 | return found == _loggers.end() ? nullptr : found->second; 47 | } 48 | 49 | template 50 | std::shared_ptr create(const std::string& logger_name, const It& sinks_begin, const It& sinks_end) 51 | { 52 | std::lock_guard lock(_mutex); 53 | throw_if_exists(logger_name); 54 | std::shared_ptr new_logger; 55 | if (_async_mode) 56 | new_logger = std::make_shared(logger_name, sinks_begin, sinks_end, _async_q_size, _overflow_policy, _worker_warmup_cb, _flush_interval_ms, _worker_teardown_cb); 57 | else 58 | new_logger = std::make_shared(logger_name, sinks_begin, sinks_end); 59 | 60 | if (_formatter) 61 | new_logger->set_formatter(_formatter); 62 | 63 | if (_err_handler) 64 | new_logger->set_error_handler(_err_handler); 65 | 66 | new_logger->set_level(_level); 67 | 68 | 69 | //Add to registry 70 | _loggers[logger_name] = new_logger; 71 | return new_logger; 72 | } 73 | 74 | template 75 | std::shared_ptr create_async(const std::string& logger_name, size_t queue_size, const async_overflow_policy overflow_policy, const std::function& worker_warmup_cb, const std::chrono::milliseconds& flush_interval_ms, const std::function& worker_teardown_cb, const It& sinks_begin, const It& sinks_end) 76 | { 77 | std::lock_guard lock(_mutex); 78 | throw_if_exists(logger_name); 79 | auto new_logger = std::make_shared(logger_name, sinks_begin, sinks_end, queue_size, overflow_policy, worker_warmup_cb, flush_interval_ms, worker_teardown_cb); 80 | 81 | if (_formatter) 82 | new_logger->set_formatter(_formatter); 83 | 84 | if (_err_handler) 85 | new_logger->set_error_handler(_err_handler); 86 | 87 | new_logger->set_level(_level); 88 | 89 | //Add to registry 90 | _loggers[logger_name] = new_logger; 91 | return new_logger; 92 | } 93 | 94 | void apply_all(std::function)> fun) 95 | { 96 | std::lock_guard lock(_mutex); 97 | for (auto &l : _loggers) 98 | fun(l.second); 99 | } 100 | 101 | void drop(const std::string& logger_name) 102 | { 103 | std::lock_guard lock(_mutex); 104 | _loggers.erase(logger_name); 105 | } 106 | 107 | void drop_all() 108 | { 109 | std::lock_guard lock(_mutex); 110 | _loggers.clear(); 111 | } 112 | std::shared_ptr create(const std::string& logger_name, sinks_init_list sinks) 113 | { 114 | return create(logger_name, sinks.begin(), sinks.end()); 115 | } 116 | 117 | std::shared_ptr create(const std::string& logger_name, sink_ptr sink) 118 | { 119 | return create(logger_name, { sink }); 120 | } 121 | 122 | std::shared_ptr create_async(const std::string& logger_name, size_t queue_size, const async_overflow_policy overflow_policy, const std::function& worker_warmup_cb, const std::chrono::milliseconds& flush_interval_ms, const std::function& worker_teardown_cb, sinks_init_list sinks) 123 | { 124 | return create_async(logger_name, queue_size, overflow_policy, worker_warmup_cb, flush_interval_ms, worker_teardown_cb, sinks.begin(), sinks.end()); 125 | } 126 | 127 | std::shared_ptr create_async(const std::string& logger_name, size_t queue_size, const async_overflow_policy overflow_policy, const std::function& worker_warmup_cb, const std::chrono::milliseconds& flush_interval_ms, const std::function& worker_teardown_cb, sink_ptr sink) 128 | { 129 | return create_async(logger_name, queue_size, overflow_policy, worker_warmup_cb, flush_interval_ms, worker_teardown_cb, { sink }); 130 | } 131 | 132 | void formatter(formatter_ptr f) 133 | { 134 | std::lock_guard lock(_mutex); 135 | _formatter = f; 136 | for (auto& l : _loggers) 137 | l.second->set_formatter(_formatter); 138 | } 139 | 140 | void set_pattern(const std::string& pattern) 141 | { 142 | std::lock_guard lock(_mutex); 143 | _formatter = std::make_shared(pattern); 144 | for (auto& l : _loggers) 145 | l.second->set_formatter(_formatter); 146 | } 147 | 148 | void set_level(level::level_enum log_level) 149 | { 150 | std::lock_guard lock(_mutex); 151 | for (auto& l : _loggers) 152 | l.second->set_level(log_level); 153 | _level = log_level; 154 | } 155 | 156 | void set_error_handler(log_err_handler handler) 157 | { 158 | for (auto& l : _loggers) 159 | l.second->set_error_handler(handler); 160 | _err_handler = handler; 161 | } 162 | 163 | void set_async_mode(size_t q_size, const async_overflow_policy overflow_policy, const std::function& worker_warmup_cb, const std::chrono::milliseconds& flush_interval_ms, const std::function& worker_teardown_cb) 164 | { 165 | std::lock_guard lock(_mutex); 166 | _async_mode = true; 167 | _async_q_size = q_size; 168 | _overflow_policy = overflow_policy; 169 | _worker_warmup_cb = worker_warmup_cb; 170 | _flush_interval_ms = flush_interval_ms; 171 | _worker_teardown_cb = worker_teardown_cb; 172 | } 173 | 174 | void set_sync_mode() 175 | { 176 | std::lock_guard lock(_mutex); 177 | _async_mode = false; 178 | } 179 | 180 | static registry_t& instance() 181 | { 182 | static registry_t s_instance; 183 | return s_instance; 184 | } 185 | 186 | private: 187 | registry_t() {} 188 | registry_t(const registry_t&) = delete; 189 | registry_t& operator=(const registry_t&) = delete; 190 | 191 | void throw_if_exists(const std::string &logger_name) 192 | { 193 | if (_loggers.find(logger_name) != _loggers.end()) 194 | throw spdlog_ex("logger with name '" + logger_name + "' already exists"); 195 | } 196 | Mutex _mutex; 197 | std::unordered_map > _loggers; 198 | formatter_ptr _formatter; 199 | level::level_enum _level = level::info; 200 | log_err_handler _err_handler; 201 | bool _async_mode = false; 202 | size_t _async_q_size = 0; 203 | async_overflow_policy _overflow_policy = async_overflow_policy::block_retry; 204 | std::function _worker_warmup_cb = nullptr; 205 | std::chrono::milliseconds _flush_interval_ms; 206 | std::function _worker_teardown_cb = nullptr; 207 | }; 208 | #ifdef SPDLOG_NO_REGISTRY_MUTEX 209 | typedef registry_t registry; 210 | #else 211 | typedef registry_t registry; 212 | #endif 213 | } 214 | } 215 | -------------------------------------------------------------------------------- /include/spdlog/details/spdlog_impl.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2015 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | // 9 | // Global registry functions 10 | // 11 | #include "spdlog/spdlog.h" 12 | #include "spdlog/details/registry.h" 13 | #include "spdlog/sinks/file_sinks.h" 14 | #include "spdlog/sinks/stdout_sinks.h" 15 | #ifdef SPDLOG_ENABLE_SYSLOG 16 | #include "spdlog/sinks/syslog_sink.h" 17 | #endif 18 | 19 | #ifdef _WIN32 20 | #include "spdlog/sinks/wincolor_sink.h" 21 | #else 22 | #include "spdlog/sinks/ansicolor_sink.h" 23 | #endif 24 | 25 | 26 | #ifdef __ANDROID__ 27 | #include "spdlog/sinks/android_sink.h" 28 | #endif 29 | 30 | #include 31 | #include 32 | #include 33 | #include 34 | 35 | inline void spdlog::register_logger(std::shared_ptr logger) 36 | { 37 | return details::registry::instance().register_logger(logger); 38 | } 39 | 40 | inline std::shared_ptr spdlog::get(const std::string& name) 41 | { 42 | return details::registry::instance().get(name); 43 | } 44 | 45 | inline void spdlog::drop(const std::string &name) 46 | { 47 | details::registry::instance().drop(name); 48 | } 49 | 50 | // Create multi/single threaded simple file logger 51 | inline std::shared_ptr spdlog::basic_logger_mt(const std::string& logger_name, const filename_t& filename, bool truncate) 52 | { 53 | return create(logger_name, filename, truncate); 54 | } 55 | 56 | inline std::shared_ptr spdlog::basic_logger_st(const std::string& logger_name, const filename_t& filename, bool truncate) 57 | { 58 | return create(logger_name, filename, truncate); 59 | } 60 | 61 | // Create multi/single threaded rotating file logger 62 | inline std::shared_ptr spdlog::rotating_logger_mt(const std::string& logger_name, const filename_t& filename, size_t max_file_size, size_t max_files) 63 | { 64 | return create(logger_name, filename, max_file_size, max_files); 65 | } 66 | 67 | inline std::shared_ptr spdlog::rotating_logger_st(const std::string& logger_name, const filename_t& filename, size_t max_file_size, size_t max_files) 68 | { 69 | return create(logger_name, filename, max_file_size, max_files); 70 | } 71 | 72 | // Create file logger which creates new file at midnight): 73 | inline std::shared_ptr spdlog::daily_logger_mt(const std::string& logger_name, const filename_t& filename, int hour, int minute) 74 | { 75 | return create(logger_name, filename, hour, minute); 76 | } 77 | 78 | inline std::shared_ptr spdlog::daily_logger_st(const std::string& logger_name, const filename_t& filename, int hour, int minute) 79 | { 80 | return create(logger_name, filename, hour, minute); 81 | } 82 | 83 | 84 | // 85 | // stdout/stderr loggers 86 | // 87 | inline std::shared_ptr spdlog::stdout_logger_mt(const std::string& logger_name) 88 | { 89 | return spdlog::details::registry::instance().create(logger_name, spdlog::sinks::stdout_sink_mt::instance()); 90 | } 91 | 92 | inline std::shared_ptr spdlog::stdout_logger_st(const std::string& logger_name) 93 | { 94 | return spdlog::details::registry::instance().create(logger_name, spdlog::sinks::stdout_sink_st::instance()); 95 | } 96 | 97 | inline std::shared_ptr spdlog::stderr_logger_mt(const std::string& logger_name) 98 | { 99 | return spdlog::details::registry::instance().create(logger_name, spdlog::sinks::stderr_sink_mt::instance()); 100 | } 101 | 102 | inline std::shared_ptr spdlog::stderr_logger_st(const std::string& logger_name) 103 | { 104 | return spdlog::details::registry::instance().create(logger_name, spdlog::sinks::stderr_sink_st::instance()); 105 | } 106 | 107 | // 108 | // stdout/stderr color loggers 109 | // 110 | #ifdef _WIN32 111 | inline std::shared_ptr spdlog::stdout_color_mt(const std::string& logger_name) 112 | { 113 | auto sink = std::make_shared(); 114 | return spdlog::details::registry::instance().create(logger_name, sink); 115 | } 116 | 117 | inline std::shared_ptr spdlog::stdout_color_st(const std::string& logger_name) 118 | { 119 | auto sink = std::make_shared(); 120 | return spdlog::details::registry::instance().create(logger_name, sink); 121 | } 122 | 123 | inline std::shared_ptr spdlog::stderr_color_mt(const std::string& logger_name) 124 | { 125 | auto sink = std::make_shared(); 126 | return spdlog::details::registry::instance().create(logger_name, sink); 127 | } 128 | 129 | 130 | inline std::shared_ptr spdlog::stderr_color_st(const std::string& logger_name) 131 | { 132 | auto sink = std::make_shared(); 133 | return spdlog::details::registry::instance().create(logger_name, sink); 134 | } 135 | 136 | #else //ansi terminal colors 137 | 138 | inline std::shared_ptr spdlog::stdout_color_mt(const std::string& logger_name) 139 | { 140 | auto sink = std::make_shared(); 141 | return spdlog::details::registry::instance().create(logger_name, sink); 142 | } 143 | 144 | inline std::shared_ptr spdlog::stdout_color_st(const std::string& logger_name) 145 | { 146 | auto sink = std::make_shared(); 147 | return spdlog::details::registry::instance().create(logger_name, sink); 148 | } 149 | 150 | inline std::shared_ptr spdlog::stderr_color_mt(const std::string& logger_name) 151 | { 152 | auto sink = std::make_shared(); 153 | return spdlog::details::registry::instance().create(logger_name, sink); 154 | } 155 | 156 | inline std::shared_ptr spdlog::stderr_color_st(const std::string& logger_name) 157 | { 158 | auto sink = std::make_shared(); 159 | return spdlog::details::registry::instance().create(logger_name, sink); 160 | } 161 | #endif 162 | 163 | #ifdef SPDLOG_ENABLE_SYSLOG 164 | // Create syslog logger 165 | inline std::shared_ptr spdlog::syslog_logger(const std::string& logger_name, const std::string& syslog_ident, int syslog_option) 166 | { 167 | return create(logger_name, syslog_ident, syslog_option); 168 | } 169 | #endif 170 | 171 | #ifdef __ANDROID__ 172 | inline std::shared_ptr spdlog::android_logger(const std::string& logger_name, const std::string& tag) 173 | { 174 | return create(logger_name, tag); 175 | } 176 | #endif 177 | 178 | // Create and register a logger a single sink 179 | inline std::shared_ptr spdlog::create(const std::string& logger_name, const spdlog::sink_ptr& sink) 180 | { 181 | return details::registry::instance().create(logger_name, sink); 182 | } 183 | 184 | //Create logger with multiple sinks 185 | 186 | inline std::shared_ptr spdlog::create(const std::string& logger_name, spdlog::sinks_init_list sinks) 187 | { 188 | return details::registry::instance().create(logger_name, sinks); 189 | } 190 | 191 | 192 | template 193 | inline std::shared_ptr spdlog::create(const std::string& logger_name, Args... args) 194 | { 195 | sink_ptr sink = std::make_shared(args...); 196 | return details::registry::instance().create(logger_name, { sink }); 197 | } 198 | 199 | 200 | template 201 | inline std::shared_ptr spdlog::create(const std::string& logger_name, const It& sinks_begin, const It& sinks_end) 202 | { 203 | return details::registry::instance().create(logger_name, sinks_begin, sinks_end); 204 | } 205 | 206 | // Create and register an async logger with a single sink 207 | inline std::shared_ptr spdlog::create_async(const std::string& logger_name, const sink_ptr& sink, size_t queue_size, const async_overflow_policy overflow_policy, const std::function& worker_warmup_cb, const std::chrono::milliseconds& flush_interval_ms, const std::function& worker_teardown_cb) 208 | { 209 | return details::registry::instance().create_async(logger_name, queue_size, overflow_policy, worker_warmup_cb, flush_interval_ms, worker_teardown_cb, sink); 210 | } 211 | 212 | // Create and register an async logger with multiple sinks 213 | inline std::shared_ptr spdlog::create_async(const std::string& logger_name, sinks_init_list sinks, size_t queue_size, const async_overflow_policy overflow_policy, const std::function& worker_warmup_cb, const std::chrono::milliseconds& flush_interval_ms, const std::function& worker_teardown_cb ) 214 | { 215 | return details::registry::instance().create_async(logger_name, queue_size, overflow_policy, worker_warmup_cb, flush_interval_ms, worker_teardown_cb, sinks); 216 | } 217 | 218 | template 219 | inline std::shared_ptr spdlog::create_async(const std::string& logger_name, const It& sinks_begin, const It& sinks_end, size_t queue_size, const async_overflow_policy overflow_policy, const std::function& worker_warmup_cb, const std::chrono::milliseconds& flush_interval_ms, const std::function& worker_teardown_cb) 220 | { 221 | return details::registry::instance().create_async(logger_name, queue_size, overflow_policy, worker_warmup_cb, flush_interval_ms, worker_teardown_cb, sinks_begin, sinks_end); 222 | } 223 | 224 | inline void spdlog::set_formatter(spdlog::formatter_ptr f) 225 | { 226 | details::registry::instance().formatter(f); 227 | } 228 | 229 | inline void spdlog::set_pattern(const std::string& format_string) 230 | { 231 | return details::registry::instance().set_pattern(format_string); 232 | } 233 | 234 | inline void spdlog::set_level(level::level_enum log_level) 235 | { 236 | return details::registry::instance().set_level(log_level); 237 | } 238 | 239 | inline void spdlog::set_error_handler(log_err_handler handler) 240 | { 241 | return details::registry::instance().set_error_handler(handler); 242 | } 243 | 244 | 245 | inline void spdlog::set_async_mode(size_t queue_size, const async_overflow_policy overflow_policy, const std::function& worker_warmup_cb, const std::chrono::milliseconds& flush_interval_ms, const std::function& worker_teardown_cb) 246 | { 247 | details::registry::instance().set_async_mode(queue_size, overflow_policy, worker_warmup_cb, flush_interval_ms, worker_teardown_cb); 248 | } 249 | 250 | inline void spdlog::set_sync_mode() 251 | { 252 | details::registry::instance().set_sync_mode(); 253 | } 254 | 255 | inline void spdlog::apply_all(std::function)> fun) 256 | { 257 | details::registry::instance().apply_all(fun); 258 | } 259 | 260 | inline void spdlog::drop_all() 261 | { 262 | details::registry::instance().drop_all(); 263 | } 264 | -------------------------------------------------------------------------------- /include/spdlog/fmt/bundled/ostream.cc: -------------------------------------------------------------------------------- 1 | /* 2 | Formatting library for C++ - std::ostream support 3 | 4 | Copyright (c) 2012 - 2016, Victor Zverovich 5 | All rights reserved. 6 | 7 | For the license information refer to format.h. 8 | */ 9 | 10 | #include "ostream.h" 11 | 12 | namespace fmt { 13 | 14 | namespace internal { 15 | FMT_FUNC void write(std::ostream &os, Writer &w) { 16 | const char *data = w.data(); 17 | typedef internal::MakeUnsigned::Type UnsignedStreamSize; 18 | UnsignedStreamSize size = w.size(); 19 | UnsignedStreamSize max_size = 20 | internal::to_unsigned((std::numeric_limits::max)()); 21 | do { 22 | UnsignedStreamSize n = size <= max_size ? size : max_size; 23 | os.write(data, static_cast(n)); 24 | data += n; 25 | size -= n; 26 | } while (size != 0); 27 | } 28 | } 29 | 30 | FMT_FUNC void print(std::ostream &os, CStringRef format_str, ArgList args) { 31 | MemoryWriter w; 32 | w.write(format_str, args); 33 | internal::write(os, w); 34 | } 35 | } // namespace fmt 36 | -------------------------------------------------------------------------------- /include/spdlog/fmt/bundled/ostream.h: -------------------------------------------------------------------------------- 1 | /* 2 | Formatting library for C++ - std::ostream support 3 | 4 | Copyright (c) 2012 - 2016, Victor Zverovich 5 | All rights reserved. 6 | 7 | For the license information refer to format.h. 8 | */ 9 | 10 | #ifndef FMT_OSTREAM_H_ 11 | #define FMT_OSTREAM_H_ 12 | 13 | #include "format.h" 14 | #include 15 | 16 | namespace fmt { 17 | 18 | namespace internal { 19 | 20 | template 21 | class FormatBuf : public std::basic_streambuf { 22 | private: 23 | typedef typename std::basic_streambuf::int_type int_type; 24 | typedef typename std::basic_streambuf::traits_type traits_type; 25 | 26 | Buffer &buffer_; 27 | 28 | public: 29 | FormatBuf(Buffer &buffer) : buffer_(buffer) {} 30 | 31 | protected: 32 | // The put-area is actually always empty. This makes the implementation 33 | // simpler and has the advantage that the streambuf and the buffer are always 34 | // in sync and sputc never writes into uninitialized memory. The obvious 35 | // disadvantage is that each call to sputc always results in a (virtual) call 36 | // to overflow. There is no disadvantage here for sputn since this always 37 | // results in a call to xsputn. 38 | 39 | int_type overflow(int_type ch = traits_type::eof()) FMT_OVERRIDE { 40 | if (!traits_type::eq_int_type(ch, traits_type::eof())) 41 | buffer_.push_back(static_cast(ch)); 42 | return ch; 43 | } 44 | 45 | std::streamsize xsputn(const Char *s, std::streamsize count) FMT_OVERRIDE { 46 | buffer_.append(s, s + count); 47 | return count; 48 | } 49 | }; 50 | 51 | Yes &convert(std::ostream &); 52 | 53 | struct DummyStream : std::ostream { 54 | DummyStream(); // Suppress a bogus warning in MSVC. 55 | // Hide all operator<< overloads from std::ostream. 56 | void operator<<(Null<>); 57 | }; 58 | 59 | No &operator<<(std::ostream &, int); 60 | 61 | template 62 | struct ConvertToIntImpl { 63 | // Convert to int only if T doesn't have an overloaded operator<<. 64 | enum { 65 | value = sizeof(convert(get() << get())) == sizeof(No) 66 | }; 67 | }; 68 | 69 | // Write the content of w to os. 70 | FMT_API void write(std::ostream &os, Writer &w); 71 | } // namespace internal 72 | 73 | // Formats a value. 74 | template 75 | void format_arg(BasicFormatter &f, 76 | const Char *&format_str, const T &value) { 77 | internal::MemoryBuffer buffer; 78 | 79 | internal::FormatBuf format_buf(buffer); 80 | std::basic_ostream output(&format_buf); 81 | output << value; 82 | 83 | BasicStringRef str(&buffer[0], buffer.size()); 84 | typedef internal::MakeArg< BasicFormatter > MakeArg; 85 | format_str = f.format(format_str, MakeArg(str)); 86 | } 87 | 88 | /** 89 | \rst 90 | Prints formatted data to the stream *os*. 91 | 92 | **Example**:: 93 | 94 | print(cerr, "Don't {}!", "panic"); 95 | \endrst 96 | */ 97 | FMT_API void print(std::ostream &os, CStringRef format_str, ArgList args); 98 | FMT_VARIADIC(void, print, std::ostream &, CStringRef) 99 | } // namespace fmt 100 | 101 | #ifdef FMT_HEADER_ONLY 102 | # include "ostream.cc" 103 | #endif 104 | 105 | #endif // FMT_OSTREAM_H_ 106 | -------------------------------------------------------------------------------- /include/spdlog/fmt/bundled/posix.cc: -------------------------------------------------------------------------------- 1 | /* 2 | A C++ interface to POSIX functions. 3 | 4 | Copyright (c) 2012 - 2016, Victor Zverovich 5 | All rights reserved. 6 | 7 | For the license information refer to format.h. 8 | */ 9 | 10 | // Disable bogus MSVC warnings. 11 | #ifndef _CRT_SECURE_NO_WARNINGS 12 | # define _CRT_SECURE_NO_WARNINGS 13 | #endif 14 | 15 | #include "posix.h" 16 | 17 | #include 18 | #include 19 | #include 20 | 21 | #ifndef _WIN32 22 | # include 23 | #else 24 | # ifndef WIN32_LEAN_AND_MEAN 25 | # define WIN32_LEAN_AND_MEAN 26 | # endif 27 | # include 28 | # include 29 | 30 | # define O_CREAT _O_CREAT 31 | # define O_TRUNC _O_TRUNC 32 | 33 | # ifndef S_IRUSR 34 | # define S_IRUSR _S_IREAD 35 | # endif 36 | 37 | # ifndef S_IWUSR 38 | # define S_IWUSR _S_IWRITE 39 | # endif 40 | 41 | # ifdef __MINGW32__ 42 | # define _SH_DENYNO 0x40 43 | # endif 44 | 45 | #endif // _WIN32 46 | 47 | #ifdef fileno 48 | # undef fileno 49 | #endif 50 | 51 | namespace { 52 | #ifdef _WIN32 53 | // Return type of read and write functions. 54 | typedef int RWResult; 55 | 56 | // On Windows the count argument to read and write is unsigned, so convert 57 | // it from size_t preventing integer overflow. 58 | inline unsigned convert_rwcount(std::size_t count) { 59 | return count <= UINT_MAX ? static_cast(count) : UINT_MAX; 60 | } 61 | #else 62 | // Return type of read and write functions. 63 | typedef ssize_t RWResult; 64 | 65 | inline std::size_t convert_rwcount(std::size_t count) { return count; } 66 | #endif 67 | } 68 | 69 | fmt::BufferedFile::~BufferedFile() FMT_NOEXCEPT { 70 | if (file_ && FMT_SYSTEM(fclose(file_)) != 0) 71 | fmt::report_system_error(errno, "cannot close file"); 72 | } 73 | 74 | fmt::BufferedFile::BufferedFile( 75 | fmt::CStringRef filename, fmt::CStringRef mode) { 76 | FMT_RETRY_VAL(file_, FMT_SYSTEM(fopen(filename.c_str(), mode.c_str())), 0); 77 | if (!file_) 78 | FMT_THROW(SystemError(errno, "cannot open file {}", filename)); 79 | } 80 | 81 | void fmt::BufferedFile::close() { 82 | if (!file_) 83 | return; 84 | int result = FMT_SYSTEM(fclose(file_)); 85 | file_ = FMT_NULL; 86 | if (result != 0) 87 | FMT_THROW(SystemError(errno, "cannot close file")); 88 | } 89 | 90 | // A macro used to prevent expansion of fileno on broken versions of MinGW. 91 | #define FMT_ARGS 92 | 93 | int fmt::BufferedFile::fileno() const { 94 | int fd = FMT_POSIX_CALL(fileno FMT_ARGS(file_)); 95 | if (fd == -1) 96 | FMT_THROW(SystemError(errno, "cannot get file descriptor")); 97 | return fd; 98 | } 99 | 100 | fmt::File::File(fmt::CStringRef path, int oflag) { 101 | int mode = S_IRUSR | S_IWUSR; 102 | #if defined(_WIN32) && !defined(__MINGW32__) 103 | fd_ = -1; 104 | FMT_POSIX_CALL(sopen_s(&fd_, path.c_str(), oflag, _SH_DENYNO, mode)); 105 | #else 106 | FMT_RETRY(fd_, FMT_POSIX_CALL(open(path.c_str(), oflag, mode))); 107 | #endif 108 | if (fd_ == -1) 109 | FMT_THROW(SystemError(errno, "cannot open file {}", path)); 110 | } 111 | 112 | fmt::File::~File() FMT_NOEXCEPT { 113 | // Don't retry close in case of EINTR! 114 | // See http://linux.derkeiler.com/Mailing-Lists/Kernel/2005-09/3000.html 115 | if (fd_ != -1 && FMT_POSIX_CALL(close(fd_)) != 0) 116 | fmt::report_system_error(errno, "cannot close file"); 117 | } 118 | 119 | void fmt::File::close() { 120 | if (fd_ == -1) 121 | return; 122 | // Don't retry close in case of EINTR! 123 | // See http://linux.derkeiler.com/Mailing-Lists/Kernel/2005-09/3000.html 124 | int result = FMT_POSIX_CALL(close(fd_)); 125 | fd_ = -1; 126 | if (result != 0) 127 | FMT_THROW(SystemError(errno, "cannot close file")); 128 | } 129 | 130 | fmt::LongLong fmt::File::size() const { 131 | #ifdef _WIN32 132 | // Use GetFileSize instead of GetFileSizeEx for the case when _WIN32_WINNT 133 | // is less than 0x0500 as is the case with some default MinGW builds. 134 | // Both functions support large file sizes. 135 | DWORD size_upper = 0; 136 | HANDLE handle = reinterpret_cast(_get_osfhandle(fd_)); 137 | DWORD size_lower = FMT_SYSTEM(GetFileSize(handle, &size_upper)); 138 | if (size_lower == INVALID_FILE_SIZE) { 139 | DWORD error = GetLastError(); 140 | if (error != NO_ERROR) 141 | FMT_THROW(WindowsError(GetLastError(), "cannot get file size")); 142 | } 143 | fmt::ULongLong long_size = size_upper; 144 | return (long_size << sizeof(DWORD) * CHAR_BIT) | size_lower; 145 | #else 146 | typedef struct stat Stat; 147 | Stat file_stat = Stat(); 148 | if (FMT_POSIX_CALL(fstat(fd_, &file_stat)) == -1) 149 | FMT_THROW(SystemError(errno, "cannot get file attributes")); 150 | FMT_STATIC_ASSERT(sizeof(fmt::LongLong) >= sizeof(file_stat.st_size), 151 | "return type of File::size is not large enough"); 152 | return file_stat.st_size; 153 | #endif 154 | } 155 | 156 | std::size_t fmt::File::read(void *buffer, std::size_t count) { 157 | RWResult result = 0; 158 | FMT_RETRY(result, FMT_POSIX_CALL(read(fd_, buffer, convert_rwcount(count)))); 159 | if (result < 0) 160 | FMT_THROW(SystemError(errno, "cannot read from file")); 161 | return internal::to_unsigned(result); 162 | } 163 | 164 | std::size_t fmt::File::write(const void *buffer, std::size_t count) { 165 | RWResult result = 0; 166 | FMT_RETRY(result, FMT_POSIX_CALL(write(fd_, buffer, convert_rwcount(count)))); 167 | if (result < 0) 168 | FMT_THROW(SystemError(errno, "cannot write to file")); 169 | return internal::to_unsigned(result); 170 | } 171 | 172 | fmt::File fmt::File::dup(int fd) { 173 | // Don't retry as dup doesn't return EINTR. 174 | // http://pubs.opengroup.org/onlinepubs/009695399/functions/dup.html 175 | int new_fd = FMT_POSIX_CALL(dup(fd)); 176 | if (new_fd == -1) 177 | FMT_THROW(SystemError(errno, "cannot duplicate file descriptor {}", fd)); 178 | return File(new_fd); 179 | } 180 | 181 | void fmt::File::dup2(int fd) { 182 | int result = 0; 183 | FMT_RETRY(result, FMT_POSIX_CALL(dup2(fd_, fd))); 184 | if (result == -1) { 185 | FMT_THROW(SystemError(errno, 186 | "cannot duplicate file descriptor {} to {}", fd_, fd)); 187 | } 188 | } 189 | 190 | void fmt::File::dup2(int fd, ErrorCode &ec) FMT_NOEXCEPT { 191 | int result = 0; 192 | FMT_RETRY(result, FMT_POSIX_CALL(dup2(fd_, fd))); 193 | if (result == -1) 194 | ec = ErrorCode(errno); 195 | } 196 | 197 | void fmt::File::pipe(File &read_end, File &write_end) { 198 | // Close the descriptors first to make sure that assignments don't throw 199 | // and there are no leaks. 200 | read_end.close(); 201 | write_end.close(); 202 | int fds[2] = {}; 203 | #ifdef _WIN32 204 | // Make the default pipe capacity same as on Linux 2.6.11+. 205 | enum { DEFAULT_CAPACITY = 65536 }; 206 | int result = FMT_POSIX_CALL(pipe(fds, DEFAULT_CAPACITY, _O_BINARY)); 207 | #else 208 | // Don't retry as the pipe function doesn't return EINTR. 209 | // http://pubs.opengroup.org/onlinepubs/009696799/functions/pipe.html 210 | int result = FMT_POSIX_CALL(pipe(fds)); 211 | #endif 212 | if (result != 0) 213 | FMT_THROW(SystemError(errno, "cannot create pipe")); 214 | // The following assignments don't throw because read_fd and write_fd 215 | // are closed. 216 | read_end = File(fds[0]); 217 | write_end = File(fds[1]); 218 | } 219 | 220 | fmt::BufferedFile fmt::File::fdopen(const char *mode) { 221 | // Don't retry as fdopen doesn't return EINTR. 222 | FILE *f = FMT_POSIX_CALL(fdopen(fd_, mode)); 223 | if (!f) 224 | FMT_THROW(SystemError(errno, "cannot associate stream with file descriptor")); 225 | BufferedFile file(f); 226 | fd_ = -1; 227 | return file; 228 | } 229 | 230 | long fmt::getpagesize() { 231 | #ifdef _WIN32 232 | SYSTEM_INFO si; 233 | GetSystemInfo(&si); 234 | return si.dwPageSize; 235 | #else 236 | long size = FMT_POSIX_CALL(sysconf(_SC_PAGESIZE)); 237 | if (size < 0) 238 | FMT_THROW(SystemError(errno, "cannot get memory page size")); 239 | return size; 240 | #endif 241 | } 242 | -------------------------------------------------------------------------------- /include/spdlog/fmt/bundled/time.h: -------------------------------------------------------------------------------- 1 | /* 2 | Formatting library for C++ - time formatting 3 | 4 | Copyright (c) 2012 - 2016, Victor Zverovich 5 | All rights reserved. 6 | 7 | For the license information refer to format.h. 8 | */ 9 | 10 | #ifndef FMT_TIME_H_ 11 | #define FMT_TIME_H_ 12 | 13 | #include "format.h" 14 | #include 15 | 16 | #ifdef _MSC_VER 17 | # pragma warning(push) 18 | # pragma warning(disable: 4702) // unreachable code 19 | # pragma warning(disable: 4996) // "deprecated" functions 20 | #endif 21 | 22 | namespace fmt { 23 | template 24 | void format_arg(BasicFormatter &f, 25 | const char *&format_str, const std::tm &tm) { 26 | if (*format_str == ':') 27 | ++format_str; 28 | const char *end = format_str; 29 | while (*end && *end != '}') 30 | ++end; 31 | if (*end != '}') 32 | FMT_THROW(FormatError("missing '}' in format string")); 33 | internal::MemoryBuffer format; 34 | format.append(format_str, end + 1); 35 | format[format.size() - 1] = '\0'; 36 | Buffer &buffer = f.writer().buffer(); 37 | std::size_t start = buffer.size(); 38 | for (;;) { 39 | std::size_t size = buffer.capacity() - start; 40 | std::size_t count = std::strftime(&buffer[start], size, &format[0], &tm); 41 | if (count != 0) { 42 | buffer.resize(start + count); 43 | break; 44 | } 45 | if (size >= format.size() * 256) { 46 | // If the buffer is 256 times larger than the format string, assume 47 | // that `strftime` gives an empty result. There doesn't seem to be a 48 | // better way to distinguish the two cases: 49 | // https://github.com/fmtlib/fmt/issues/367 50 | break; 51 | } 52 | const std::size_t MIN_GROWTH = 10; 53 | buffer.reserve(buffer.capacity() + (size > MIN_GROWTH ? size : MIN_GROWTH)); 54 | } 55 | format_str = end + 1; 56 | } 57 | 58 | namespace internal{ 59 | inline Null<> localtime_r(...) { return Null<>(); } 60 | inline Null<> localtime_s(...) { return Null<>(); } 61 | inline Null<> gmtime_r(...) { return Null<>(); } 62 | inline Null<> gmtime_s(...) { return Null<>(); } 63 | } 64 | 65 | // Thread-safe replacement for std::localtime 66 | inline std::tm localtime(std::time_t time) { 67 | struct LocalTime { 68 | std::time_t time_; 69 | std::tm tm_; 70 | 71 | LocalTime(std::time_t t): time_(t) {} 72 | 73 | bool run() { 74 | using namespace fmt::internal; 75 | return handle(localtime_r(&time_, &tm_)); 76 | } 77 | 78 | bool handle(std::tm *tm) { return tm != FMT_NULL; } 79 | 80 | bool handle(internal::Null<>) { 81 | using namespace fmt::internal; 82 | return fallback(localtime_s(&tm_, &time_)); 83 | } 84 | 85 | bool fallback(int res) { return res == 0; } 86 | 87 | bool fallback(internal::Null<>) { 88 | using namespace fmt::internal; 89 | std::tm *tm = std::localtime(&time_); 90 | if (tm) tm_ = *tm; 91 | return tm != FMT_NULL; 92 | } 93 | }; 94 | LocalTime lt(time); 95 | if (lt.run()) 96 | return lt.tm_; 97 | // Too big time values may be unsupported. 98 | FMT_THROW(fmt::FormatError("time_t value out of range")); 99 | return std::tm(); 100 | } 101 | 102 | // Thread-safe replacement for std::gmtime 103 | inline std::tm gmtime(std::time_t time) { 104 | struct GMTime { 105 | std::time_t time_; 106 | std::tm tm_; 107 | 108 | GMTime(std::time_t t): time_(t) {} 109 | 110 | bool run() { 111 | using namespace fmt::internal; 112 | return handle(gmtime_r(&time_, &tm_)); 113 | } 114 | 115 | bool handle(std::tm *tm) { return tm != FMT_NULL; } 116 | 117 | bool handle(internal::Null<>) { 118 | using namespace fmt::internal; 119 | return fallback(gmtime_s(&tm_, &time_)); 120 | } 121 | 122 | bool fallback(int res) { return res == 0; } 123 | 124 | bool fallback(internal::Null<>) { 125 | std::tm *tm = std::gmtime(&time_); 126 | if (tm != FMT_NULL) tm_ = *tm; 127 | return tm != FMT_NULL; 128 | } 129 | }; 130 | GMTime gt(time); 131 | if (gt.run()) 132 | return gt.tm_; 133 | // Too big time values may be unsupported. 134 | FMT_THROW(fmt::FormatError("time_t value out of range")); 135 | return std::tm(); 136 | } 137 | } //namespace fmt 138 | 139 | #ifdef _MSC_VER 140 | # pragma warning(pop) 141 | #endif 142 | 143 | #endif // FMT_TIME_H_ 144 | -------------------------------------------------------------------------------- /include/spdlog/fmt/fmt.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2016 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | // 9 | // Include a bundled header-only copy of fmtlib or an external one. 10 | // By default spdlog include its own copy. 11 | // 12 | 13 | #if !defined(SPDLOG_FMT_EXTERNAL) 14 | 15 | #ifndef FMT_HEADER_ONLY 16 | #define FMT_HEADER_ONLY 17 | #endif 18 | #ifndef FMT_USE_WINDOWS_H 19 | #define FMT_USE_WINDOWS_H 0 20 | #endif 21 | #include "spdlog/fmt/bundled/format.h" 22 | 23 | #else //external fmtlib 24 | 25 | #include 26 | 27 | #endif 28 | 29 | -------------------------------------------------------------------------------- /include/spdlog/fmt/ostr.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2016 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | // include external or bundled copy of fmtlib's ostream support 9 | // 10 | #if !defined(SPDLOG_FMT_EXTERNAL) 11 | #include "spdlog/fmt/fmt.h" 12 | #include "spdlog/fmt/bundled/ostream.h" 13 | #else 14 | #include 15 | #endif 16 | 17 | 18 | -------------------------------------------------------------------------------- /include/spdlog/formatter.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2015 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | #include "spdlog/details/log_msg.h" 9 | 10 | #include 11 | #include 12 | #include 13 | 14 | namespace spdlog 15 | { 16 | namespace details 17 | { 18 | class flag_formatter; 19 | } 20 | 21 | class formatter 22 | { 23 | public: 24 | virtual ~formatter() {} 25 | virtual void format(details::log_msg& msg) = 0; 26 | }; 27 | 28 | class pattern_formatter SPDLOG_FINAL : public formatter 29 | { 30 | 31 | public: 32 | explicit pattern_formatter(const std::string& pattern, pattern_time_type pattern_time = pattern_time_type::local); 33 | pattern_formatter(const pattern_formatter&) = delete; 34 | pattern_formatter& operator=(const pattern_formatter&) = delete; 35 | void format(details::log_msg& msg) override; 36 | private: 37 | const std::string _pattern; 38 | const pattern_time_type _pattern_time; 39 | std::vector> _formatters; 40 | std::tm get_time(details::log_msg& msg); 41 | void handle_flag(char flag); 42 | void compile_pattern(const std::string& pattern); 43 | }; 44 | } 45 | 46 | #include "spdlog/details/pattern_formatter_impl.h" 47 | 48 | -------------------------------------------------------------------------------- /include/spdlog/logger.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2015 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | // Thread safe logger (except for set_pattern(..), set_formatter(..) and set_error_handler()) 9 | // Has name, log level, vector of std::shared sink pointers and formatter 10 | // Upon each log write the logger: 11 | // 1. Checks if its log level is enough to log the message 12 | // 2. Format the message using the formatter function 13 | // 3. Pass the formatted message to its sinks to performa the actual logging 14 | 15 | #include "spdlog/sinks/base_sink.h" 16 | #include "spdlog/common.h" 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | namespace spdlog 23 | { 24 | 25 | class logger 26 | { 27 | public: 28 | logger(const std::string& logger_name, sink_ptr single_sink); 29 | logger(const std::string& name, sinks_init_list); 30 | template 31 | logger(const std::string& name, const It& begin, const It& end); 32 | 33 | virtual ~logger(); 34 | logger(const logger&) = delete; 35 | logger& operator=(const logger&) = delete; 36 | 37 | 38 | template void log(level::level_enum lvl, const char* fmt, const Args&... args); 39 | template void log(level::level_enum lvl, const char* msg); 40 | template void trace(const char* fmt, const Arg1&, const Args&... args); 41 | template void debug(const char* fmt, const Arg1&, const Args&... args); 42 | template void info(const char* fmt, const Arg1&, const Args&... args); 43 | template void warn(const char* fmt, const Arg1&, const Args&... args); 44 | template void error(const char* fmt, const Arg1&, const Args&... args); 45 | template void critical(const char* fmt, const Arg1&, const Args&... args); 46 | 47 | template void log_if(const bool flag, level::level_enum lvl, const char* fmt, const Args&... args); 48 | template void log_if(const bool flag, level::level_enum lvl, const char* msg); 49 | template void trace_if(const bool flag, const char* fmt, const Arg1&, const Args&... args); 50 | template void debug_if(const bool flag, const char* fmt, const Arg1&, const Args&... args); 51 | template void info_if(const bool flag, const char* fmt, const Arg1&, const Args&... args); 52 | template void warn_if(const bool flag, const char* fmt, const Arg1&, const Args&... args); 53 | template void error_if(const bool flag, const char* fmt, const Arg1&, const Args&... args); 54 | template void critical_if(const bool flag, const char* fmt, const Arg1&, const Args&... args); 55 | 56 | #ifdef SPDLOG_WCHAR_TO_UTF8_SUPPORT 57 | template void log(level::level_enum lvl, const wchar_t* msg); 58 | template void log(level::level_enum lvl, const wchar_t* fmt, const Args&... args); 59 | template void trace(const wchar_t* fmt, const Args&... args); 60 | template void debug(const wchar_t* fmt, const Args&... args); 61 | template void info(const wchar_t* fmt, const Args&... args); 62 | template void warn(const wchar_t* fmt, const Args&... args); 63 | template void error(const wchar_t* fmt, const Args&... args); 64 | template void critical(const wchar_t* fmt, const Args&... args); 65 | 66 | template void log_if(const bool flag, level::level_enum lvl, const wchar_t* msg); 67 | template void log_if(const bool flag, level::level_enum lvl, const wchar_t* fmt, const Args&... args); 68 | template void trace_if(const bool flag, const wchar_t* fmt, const Args&... args); 69 | template void debug_if(const bool flag, const wchar_t* fmt, const Args&... args); 70 | template void info_if(const bool flag, const wchar_t* fmt, const Args&... args); 71 | template void warn_if(const bool flag, const wchar_t* fmt, const Args&... args); 72 | template void error_if(const bool flag, const wchar_t* fmt, const Args&... args); 73 | template void critical_if(const bool flag, const wchar_t* fmt, const Args&... args); 74 | #endif // SPDLOG_WCHAR_TO_UTF8_SUPPORT 75 | 76 | template void log(level::level_enum lvl, const T&); 77 | template void trace(const T&); 78 | template void debug(const T&); 79 | template void info(const T&); 80 | template void warn(const T&); 81 | template void error(const T&); 82 | template void critical(const T&); 83 | 84 | template void log_if(const bool flag, level::level_enum lvl, const T&); 85 | template void trace_if(const bool flag, const T&); 86 | template void debug_if(const bool flag, const T&); 87 | template void info_if(const bool flag, const T&); 88 | template void warn_if(const bool flag, const T&); 89 | template void error_if(const bool flag, const T&); 90 | template void critical_if(const bool flag, const T&); 91 | 92 | bool should_log(level::level_enum) const; 93 | void set_level(level::level_enum); 94 | level::level_enum level() const; 95 | const std::string& name() const; 96 | void set_pattern(const std::string&, pattern_time_type = pattern_time_type::local); 97 | void set_formatter(formatter_ptr); 98 | 99 | // automatically call flush() if message level >= log_level 100 | void flush_on(level::level_enum log_level); 101 | 102 | virtual void flush(); 103 | 104 | const std::vector& sinks() const; 105 | 106 | // error handler 107 | virtual void set_error_handler(log_err_handler); 108 | virtual log_err_handler error_handler(); 109 | 110 | protected: 111 | virtual void _sink_it(details::log_msg&); 112 | virtual void _set_pattern(const std::string&, pattern_time_type); 113 | virtual void _set_formatter(formatter_ptr); 114 | 115 | // default error handler: print the error to stderr with the max rate of 1 message/minute 116 | virtual void _default_err_handler(const std::string &msg); 117 | 118 | // return true if the given message level should trigger a flush 119 | bool _should_flush_on(const details::log_msg&); 120 | 121 | const std::string _name; 122 | std::vector _sinks; 123 | formatter_ptr _formatter; 124 | spdlog::level_t _level; 125 | spdlog::level_t _flush_level; 126 | log_err_handler _err_handler; 127 | std::atomic _last_err_time; 128 | std::atomic _msg_counter; 129 | }; 130 | } 131 | 132 | #include "spdlog/details/logger_impl.h" 133 | -------------------------------------------------------------------------------- /include/spdlog/sinks/android_sink.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2015 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | #if defined(__ANDROID__) 9 | 10 | #include "spdlog/sinks/sink.h" 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #if !defined(SPDLOG_ANDROID_RETRIES) 19 | #define SPDLOG_ANDROID_RETRIES 2 20 | #endif 21 | 22 | namespace spdlog 23 | { 24 | namespace sinks 25 | { 26 | 27 | /* 28 | * Android sink (logging using __android_log_write) 29 | * __android_log_write is thread-safe. No lock is needed. 30 | */ 31 | class android_sink : public sink 32 | { 33 | public: 34 | explicit android_sink(const std::string& tag = "spdlog", bool use_raw_msg = false): _tag(tag), _use_raw_msg(use_raw_msg) {} 35 | 36 | void log(const details::log_msg& msg) override 37 | { 38 | const android_LogPriority priority = convert_to_android(msg.level); 39 | const char *msg_output = (_use_raw_msg ? msg.raw.c_str() : msg.formatted.c_str()); 40 | 41 | // See system/core/liblog/logger_write.c for explanation of return value 42 | int ret = __android_log_write(priority, _tag.c_str(), msg_output); 43 | int retry_count = 0; 44 | while ((ret == -11/*EAGAIN*/) && (retry_count < SPDLOG_ANDROID_RETRIES)) 45 | { 46 | std::this_thread::sleep_for(std::chrono::milliseconds(5)); 47 | ret = __android_log_write(priority, _tag.c_str(), msg_output); 48 | retry_count++; 49 | } 50 | 51 | if (ret < 0) 52 | { 53 | throw spdlog_ex("__android_log_write() failed", ret); 54 | } 55 | } 56 | 57 | void flush() override 58 | { 59 | } 60 | 61 | private: 62 | static android_LogPriority convert_to_android(spdlog::level::level_enum level) 63 | { 64 | switch(level) 65 | { 66 | case spdlog::level::trace: 67 | return ANDROID_LOG_VERBOSE; 68 | case spdlog::level::debug: 69 | return ANDROID_LOG_DEBUG; 70 | case spdlog::level::info: 71 | return ANDROID_LOG_INFO; 72 | case spdlog::level::warn: 73 | return ANDROID_LOG_WARN; 74 | case spdlog::level::err: 75 | return ANDROID_LOG_ERROR; 76 | case spdlog::level::critical: 77 | return ANDROID_LOG_FATAL; 78 | default: 79 | return ANDROID_LOG_DEFAULT; 80 | } 81 | } 82 | 83 | std::string _tag; 84 | bool _use_raw_msg; 85 | }; 86 | 87 | } 88 | } 89 | 90 | #endif 91 | -------------------------------------------------------------------------------- /include/spdlog/sinks/ansicolor_sink.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2017 spdlog authors. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | #include "spdlog/sinks/base_sink.h" 9 | #include "spdlog/common.h" 10 | #include "spdlog/details/os.h" 11 | 12 | #include 13 | #include 14 | 15 | namespace spdlog 16 | { 17 | namespace sinks 18 | { 19 | 20 | /** 21 | * This sink prefixes the output with an ANSI escape sequence color code depending on the severity 22 | * of the message. 23 | * If no color terminal detected, omit the escape codes. 24 | */ 25 | template 26 | class ansicolor_sink: public base_sink 27 | { 28 | public: 29 | ansicolor_sink(FILE* file): target_file_(file) 30 | { 31 | should_do_colors_ = details::os::in_terminal(file) && details::os::is_color_terminal(); 32 | colors_[level::trace] = cyan; 33 | colors_[level::debug] = cyan; 34 | colors_[level::info] = reset; 35 | colors_[level::warn] = yellow + bold; 36 | colors_[level::err] = red + bold; 37 | colors_[level::critical] = bold + on_red; 38 | colors_[level::off] = reset; 39 | } 40 | virtual ~ansicolor_sink() 41 | { 42 | _flush(); 43 | } 44 | 45 | void set_color(level::level_enum color_level, const std::string& color) 46 | { 47 | std::lock_guard lock(base_sink::_mutex); 48 | colors_[color_level] = color; 49 | } 50 | 51 | /// Formatting codes 52 | const std::string reset = "\033[00m"; 53 | const std::string bold = "\033[1m"; 54 | const std::string dark = "\033[2m"; 55 | const std::string underline = "\033[4m"; 56 | const std::string blink = "\033[5m"; 57 | const std::string reverse = "\033[7m"; 58 | const std::string concealed = "\033[8m"; 59 | 60 | // Foreground colors 61 | const std::string grey = "\033[30m"; 62 | const std::string red = "\033[31m"; 63 | const std::string green = "\033[32m"; 64 | const std::string yellow = "\033[33m"; 65 | const std::string blue = "\033[34m"; 66 | const std::string magenta = "\033[35m"; 67 | const std::string cyan = "\033[36m"; 68 | const std::string white = "\033[37m"; 69 | 70 | /// Background colors 71 | const std::string on_grey = "\033[40m"; 72 | const std::string on_red = "\033[41m"; 73 | const std::string on_green = "\033[42m"; 74 | const std::string on_yellow = "\033[43m"; 75 | const std::string on_blue = "\033[44m"; 76 | const std::string on_magenta = "\033[45m"; 77 | const std::string on_cyan = "\033[46m"; 78 | const std::string on_white = "\033[47m"; 79 | 80 | protected: 81 | virtual void _sink_it(const details::log_msg& msg) override 82 | { 83 | // Wrap the originally formatted message in color codes. 84 | // If color is not supported in the terminal, log as is instead. 85 | if (should_do_colors_) 86 | { 87 | const std::string& prefix = colors_[msg.level]; 88 | fwrite(prefix.data(), sizeof(char), prefix.size(), target_file_); 89 | fwrite(msg.formatted.data(), sizeof(char), msg.formatted.size(), target_file_); 90 | fwrite(reset.data(), sizeof(char), reset.size(), target_file_); 91 | } 92 | else 93 | { 94 | fwrite(msg.formatted.data(), sizeof(char), msg.formatted.size(), target_file_); 95 | } 96 | _flush(); 97 | } 98 | 99 | void _flush() override 100 | { 101 | fflush(target_file_); 102 | } 103 | FILE* target_file_; 104 | bool should_do_colors_; 105 | std::map colors_; 106 | }; 107 | 108 | 109 | template 110 | class ansicolor_stdout_sink: public ansicolor_sink 111 | { 112 | public: 113 | ansicolor_stdout_sink(): ansicolor_sink(stdout) 114 | {} 115 | }; 116 | 117 | template 118 | class ansicolor_stderr_sink: public ansicolor_sink 119 | { 120 | public: 121 | ansicolor_stderr_sink(): ansicolor_sink(stderr) 122 | {} 123 | }; 124 | 125 | typedef ansicolor_stdout_sink ansicolor_stdout_sink_mt; 126 | typedef ansicolor_stdout_sink ansicolor_stdout_sink_st; 127 | 128 | typedef ansicolor_stderr_sink ansicolor_stderr_sink_mt; 129 | typedef ansicolor_stderr_sink ansicolor_stderr_sink_st; 130 | 131 | } // namespace sinks 132 | } // namespace spdlog 133 | 134 | -------------------------------------------------------------------------------- /include/spdlog/sinks/base_sink.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2015 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | // 8 | // base sink templated over a mutex (either dummy or real) 9 | // concrete implementation should only override the _sink_it method. 10 | // all locking is taken care of here so no locking needed by the implementers.. 11 | // 12 | 13 | #include "spdlog/sinks/sink.h" 14 | #include "spdlog/formatter.h" 15 | #include "spdlog/common.h" 16 | #include "spdlog/details/log_msg.h" 17 | 18 | #include 19 | 20 | namespace spdlog 21 | { 22 | namespace sinks 23 | { 24 | template 25 | class base_sink:public sink 26 | { 27 | public: 28 | base_sink():_mutex() {} 29 | virtual ~base_sink() = default; 30 | 31 | base_sink(const base_sink&) = delete; 32 | base_sink& operator=(const base_sink&) = delete; 33 | 34 | void log(const details::log_msg& msg) SPDLOG_FINAL override 35 | { 36 | std::lock_guard lock(_mutex); 37 | _sink_it(msg); 38 | } 39 | void flush() SPDLOG_FINAL override 40 | { 41 | _flush(); 42 | } 43 | 44 | protected: 45 | virtual void _sink_it(const details::log_msg& msg) = 0; 46 | virtual void _flush() = 0; 47 | Mutex _mutex; 48 | }; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /include/spdlog/sinks/dist_sink.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2015 David Schury, Gabi Melman 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | #include "spdlog/details/log_msg.h" 9 | #include "spdlog/details/null_mutex.h" 10 | #include "spdlog/sinks/base_sink.h" 11 | #include "spdlog/sinks/sink.h" 12 | 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | // Distribution sink (mux). Stores a vector of sinks which get called when log is called 19 | 20 | namespace spdlog 21 | { 22 | namespace sinks 23 | { 24 | template 25 | class dist_sink: public base_sink 26 | { 27 | public: 28 | explicit dist_sink() :_sinks() {} 29 | dist_sink(const dist_sink&) = delete; 30 | dist_sink& operator=(const dist_sink&) = delete; 31 | virtual ~dist_sink() = default; 32 | 33 | protected: 34 | std::vector> _sinks; 35 | 36 | void _sink_it(const details::log_msg& msg) override 37 | { 38 | for (auto &sink : _sinks) 39 | { 40 | if( sink->should_log( msg.level)) 41 | { 42 | sink->log(msg); 43 | } 44 | } 45 | } 46 | 47 | void _flush() override 48 | { 49 | std::lock_guard lock(base_sink::_mutex); 50 | for (auto &sink : _sinks) 51 | sink->flush(); 52 | } 53 | 54 | public: 55 | 56 | 57 | void add_sink(std::shared_ptr sink) 58 | { 59 | std::lock_guard lock(base_sink::_mutex); 60 | _sinks.push_back(sink); 61 | } 62 | 63 | void remove_sink(std::shared_ptr sink) 64 | { 65 | std::lock_guard lock(base_sink::_mutex); 66 | _sinks.erase(std::remove(_sinks.begin(), _sinks.end(), sink), _sinks.end()); 67 | } 68 | }; 69 | 70 | typedef dist_sink dist_sink_mt; 71 | typedef dist_sink dist_sink_st; 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /include/spdlog/sinks/file_sinks.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2015 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | #include "spdlog/sinks/base_sink.h" 9 | #include "spdlog/details/null_mutex.h" 10 | #include "spdlog/details/file_helper.h" 11 | #include "spdlog/fmt/fmt.h" 12 | 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | namespace spdlog 22 | { 23 | namespace sinks 24 | { 25 | /* 26 | * Trivial file sink with single file as target 27 | */ 28 | template 29 | class simple_file_sink SPDLOG_FINAL : public base_sink < Mutex > 30 | { 31 | public: 32 | explicit simple_file_sink(const filename_t &filename, bool truncate = false):_force_flush(false) 33 | { 34 | _file_helper.open(filename, truncate); 35 | } 36 | 37 | void set_force_flush(bool force_flush) 38 | { 39 | _force_flush = force_flush; 40 | } 41 | 42 | protected: 43 | void _sink_it(const details::log_msg& msg) override 44 | { 45 | _file_helper.write(msg); 46 | if(_force_flush) 47 | _file_helper.flush(); 48 | } 49 | void _flush() override 50 | { 51 | _file_helper.flush(); 52 | } 53 | private: 54 | details::file_helper _file_helper; 55 | bool _force_flush; 56 | }; 57 | 58 | typedef simple_file_sink simple_file_sink_mt; 59 | typedef simple_file_sink simple_file_sink_st; 60 | 61 | /* 62 | * Rotating file sink based on size 63 | */ 64 | template 65 | class rotating_file_sink SPDLOG_FINAL : public base_sink < Mutex > 66 | { 67 | public: 68 | rotating_file_sink(const filename_t &base_filename, 69 | std::size_t max_size, std::size_t max_files) : 70 | _base_filename(base_filename), 71 | _max_size(max_size), 72 | _max_files(max_files), 73 | _current_size(0), 74 | _file_helper() 75 | { 76 | _file_helper.open(calc_filename(_base_filename, 0)); 77 | _current_size = _file_helper.size(); //expensive. called only once 78 | } 79 | 80 | 81 | protected: 82 | void _sink_it(const details::log_msg& msg) override 83 | { 84 | _current_size += msg.formatted.size(); 85 | if (_current_size > _max_size) 86 | { 87 | _rotate(); 88 | _current_size = msg.formatted.size(); 89 | } 90 | _file_helper.write(msg); 91 | } 92 | 93 | void _flush() override 94 | { 95 | _file_helper.flush(); 96 | } 97 | 98 | private: 99 | static filename_t calc_filename(const filename_t& filename, std::size_t index) 100 | { 101 | std::conditional::value, fmt::MemoryWriter, fmt::WMemoryWriter>::type w; 102 | if (index) 103 | w.write(SPDLOG_FILENAME_T("{}.{}"), filename, index); 104 | else 105 | w.write(SPDLOG_FILENAME_T("{}"), filename); 106 | return w.str(); 107 | } 108 | 109 | // Rotate files: 110 | // log.txt -> log.txt.1 111 | // log.txt.1 -> log.txt.2 112 | // log.txt.2 -> log.txt.3 113 | // lo3.txt.3 -> delete 114 | 115 | void _rotate() 116 | { 117 | using details::os::filename_to_str; 118 | _file_helper.close(); 119 | for (auto i = _max_files; i > 0; --i) 120 | { 121 | filename_t src = calc_filename(_base_filename, i - 1); 122 | filename_t target = calc_filename(_base_filename, i); 123 | 124 | if (details::file_helper::file_exists(target)) 125 | { 126 | if (details::os::remove(target) != 0) 127 | { 128 | throw spdlog_ex("rotating_file_sink: failed removing " + filename_to_str(target), errno); 129 | } 130 | } 131 | if (details::file_helper::file_exists(src) && details::os::rename(src, target)) 132 | { 133 | throw spdlog_ex("rotating_file_sink: failed renaming " + filename_to_str(src) + " to " + filename_to_str(target), errno); 134 | } 135 | } 136 | _file_helper.reopen(true); 137 | } 138 | filename_t _base_filename; 139 | std::size_t _max_size; 140 | std::size_t _max_files; 141 | std::size_t _current_size; 142 | details::file_helper _file_helper; 143 | }; 144 | 145 | typedef rotating_file_sink rotating_file_sink_mt; 146 | typedef rotating_file_sinkrotating_file_sink_st; 147 | 148 | /* 149 | * Default generator of daily log file names. 150 | */ 151 | struct default_daily_file_name_calculator 152 | { 153 | // Create filename for the form basename.YYYY-MM-DD_hh-mm 154 | static filename_t calc_filename(const filename_t& basename) 155 | { 156 | std::tm tm = spdlog::details::os::localtime(); 157 | std::conditional::value, fmt::MemoryWriter, fmt::WMemoryWriter>::type w; 158 | w.write(SPDLOG_FILENAME_T("{}_{:04d}-{:02d}-{:02d}_{:02d}-{:02d}"), basename, tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min); 159 | return w.str(); 160 | } 161 | }; 162 | 163 | /* 164 | * Generator of daily log file names in format basename.YYYY-MM-DD 165 | */ 166 | struct dateonly_daily_file_name_calculator 167 | { 168 | // Create filename for the form basename.YYYY-MM-DD 169 | static filename_t calc_filename(const filename_t& basename) 170 | { 171 | std::tm tm = spdlog::details::os::localtime(); 172 | std::conditional::value, fmt::MemoryWriter, fmt::WMemoryWriter>::type w; 173 | w.write(SPDLOG_FILENAME_T("{}_{:04d}-{:02d}-{:02d}"), basename, tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday); 174 | return w.str(); 175 | } 176 | }; 177 | 178 | /* 179 | * Rotating file sink based on date. rotates at midnight 180 | */ 181 | template 182 | class daily_file_sink SPDLOG_FINAL :public base_sink < Mutex > 183 | { 184 | public: 185 | //create daily file sink which rotates on given time 186 | daily_file_sink( 187 | const filename_t& base_filename, 188 | int rotation_hour, 189 | int rotation_minute) : _base_filename(base_filename), 190 | _rotation_h(rotation_hour), 191 | _rotation_m(rotation_minute) 192 | { 193 | if (rotation_hour < 0 || rotation_hour > 23 || rotation_minute < 0 || rotation_minute > 59) 194 | throw spdlog_ex("daily_file_sink: Invalid rotation time in ctor"); 195 | _rotation_tp = _next_rotation_tp(); 196 | _file_helper.open(FileNameCalc::calc_filename(_base_filename)); 197 | } 198 | 199 | 200 | protected: 201 | void _sink_it(const details::log_msg& msg) override 202 | { 203 | if (std::chrono::system_clock::now() >= _rotation_tp) 204 | { 205 | _file_helper.open(FileNameCalc::calc_filename(_base_filename)); 206 | _rotation_tp = _next_rotation_tp(); 207 | } 208 | _file_helper.write(msg); 209 | } 210 | 211 | void _flush() override 212 | { 213 | _file_helper.flush(); 214 | } 215 | 216 | private: 217 | std::chrono::system_clock::time_point _next_rotation_tp() 218 | { 219 | auto now = std::chrono::system_clock::now(); 220 | time_t tnow = std::chrono::system_clock::to_time_t(now); 221 | tm date = spdlog::details::os::localtime(tnow); 222 | date.tm_hour = _rotation_h; 223 | date.tm_min = _rotation_m; 224 | date.tm_sec = 0; 225 | auto rotation_time = std::chrono::system_clock::from_time_t(std::mktime(&date)); 226 | if (rotation_time > now) 227 | return rotation_time; 228 | else 229 | return std::chrono::system_clock::time_point(rotation_time + std::chrono::hours(24)); 230 | } 231 | 232 | filename_t _base_filename; 233 | int _rotation_h; 234 | int _rotation_m; 235 | std::chrono::system_clock::time_point _rotation_tp; 236 | details::file_helper _file_helper; 237 | }; 238 | 239 | typedef daily_file_sink daily_file_sink_mt; 240 | typedef daily_file_sink daily_file_sink_st; 241 | } 242 | } 243 | -------------------------------------------------------------------------------- /include/spdlog/sinks/msvc_sink.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2016 Alexander Dalshov. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | #if defined(_MSC_VER) 9 | 10 | #include "spdlog/sinks/base_sink.h" 11 | #include "spdlog/details/null_mutex.h" 12 | 13 | #include 14 | 15 | #include 16 | #include 17 | 18 | namespace spdlog 19 | { 20 | namespace sinks 21 | { 22 | /* 23 | * MSVC sink (logging using OutputDebugStringA) 24 | */ 25 | template 26 | class msvc_sink : public base_sink < Mutex > 27 | { 28 | public: 29 | explicit msvc_sink() 30 | { 31 | } 32 | 33 | 34 | 35 | protected: 36 | void _sink_it(const details::log_msg& msg) override 37 | { 38 | OutputDebugStringA(msg.formatted.c_str()); 39 | } 40 | 41 | void _flush() override 42 | {} 43 | }; 44 | 45 | typedef msvc_sink msvc_sink_mt; 46 | typedef msvc_sink msvc_sink_st; 47 | 48 | } 49 | } 50 | 51 | #endif 52 | -------------------------------------------------------------------------------- /include/spdlog/sinks/null_sink.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2015 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | #include "spdlog/sinks/base_sink.h" 9 | #include "spdlog/details/null_mutex.h" 10 | 11 | #include 12 | 13 | namespace spdlog 14 | { 15 | namespace sinks 16 | { 17 | 18 | template 19 | class null_sink : public base_sink < Mutex > 20 | { 21 | protected: 22 | void _sink_it(const details::log_msg&) override 23 | {} 24 | 25 | void _flush() override 26 | {} 27 | 28 | }; 29 | typedef null_sink null_sink_st; 30 | typedef null_sink null_sink_mt; 31 | 32 | } 33 | } 34 | 35 | -------------------------------------------------------------------------------- /include/spdlog/sinks/ostream_sink.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2015 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | #include "spdlog/details/null_mutex.h" 9 | #include "spdlog/sinks/base_sink.h" 10 | 11 | #include 12 | #include 13 | 14 | namespace spdlog 15 | { 16 | namespace sinks 17 | { 18 | template 19 | class ostream_sink: public base_sink 20 | { 21 | public: 22 | explicit ostream_sink(std::ostream& os, bool force_flush=false) :_ostream(os), _force_flush(force_flush) {} 23 | ostream_sink(const ostream_sink&) = delete; 24 | ostream_sink& operator=(const ostream_sink&) = delete; 25 | virtual ~ostream_sink() = default; 26 | 27 | protected: 28 | void _sink_it(const details::log_msg& msg) override 29 | { 30 | _ostream.write(msg.formatted.data(), msg.formatted.size()); 31 | if (_force_flush) 32 | _ostream.flush(); 33 | } 34 | 35 | void _flush() override 36 | { 37 | _ostream.flush(); 38 | } 39 | 40 | std::ostream& _ostream; 41 | bool _force_flush; 42 | }; 43 | 44 | typedef ostream_sink ostream_sink_mt; 45 | typedef ostream_sink ostream_sink_st; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /include/spdlog/sinks/sink.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2015 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | 7 | #pragma once 8 | 9 | #include "spdlog/details/log_msg.h" 10 | 11 | namespace spdlog 12 | { 13 | namespace sinks 14 | { 15 | class sink 16 | { 17 | public: 18 | sink() 19 | { 20 | _level = level::trace; 21 | } 22 | 23 | virtual ~sink() {} 24 | virtual void log(const details::log_msg& msg) = 0; 25 | virtual void flush() = 0; 26 | 27 | bool should_log(level::level_enum msg_level) const; 28 | void set_level(level::level_enum log_level); 29 | level::level_enum level() const; 30 | 31 | private: 32 | level_t _level; 33 | 34 | }; 35 | 36 | inline bool sink::should_log(level::level_enum msg_level) const 37 | { 38 | return msg_level >= _level.load(std::memory_order_relaxed); 39 | } 40 | 41 | inline void sink::set_level(level::level_enum log_level) 42 | { 43 | _level.store(log_level); 44 | } 45 | 46 | inline level::level_enum sink::level() const 47 | { 48 | return static_cast(_level.load(std::memory_order_relaxed)); 49 | } 50 | 51 | } 52 | } 53 | 54 | -------------------------------------------------------------------------------- /include/spdlog/sinks/stdout_sinks.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2015 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | #include "spdlog/details/null_mutex.h" 9 | #include "spdlog/sinks/base_sink.h" 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | namespace spdlog 16 | { 17 | namespace sinks 18 | { 19 | 20 | template 21 | class stdout_sink SPDLOG_FINAL : public base_sink 22 | { 23 | using MyType = stdout_sink; 24 | public: 25 | stdout_sink() 26 | {} 27 | static std::shared_ptr instance() 28 | { 29 | static std::shared_ptr instance = std::make_shared(); 30 | return instance; 31 | } 32 | protected: 33 | void _sink_it(const details::log_msg& msg) override 34 | { 35 | fwrite(msg.formatted.data(), sizeof(char), msg.formatted.size(), stdout); 36 | _flush(); 37 | } 38 | 39 | void _flush() override 40 | { 41 | fflush(stdout); 42 | } 43 | }; 44 | 45 | typedef stdout_sink stdout_sink_st; 46 | typedef stdout_sink stdout_sink_mt; 47 | 48 | 49 | template 50 | class stderr_sink SPDLOG_FINAL : public base_sink 51 | { 52 | using MyType = stderr_sink; 53 | public: 54 | stderr_sink() 55 | {} 56 | static std::shared_ptr instance() 57 | { 58 | static std::shared_ptr instance = std::make_shared(); 59 | return instance; 60 | } 61 | protected: 62 | void _sink_it(const details::log_msg& msg) override 63 | { 64 | fwrite(msg.formatted.data(), sizeof(char), msg.formatted.size(), stderr); 65 | _flush(); 66 | } 67 | 68 | void _flush() override 69 | { 70 | fflush(stderr); 71 | } 72 | }; 73 | 74 | typedef stderr_sink stderr_sink_mt; 75 | typedef stderr_sink stderr_sink_st; 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /include/spdlog/sinks/syslog_sink.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2015 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | #include "spdlog/common.h" 9 | 10 | #ifdef SPDLOG_ENABLE_SYSLOG 11 | 12 | #include "spdlog/sinks/sink.h" 13 | #include "spdlog/details/log_msg.h" 14 | 15 | #include 16 | #include 17 | #include 18 | 19 | 20 | namespace spdlog 21 | { 22 | namespace sinks 23 | { 24 | /** 25 | * Sink that write to syslog using the `syscall()` library call. 26 | * 27 | * Locking is not needed, as `syslog()` itself is thread-safe. 28 | */ 29 | class syslog_sink : public sink 30 | { 31 | public: 32 | // 33 | syslog_sink(const std::string& ident = "", int syslog_option=0, int syslog_facility=LOG_USER): 34 | _ident(ident) 35 | { 36 | _priorities[static_cast(level::trace)] = LOG_DEBUG; 37 | _priorities[static_cast(level::debug)] = LOG_DEBUG; 38 | _priorities[static_cast(level::info)] = LOG_INFO; 39 | _priorities[static_cast(level::warn)] = LOG_WARNING; 40 | _priorities[static_cast(level::err)] = LOG_ERR; 41 | _priorities[static_cast(level::critical)] = LOG_CRIT; 42 | _priorities[static_cast(level::off)] = LOG_INFO; 43 | 44 | //set ident to be program name if empty 45 | ::openlog(_ident.empty()? nullptr:_ident.c_str(), syslog_option, syslog_facility); 46 | } 47 | ~syslog_sink() 48 | { 49 | ::closelog(); 50 | } 51 | 52 | syslog_sink(const syslog_sink&) = delete; 53 | syslog_sink& operator=(const syslog_sink&) = delete; 54 | 55 | void log(const details::log_msg &msg) override 56 | { 57 | ::syslog(syslog_prio_from_level(msg), "%s", msg.raw.str().c_str()); 58 | } 59 | 60 | void flush() override 61 | { 62 | } 63 | 64 | 65 | private: 66 | std::array _priorities; 67 | //must store the ident because the man says openlog might use the pointer as is and not a string copy 68 | const std::string _ident; 69 | 70 | // 71 | // Simply maps spdlog's log level to syslog priority level. 72 | // 73 | int syslog_prio_from_level(const details::log_msg &msg) const 74 | { 75 | return _priorities[static_cast(msg.level)]; 76 | } 77 | }; 78 | } 79 | } 80 | 81 | #endif 82 | -------------------------------------------------------------------------------- /include/spdlog/sinks/wincolor_sink.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2016 spdlog 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | #include "spdlog/sinks/base_sink.h" 9 | #include "spdlog/details/null_mutex.h" 10 | #include "spdlog/common.h" 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | namespace spdlog 18 | { 19 | namespace sinks 20 | { 21 | /* 22 | * Windows color console sink. Uses WriteConsoleA to write to the console with colors 23 | */ 24 | template 25 | class wincolor_sink: public base_sink 26 | { 27 | public: 28 | const WORD BOLD = FOREGROUND_INTENSITY; 29 | const WORD RED = FOREGROUND_RED; 30 | const WORD CYAN = FOREGROUND_GREEN | FOREGROUND_BLUE; 31 | const WORD WHITE = FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE; 32 | const WORD YELLOW = FOREGROUND_RED | FOREGROUND_GREEN; 33 | 34 | wincolor_sink(HANDLE std_handle): out_handle_(std_handle) 35 | { 36 | colors_[level::trace] = CYAN; 37 | colors_[level::debug] = CYAN; 38 | colors_[level::info] = WHITE | BOLD; 39 | colors_[level::warn] = YELLOW | BOLD; 40 | colors_[level::err] = RED | BOLD; // red bold 41 | colors_[level::critical] = BACKGROUND_RED | WHITE | BOLD; // white bold on red background 42 | colors_[level::off] = 0; 43 | } 44 | 45 | virtual ~wincolor_sink() 46 | { 47 | this->flush(); 48 | } 49 | 50 | wincolor_sink(const wincolor_sink& other) = delete; 51 | wincolor_sink& operator=(const wincolor_sink& other) = delete; 52 | 53 | protected: 54 | virtual void _sink_it(const details::log_msg& msg) override 55 | { 56 | auto color = colors_[msg.level]; 57 | auto orig_attribs = set_console_attribs(color); 58 | WriteConsoleA(out_handle_, msg.formatted.data(), static_cast(msg.formatted.size()), nullptr, nullptr); 59 | SetConsoleTextAttribute(out_handle_, orig_attribs); //reset to orig colors 60 | } 61 | 62 | virtual void _flush() override 63 | { 64 | // windows console always flushed? 65 | } 66 | 67 | // change the color for the given level 68 | void set_color(level::level_enum level, WORD color) 69 | { 70 | std::lock_guard lock(base_sink::_mutex); 71 | colors_[level] = color; 72 | } 73 | 74 | private: 75 | HANDLE out_handle_; 76 | std::map colors_; 77 | 78 | // set color and return the orig console attributes (for resetting later) 79 | WORD set_console_attribs(WORD attribs) 80 | { 81 | CONSOLE_SCREEN_BUFFER_INFO orig_buffer_info; 82 | GetConsoleScreenBufferInfo(out_handle_, &orig_buffer_info); 83 | WORD back_color = orig_buffer_info.wAttributes; 84 | // retrieve the current background color 85 | back_color &= ~(FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE | FOREGROUND_INTENSITY); 86 | // keep the background color unchanged 87 | SetConsoleTextAttribute(out_handle_, attribs | back_color); 88 | return orig_buffer_info.wAttributes; //return orig attribs 89 | } 90 | }; 91 | 92 | // 93 | // windows color console to stdout 94 | // 95 | template 96 | class wincolor_stdout_sink: public wincolor_sink 97 | { 98 | public: 99 | wincolor_stdout_sink() : wincolor_sink(GetStdHandle(STD_OUTPUT_HANDLE)) 100 | {} 101 | }; 102 | 103 | typedef wincolor_stdout_sink wincolor_stdout_sink_mt; 104 | typedef wincolor_stdout_sink wincolor_stdout_sink_st; 105 | 106 | // 107 | // windows color console to stderr 108 | // 109 | template 110 | class wincolor_stderr_sink: public wincolor_sink 111 | { 112 | public: 113 | wincolor_stderr_sink() : wincolor_sink(GetStdHandle(STD_ERROR_HANDLE)) 114 | {} 115 | }; 116 | 117 | typedef wincolor_stderr_sink wincolor_stderr_sink_mt; 118 | typedef wincolor_stderr_sink wincolor_stderr_sink_st; 119 | 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /include/spdlog/spdlog.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2015 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | // spdlog main header file. 6 | // see example.cpp for usage example 7 | 8 | #pragma once 9 | 10 | #define SPDLOG_VERSION "0.14.0" 11 | 12 | #include "spdlog/tweakme.h" 13 | #include "spdlog/common.h" 14 | #include "spdlog/logger.h" 15 | 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | namespace spdlog 22 | { 23 | 24 | // 25 | // Return an existing logger or nullptr if a logger with such name doesn't exist. 26 | // example: spdlog::get("my_logger")->info("hello {}", "world"); 27 | // 28 | std::shared_ptr get(const std::string& name); 29 | 30 | 31 | // 32 | // Set global formatting 33 | // example: spdlog::set_pattern("%Y-%m-%d %H:%M:%S.%e %l : %v"); 34 | // 35 | void set_pattern(const std::string& format_string); 36 | void set_formatter(formatter_ptr f); 37 | 38 | // 39 | // Set global logging level for 40 | // 41 | void set_level(level::level_enum log_level); 42 | 43 | // 44 | // Set global error handler 45 | // 46 | void set_error_handler(log_err_handler); 47 | 48 | // 49 | // Turn on async mode (off by default) and set the queue size for each async_logger. 50 | // effective only for loggers created after this call. 51 | // queue_size: size of queue (must be power of 2): 52 | // Each logger will pre-allocate a dedicated queue with queue_size entries upon construction. 53 | // 54 | // async_overflow_policy (optional, block_retry by default): 55 | // async_overflow_policy::block_retry - if queue is full, block until queue has room for the new log entry. 56 | // async_overflow_policy::discard_log_msg - never block and discard any new messages when queue overflows. 57 | // 58 | // worker_warmup_cb (optional): 59 | // callback function that will be called in worker thread upon start (can be used to init stuff like thread affinity) 60 | // 61 | // worker_teardown_cb (optional): 62 | // callback function that will be called in worker thread upon exit 63 | // 64 | void set_async_mode(size_t queue_size, const async_overflow_policy overflow_policy = async_overflow_policy::block_retry, const std::function& worker_warmup_cb = nullptr, const std::chrono::milliseconds& flush_interval_ms = std::chrono::milliseconds::zero(), const std::function& worker_teardown_cb = nullptr); 65 | 66 | // Turn off async mode 67 | void set_sync_mode(); 68 | 69 | 70 | // 71 | // Create and register multi/single threaded basic file logger. 72 | // Basic logger simply writes to given file without any limitations or rotations. 73 | // 74 | std::shared_ptr basic_logger_mt(const std::string& logger_name, const filename_t& filename, bool truncate = false); 75 | std::shared_ptr basic_logger_st(const std::string& logger_name, const filename_t& filename, bool truncate = false); 76 | 77 | // 78 | // Create and register multi/single threaded rotating file logger 79 | // 80 | std::shared_ptr rotating_logger_mt(const std::string& logger_name, const filename_t& filename, size_t max_file_size, size_t max_files); 81 | std::shared_ptr rotating_logger_st(const std::string& logger_name, const filename_t& filename, size_t max_file_size, size_t max_files); 82 | 83 | // 84 | // Create file logger which creates new file on the given time (default in midnight): 85 | // 86 | std::shared_ptr daily_logger_mt(const std::string& logger_name, const filename_t& filename, int hour=0, int minute=0); 87 | std::shared_ptr daily_logger_st(const std::string& logger_name, const filename_t& filename, int hour=0, int minute=0); 88 | 89 | // 90 | // Create and register stdout/stderr loggers 91 | // 92 | std::shared_ptr stdout_logger_mt(const std::string& logger_name); 93 | std::shared_ptr stdout_logger_st(const std::string& logger_name); 94 | std::shared_ptr stderr_logger_mt(const std::string& logger_name); 95 | std::shared_ptr stderr_logger_st(const std::string& logger_name); 96 | // 97 | // Create and register colored stdout/stderr loggers 98 | // 99 | std::shared_ptr stdout_color_mt(const std::string& logger_name); 100 | std::shared_ptr stdout_color_st(const std::string& logger_name); 101 | std::shared_ptr stderr_color_mt(const std::string& logger_name); 102 | std::shared_ptr stderr_color_st(const std::string& logger_name); 103 | 104 | 105 | // 106 | // Create and register a syslog logger 107 | // 108 | #ifdef SPDLOG_ENABLE_SYSLOG 109 | std::shared_ptr syslog_logger(const std::string& logger_name, const std::string& ident = "", int syslog_option = 0); 110 | #endif 111 | 112 | #if defined(__ANDROID__) 113 | std::shared_ptr android_logger(const std::string& logger_name, const std::string& tag = "spdlog"); 114 | #endif 115 | 116 | // Create and register a logger with a single sink 117 | std::shared_ptr create(const std::string& logger_name, const sink_ptr& sink); 118 | 119 | // Create and register a logger with multiple sinks 120 | std::shared_ptr create(const std::string& logger_name, sinks_init_list sinks); 121 | template 122 | std::shared_ptr create(const std::string& logger_name, const It& sinks_begin, const It& sinks_end); 123 | 124 | 125 | // Create and register a logger with templated sink type 126 | // Example: 127 | // spdlog::create("mylog", "dailylog_filename"); 128 | template 129 | std::shared_ptr create(const std::string& logger_name, Args...); 130 | 131 | // Create and register an async logger with a single sink 132 | std::shared_ptr create_async(const std::string& logger_name, const sink_ptr& sink, size_t queue_size, const async_overflow_policy overflow_policy = async_overflow_policy::block_retry, const std::function& worker_warmup_cb = nullptr, const std::chrono::milliseconds& flush_interval_ms = std::chrono::milliseconds::zero(), const std::function& worker_teardown_cb = nullptr); 133 | 134 | // Create and register an async logger with multiple sinks 135 | std::shared_ptr create_async(const std::string& logger_name, sinks_init_list sinks, size_t queue_size, const async_overflow_policy overflow_policy = async_overflow_policy::block_retry, const std::function& worker_warmup_cb = nullptr, const std::chrono::milliseconds& flush_interval_ms = std::chrono::milliseconds::zero(), const std::function& worker_teardown_cb = nullptr); 136 | template 137 | std::shared_ptr create_async(const std::string& logger_name, const It& sinks_begin, const It& sinks_end, size_t queue_size, const async_overflow_policy overflow_policy = async_overflow_policy::block_retry, const std::function& worker_warmup_cb = nullptr, const std::chrono::milliseconds& flush_interval_ms = std::chrono::milliseconds::zero(), const std::function& worker_teardown_cb = nullptr); 138 | 139 | // Register the given logger with the given name 140 | void register_logger(std::shared_ptr logger); 141 | 142 | // Apply a user defined function on all registered loggers 143 | // Example: 144 | // spdlog::apply_all([&](std::shared_ptr l) {l->flush();}); 145 | void apply_all(std::function)> fun); 146 | 147 | // Drop the reference to the given logger 148 | void drop(const std::string &name); 149 | 150 | // Drop all references from the registry 151 | void drop_all(); 152 | 153 | 154 | /////////////////////////////////////////////////////////////////////////////// 155 | // 156 | // Trace & Debug can be switched on/off at compile time for zero cost debug statements. 157 | // Uncomment SPDLOG_DEBUG_ON/SPDLOG_TRACE_ON in teakme.h to enable. 158 | // SPDLOG_TRACE(..) will also print current file and line. 159 | // 160 | // Example: 161 | // spdlog::set_level(spdlog::level::trace); 162 | // SPDLOG_TRACE(my_logger, "some trace message"); 163 | // SPDLOG_TRACE(my_logger, "another trace message {} {}", 1, 2); 164 | // SPDLOG_DEBUG(my_logger, "some debug message {} {}", 3, 4); 165 | // SPDLOG_DEBUG_IF(my_logger, true, "some debug message {} {}", 3, 4); 166 | /////////////////////////////////////////////////////////////////////////////// 167 | 168 | #ifdef SPDLOG_TRACE_ON 169 | #define SPDLOG_STR_H(x) #x 170 | #define SPDLOG_STR_HELPER(x) SPDLOG_STR_H(x) 171 | #define SPDLOG_TRACE(logger, ...) logger->trace("[" __FILE__ " line #" SPDLOG_STR_HELPER(__LINE__) "] " __VA_ARGS__) 172 | #define SPDLOG_TRACE_IF(logger, flag, ...) logger->trace_if(flag, "[" __FILE__ " line #" SPDLOG_STR_HELPER(__LINE__) "] " __VA_ARGS__) 173 | #else 174 | #define SPDLOG_TRACE(logger, ...) 175 | #define SPDLOG_TRACE_IF(logger, flag, ...) 176 | #endif 177 | 178 | #ifdef SPDLOG_DEBUG_ON 179 | #define SPDLOG_DEBUG(logger, ...) logger->debug(__VA_ARGS__) 180 | #define SPDLOG_DEBUG_IF(logger, flag, ...) logger->debug_if(flag, __VA_ARGS__) 181 | #else 182 | #define SPDLOG_DEBUG(logger, ...) 183 | #define SPDLOG_DEBUG_IF(logger, flag, ...) 184 | #endif 185 | 186 | } 187 | 188 | 189 | #include "spdlog/details/spdlog_impl.h" 190 | -------------------------------------------------------------------------------- /include/spdlog/tweakme.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright(c) 2015 Gabi Melman. 3 | // Distributed under the MIT License (http://opensource.org/licenses/MIT) 4 | // 5 | 6 | #pragma once 7 | 8 | /////////////////////////////////////////////////////////////////////////////// 9 | // 10 | // Edit this file to squeeze more performance, and to customize supported features 11 | // 12 | /////////////////////////////////////////////////////////////////////////////// 13 | 14 | 15 | /////////////////////////////////////////////////////////////////////////////// 16 | // Under Linux, the much faster CLOCK_REALTIME_COARSE clock can be used. 17 | // This clock is less accurate - can be off by dozens of millis - depending on the kernel HZ. 18 | // Uncomment to use it instead of the regular clock. 19 | // 20 | // #define SPDLOG_CLOCK_COARSE 21 | /////////////////////////////////////////////////////////////////////////////// 22 | 23 | 24 | /////////////////////////////////////////////////////////////////////////////// 25 | // Uncomment if date/time logging is not needed and never appear in the log pattern. 26 | // This will prevent spdlog from quering the clock on each log call. 27 | // 28 | // WARNING: If the log pattern contains any date/time while this flag is on, the result is undefined. 29 | // You must set new pattern(spdlog::set_pattern(..") without any date/time in it 30 | // 31 | // #define SPDLOG_NO_DATETIME 32 | /////////////////////////////////////////////////////////////////////////////// 33 | 34 | 35 | /////////////////////////////////////////////////////////////////////////////// 36 | // Uncomment if thread id logging is not needed (i.e. no %t in the log pattern). 37 | // This will prevent spdlog from quering the thread id on each log call. 38 | // 39 | // WARNING: If the log pattern contains thread id (i.e, %t) while this flag is on, the result is undefined. 40 | // 41 | // #define SPDLOG_NO_THREAD_ID 42 | /////////////////////////////////////////////////////////////////////////////// 43 | 44 | 45 | /////////////////////////////////////////////////////////////////////////////// 46 | // Uncomment if logger name logging is not needed. 47 | // This will prevent spdlog from copying the logger name on each log call. 48 | // 49 | // #define SPDLOG_NO_NAME 50 | /////////////////////////////////////////////////////////////////////////////// 51 | 52 | /////////////////////////////////////////////////////////////////////////////// 53 | // Uncomment to enable the SPDLOG_DEBUG/SPDLOG_TRACE macros. 54 | // 55 | // #define SPDLOG_DEBUG_ON 56 | // #define SPDLOG_TRACE_ON 57 | /////////////////////////////////////////////////////////////////////////////// 58 | 59 | 60 | /////////////////////////////////////////////////////////////////////////////// 61 | // Uncomment to avoid locking in the registry operations (spdlog::get(), spdlog::drop() spdlog::register()). 62 | // Use only if your code never modifes concurrently the registry. 63 | // Note that upon creating a logger the registry is modified by spdlog.. 64 | // 65 | // #define SPDLOG_NO_REGISTRY_MUTEX 66 | /////////////////////////////////////////////////////////////////////////////// 67 | 68 | 69 | /////////////////////////////////////////////////////////////////////////////// 70 | // Uncomment to avoid spdlog's usage of atomic log levels 71 | // Use only if your code never modifies a logger's log levels concurrently by different threads. 72 | // 73 | // #define SPDLOG_NO_ATOMIC_LEVELS 74 | /////////////////////////////////////////////////////////////////////////////// 75 | 76 | 77 | /////////////////////////////////////////////////////////////////////////////// 78 | // Uncomment to enable usage of wchar_t for file names on Windows. 79 | // 80 | // #define SPDLOG_WCHAR_FILENAMES 81 | /////////////////////////////////////////////////////////////////////////////// 82 | 83 | 84 | /////////////////////////////////////////////////////////////////////////////// 85 | // Uncomment to override default eol ("\n" or "\r\n" under Linux/Windows) 86 | // 87 | // #define SPDLOG_EOL ";-)\n" 88 | /////////////////////////////////////////////////////////////////////////////// 89 | 90 | 91 | /////////////////////////////////////////////////////////////////////////////// 92 | // Uncomment to use your own copy of the fmt library instead of spdlog's copy. 93 | // In this case spdlog will try to include so set your -I flag accordingly. 94 | // 95 | // #define SPDLOG_FMT_EXTERNAL 96 | /////////////////////////////////////////////////////////////////////////////// 97 | 98 | 99 | /////////////////////////////////////////////////////////////////////////////// 100 | // Uncomment to enable syslog (disabled by default) 101 | // 102 | // #define SPDLOG_ENABLE_SYSLOG 103 | /////////////////////////////////////////////////////////////////////////////// 104 | 105 | 106 | /////////////////////////////////////////////////////////////////////////////// 107 | // Uncomment to enable wchar_t support (convert to utf8) 108 | // 109 | // #define SPDLOG_WCHAR_TO_UTF8_SUPPORT 110 | /////////////////////////////////////////////////////////////////////////////// 111 | 112 | 113 | /////////////////////////////////////////////////////////////////////////////// 114 | // Uncomment to prevent child processes from inheriting log file descriptors 115 | // 116 | // #define SPDLOG_PREVENT_CHILD_FD 117 | /////////////////////////////////////////////////////////////////////////////// 118 | 119 | 120 | /////////////////////////////////////////////////////////////////////////////// 121 | // Uncomment to mark some types as final, allowing more optimizations in release 122 | // mode with recent compilers. See GCC's documentation for -Wsuggest-final-types 123 | // for instance. 124 | // 125 | // #define SPDLOG_FINAL final 126 | /////////////////////////////////////////////////////////////////////////////// 127 | 128 | 129 | /////////////////////////////////////////////////////////////////////////////// 130 | // Uncomment to enable message counting feature. Adds %i logger pattern that 131 | // prints log message sequence id. 132 | // 133 | // #define SPDLOG_ENABLE_MESSAGE_COUNTER 134 | /////////////////////////////////////////////////////////////////////////////// 135 | 136 | /////////////////////////////////////////////////////////////////////////////// 137 | // Uncomment to enable user defined tag names 138 | // 139 | // #define SPDLOG_LEVEL_NAMES { " TRACE", " DEBUG", " INFO", 140 | // " WARNING", " ERROR", "CRITICAL", "OFF" }; 141 | /////////////////////////////////////////////////////////////////////////////// -------------------------------------------------------------------------------- /include/squeakrconfig.h: -------------------------------------------------------------------------------- 1 | /* 2 | * ============================================================================ 3 | * 4 | * Authors: Prashant Pandey 5 | * Rob Johnson 6 | * Rob Patro (rob.patro@cs.stonybrook.edu) 7 | * 8 | * ============================================================================ 9 | */ 10 | 11 | #ifndef _SQUEAKR_CONFIG_H_ 12 | #define _SQUEAKR_CONFIG_H_ 13 | 14 | #include 15 | 16 | #define VERSION 2 17 | 18 | typedef struct __attribute__ ((__packed__)) squeakrconfig { 19 | uint64_t kmer_size; 20 | uint64_t cutoff; 21 | uint64_t contains_counts; 22 | uint64_t endianness{0x0102030405060708ULL}; 23 | uint32_t version{VERSION}; 24 | } squeakrconfig; 25 | 26 | #endif 27 | -------------------------------------------------------------------------------- /include/util.h: -------------------------------------------------------------------------------- 1 | /* 2 | * ============================================================================ 3 | * 4 | * Authors: Prashant Pandey 5 | * Rob Johnson 6 | * Rob Patro (rob.patro@cs.stonybrook.edu) 7 | * 8 | * ============================================================================ 9 | */ 10 | 11 | #ifndef _UTIL_H_ 12 | #define _UTIL_H_ 13 | 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | #include "spdlog/spdlog.h" 26 | 27 | #ifdef DEBUG 28 | #define PRINT_DEBUG 1 29 | #else 30 | #define PRINT_DEBUG 0 31 | #endif 32 | 33 | #define DEBUG(x) do { \ 34 | if (PRINT_DEBUG) { std::cerr << x << std::endl; } \ 35 | } while (0) 36 | 37 | #define ERROR(x) do { \ 38 | { std::cerr << x << std::endl; } \ 39 | } while (0) 40 | 41 | #define PRINT(x) do { \ 42 | { std::cout << x << std::endl; } \ 43 | } while (0) 44 | 45 | void print_time_elapsed(std::string desc, struct timeval* start, struct 46 | timeval* end, spdlog::logger* console); 47 | 48 | std::string last_part(std::string str, char c); 49 | 50 | #endif 51 | -------------------------------------------------------------------------------- /scripts/lognumslots.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script takes two input arguments: 4 | # 1. The path to the output file of 'ntCard' 5 | # 2. F0 value output by ntCard. 6 | # It then calculates log of the number of slots needed by Squeakr to count 7 | # k-mers. 8 | 9 | F0=$2 10 | f1=`grep -P "\t1\t" $1 | cut -f3- -d$'\t'` 11 | f2=`grep -P "\t2\t" $1 | cut -f3- -d$'\t'` 12 | fgt2=$[ $F0 - $f1 - $f2 ] 13 | ns=$[ $f1 + 2 * $f2 + 3 * $fgt2 ] 14 | echo "x=$ns;l2=l(x)/l(2);s=scale;scale=0;l2ru=l2+1-(l2%1);np2=2^(l2ru/1);scale=s;if(x > (0.9*np2)) { l2ru=l2ru+1; np2=2*np2; }; s=scale; scale=0; print l2ru/1; scale=s;" | bc -l 15 | echo 16 | -------------------------------------------------------------------------------- /scripts/merge_into_develop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ $# -eq 0 ] 4 | then 5 | echo "No input arguments provided. Usage is merge_into_development.sh " 6 | exit 1 7 | fi 8 | 9 | feature=$1 10 | 11 | # from https://stackoverflow.com/questions/173919/is-there-a-theirs-version-of-git-merge-s-ours 12 | # in case branchA is not our current branch 13 | git checkout development 14 | 15 | # make merge commit but without conflicts!! 16 | # the contents of 'ours' will be discarded later 17 | git merge -s ours ${feature} 18 | 19 | # make temporary branch to merged commit 20 | git branch branchTEMP 21 | 22 | # get contents of working tree and index to the one of branchB 23 | git reset --hard ${feature} 24 | 25 | # reset to our merged commit but 26 | # keep contents of working tree and index 27 | git reset --soft branchTEMP 28 | 29 | # change the contents of the merged commit 30 | # with the contents of branchB 31 | git commit --amend 32 | 33 | # get rid off our temporary branch 34 | git branch -D branchTEMP 35 | 36 | # verify that the merge commit contains only contents of branchB 37 | git diff HEAD ${feature} 38 | -------------------------------------------------------------------------------- /scripts/merge_into_master.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ $# -eq 0 ] 4 | then 5 | echo "No input arguments provided. Usage is merge_into_master.sh " 6 | exit 1 7 | fi 8 | 9 | feature=$1 10 | 11 | # from https://stackoverflow.com/questions/173919/is-there-a-theirs-version-of-git-merge-s-ours 12 | # in case branchA is not our current branch 13 | git checkout master 14 | 15 | # make merge commit but without conflicts!! 16 | # the contents of 'ours' will be discarded later 17 | git merge -s ours ${feature} 18 | 19 | # make temporary branch to merged commit 20 | git branch branchTEMP 21 | 22 | # get contents of working tree and index to the one of branchB 23 | git reset --hard ${feature} 24 | 25 | # reset to our merged commit but 26 | # keep contents of working tree and index 27 | git reset --soft branchTEMP 28 | 29 | # change the contents of the merged commit 30 | # with the contents of branchB 31 | git commit --amend 32 | 33 | # get rid off our temporary branch 34 | git branch -D branchTEMP 35 | 36 | # verify that the merge commit contains only contents of branchB 37 | git diff HEAD ${feature} 38 | -------------------------------------------------------------------------------- /src/SqueakrFS.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * ============================================================================ 3 | * 4 | * Authors: Prashant Pandey 5 | * Rob Johnson 6 | * Rob Patro (rob.patro@cs.stonybrook.edu) 7 | * 8 | * ============================================================================ 9 | */ 10 | 11 | #include "SqueakrFS.h" 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | namespace squeakr { 18 | namespace fs { 19 | 20 | bool has_suffix(const std::string& s, const std::string& suffix) 21 | { 22 | return (s.size() >= suffix.size()) && equal(suffix.rbegin(), 23 | suffix.rend(), s.rbegin()); 24 | } 25 | 26 | std::string GetDir(std::string str) { 27 | uint64_t found = str.find_last_of('/'); 28 | return str.substr(0, found); 29 | } 30 | 31 | // Taken from 32 | // http://stackoverflow.com/questions/12774207/fastest-way-to-check-if-a-file-exist-using-standard-c-c11-c 33 | bool FileExists(const char* path) { 34 | struct stat fileStat; 35 | if (stat(path, &fileStat)) { 36 | return false; 37 | } 38 | if (!S_ISREG(fileStat.st_mode)) { 39 | return false; 40 | } 41 | return true; 42 | } 43 | 44 | // Taken from 45 | // http://stackoverflow.com/questions/12774207/fastest-way-to-check-if-a-file-exist-using-standard-c-c11-c 46 | bool DirExists(const char* path) { 47 | struct stat fileStat; 48 | if (stat(path, &fileStat)) { 49 | return false; 50 | } 51 | if (!S_ISDIR(fileStat.st_mode)) { 52 | return false; 53 | } 54 | return true; 55 | } 56 | 57 | void MakeDir(const char* path) { mkdir(path, ACCESSPERMS); } 58 | 59 | std::vector GetFilesExt(const char *dir, const char *ext) { 60 | DIR *folder = opendir(dir); 61 | 62 | if (!folder) { 63 | std::cerr << "Directory doesn't exist " << dir << std::endl; 64 | exit(1); 65 | } 66 | 67 | std::vector ret; 68 | dirent *entry; 69 | while((entry = readdir(folder)) != NULL) 70 | { 71 | if(has_suffix(entry->d_name, ext)) 72 | { 73 | std::string filename(entry->d_name); 74 | std::string dirname(dir); 75 | ret.push_back(std::string(dirname + filename)); 76 | } 77 | } 78 | 79 | return ret; 80 | } 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/gqf/hashutil.c: -------------------------------------------------------------------------------- 1 | /* 2 | * ============================================================================ 3 | * 4 | * Authors: Prashant Pandey 5 | * Rob Johnson 6 | * 7 | * ============================================================================ 8 | */ 9 | 10 | #include "gqf/hashutil.h" 11 | 12 | 13 | 14 | //----------------------------------------------------------------------------- 15 | // MurmurHash2, 64-bit versions, by Austin Appleby 16 | 17 | // The same caveats as 32-bit MurmurHash2 apply here - beware of alignment 18 | // and endian-ness issues if used across multiple platforms. 19 | 20 | 21 | // 64-bit hash for 64-bit platforms 22 | 23 | uint64_t MurmurHash64A ( const void * key, int len, unsigned int seed ) 24 | { 25 | const uint64_t m = 0xc6a4a7935bd1e995; 26 | const int r = 47; 27 | 28 | uint64_t h = seed ^ (len * m); 29 | 30 | const uint64_t * data = (const uint64_t *)key; 31 | const uint64_t * end = data + (len/8); 32 | 33 | while(data != end) 34 | { 35 | uint64_t k = *data++; 36 | 37 | k *= m; 38 | k ^= k >> r; 39 | k *= m; 40 | 41 | h ^= k; 42 | h *= m; 43 | } 44 | 45 | const unsigned char * data2 = (const unsigned char*)data; 46 | 47 | switch(len & 7) 48 | { 49 | case 7: h ^= (uint64_t)data2[6] << 48; 50 | case 6: h ^= (uint64_t)data2[5] << 40; 51 | case 5: h ^= (uint64_t)data2[4] << 32; 52 | case 4: h ^= (uint64_t)data2[3] << 24; 53 | case 3: h ^= (uint64_t)data2[2] << 16; 54 | case 2: h ^= (uint64_t)data2[1] << 8; 55 | case 1: h ^= (uint64_t)data2[0]; 56 | h *= m; 57 | }; 58 | 59 | h ^= h >> r; 60 | h *= m; 61 | h ^= h >> r; 62 | 63 | return h; 64 | } 65 | 66 | 67 | // 64-bit hash for 32-bit platforms 68 | 69 | uint64_t MurmurHash64B ( const void * key, int len, unsigned int seed ) 70 | { 71 | const unsigned int m = 0x5bd1e995; 72 | const int r = 24; 73 | 74 | unsigned int h1 = seed ^ len; 75 | unsigned int h2 = 0; 76 | 77 | const unsigned int * data = (const unsigned int *)key; 78 | 79 | while(len >= 8) 80 | { 81 | unsigned int k1 = *data++; 82 | k1 *= m; k1 ^= k1 >> r; k1 *= m; 83 | h1 *= m; h1 ^= k1; 84 | len -= 4; 85 | 86 | unsigned int k2 = *data++; 87 | k2 *= m; k2 ^= k2 >> r; k2 *= m; 88 | h2 *= m; h2 ^= k2; 89 | len -= 4; 90 | } 91 | 92 | if(len >= 4) 93 | { 94 | unsigned int k1 = *data++; 95 | k1 *= m; k1 ^= k1 >> r; k1 *= m; 96 | h1 *= m; h1 ^= k1; 97 | len -= 4; 98 | } 99 | 100 | switch(len) 101 | { 102 | case 3: h2 ^= ((unsigned char*)data)[2] << 16; 103 | case 2: h2 ^= ((unsigned char*)data)[1] << 8; 104 | case 1: h2 ^= ((unsigned char*)data)[0]; 105 | h2 *= m; 106 | }; 107 | 108 | h1 ^= h2 >> 18; h1 *= m; 109 | h2 ^= h1 >> 22; h2 *= m; 110 | h1 ^= h2 >> 17; h1 *= m; 111 | h2 ^= h1 >> 19; h2 *= m; 112 | 113 | uint64_t h = h1; 114 | 115 | h = (h << 32) | h2; 116 | 117 | return h; 118 | } 119 | 120 | /* 121 | * For any 1 for a snapshot. 131 | 132 | uint64_t hash_64(uint64_t key, uint64_t mask) 133 | { 134 | key = (~key + (key << 21)) & mask; // key = (key << 21) - key - 1; 135 | key = key ^ key >> 24; 136 | key = ((key + (key << 3)) + (key << 8)) & mask; // key * 265 137 | key = key ^ key >> 14; 138 | key = ((key + (key << 2)) + (key << 4)) & mask; // key * 21 139 | key = key ^ key >> 28; 140 | key = (key + (key << 31)) & mask; 141 | return key; 142 | } 143 | 144 | // The inversion of hash_64(). Modified from 145 | // 146 | uint64_t hash_64i(uint64_t key, uint64_t mask) 147 | { 148 | uint64_t tmp; 149 | 150 | // Invert key = key + (key << 31) 151 | tmp = (key - (key << 31)); 152 | key = (key - (tmp << 31)) & mask; 153 | 154 | // Invert key = key ^ (key >> 28) 155 | tmp = key ^ key >> 28; 156 | key = key ^ tmp >> 28; 157 | 158 | // Invert key *= 21 159 | key = (key * 14933078535860113213ull) & mask; 160 | 161 | // Invert key = key ^ (key >> 14) 162 | tmp = key ^ key >> 14; 163 | tmp = key ^ tmp >> 14; 164 | tmp = key ^ tmp >> 14; 165 | key = key ^ tmp >> 14; 166 | 167 | // Invert key *= 265 168 | key = (key * 15244667743933553977ull) & mask; 169 | 170 | // Invert key = key ^ (key >> 24) 171 | tmp = key ^ key >> 24; 172 | key = key ^ tmp >> 24; 173 | 174 | // Invert key = (~key) + (key << 21) 175 | tmp = ~key; 176 | tmp = ~(key - (tmp << 21)); 177 | tmp = ~(key - (tmp << 21)); 178 | key = ~(key - (tmp << 21)) & mask; 179 | 180 | return key; 181 | } 182 | 183 | -------------------------------------------------------------------------------- /src/gqf/partitioned_counter.c: -------------------------------------------------------------------------------- 1 | /* 2 | * ============================================================================ 3 | * 4 | * Author: Prashant Pandey (), ppandey@cs.stonybrook.edu 5 | * Organization: Stony Brook University 6 | * 7 | * ============================================================================ 8 | */ 9 | 10 | /*#define _GNU_SOURCE*/ 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include "gqf/partitioned_counter.h" 21 | 22 | #define min(a,b) ((a) < (b) ? (a) : (b)) 23 | 24 | int pc_init(pc_t *pc, int64_t *global_counter, uint32_t num_counters, 25 | int32_t threshold) { 26 | uint32_t num_cpus = (int)sysconf( _SC_NPROCESSORS_ONLN ); 27 | if (num_cpus < 0) { 28 | perror( "sysconf" ); 29 | return PC_ERROR; 30 | } 31 | pc->num_counters = num_counters == 0 ? num_cpus : min(num_cpus, 32 | num_counters); 33 | 34 | pc->local_counters = (lctr_t *)calloc(pc->num_counters, 35 | sizeof(*pc->local_counters)); 36 | if (pc->local_counters == NULL) { 37 | perror("Couldn't allocate memory for local counters."); 38 | return PC_ERROR; 39 | } 40 | /*printf("Padding check: 0: %p 1: %p\n", (void*)&pc->local_counters[0],*/ 41 | /*(void*)&pc->local_counters[1]);*/ 42 | pc->global_counter = global_counter; 43 | pc->threshold = threshold; 44 | 45 | return 0; 46 | } 47 | 48 | void pc_destructor(pc_t *pc) 49 | { 50 | pc_sync(pc); 51 | lctr_t *lc = pc->local_counters; 52 | pc->local_counters = NULL; 53 | free(lc); 54 | } 55 | 56 | void pc_add(pc_t *pc, int64_t count) { 57 | int cpuid = sched_getcpu(); 58 | uint32_t counter_id = cpuid % pc->num_counters; 59 | int64_t cur_count = 60 | __atomic_add_fetch(&pc->local_counters[counter_id].counter, count, 61 | __ATOMIC_SEQ_CST); 62 | if (cur_count > pc->threshold || cur_count < -pc->threshold) { 63 | int64_t new_count = 64 | __atomic_exchange_n(&pc->local_counters[counter_id].counter, 0, 65 | __ATOMIC_SEQ_CST); 66 | __atomic_fetch_add(pc->global_counter, new_count, __ATOMIC_SEQ_CST); 67 | } 68 | } 69 | 70 | void pc_sync(pc_t *pc) { 71 | for (uint32_t i = 0; i < pc->num_counters; i++) { 72 | int64_t c = __atomic_exchange_n(&pc->local_counters[i].counter, 0, 73 | __ATOMIC_SEQ_CST); 74 | __atomic_fetch_add(pc->global_counter, c, __ATOMIC_SEQ_CST); 75 | } 76 | } 77 | 78 | -------------------------------------------------------------------------------- /src/info.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * ============================================================================ 3 | * 4 | * Authors: Prashant Pandey 5 | * Rob Johnson 6 | * Rob Patro (rob.patro@cs.stonybrook.edu) 7 | * 8 | * ============================================================================ 9 | */ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | #include "clipp.h" 28 | #include "ProgOpts.h" 29 | #include "squeakrconfig.h" 30 | #include "gqf_cpp.h" 31 | #include "kmer.h" 32 | 33 | /* 34 | * === FUNCTION ============================================================== 35 | * Name: main 36 | * Description: 37 | * ============================================================================= 38 | */ 39 | int info_main(InfoOpts& opts) 40 | { 41 | spdlog::logger* console = opts.console.get(); 42 | 43 | //Initialize the QF 44 | console->info("Reading squeakr file off the disk."); 45 | CQF cqf(opts.squeakr_file, MMAP); 46 | 47 | // seek to the end of the file and read the k-mer size 48 | std::ifstream squeakr_file(opts.squeakr_file, std::ofstream::in); 49 | squeakr_file.seekg(0, squeakr_file.end); 50 | uint64_t file_size = squeakr_file.tellg(); 51 | squeakrconfig config; 52 | squeakr_file.seekg(file_size - sizeof(squeakrconfig)); 53 | squeakr_file.read((char*)&config, sizeof(config)); 54 | squeakr_file.close(); 55 | if (config.version != VERSION) { 56 | console->error("Squeakr index version is invalid. Expected: {} Available: {}", 57 | VERSION, config.version); 58 | exit(1); 59 | } 60 | 61 | std::string contains_counts = config.contains_counts ? "YES" : "NO"; 62 | std::string exact = cqf.is_exact() ? "YES" : "NO"; 63 | 64 | console->info("version: {}", config.version); 65 | console->info("kmer size: {}", config.kmer_size); 66 | console->info("Cutoff: {}", config.cutoff); 67 | console->info("Contains Counts: {}", contains_counts); 68 | console->info("CQF info....."); 69 | console->info("Is Exact: {}", exact); 70 | console->info("Total slots: {}", cqf.numslots()); 71 | console->info("Seed: {}", cqf.seed()); 72 | console->info("Keybits: {}", cqf.keybits()); 73 | console->info("Total elements: {}", cqf.total_elts()); 74 | console->info("Distinct elements: {}", cqf.dist_elts()); 75 | 76 | return EXIT_SUCCESS; 77 | } /* ---------- end of function main ---------- */ 78 | -------------------------------------------------------------------------------- /src/innerprod.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * ============================================================================ 3 | * 4 | * Authors: Prashant Pandey 5 | * Rob Johnson 6 | * Rob Patro (rob.patro@cs.stonybrook.edu) 7 | * 8 | * ============================================================================ 9 | */ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | #include "clipp.h" 27 | #include "ProgOpts.h" 28 | #include "gqf_cpp.h" 29 | 30 | /* 31 | * === FUNCTION ====================================================================== 32 | * Name: main 33 | * Description: 34 | * ===================================================================================== 35 | */ 36 | int inner_prod_main(InnerProdOpts& opts) 37 | { 38 | uint64_t inner_prod; 39 | struct timeval start, end; 40 | struct timezone tzp; 41 | 42 | srand(time(NULL)); 43 | spdlog::logger* console = opts.console.get(); 44 | 45 | //Initialize the QF 46 | console->info("Mmaping Squeakr files from disk."); 47 | CQF cfa(opts.squeakr_filea, FREAD); 48 | CQF cfb(opts.squeakr_fileb, FREAD); 49 | 50 | if (cfa.seed() != cfb.seed()) { 51 | console->error("Input CQFs do not have the same seed."); 52 | return 1; 53 | } 54 | 55 | console->info("Performing inner product querries."); 56 | 57 | gettimeofday(&start, &tzp); 58 | inner_prod = cfa.inner_prod(cfb); 59 | gettimeofday(&end, &tzp); 60 | console->info("Inner product: {}", inner_prod); 61 | print_time_elapsed("", &start, &end, console); 62 | 63 | return EXIT_SUCCESS; 64 | } /* ---------- end of function main ---------- */ 65 | 66 | -------------------------------------------------------------------------------- /src/kmer.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * ============================================================================ 3 | * 4 | * Authors: Prashant Pandey 5 | * Rob Johnson 6 | * Rob Patro (rob.patro@cs.stonybrook.edu) 7 | * 8 | * ============================================================================ 9 | */ 10 | 11 | #include 12 | #include "kmer.h" 13 | #include 14 | #include 15 | 16 | /*return the integer representation of the base */ 17 | char Kmer::map_int(uint8_t base) 18 | { 19 | switch(base) { 20 | case DNA_MAP::A: { return 'A'; } 21 | case DNA_MAP::T: { return 'T'; } 22 | case DNA_MAP::C: { return 'C'; } 23 | case DNA_MAP::G: { return 'G'; } 24 | default: { return DNA_MAP::G+1; } 25 | } 26 | } 27 | 28 | /*return the integer representation of the base */ 29 | uint8_t Kmer::map_base(char base) 30 | { 31 | switch(base) { 32 | case 'A': { return DNA_MAP::A; } 33 | case 'T': { return DNA_MAP::T; } 34 | case 'C': { return DNA_MAP::C; } 35 | case 'G': { return DNA_MAP::G; } 36 | default: { return DNA_MAP::G+1; } 37 | } 38 | } 39 | 40 | /** 41 | * Converts a string of "ATCG" to a uint64_t 42 | * where each character is represented by using only two bits 43 | */ 44 | __int128_t Kmer::str_to_int(std::string str) 45 | { 46 | __int128_t strint = 0; 47 | for (auto it = str.begin(); it != str.end(); it++) { 48 | uint8_t curr = 0; 49 | switch (*it) { 50 | case 'A': { curr = DNA_MAP::A; break; } 51 | case 'T': { curr = DNA_MAP::T; break; } 52 | case 'C': { curr = DNA_MAP::C; break; } 53 | case 'G': { curr = DNA_MAP::G; break; } 54 | } 55 | strint = strint | curr; 56 | strint = strint << 2; 57 | } 58 | return strint >> 2; 59 | } 60 | 61 | /** 62 | * Converts a uint64_t to a string of "ACTG" 63 | * where each character is represented by using only two bits 64 | */ 65 | std::string Kmer::int_to_str(__int128_t kmer, uint64_t kmer_size) 66 | { 67 | uint8_t base; 68 | std::string str; 69 | for (uint32_t i = kmer_size; i > 0; i--) { 70 | base = (kmer >> (i*2-2)) & 3ULL; 71 | char chr = Kmer::map_int(base); 72 | str.push_back(chr); 73 | } 74 | return str; 75 | } 76 | 77 | /* Return the reverse complement of a base */ 78 | int Kmer::reverse_complement_base(int x) { return 3 - x; } 79 | 80 | /* Calculate the revsese complement of a kmer */ 81 | __int128_t Kmer::reverse_complement(__int128_t kmer, uint64_t kmer_size) 82 | { 83 | __int128_t rc = 0; 84 | uint8_t base = 0; 85 | for (uint32_t i = 0; i < kmer_size; i++) { 86 | base = kmer & 3ULL; 87 | base = reverse_complement_base(base); 88 | kmer >>= 2; 89 | rc |= base; 90 | rc <<= 2; 91 | } 92 | rc >>=2; 93 | return rc; 94 | } 95 | 96 | /* Compare the kmer and its reverse complement and return the result 97 | * Return true if the kmer is greater than or equal to its 98 | * reverse complement. 99 | * */ 100 | bool Kmer::compare_kmers(__int128_t kmer, __int128_t kmer_rev) 101 | { 102 | return kmer >= kmer_rev; 103 | } 104 | 105 | void Kmer::parse_kmers(const char *filename, uint64_t kmer_size, 106 | std::unordered_set& kmerset) { 107 | std::ifstream ipfile(filename); 108 | std::string read; 109 | while (ipfile >> read) { 110 | 111 | start_read: 112 | if (read.length() < kmer_size) 113 | continue; 114 | { 115 | uint64_t first = 0; 116 | uint64_t first_rev = 0; 117 | uint64_t item = 0; 118 | bool flag = false; 119 | for(uint32_t i = 0; i < kmer_size; i++) { //First kmer 120 | uint8_t curr = Kmer::map_base(read[i]); 121 | if (curr > DNA_MAP::G) { // 'N' is encountered 122 | if (i + 1 < read.length()) 123 | read = read.substr(i + 1, read.length()); 124 | else { 125 | flag = true; 126 | break; 127 | } 128 | goto start_read; 129 | } 130 | first = first | curr; 131 | first = first << 2; 132 | } 133 | if (flag) continue; 134 | first = first >> 2; 135 | first_rev = Kmer::reverse_complement(first, kmer_size); 136 | 137 | if (Kmer::compare_kmers(first, first_rev)) 138 | item = first; 139 | else 140 | item = first_rev; 141 | 142 | kmerset.insert(item); 143 | 144 | uint64_t next = (first << 2) & BITMASK(2*kmer_size); 145 | uint64_t next_rev = first_rev >> 2; 146 | 147 | for(uint32_t i=kmer_size; i DNA_MAP::G) { // 'N' is encountered 150 | if (i + 1 < read.length()) 151 | read = read.substr(i + 1, read.length()); 152 | else { 153 | flag = true; 154 | break; 155 | } 156 | goto start_read; 157 | } 158 | if (flag) continue; 159 | next |= curr; 160 | uint64_t tmp = Kmer::reverse_complement_base(curr); 161 | tmp <<= (kmer_size*2-2); 162 | next_rev = next_rev | tmp; 163 | if (Kmer::compare_kmers(next, next_rev)) 164 | item = next; 165 | else 166 | item = next_rev; 167 | 168 | kmerset.insert(item); 169 | 170 | next = (next << 2) & BITMASK(2*kmer_size); 171 | next_rev = next_rev >> 2; 172 | } 173 | } 174 | } 175 | } 176 | -------------------------------------------------------------------------------- /src/list.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * ============================================================================ 3 | * 4 | * Authors: Prashant Pandey 5 | * Rob Johnson 6 | * Rob Patro (rob.patro@cs.stonybrook.edu) 7 | * 8 | * ============================================================================ 9 | */ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | #include "clipp.h" 28 | #include "ProgOpts.h" 29 | #include "gqf_cpp.h" 30 | #include "kmer.h" 31 | 32 | /* 33 | * === FUNCTION ============================================================== 34 | * Name: main 35 | * Description: 36 | * ============================================================================= 37 | */ 38 | int list_main(ListOpts& opts) 39 | { 40 | struct timeval start, end; 41 | struct timezone tzp; 42 | spdlog::logger* console = opts.console.get(); 43 | 44 | //Initialize the QF 45 | console->info("Reading kmers into the QF off the disk."); 46 | CQF cqf(opts.squeakr_file, FREAD); 47 | 48 | if (!cqf.is_exact()) { 49 | console->error("This is not a Squeakr exact representation."); 50 | return 1; 51 | } 52 | 53 | uint64_t kmer_size = cqf.keybits() / 2; 54 | std::ofstream opfile(opts.output_file.c_str(), std::ofstream::out); 55 | 56 | CQF::Iterator it = cqf.begin(); 57 | gettimeofday(&start, &tzp); 58 | do { 59 | KeyObject k = *it; 60 | opfile << Kmer::int_to_str(k.key, kmer_size) << "\t" << k.count << std::endl; 61 | ++it; 62 | } while (!it.done()); 63 | gettimeofday(&end, &tzp); 64 | opfile.close(); 65 | print_time_elapsed("", &start, &end, console); 66 | 67 | 68 | return EXIT_SUCCESS; 69 | } /* ---------- end of function main ---------- */ 70 | -------------------------------------------------------------------------------- /src/query.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * ============================================================================ 3 | * 4 | * Authors: Prashant Pandey 5 | * Rob Johnson 6 | * Rob Patro (rob.patro@cs.stonybrook.edu) 7 | * 8 | * ============================================================================ 9 | */ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | 34 | #include "clipp.h" 35 | #include "ProgOpts.h" 36 | #include "gqf_cpp.h" 37 | #include "kmer.h" 38 | #include "squeakrconfig.h" 39 | 40 | /* 41 | * === FUNCTION ============================================================= 42 | * Name: main 43 | * Description: 44 | * ============================================================================ 45 | */ 46 | int query_main(QueryOpts& opts) 47 | { 48 | struct timeval start, end; 49 | struct timezone tzp; 50 | std::unordered_set kmers; 51 | 52 | srand(time(NULL)); 53 | spdlog::logger* console = opts.console.get(); 54 | 55 | //Initialize the QF 56 | console->info("Reading kmers into the QF off the disk."); 57 | CQF cqf(opts.squeakr_file, FREAD); 58 | 59 | // seek to the end of the file and read the k-mer size 60 | std::ifstream squeakr_file(opts.squeakr_file, std::ofstream::in); 61 | squeakr_file.seekg(0, squeakr_file.end); 62 | uint64_t file_size = squeakr_file.tellg(); 63 | squeakrconfig config; 64 | squeakr_file.seekg(file_size - sizeof(squeakrconfig)); 65 | squeakr_file.read((char*)&config, sizeof(config)); 66 | squeakr_file.close(); 67 | if (config.version != VERSION) { 68 | console->error("Squeakr index version is invalid. Expected: {} Available: {}", 69 | VERSION, config.version); 70 | exit(1); 71 | } 72 | console->info("kmer size: {}, version: {}", config.kmer_size, config.version); 73 | 74 | if (cqf.is_exact() && config.kmer_size*2 != cqf.keybits()) { 75 | console->error("K-mer size is not correct."); 76 | return 1; 77 | } 78 | 79 | console->info("Parsing query file for {}-mers.", config.kmer_size); 80 | Kmer::parse_kmers(opts.queryfile.c_str(), config.kmer_size, kmers); 81 | console->info("Found {} k-mers", kmers.size()); 82 | 83 | std::ofstream opfile(opts.output_file.c_str(), std::ofstream::out); 84 | 85 | console->info("Querying kmers in the QF."); 86 | uint64_t num_not_found = 0; 87 | gettimeofday(&start, &tzp); 88 | for (auto it = kmers.begin(); it != kmers.end(); ++it) { 89 | uint64_t count = cqf.query(KeyObject(*it, 0, 0), 0); 90 | if (count == 0) { 91 | console->error(Kmer::int_to_str(*it, config.kmer_size)); 92 | num_not_found++; 93 | } 94 | else 95 | opfile << Kmer::int_to_str(*it, config.kmer_size) << "\t" << count << std::endl; 96 | } 97 | gettimeofday(&end, &tzp); 98 | opfile.close(); 99 | print_time_elapsed("", &start, &end, console); 100 | 101 | console->info("Not found: {}", num_not_found); 102 | 103 | return EXIT_SUCCESS; 104 | } /* ---------- end of function main ---------- */ 105 | 106 | -------------------------------------------------------------------------------- /src/squeakr.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * ============================================================================ 3 | * 4 | * Authors: Prashant Pandey 5 | * Rob Johnson 6 | * Rob Patro (rob.patro@cs.stonybrook.edu) 7 | * 8 | * ============================================================================ 9 | */ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include "ProgOpts.h" 18 | #include "clipp.h" 19 | #include "SqueakrFS.h" 20 | 21 | template 22 | void explore_options_verbose(T& res) { 23 | if(res.any_error()) { std::cerr << "error\n"; } 24 | 25 | //aggregated errors 26 | if(res.unmapped_args_count()) { std::cerr << "error unmapped args count\n"; /* ... */ } 27 | if(res.any_bad_repeat()) { std::cerr << "error bad repeat \n"; /* ... */ } 28 | if(res.any_blocked()) { std::cerr << "error blocked \n"; /* ... */ } 29 | if(res.any_conflict()) { std::cerr << "error conflict\n"; /* ... */ } 30 | 31 | for(const auto& m : res.missing()) { 32 | std::cerr << "missing " << m.param() << " after index " << m.after_index() << '\n'; 33 | } 34 | 35 | //per-argument mapping 36 | for(const auto& m : res) { 37 | std::cerr << m.index() << ": " << m.arg() << " -> " << m.param(); 38 | std::cerr << " repeat #" << m.repeat(); 39 | if(m.blocked()) std::cerr << " blocked"; 40 | if(m.conflict()) std::cerr << " conflict"; 41 | std::cerr<< '\n'; 42 | } 43 | } 44 | 45 | int query_main (QueryOpts& opt); 46 | int count_main (CountOpts& opt); 47 | int inner_prod_main (InnerProdOpts& opt); 48 | int list_main (ListOpts& opt); 49 | int info_main (InfoOpts& opt); 50 | 51 | /* 52 | * === FUNCTION ============================================================= 53 | * Name: main 54 | * Description: 55 | * ============================================================================ 56 | */ 57 | int main ( int argc, char *argv[] ) { 58 | using namespace clipp; 59 | enum class mode {count, query, inner_prod, list, info, help}; 60 | mode selected = mode::help; 61 | 62 | auto console = spdlog::stdout_color_mt("squeakr_console"); 63 | 64 | CountOpts countopt; 65 | QueryOpts queryopt; 66 | InnerProdOpts innerprodopt; 67 | ListOpts listopt; 68 | InfoOpts infoopt; 69 | countopt.console = console; 70 | queryopt.console = console; 71 | innerprodopt.console = console; 72 | listopt.console = console; 73 | infoopt.console = console; 74 | 75 | auto ensure_file_exists = [](const std::string& s) -> bool { 76 | bool exists = squeakr::fs::FileExists(s.c_str()); 77 | if (!exists) { 78 | std::string e = "The required input file " + s + " does not seem to exist."; 79 | throw std::runtime_error{e}; 80 | } 81 | return true; 82 | }; 83 | 84 | auto ensure_parent_dir_exists = [](const std::string& s) -> bool { 85 | std::string parent_dir = squeakr::fs::GetDir(s); 86 | bool exists = squeakr::fs::DirExists(parent_dir.c_str()); 87 | if (!exists) { 88 | std::string e = "The required input directory " + parent_dir + " does not seem to exist."; 89 | throw std::runtime_error{e}; 90 | } 91 | return true; 92 | }; 93 | 94 | auto enusure_size_is_specified = [](const CountOpts countopt) -> bool { 95 | if (!countopt.setqbits && countopt.numthreads == 0) { 96 | std::string e = "Size option is required if the thread count is greater than 1."; 97 | throw std::runtime_error{e}; 98 | } 99 | return true; 100 | }; 101 | 102 | auto count_mode = ( 103 | command("count").set(selected, mode::count), 104 | option("-e", "--exact").set(countopt.exact, 1) % 105 | "squeakr-exact (default is Squeakr approximate)", 106 | required("-k","--kmer") & value("k-size", countopt.ksize) % 107 | "length of k-mers to count", 108 | option("-c","--cutoff") & value("cutoff", countopt.cutoff) % 109 | "only output k-mers with count greater than or equal to cutoff (default = 1)", 110 | option("-n","--no-counts").set(countopt.contains_counts, 0) % 111 | "only output k-mers and no counts (default = false)", 112 | option("-s","--log-slots").set(countopt.setqbits, true) & 113 | value("log-slots", countopt.qbits) % "log of number of slots in the CQF. (Size argument is only optional when numthreads is exactly 1.)", 114 | option("-t","--threads") & value("num-threads", 115 | countopt.numthreads) % 116 | "number of threads to use to count (default = number of hardware threads)", 117 | required("-o","--output-file") & 118 | value(ensure_parent_dir_exists, "out-file", 119 | countopt.output_file) % 120 | "file in which output should be written", 121 | values(ensure_file_exists, "files", countopt.filenames) % "list of files to be counted (supported files: fastq and compressed gzip or bzip2 fastq files)" 122 | //option("-h", "--help") % "show help" 123 | ); 124 | 125 | auto query_mode = ( 126 | command("query").set(selected, mode::query), 127 | required("-f", "--squeakr-file") & value(ensure_file_exists, "squeakr-file", 128 | queryopt.squeakr_file) % "input squeakr file", 129 | required("-q","--query-file") & value(ensure_file_exists, "query-file", 130 | queryopt.queryfile) % "input query file", 131 | required("-o", "--output-file") & 132 | value(ensure_parent_dir_exists, "output-file", 133 | queryopt.output_file) % "output file" 134 | //option("-h", "--help") % "show help" 135 | ); 136 | 137 | auto inner_prod_mode = ( 138 | command("inner_prod").set(selected, mode::inner_prod), 139 | value(ensure_file_exists, "first-input", 140 | innerprodopt.squeakr_filea) 141 | % "first input squeakr file", 142 | value(ensure_file_exists, "second-input", 143 | innerprodopt.squeakr_fileb) 144 | % "second input squeakr file" 145 | //option("-h", "--help") % "show help" 146 | ); 147 | 148 | auto list_mode = ( 149 | command("list").set(selected, mode::list), 150 | required("-f", "--squeakr-file-file") & value(ensure_file_exists, "squeakr-file", 151 | listopt.squeakr_file) % "input squeakr file", 152 | required("-o", "--output-file") & 153 | value(ensure_parent_dir_exists, "output-file", 154 | listopt.output_file) % "output file" 155 | //option("-h", "--help") % "show help" 156 | ); 157 | 158 | auto info_mode = ( 159 | command("info").set(selected, mode::info), 160 | required("-f", "--squeakr-file-file") & value(ensure_file_exists, "squeakr-file", 161 | infoopt.squeakr_file) % "input squeakr file" 162 | //option("-h", "--help") % "show help" 163 | ); 164 | 165 | auto cli = ( 166 | (count_mode | query_mode | inner_prod_mode | list_mode | 167 | info_mode | 168 | command("help").set(selected,mode::help) ), 169 | option("-v", "--version").call([]{std::cout << "version 0.7\n\n";}).doc("show version") 170 | ); 171 | 172 | assert(count_mode.flags_are_prefix_free()); 173 | assert(query_mode.flags_are_prefix_free()); 174 | assert(inner_prod_mode.flags_are_prefix_free()); 175 | assert(list_mode.flags_are_prefix_free()); 176 | assert(info_mode.flags_are_prefix_free()); 177 | 178 | decltype(parse(argc, argv, cli)) res; 179 | try { 180 | res = parse(argc, argv, cli); 181 | } catch (std::exception& e) { 182 | std::cout << "\n\nParsing command line failed with exception: " << 183 | e.what() << "\n"; 184 | std::cout << "\n\n"; 185 | std::cout << make_man_page(cli, "squeakr"); 186 | return 1; 187 | } 188 | 189 | if(res) { 190 | switch(selected) { 191 | case mode::count: 192 | try { 193 | enusure_size_is_specified(countopt); 194 | } catch (std::exception& e) { 195 | std::cout << "\n\nParsing command line failed with exception: " << 196 | e.what() << "\n"; 197 | std::cout << "\n\n"; 198 | std::cout << make_man_page(cli, "squeakr"); 199 | return 1; 200 | } 201 | count_main(countopt); 202 | break; 203 | case mode::query: query_main(queryopt); break; 204 | case mode::inner_prod: inner_prod_main(innerprodopt); break; 205 | case mode::list: list_main(listopt); break; 206 | case mode::info: info_main(infoopt); break; 207 | case mode::help: break; 208 | } 209 | } else { 210 | auto b = res.begin(); 211 | auto e = res.end(); 212 | if (std::distance(b,e) > 0) { 213 | if (b->arg() == "count") { 214 | std::cout << make_man_page(count_mode, "squeakr"); 215 | } else if (b->arg() == "query") { 216 | std::cout << make_man_page(query_mode, "squeakr"); 217 | } else if (b->arg() == "inner_prod") { 218 | std::cout << make_man_page(inner_prod_mode, "squeakr"); 219 | } else if (b->arg() == "list") { 220 | std::cout << make_man_page(list_mode, "squeakr"); 221 | } else if (b->arg() == "info") { 222 | std::cout << make_man_page(info_mode, "squeakr"); 223 | } else { 224 | std::cout << "There is no command \"" << b->arg() << "\"\n"; 225 | std::cout << usage_lines(cli, "squeakr") << '\n'; 226 | } 227 | } else { 228 | std::cout << usage_lines(cli, "squeakr") << '\n'; 229 | } 230 | } 231 | 232 | return 0; 233 | } 234 | -------------------------------------------------------------------------------- /src/util.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * ============================================================================ 3 | * 4 | * Authors: Prashant Pandey 5 | * Rob Johnson 6 | * Rob Patro (rob.patro@cs.stonybrook.edu) 7 | * 8 | * ============================================================================ 9 | */ 10 | 11 | #include "util.h" 12 | 13 | /* Count distinct items in a sorted list */ 14 | uint64_t count_distinct_kmers(std::multiset<__int128_t> kmers) 15 | { 16 | uint64_t cnt = 0; 17 | __int128_t curr_kmer = 0; 18 | 19 | for(__int128_t kmer: kmers) { 20 | if (kmer != curr_kmer) { 21 | curr_kmer = kmer; 22 | cnt++; 23 | } 24 | } 25 | return cnt; 26 | } 27 | 28 | /* Print elapsed time using the start and end timeval */ 29 | void print_time_elapsed(std::string desc, struct timeval* start, struct 30 | timeval* end, spdlog::logger* console) 31 | { 32 | struct timeval elapsed; 33 | if (start->tv_usec > end->tv_usec) { 34 | end->tv_usec += 1000000; 35 | end->tv_sec--; 36 | } 37 | elapsed.tv_usec = end->tv_usec - start->tv_usec; 38 | elapsed.tv_sec = end->tv_sec - start->tv_sec; 39 | float time_elapsed = (elapsed.tv_sec * 1000000 + elapsed.tv_usec)/1000000.f; 40 | console->info("{} Total Time Elapsed: {} seconds", desc, 41 | std::to_string(time_elapsed)); 42 | } 43 | 44 | std::string last_part(std::string str, char c) { 45 | uint64_t found = str.find_last_of(c); 46 | return str.substr(found + 1); 47 | } 48 | 49 | // A=1, C=0, T=2, G=3 50 | void getRandomKmers(int n, uint64_t range, std::vector& kmers, 51 | uint32_t K) 52 | { 53 | uint64_t kmer; 54 | for (int j = 0; j < n; j++) { 55 | kmer = 0; 56 | for (uint i = 0; i < K; i++) { 57 | uint8_t c = rand()%4; 58 | kmer = kmer | c; 59 | kmer = kmer << 2; 60 | } 61 | kmer = kmer >> 2; 62 | kmers.push_back(kmer%range); 63 | } 64 | } 65 | 66 | --------------------------------------------------------------------------------