├── .autotools ├── .codecov.yml ├── .cproject ├── .gitattributes ├── .gitignore ├── .idea ├── .gitignore ├── .name ├── codeStyleSettings.xml ├── codeStyles │ ├── Project.xml │ └── codeStyleConfig.xml ├── encodings.xml ├── misc.xml ├── modules.xml ├── vcs.xml └── zindex.iml ├── .project ├── .settings ├── language.settings.xml ├── org.eclipse.cdt.codan.core.prefs └── org.eclipse.cdt.managedbuilder.core.prefs ├── .travis.yml ├── AUTHORS ├── CMakeLists.txt ├── LICENSE ├── Makefile ├── README.md ├── ext ├── cJSON │ ├── cJSON.c │ └── cJSON.h ├── sqlite │ ├── shell.c │ ├── sqlite3.c │ ├── sqlite3.h │ └── sqlite3ext.h └── tclap │ ├── Arg.h │ ├── ArgException.h │ ├── ArgTraits.h │ ├── CmdLine.h │ ├── CmdLineInterface.h │ ├── CmdLineOutput.h │ ├── Constraint.h │ ├── DocBookOutput.h │ ├── HelpVisitor.h │ ├── IgnoreRestVisitor.h │ ├── MultiArg.h │ ├── MultiSwitchArg.h │ ├── OptionalUnlabeledTracker.h │ ├── StandardTraits.h │ ├── StdOutput.h │ ├── SwitchArg.h │ ├── UnlabeledMultiArg.h │ ├── UnlabeledValueArg.h │ ├── ValueArg.h │ ├── ValuesConstraint.h │ ├── VersionVisitor.h │ ├── Visitor.h │ ├── XorHandler.h │ └── ZshCompletionOutput.h ├── scripts ├── .gitignore ├── build-lto-cloudflare-zlib.sh ├── build-lto-zlib.sh └── pgo.sh ├── src ├── ConsoleLog.cpp ├── ConsoleLog.h ├── ExternalIndexer.cpp ├── ExternalIndexer.h ├── FieldIndexer.cpp ├── FieldIndexer.h ├── File.h ├── Index.cpp ├── Index.h ├── IndexParser.cpp ├── IndexParser.h ├── IndexSink.h ├── LineFinder.cpp ├── LineFinder.h ├── LineIndexer.h ├── LineSink.h ├── Log.h ├── Pipe.cpp ├── Pipe.h ├── PrettyBytes.cpp ├── PrettyBytes.h ├── RangeFetcher.cpp ├── RangeFetcher.h ├── RegExp.cpp ├── RegExp.h ├── RegExpIndexer.cpp ├── RegExpIndexer.h ├── Sqlite.cpp ├── Sqlite.h ├── SqliteError.h ├── StringView.cpp ├── StringView.h ├── zindex.cpp └── zq.cpp └── tests ├── .gitignore ├── CaptureLog.h ├── CaptureSink.h ├── ExternalIndexerTest.cpp ├── FieldIndexerTest.cpp ├── IndexTest.cpp ├── LineFinderTest.cpp ├── LogTest.cpp ├── PrettyBytesTest.cpp ├── RangeFetcherTest.cpp ├── RegExpIndexerTest.cpp ├── RegExpTest.cpp ├── SqliteTest.cpp ├── TempDir.cpp ├── TempDir.h ├── catch.hpp ├── files └── config.json └── test_main.cpp /.autotools: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 42 | 43 | -------------------------------------------------------------------------------- /.codecov.yml: -------------------------------------------------------------------------------- 1 | ignore: 2 | - "ext/**/*" 3 | - "tests/**/*" 4 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | *.h linguist-language=C++ 2 | ext/* linguist-vendored 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.o 2 | *.swp 3 | /build 4 | core 5 | .fig 6 | cmake-build-* 7 | -------------------------------------------------------------------------------- /.idea/.gitignore: -------------------------------------------------------------------------------- 1 | workspace.xml 2 | -------------------------------------------------------------------------------- /.idea/.name: -------------------------------------------------------------------------------- 1 | zindex -------------------------------------------------------------------------------- /.idea/codeStyleSettings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 51 | 54 | -------------------------------------------------------------------------------- /.idea/codeStyles/Project.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 44 | -------------------------------------------------------------------------------- /.idea/codeStyles/codeStyleConfig.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | -------------------------------------------------------------------------------- /.idea/encodings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.idea/zindex.iml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | zindex 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.cdt.autotools.core.genmakebuilderV2 10 | 11 | 12 | 13 | 14 | org.eclipse.cdt.managedbuilder.core.genmakebuilder 15 | clean,full,incremental, 16 | 17 | 18 | 19 | 20 | org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder 21 | full,incremental, 22 | 23 | 24 | 25 | 26 | 27 | org.eclipse.cdt.core.cnature 28 | org.eclipse.cdt.core.ccnature 29 | org.eclipse.cdt.managedbuilder.core.managedBuildNature 30 | org.eclipse.cdt.managedbuilder.core.ScannerConfigNature 31 | org.eclipse.cdt.autotools.core.autotoolsNatureV2 32 | 33 | 34 | -------------------------------------------------------------------------------- /.settings/language.settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /.settings/org.eclipse.cdt.managedbuilder.core.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | environment/buildEnvironmentInclude/org.eclipse.linuxtools.cdt.autotools.core.configuration.build.1821571161/CPATH/delimiter=\: 3 | environment/buildEnvironmentInclude/org.eclipse.linuxtools.cdt.autotools.core.configuration.build.1821571161/CPATH/operation=remove 4 | environment/buildEnvironmentInclude/org.eclipse.linuxtools.cdt.autotools.core.configuration.build.1821571161/CPLUS_INCLUDE_PATH/delimiter=\: 5 | environment/buildEnvironmentInclude/org.eclipse.linuxtools.cdt.autotools.core.configuration.build.1821571161/CPLUS_INCLUDE_PATH/operation=remove 6 | environment/buildEnvironmentInclude/org.eclipse.linuxtools.cdt.autotools.core.configuration.build.1821571161/C_INCLUDE_PATH/delimiter=\: 7 | environment/buildEnvironmentInclude/org.eclipse.linuxtools.cdt.autotools.core.configuration.build.1821571161/C_INCLUDE_PATH/operation=remove 8 | environment/buildEnvironmentInclude/org.eclipse.linuxtools.cdt.autotools.core.configuration.build.1821571161/append=true 9 | environment/buildEnvironmentInclude/org.eclipse.linuxtools.cdt.autotools.core.configuration.build.1821571161/appendContributed=true 10 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: cpp 3 | compiler: 4 | - gcc 5 | - clang 6 | env: 7 | - BUILD_TYPE=Debug COVERAGE=On 8 | - BUILD_TYPE=Release COVERAGE=Off 9 | install: 10 | - if [ "$CXX" = "g++" ]; then export CXX="g++-4.7" CC="gcc-4.7"; fi 11 | script: 12 | - mkdir build 13 | - cd build 14 | - cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DCoverage=${COVERAGE} .. 15 | - cmake --build . -- -j2 16 | - ctest -D ExperimentalBuild -j2 17 | - ctest -D ExperimentalTest -j2 18 | - if [ "${COVERAGE}" = "On" ]; then bash <(curl -s https://codecov.io/bash); fi 19 | #- ctest -D ExperimentalMemCheck 20 | addons: 21 | apt: 22 | sources: 23 | - ubuntu-toolchain-r-test 24 | - kalakris-cmake 25 | packages: 26 | - gcc-4.7 27 | - g++-4.7 28 | - valgrind 29 | - cmake 30 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Matt Godbolt 2 | Andrew Clarke 3 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8.4) 2 | project(zindex) 3 | 4 | if (APPLE AND CMAKE_CXX_COMPILER_ID MATCHES "Clang") 5 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -stdlib=libc++ -Wall -Werror -Wextra") 6 | else () 7 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -pthread -Wall -Werror -Wextra") 8 | endif () 9 | 10 | if (APPLE AND CMAKE_C_COMPILER_ID MATCHES "Clang") 11 | set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra") 12 | else () 13 | set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -pthread -Wall -Wextra") 14 | endif () 15 | 16 | option(UseLTO "Use link-time optimization" OFF) 17 | option(Static "Statically link" OFF) 18 | option(BuildSqlShell "Build the sqlite shell" OFF) 19 | option(ArchNative "Target the computer being built on (march=native)" OFF) 20 | option(PGO "Set PGO flags" "") 21 | option(Coverage "Enable coverage reporting" OFF) 22 | 23 | if (Coverage) 24 | add_compile_options(--coverage -O0) 25 | set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --coverage") 26 | endif (Coverage) 27 | 28 | if (PGO) 29 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${PGO}") 30 | set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${PGO}") 31 | endif (PGO) 32 | 33 | if (UseLTO) 34 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -flto -Wno-maybe-uninitialized") 35 | set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -flto -Wno-maybe-uninitialized") 36 | set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-linker-plugin") 37 | endif (UseLTO) 38 | 39 | set(COMMON_LIBS "") 40 | if (Static) 41 | set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static -static-libgcc -static-libstdc++") 42 | set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DSQLITE_OMIT_LOAD_EXTENSION=1") 43 | set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") 44 | set(BUILD_SHARED_LIBRARIES OFF) 45 | set(CMAKE_EXE_LINK_DYNAMIC_C_FLAGS) # remove -Wl,-Bdynamic 46 | set(CMAKE_EXE_LINK_DYNAMIC_CXX_FLAGS) 47 | else (Static) 48 | set(COMMON_LIBS "dl") 49 | endif (Static) 50 | 51 | if (ArchNative) 52 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native") 53 | set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=native") 54 | endif (ArchNative) 55 | 56 | include(CTest) 57 | 58 | find_package(ZLIB REQUIRED) 59 | include_directories(BEFORE SYSTEM ext/sqlite) 60 | include_directories(${ZLIB_INCLUDE_DIRS} src ext) 61 | 62 | set(SOURCE_FILES 63 | src/File.h 64 | src/Index.cpp 65 | src/Index.h 66 | src/IndexParser.cpp 67 | src/IndexParser.h 68 | src/LineFinder.cpp 69 | src/LineFinder.h 70 | src/LineSink.h 71 | src/Sqlite.cpp 72 | src/Sqlite.h 73 | src/SqliteError.h 74 | src/RegExp.h 75 | src/RegExp.cpp 76 | src/RegExpIndexer.cpp 77 | src/RegExpIndexer.h 78 | src/LineIndexer.h 79 | src/IndexSink.h 80 | src/Log.h 81 | ext/cJSON/cJSON.h 82 | ext/cJSON/cJSON.c 83 | src/ConsoleLog.h 84 | src/ConsoleLog.cpp 85 | src/StringView.cpp 86 | src/StringView.h 87 | src/PrettyBytes.h 88 | src/PrettyBytes.cpp 89 | src/RangeFetcher.cpp 90 | src/RangeFetcher.h 91 | src/FieldIndexer.cpp 92 | src/FieldIndexer.h 93 | src/ExternalIndexer.cpp 94 | src/ExternalIndexer.h 95 | src/Pipe.cpp 96 | src/Pipe.h 97 | ext/sqlite/sqlite3.c) 98 | 99 | set(TEST_FILES 100 | tests/catch.hpp 101 | tests/LineFinderTest.cpp 102 | tests/test_main.cpp 103 | tests/SqliteTest.cpp 104 | tests/RegExpTest.cpp 105 | tests/RegExpIndexerTest.cpp 106 | tests/TempDir.h 107 | tests/TempDir.cpp 108 | tests/PrettyBytesTest.cpp 109 | tests/IndexTest.cpp 110 | tests/RangeFetcherTest.cpp 111 | tests/FieldIndexerTest.cpp 112 | tests/ExternalIndexerTest.cpp 113 | tests/LogTest.cpp) 114 | 115 | add_library(libzindex ${SOURCE_FILES}) 116 | set_target_properties(libzindex PROPERTIES OUTPUT_NAME zindex) 117 | 118 | add_executable(zindex src/zindex.cpp) 119 | target_link_libraries(zindex libzindex ${ZLIB_LIBRARIES} ${COMMON_LIBS}) 120 | add_executable(zq src/zq.cpp) 121 | target_link_libraries(zq libzindex ${ZLIB_LIBRARIES} ${COMMON_LIBS}) 122 | 123 | add_executable(unit-tests ${TEST_FILES}) 124 | target_link_libraries(unit-tests libzindex ${ZLIB_LIBRARIES} ${COMMON_LIBS}) 125 | 126 | if (BuildSqlShell) 127 | add_executable(sql-shell ext/sqlite/shell.c ext/sqlite/sqlite3.c) 128 | target_link_libraries(sql-shell ${COMMON_LIBS}) 129 | endif (BuildSqlShell) 130 | 131 | find_program(MEMORYCHECK_COMMAND valgrind) 132 | enable_testing() 133 | add_test(NAME unit-tests 134 | COMMAND unit-tests) 135 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012-2016, Matt Godbolt 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, 8 | this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 14 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 17 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 18 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 19 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 20 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 21 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 22 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 23 | POSSIBILITY OF SUCH DAMAGE. 24 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | default: all 2 | 3 | BUILD_TYPE ?= Release 4 | BUILD_DIR := build/${BUILD_TYPE} 5 | ROOT_DIR := $(shell pwd) 6 | 7 | ${BUILD_DIR}/Makefile: CMakeLists.txt 8 | mkdir -p ${BUILD_DIR} 9 | cd ${BUILD_DIR} && cmake ${ROOT_DIR} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} 10 | 11 | all: ${BUILD_DIR}/Makefile 12 | $(MAKE) -C ${BUILD_DIR} all 13 | 14 | test: ${BUILD_DIR}/Makefile all 15 | $(MAKE) -C ${BUILD_DIR} test 16 | 17 | clean: 18 | rm -rf build 19 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/mattgodbolt/zindex.svg?branch=master)](https://travis-ci.org/mattgodbolt/zindex) [![codecov](https://codecov.io/gh/mattgodbolt/zindex/branch/master/graph/badge.svg)](https://codecov.io/gh/mattgodbolt/zindex) 2 | 3 | 4 | `zindex` creates and queries an index on a compressed, line-based text file in a 5 | time- and space-efficient way. 6 | 7 | ### The itch I had 8 | 9 | I have many multigigabyte text gzipped log files and I'd like to be able to find data in them by an index. 10 | There's a key on each line that a simple regex can pull out. However, to find a 11 | particular record requires `zgrep`, which takes ages as it has to seek through 12 | gigabytes of previous data to get to each record. 13 | 14 | Enter `zindex` which builds an index and also stores decompression checkpoints along the way 15 | which allows lightning fast random access. Pulling out single lines by either 16 | line number of by an index entry is then almost instant, even for huge files. The indices 17 | themselves are small too, typically ~10% of the compressed file size for a simple unique 18 | numeric index. 19 | 20 | ## Creating an index 21 | 22 | `zindex` needs to be told what part of each line constitutes the index. This can be done by 23 | a regular expression, by field, or by piping each line through an external program. 24 | 25 | By default zindex creates an index of `file.gz.zindex` when asked to index `file.gz`. 26 | 27 | Example: create an index on lines matching a numeric regular expression. The capture group 28 | indicates the part that's to be indexed, and the options show each line has a unique, numeric index. 29 | 30 | ```bash 31 | $ zindex file.gz --regex 'id:([0-9]+)' --numeric --unique 32 | ``` 33 | 34 | Example: create an index on the second field of a CSV file: 35 | 36 | ```bash 37 | $ zindex file.gz --delimiter , --field 2 38 | ``` 39 | 40 | Example: create an index on a JSON field `orderId.id` in any of the items in the document root's `actions` array (requires [jq](http://stedolan.github.io/jq/)). 41 | The `jq` query creates an array of all the `orderId.id`s, then `join`s them with a space to ensure each individual line piped to jq creates a single line of output, 42 | with multiple matches separated by spaces (which is the default separator). 43 | 44 | ```bash 45 | $ zindex file.gz --pipe "jq --raw-output --unbuffered '[.actions[].orderId.id] | join(\" \")'" 46 | ``` 47 | 48 | Multiple indices, and configuration of the index creation by JSON configuration file are supported, see below. 49 | 50 | ## Querying the index 51 | 52 | The `zq` program is used to query an index. It's given the name of the compressed file and a list of queries. For example: 53 | 54 | ```bash 55 | $ zq file.gz 1023 4443 554 56 | ``` 57 | 58 | It's also possible to output by line number, so to print lines 1 and 1000 from a file: 59 | 60 | ```bash 61 | $ zq file.gz --line 1 1000 62 | ``` 63 | 64 | ## Building from source 65 | 66 | `zindex` uses CMake for its basic building (though has a bootstrapping `Makefile`), and requires a C++11 compatible compiler (GCC 4.8 or above and clang 3.4 and above). It also requires `zlib`. With the relevant compiler available, building ought to be as simple as: 67 | 68 | ```bash 69 | $ git clone https://github.com/mattgodbolt/zindex.git 70 | $ cd zindex 71 | $ make 72 | ``` 73 | 74 | Binaries are left in `build/Release`. 75 | 76 | Additionally a static binary can be built if you're happy to dip your toe into CMake: 77 | 78 | ```bash 79 | $ cd path/to/build/directory 80 | $ cmake path/to/zindex/checkout/dir -DStatic:BOOL=On -DCMAKE_BUILD_TYPE=Release 81 | $ make 82 | ``` 83 | 84 | ## Multiple indices 85 | 86 | To support more than one index, or for easier configuration than all the command-line flags that might be 87 | needed, there is a JSON configuration format. Pass the `--config .json` option and put something like this in the configuration file: 88 | 89 | { 90 | "indexes": [ 91 | { 92 | "type": "field", 93 | "delimiter": "\t", 94 | "fieldNum": 1 95 | }, 96 | { 97 | "name": "secondary", 98 | "type": "field", 99 | "delimiter": "\t", 100 | "fieldNum": 2 101 | } 102 | ] 103 | } 104 | 105 | This creates two indices, one on the first field and one on the second field, as delimited by tabs. One can 106 | then specify which index to query with the `-i ` option of `zq`. 107 | 108 | ### Issues and feature requests 109 | 110 | See the [issue tracker](https://github.com/mattgodbolt/zindex/issues) for TODOs and known bugs. Please raise bugs there, and feel free to submit suggestions there also. 111 | 112 | Feel free to [contact me](mailto:matt@godbolt.org) if you prefer email over bug trackers. 113 | -------------------------------------------------------------------------------- /ext/cJSON/cJSON.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2009 Dave Gamble 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to deal 6 | in the Software without restriction, including without limitation the rights 7 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | THE SOFTWARE. 21 | */ 22 | 23 | #ifndef cJSON__h 24 | #define cJSON__h 25 | 26 | #ifdef __cplusplus 27 | extern "C" 28 | { 29 | #endif 30 | 31 | /* cJSON Types: */ 32 | #define cJSON_False (1 << 0) 33 | #define cJSON_True (1 << 1) 34 | #define cJSON_NULL (1 << 2) 35 | #define cJSON_Number (1 << 3) 36 | #define cJSON_String (1 << 4) 37 | #define cJSON_Array (1 << 5) 38 | #define cJSON_Object (1 << 6) 39 | 40 | #define cJSON_IsReference 256 41 | #define cJSON_StringIsConst 512 42 | 43 | /* The cJSON structure: */ 44 | typedef struct cJSON { 45 | struct cJSON *next,*prev; /* next/prev allow you to walk array/object chains. Alternatively, use GetArraySize/GetArrayItem/GetObjectItem */ 46 | struct cJSON *child; /* An array or object item will have a child pointer pointing to a chain of the items in the array/object. */ 47 | 48 | int type; /* The type of the item, as above. */ 49 | 50 | char *valuestring; /* The item's string, if type==cJSON_String */ 51 | int valueint; /* The item's number, if type==cJSON_Number */ 52 | double valuedouble; /* The item's number, if type==cJSON_Number */ 53 | 54 | char *string; /* The item's name string, if this item is the child of, or is in the list of subitems of an object. */ 55 | } cJSON; 56 | 57 | typedef struct cJSON_Hooks { 58 | void *(*malloc_fn)(size_t sz); 59 | void (*free_fn)(void *ptr); 60 | } cJSON_Hooks; 61 | 62 | /* Supply malloc, realloc and free functions to cJSON */ 63 | extern void cJSON_InitHooks(cJSON_Hooks* hooks); 64 | 65 | 66 | /* Supply a block of JSON, and this returns a cJSON object you can interrogate. Call cJSON_Delete when finished. */ 67 | extern cJSON *cJSON_Parse(const char *value); 68 | /* Render a cJSON entity to text for transfer/storage. Free the char* when finished. */ 69 | extern char *cJSON_Print(cJSON *item); 70 | /* Render a cJSON entity to text for transfer/storage without any formatting. Free the char* when finished. */ 71 | extern char *cJSON_PrintUnformatted(cJSON *item); 72 | /* Render a cJSON entity to text using a buffered strategy. prebuffer is a guess at the final size. guessing well reduces reallocation. fmt=0 gives unformatted, =1 gives formatted */ 73 | extern char *cJSON_PrintBuffered(cJSON *item,int prebuffer,int fmt); 74 | /* Delete a cJSON entity and all subentities. */ 75 | extern void cJSON_Delete(cJSON *c); 76 | 77 | /* Returns the number of items in an array (or object). */ 78 | extern int cJSON_GetArraySize(cJSON *array); 79 | /* Retrieve item number "item" from array "array". Returns NULL if unsuccessful. */ 80 | extern cJSON *cJSON_GetArrayItem(cJSON *array,int item); 81 | /* Get item "string" from object. Case insensitive. */ 82 | extern cJSON *cJSON_GetObjectItem(cJSON *object,const char *string); 83 | extern int cJSON_HasObjectItem(cJSON *object,const char *string); 84 | /* For analysing failed parses. This returns a pointer to the parse error. You'll probably need to look a few chars back to make sense of it. Defined when cJSON_Parse() returns 0. 0 when cJSON_Parse() succeeds. */ 85 | extern const char *cJSON_GetErrorPtr(void); 86 | 87 | /* These calls create a cJSON item of the appropriate type. */ 88 | extern cJSON *cJSON_CreateNull(void); 89 | extern cJSON *cJSON_CreateTrue(void); 90 | extern cJSON *cJSON_CreateFalse(void); 91 | extern cJSON *cJSON_CreateBool(int b); 92 | extern cJSON *cJSON_CreateNumber(double num); 93 | extern cJSON *cJSON_CreateString(const char *string); 94 | extern cJSON *cJSON_CreateArray(void); 95 | extern cJSON *cJSON_CreateObject(void); 96 | 97 | /* These utilities create an Array of count items. */ 98 | extern cJSON *cJSON_CreateIntArray(const int *numbers,int count); 99 | extern cJSON *cJSON_CreateFloatArray(const float *numbers,int count); 100 | extern cJSON *cJSON_CreateDoubleArray(const double *numbers,int count); 101 | extern cJSON *cJSON_CreateStringArray(const char **strings,int count); 102 | 103 | /* Append item to the specified array/object. */ 104 | extern void cJSON_AddItemToArray(cJSON *array, cJSON *item); 105 | extern void cJSON_AddItemToObject(cJSON *object,const char *string,cJSON *item); 106 | extern void cJSON_AddItemToObjectCS(cJSON *object,const char *string,cJSON *item); /* Use this when string is definitely const (i.e. a literal, or as good as), and will definitely survive the cJSON object */ 107 | /* Append reference to item to the specified array/object. Use this when you want to add an existing cJSON to a new cJSON, but don't want to corrupt your existing cJSON. */ 108 | extern void cJSON_AddItemReferenceToArray(cJSON *array, cJSON *item); 109 | extern void cJSON_AddItemReferenceToObject(cJSON *object,const char *string,cJSON *item); 110 | 111 | /* Remove/Detatch items from Arrays/Objects. */ 112 | extern cJSON *cJSON_DetachItemFromArray(cJSON *array,int which); 113 | extern void cJSON_DeleteItemFromArray(cJSON *array,int which); 114 | extern cJSON *cJSON_DetachItemFromObject(cJSON *object,const char *string); 115 | extern void cJSON_DeleteItemFromObject(cJSON *object,const char *string); 116 | 117 | /* Update array items. */ 118 | extern void cJSON_InsertItemInArray(cJSON *array,int which,cJSON *newitem); /* Shifts pre-existing items to the right. */ 119 | extern void cJSON_ReplaceItemInArray(cJSON *array,int which,cJSON *newitem); 120 | extern void cJSON_ReplaceItemInObject(cJSON *object,const char *string,cJSON *newitem); 121 | 122 | /* Duplicate a cJSON item */ 123 | extern cJSON *cJSON_Duplicate(cJSON *item,int recurse); 124 | /* Duplicate will create a new, identical cJSON item to the one you pass, in new memory that will 125 | need to be released. With recurse!=0, it will duplicate any children connected to the item. 126 | The item->next and ->prev pointers are always zero on return from Duplicate. */ 127 | 128 | /* ParseWithOpts allows you to require (and check) that the JSON is null terminated, and to retrieve the pointer to the final byte parsed. */ 129 | /* If you supply a ptr in return_parse_end and parsing fails, then return_parse_end will contain a pointer to the error. If not, then cJSON_GetErrorPtr() does the job. */ 130 | extern cJSON *cJSON_ParseWithOpts(const char *value,const char **return_parse_end,int require_null_terminated); 131 | 132 | extern void cJSON_Minify(char *json); 133 | 134 | /* Macros for creating things quickly. */ 135 | #define cJSON_AddNullToObject(object,name) cJSON_AddItemToObject(object, name, cJSON_CreateNull()) 136 | #define cJSON_AddTrueToObject(object,name) cJSON_AddItemToObject(object, name, cJSON_CreateTrue()) 137 | #define cJSON_AddFalseToObject(object,name) cJSON_AddItemToObject(object, name, cJSON_CreateFalse()) 138 | #define cJSON_AddBoolToObject(object,name,b) cJSON_AddItemToObject(object, name, cJSON_CreateBool(b)) 139 | #define cJSON_AddNumberToObject(object,name,n) cJSON_AddItemToObject(object, name, cJSON_CreateNumber(n)) 140 | #define cJSON_AddStringToObject(object,name,s) cJSON_AddItemToObject(object, name, cJSON_CreateString(s)) 141 | 142 | /* When assigning an integer value, it needs to be propagated to valuedouble too. */ 143 | #define cJSON_SetIntValue(object,val) ((object)?(object)->valueint=(object)->valuedouble=(val):(val)) 144 | #define cJSON_SetNumberValue(object,val) ((object)?(object)->valueint=(object)->valuedouble=(val):(val)) 145 | 146 | /* Macro for iterating over an array */ 147 | #define cJSON_ArrayForEach(pos, head) for(pos = (head)->child; pos != NULL; pos = pos->next) 148 | 149 | #ifdef __cplusplus 150 | } 151 | #endif 152 | 153 | #endif 154 | -------------------------------------------------------------------------------- /ext/tclap/ArgException.h: -------------------------------------------------------------------------------- 1 | // -*- Mode: c++; c-basic-offset: 4; tab-width: 4; -*- 2 | 3 | /****************************************************************************** 4 | * 5 | * file: ArgException.h 6 | * 7 | * Copyright (c) 2003, Michael E. Smoot . 8 | * All rights reverved. 9 | * 10 | * See the file COPYING in the top directory of this distribution for 11 | * more information. 12 | * 13 | * THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS 14 | * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 16 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 19 | * DEALINGS IN THE SOFTWARE. 20 | * 21 | *****************************************************************************/ 22 | 23 | 24 | #ifndef TCLAP_ARG_EXCEPTION_H 25 | #define TCLAP_ARG_EXCEPTION_H 26 | 27 | #include 28 | #include 29 | 30 | namespace TCLAP { 31 | 32 | /** 33 | * A simple class that defines and argument exception. Should be caught 34 | * whenever a CmdLine is created and parsed. 35 | */ 36 | class ArgException : public std::exception 37 | { 38 | public: 39 | 40 | /** 41 | * Constructor. 42 | * \param text - The text of the exception. 43 | * \param id - The text identifying the argument source. 44 | * \param td - Text describing the type of ArgException it is. 45 | * of the exception. 46 | */ 47 | ArgException( const std::string& text = "undefined exception", 48 | const std::string& id = "undefined", 49 | const std::string& td = "Generic ArgException") 50 | : std::exception(), 51 | _errorText(text), 52 | _argId( id ), 53 | _typeDescription(td) 54 | { } 55 | 56 | /** 57 | * Destructor. 58 | */ 59 | virtual ~ArgException() throw() { } 60 | 61 | /** 62 | * Returns the error text. 63 | */ 64 | std::string error() const { return ( _errorText ); } 65 | 66 | /** 67 | * Returns the argument id. 68 | */ 69 | std::string argId() const 70 | { 71 | if ( _argId == "undefined" ) 72 | return " "; 73 | else 74 | return ( "Argument: " + _argId ); 75 | } 76 | 77 | /** 78 | * Returns the arg id and error text. 79 | */ 80 | const char* what() const throw() 81 | { 82 | static std::string ex; 83 | ex = _argId + " -- " + _errorText; 84 | return ex.c_str(); 85 | } 86 | 87 | /** 88 | * Returns the type of the exception. Used to explain and distinguish 89 | * between different child exceptions. 90 | */ 91 | std::string typeDescription() const 92 | { 93 | return _typeDescription; 94 | } 95 | 96 | 97 | private: 98 | 99 | /** 100 | * The text of the exception message. 101 | */ 102 | std::string _errorText; 103 | 104 | /** 105 | * The argument related to this exception. 106 | */ 107 | std::string _argId; 108 | 109 | /** 110 | * Describes the type of the exception. Used to distinguish 111 | * between different child exceptions. 112 | */ 113 | std::string _typeDescription; 114 | 115 | }; 116 | 117 | /** 118 | * Thrown from within the child Arg classes when it fails to properly 119 | * parse the argument it has been passed. 120 | */ 121 | class ArgParseException : public ArgException 122 | { 123 | public: 124 | /** 125 | * Constructor. 126 | * \param text - The text of the exception. 127 | * \param id - The text identifying the argument source 128 | * of the exception. 129 | */ 130 | ArgParseException( const std::string& text = "undefined exception", 131 | const std::string& id = "undefined" ) 132 | : ArgException( text, 133 | id, 134 | std::string( "Exception found while parsing " ) + 135 | std::string( "the value the Arg has been passed." )) 136 | { } 137 | }; 138 | 139 | /** 140 | * Thrown from CmdLine when the arguments on the command line are not 141 | * properly specified, e.g. too many arguments, required argument missing, etc. 142 | */ 143 | class CmdLineParseException : public ArgException 144 | { 145 | public: 146 | /** 147 | * Constructor. 148 | * \param text - The text of the exception. 149 | * \param id - The text identifying the argument source 150 | * of the exception. 151 | */ 152 | CmdLineParseException( const std::string& text = "undefined exception", 153 | const std::string& id = "undefined" ) 154 | : ArgException( text, 155 | id, 156 | std::string( "Exception found when the values ") + 157 | std::string( "on the command line do not meet ") + 158 | std::string( "the requirements of the defined ") + 159 | std::string( "Args." )) 160 | { } 161 | }; 162 | 163 | /** 164 | * Thrown from Arg and CmdLine when an Arg is improperly specified, e.g. 165 | * same flag as another Arg, same name, etc. 166 | */ 167 | class SpecificationException : public ArgException 168 | { 169 | public: 170 | /** 171 | * Constructor. 172 | * \param text - The text of the exception. 173 | * \param id - The text identifying the argument source 174 | * of the exception. 175 | */ 176 | SpecificationException( const std::string& text = "undefined exception", 177 | const std::string& id = "undefined" ) 178 | : ArgException( text, 179 | id, 180 | std::string("Exception found when an Arg object ")+ 181 | std::string("is improperly defined by the ") + 182 | std::string("developer." )) 183 | { } 184 | 185 | }; 186 | 187 | class ExitException { 188 | public: 189 | ExitException(int estat) : _estat(estat) {} 190 | 191 | int getExitStatus() const { return _estat; } 192 | 193 | private: 194 | int _estat; 195 | }; 196 | 197 | } // namespace TCLAP 198 | 199 | #endif 200 | 201 | -------------------------------------------------------------------------------- /ext/tclap/ArgTraits.h: -------------------------------------------------------------------------------- 1 | // -*- Mode: c++; c-basic-offset: 4; tab-width: 4; -*- 2 | 3 | /****************************************************************************** 4 | * 5 | * file: ArgTraits.h 6 | * 7 | * Copyright (c) 2007, Daniel Aarno, Michael E. Smoot . 8 | * All rights reverved. 9 | * 10 | * See the file COPYING in the top directory of this distribution for 11 | * more information. 12 | * 13 | * THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS 14 | * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 16 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 19 | * DEALINGS IN THE SOFTWARE. 20 | * 21 | *****************************************************************************/ 22 | 23 | // This is an internal tclap file, you should probably not have to 24 | // include this directly 25 | 26 | #ifndef TCLAP_ARGTRAITS_H 27 | #define TCLAP_ARGTRAITS_H 28 | 29 | namespace TCLAP { 30 | 31 | // We use two empty structs to get compile type specialization 32 | // function to work 33 | 34 | /** 35 | * A value like argument value type is a value that can be set using 36 | * operator>>. This is the default value type. 37 | */ 38 | struct ValueLike { 39 | typedef ValueLike ValueCategory; 40 | }; 41 | 42 | /** 43 | * A string like argument value type is a value that can be set using 44 | * operator=(string). Usefull if the value type contains spaces which 45 | * will be broken up into individual tokens by operator>>. 46 | */ 47 | struct StringLike {}; 48 | 49 | /** 50 | * A class can inherit from this object to make it have string like 51 | * traits. This is a compile time thing and does not add any overhead 52 | * to the inherenting class. 53 | */ 54 | struct StringLikeTrait { 55 | typedef StringLike ValueCategory; 56 | }; 57 | 58 | /** 59 | * A class can inherit from this object to make it have value like 60 | * traits. This is a compile time thing and does not add any overhead 61 | * to the inherenting class. 62 | */ 63 | struct ValueLikeTrait { 64 | typedef ValueLike ValueCategory; 65 | }; 66 | 67 | /** 68 | * Arg traits are used to get compile type specialization when parsing 69 | * argument values. Using an ArgTraits you can specify the way that 70 | * values gets assigned to any particular type during parsing. The two 71 | * supported types are string like and value like. 72 | */ 73 | template 74 | struct ArgTraits { 75 | typedef typename T::ValueCategory ValueCategory; 76 | //typedef ValueLike ValueCategory; 77 | }; 78 | 79 | #endif 80 | 81 | } // namespace 82 | -------------------------------------------------------------------------------- /ext/tclap/CmdLineInterface.h: -------------------------------------------------------------------------------- 1 | 2 | /****************************************************************************** 3 | * 4 | * file: CmdLineInterface.h 5 | * 6 | * Copyright (c) 2003, Michael E. Smoot . 7 | * Copyright (c) 2004, Michael E. Smoot, Daniel Aarno. 8 | * All rights reverved. 9 | * 10 | * See the file COPYING in the top directory of this distribution for 11 | * more information. 12 | * 13 | * THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS 14 | * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 16 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 19 | * DEALINGS IN THE SOFTWARE. 20 | * 21 | *****************************************************************************/ 22 | 23 | #ifndef TCLAP_COMMANDLINE_INTERFACE_H 24 | #define TCLAP_COMMANDLINE_INTERFACE_H 25 | 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | 32 | 33 | namespace TCLAP { 34 | 35 | class Arg; 36 | class CmdLineOutput; 37 | class XorHandler; 38 | 39 | /** 40 | * The base class that manages the command line definition and passes 41 | * along the parsing to the appropriate Arg classes. 42 | */ 43 | class CmdLineInterface 44 | { 45 | public: 46 | 47 | /** 48 | * Destructor 49 | */ 50 | virtual ~CmdLineInterface() {} 51 | 52 | /** 53 | * Adds an argument to the list of arguments to be parsed. 54 | * \param a - Argument to be added. 55 | */ 56 | virtual void add( Arg& a )=0; 57 | 58 | /** 59 | * An alternative add. Functionally identical. 60 | * \param a - Argument to be added. 61 | */ 62 | virtual void add( Arg* a )=0; 63 | 64 | /** 65 | * Add two Args that will be xor'd. 66 | * If this method is used, add does 67 | * not need to be called. 68 | * \param a - Argument to be added and xor'd. 69 | * \param b - Argument to be added and xor'd. 70 | */ 71 | virtual void xorAdd( Arg& a, Arg& b )=0; 72 | 73 | /** 74 | * Add a list of Args that will be xor'd. If this method is used, 75 | * add does not need to be called. 76 | * \param xors - List of Args to be added and xor'd. 77 | */ 78 | virtual void xorAdd( std::vector& xors )=0; 79 | 80 | /** 81 | * Parses the command line. 82 | * \param argc - Number of arguments. 83 | * \param argv - Array of arguments. 84 | */ 85 | virtual void parse(int argc, const char * const * argv)=0; 86 | 87 | /** 88 | * Parses the command line. 89 | * \param args - A vector of strings representing the args. 90 | * args[0] is still the program name. 91 | */ 92 | void parse(std::vector& args); 93 | 94 | /** 95 | * Returns the CmdLineOutput object. 96 | */ 97 | virtual CmdLineOutput* getOutput()=0; 98 | 99 | /** 100 | * \param co - CmdLineOutput object that we want to use instead. 101 | */ 102 | virtual void setOutput(CmdLineOutput* co)=0; 103 | 104 | /** 105 | * Returns the version string. 106 | */ 107 | virtual std::string& getVersion()=0; 108 | 109 | /** 110 | * Returns the program name string. 111 | */ 112 | virtual std::string& getProgramName()=0; 113 | 114 | /** 115 | * Returns the argList. 116 | */ 117 | virtual std::list& getArgList()=0; 118 | 119 | /** 120 | * Returns the XorHandler. 121 | */ 122 | virtual XorHandler& getXorHandler()=0; 123 | 124 | /** 125 | * Returns the delimiter string. 126 | */ 127 | virtual char getDelimiter()=0; 128 | 129 | /** 130 | * Returns the message string. 131 | */ 132 | virtual std::string& getMessage()=0; 133 | 134 | /** 135 | * Indicates whether or not the help and version switches were created 136 | * automatically. 137 | */ 138 | virtual bool hasHelpAndVersion()=0; 139 | 140 | /** 141 | * Resets the instance as if it had just been constructed so that the 142 | * instance can be reused. 143 | */ 144 | virtual void reset()=0; 145 | }; 146 | 147 | } //namespace 148 | 149 | 150 | #endif 151 | -------------------------------------------------------------------------------- /ext/tclap/CmdLineOutput.h: -------------------------------------------------------------------------------- 1 | 2 | 3 | /****************************************************************************** 4 | * 5 | * file: CmdLineOutput.h 6 | * 7 | * Copyright (c) 2004, Michael E. Smoot 8 | * All rights reverved. 9 | * 10 | * See the file COPYING in the top directory of this distribution for 11 | * more information. 12 | * 13 | * THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS 14 | * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 16 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 19 | * DEALINGS IN THE SOFTWARE. 20 | * 21 | *****************************************************************************/ 22 | 23 | #ifndef TCLAP_CMDLINEOUTPUT_H 24 | #define TCLAP_CMDLINEOUTPUT_H 25 | 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | 33 | namespace TCLAP { 34 | 35 | class CmdLineInterface; 36 | class ArgException; 37 | 38 | /** 39 | * The interface that any output object must implement. 40 | */ 41 | class CmdLineOutput 42 | { 43 | 44 | public: 45 | 46 | /** 47 | * Virtual destructor. 48 | */ 49 | virtual ~CmdLineOutput() {} 50 | 51 | /** 52 | * Generates some sort of output for the USAGE. 53 | * \param c - The CmdLine object the output is generated for. 54 | */ 55 | virtual void usage(CmdLineInterface& c)=0; 56 | 57 | /** 58 | * Generates some sort of output for the version. 59 | * \param c - The CmdLine object the output is generated for. 60 | */ 61 | virtual void version(CmdLineInterface& c)=0; 62 | 63 | /** 64 | * Generates some sort of output for a failure. 65 | * \param c - The CmdLine object the output is generated for. 66 | * \param e - The ArgException that caused the failure. 67 | */ 68 | virtual void failure( CmdLineInterface& c, 69 | ArgException& e )=0; 70 | 71 | }; 72 | 73 | } //namespace TCLAP 74 | #endif 75 | -------------------------------------------------------------------------------- /ext/tclap/Constraint.h: -------------------------------------------------------------------------------- 1 | 2 | /****************************************************************************** 3 | * 4 | * file: Constraint.h 5 | * 6 | * Copyright (c) 2005, Michael E. Smoot 7 | * All rights reverved. 8 | * 9 | * See the file COPYING in the top directory of this distribution for 10 | * more information. 11 | * 12 | * THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS 13 | * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 14 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 15 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 16 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 17 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 18 | * DEALINGS IN THE SOFTWARE. 19 | * 20 | *****************************************************************************/ 21 | 22 | #ifndef TCLAP_CONSTRAINT_H 23 | #define TCLAP_CONSTRAINT_H 24 | 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | 32 | namespace TCLAP { 33 | 34 | /** 35 | * The interface that defines the interaction between the Arg and Constraint. 36 | */ 37 | template 38 | class Constraint 39 | { 40 | 41 | public: 42 | /** 43 | * Returns a description of the Constraint. 44 | */ 45 | virtual std::string description() const =0; 46 | 47 | /** 48 | * Returns the short ID for the Constraint. 49 | */ 50 | virtual std::string shortID() const =0; 51 | 52 | /** 53 | * The method used to verify that the value parsed from the command 54 | * line meets the constraint. 55 | * \param value - The value that will be checked. 56 | */ 57 | virtual bool check(const T& value) const =0; 58 | 59 | /** 60 | * Destructor. 61 | * Silences warnings about Constraint being a base class with virtual 62 | * functions but without a virtual destructor. 63 | */ 64 | virtual ~Constraint() { ; } 65 | }; 66 | 67 | } //namespace TCLAP 68 | #endif 69 | -------------------------------------------------------------------------------- /ext/tclap/DocBookOutput.h: -------------------------------------------------------------------------------- 1 | // -*- Mode: c++; c-basic-offset: 4; tab-width: 4; -*- 2 | 3 | /****************************************************************************** 4 | * 5 | * file: DocBookOutput.h 6 | * 7 | * Copyright (c) 2004, Michael E. Smoot 8 | * All rights reverved. 9 | * 10 | * See the file COPYING in the top directory of this distribution for 11 | * more information. 12 | * 13 | * THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS 14 | * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 16 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 19 | * DEALINGS IN THE SOFTWARE. 20 | * 21 | *****************************************************************************/ 22 | 23 | #ifndef TCLAP_DOCBOOKOUTPUT_H 24 | #define TCLAP_DOCBOOKOUTPUT_H 25 | 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | 32 | #include 33 | #include 34 | #include 35 | #include 36 | 37 | namespace TCLAP { 38 | 39 | /** 40 | * A class that generates DocBook output for usage() method for the 41 | * given CmdLine and its Args. 42 | */ 43 | class DocBookOutput : public CmdLineOutput 44 | { 45 | 46 | public: 47 | 48 | /** 49 | * Prints the usage to stdout. Can be overridden to 50 | * produce alternative behavior. 51 | * \param c - The CmdLine object the output is generated for. 52 | */ 53 | virtual void usage(CmdLineInterface& c); 54 | 55 | /** 56 | * Prints the version to stdout. Can be overridden 57 | * to produce alternative behavior. 58 | * \param c - The CmdLine object the output is generated for. 59 | */ 60 | virtual void version(CmdLineInterface& c); 61 | 62 | /** 63 | * Prints (to stderr) an error message, short usage 64 | * Can be overridden to produce alternative behavior. 65 | * \param c - The CmdLine object the output is generated for. 66 | * \param e - The ArgException that caused the failure. 67 | */ 68 | virtual void failure(CmdLineInterface& c, 69 | ArgException& e ); 70 | 71 | protected: 72 | 73 | /** 74 | * Substitutes the char r for string x in string s. 75 | * \param s - The string to operate on. 76 | * \param r - The char to replace. 77 | * \param x - What to replace r with. 78 | */ 79 | void substituteSpecialChars( std::string& s, char r, std::string& x ); 80 | void removeChar( std::string& s, char r); 81 | void basename( std::string& s ); 82 | 83 | void printShortArg(Arg* it); 84 | void printLongArg(Arg* it); 85 | 86 | char theDelimiter; 87 | }; 88 | 89 | 90 | inline void DocBookOutput::version(CmdLineInterface& _cmd) 91 | { 92 | std::cout << _cmd.getVersion() << std::endl; 93 | } 94 | 95 | inline void DocBookOutput::usage(CmdLineInterface& _cmd ) 96 | { 97 | std::list argList = _cmd.getArgList(); 98 | std::string progName = _cmd.getProgramName(); 99 | std::string version = _cmd.getVersion(); 100 | theDelimiter = _cmd.getDelimiter(); 101 | XorHandler xorHandler = _cmd.getXorHandler(); 102 | std::vector< std::vector > xorList = xorHandler.getXorList(); 103 | basename(progName); 104 | 105 | std::cout << "" << std::endl; 106 | std::cout << "" << std::endl << std::endl; 108 | 109 | std::cout << "" << std::endl; 110 | 111 | std::cout << "" << std::endl; 112 | std::cout << "" << progName << "" << std::endl; 113 | std::cout << "1" << std::endl; 114 | std::cout << "" << std::endl; 115 | 116 | std::cout << "" << std::endl; 117 | std::cout << "" << progName << "" << std::endl; 118 | std::cout << "" << _cmd.getMessage() << "" << std::endl; 119 | std::cout << "" << std::endl; 120 | 121 | std::cout << "" << std::endl; 122 | std::cout << "" << std::endl; 123 | 124 | std::cout << "" << progName << "" << std::endl; 125 | 126 | // xor 127 | for ( int i = 0; (unsigned int)i < xorList.size(); i++ ) 128 | { 129 | std::cout << "" << std::endl; 130 | for ( ArgVectorIterator it = xorList[i].begin(); 131 | it != xorList[i].end(); it++ ) 132 | printShortArg((*it)); 133 | 134 | std::cout << "" << std::endl; 135 | } 136 | 137 | // rest of args 138 | for (ArgListIterator it = argList.begin(); it != argList.end(); it++) 139 | if ( !xorHandler.contains( (*it) ) ) 140 | printShortArg((*it)); 141 | 142 | std::cout << "" << std::endl; 143 | std::cout << "" << std::endl; 144 | 145 | std::cout << "" << std::endl; 146 | std::cout << "Description" << std::endl; 147 | std::cout << "" << std::endl; 148 | std::cout << _cmd.getMessage() << std::endl; 149 | std::cout << "" << std::endl; 150 | std::cout << "" << std::endl; 151 | 152 | std::cout << "" << std::endl; 153 | std::cout << "Options" << std::endl; 154 | 155 | std::cout << "" << std::endl; 156 | 157 | for (ArgListIterator it = argList.begin(); it != argList.end(); it++) 158 | printLongArg((*it)); 159 | 160 | std::cout << "" << std::endl; 161 | std::cout << "" << std::endl; 162 | 163 | std::cout << "" << std::endl; 164 | std::cout << "Version" << std::endl; 165 | std::cout << "" << std::endl; 166 | std::cout << version << std::endl; 167 | std::cout << "" << std::endl; 168 | std::cout << "" << std::endl; 169 | 170 | std::cout << "" << std::endl; 171 | 172 | } 173 | 174 | inline void DocBookOutput::failure( CmdLineInterface& _cmd, 175 | ArgException& e ) 176 | { 177 | static_cast(_cmd); // unused 178 | std::cout << e.what() << std::endl; 179 | throw ExitException(1); 180 | } 181 | 182 | inline void DocBookOutput::substituteSpecialChars( std::string& s, 183 | char r, 184 | std::string& x ) 185 | { 186 | size_t p; 187 | while ( (p = s.find_first_of(r)) != std::string::npos ) 188 | { 189 | s.erase(p,1); 190 | s.insert(p,x); 191 | } 192 | } 193 | 194 | inline void DocBookOutput::removeChar( std::string& s, char r) 195 | { 196 | size_t p; 197 | while ( (p = s.find_first_of(r)) != std::string::npos ) 198 | { 199 | s.erase(p,1); 200 | } 201 | } 202 | 203 | inline void DocBookOutput::basename( std::string& s ) 204 | { 205 | size_t p = s.find_last_of('/'); 206 | if ( p != std::string::npos ) 207 | { 208 | s.erase(0, p + 1); 209 | } 210 | } 211 | 212 | inline void DocBookOutput::printShortArg(Arg* a) 213 | { 214 | std::string lt = "<"; 215 | std::string gt = ">"; 216 | 217 | std::string id = a->shortID(); 218 | substituteSpecialChars(id,'<',lt); 219 | substituteSpecialChars(id,'>',gt); 220 | removeChar(id,'['); 221 | removeChar(id,']'); 222 | 223 | std::string choice = "opt"; 224 | if ( a->isRequired() ) 225 | choice = "plain"; 226 | 227 | std::cout << "acceptsMultipleValues() ) 229 | std::cout << " rep='repeat'"; 230 | 231 | 232 | std::cout << '>'; 233 | if ( !a->getFlag().empty() ) 234 | std::cout << a->flagStartChar() << a->getFlag(); 235 | else 236 | std::cout << a->nameStartString() << a->getName(); 237 | if ( a->isValueRequired() ) 238 | { 239 | std::string arg = a->shortID(); 240 | removeChar(arg,'['); 241 | removeChar(arg,']'); 242 | removeChar(arg,'<'); 243 | removeChar(arg,'>'); 244 | arg.erase(0, arg.find_last_of(theDelimiter) + 1); 245 | std::cout << theDelimiter; 246 | std::cout << "" << arg << ""; 247 | } 248 | std::cout << "" << std::endl; 249 | 250 | } 251 | 252 | inline void DocBookOutput::printLongArg(Arg* a) 253 | { 254 | std::string lt = "<"; 255 | std::string gt = ">"; 256 | 257 | std::string desc = a->getDescription(); 258 | substituteSpecialChars(desc,'<',lt); 259 | substituteSpecialChars(desc,'>',gt); 260 | 261 | std::cout << "" << std::endl; 262 | 263 | if ( !a->getFlag().empty() ) 264 | { 265 | std::cout << "" << std::endl; 266 | std::cout << "" << std::endl; 269 | std::cout << "" << std::endl; 270 | } 271 | 272 | std::cout << "" << std::endl; 273 | std::cout << "" << std::endl; 287 | std::cout << "" << std::endl; 288 | 289 | std::cout << "" << std::endl; 290 | std::cout << "" << std::endl; 291 | std::cout << desc << std::endl; 292 | std::cout << "" << std::endl; 293 | std::cout << "" << std::endl; 294 | 295 | std::cout << "" << std::endl; 296 | } 297 | 298 | } //namespace TCLAP 299 | #endif 300 | -------------------------------------------------------------------------------- /ext/tclap/HelpVisitor.h: -------------------------------------------------------------------------------- 1 | 2 | /****************************************************************************** 3 | * 4 | * file: HelpVisitor.h 5 | * 6 | * Copyright (c) 2003, Michael E. Smoot . 7 | * All rights reverved. 8 | * 9 | * See the file COPYING in the top directory of this distribution for 10 | * more information. 11 | * 12 | * THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS 13 | * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 14 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 15 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 16 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 17 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 18 | * DEALINGS IN THE SOFTWARE. 19 | * 20 | *****************************************************************************/ 21 | 22 | #ifndef TCLAP_HELP_VISITOR_H 23 | #define TCLAP_HELP_VISITOR_H 24 | 25 | #include 26 | #include 27 | #include 28 | 29 | namespace TCLAP { 30 | 31 | /** 32 | * A Visitor object that calls the usage method of the given CmdLineOutput 33 | * object for the specified CmdLine object. 34 | */ 35 | class HelpVisitor: public Visitor 36 | { 37 | protected: 38 | 39 | /** 40 | * The CmdLine the output will be generated for. 41 | */ 42 | CmdLineInterface* _cmd; 43 | 44 | /** 45 | * The output object. 46 | */ 47 | CmdLineOutput** _out; 48 | 49 | public: 50 | 51 | /** 52 | * Constructor. 53 | * \param cmd - The CmdLine the output will be generated for. 54 | * \param out - The type of output. 55 | */ 56 | HelpVisitor(CmdLineInterface* cmd, CmdLineOutput** out) 57 | : Visitor(), _cmd( cmd ), _out( out ) { } 58 | 59 | /** 60 | * Calls the usage method of the CmdLineOutput for the 61 | * specified CmdLine. 62 | */ 63 | void visit() { (*_out)->usage(*_cmd); throw ExitException(0); } 64 | 65 | }; 66 | 67 | } 68 | 69 | #endif 70 | -------------------------------------------------------------------------------- /ext/tclap/IgnoreRestVisitor.h: -------------------------------------------------------------------------------- 1 | 2 | /****************************************************************************** 3 | * 4 | * file: IgnoreRestVisitor.h 5 | * 6 | * Copyright (c) 2003, Michael E. Smoot . 7 | * All rights reverved. 8 | * 9 | * See the file COPYING in the top directory of this distribution for 10 | * more information. 11 | * 12 | * THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS 13 | * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 14 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 15 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 16 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 17 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 18 | * DEALINGS IN THE SOFTWARE. 19 | * 20 | *****************************************************************************/ 21 | 22 | 23 | #ifndef TCLAP_IGNORE_REST_VISITOR_H 24 | #define TCLAP_IGNORE_REST_VISITOR_H 25 | 26 | #include 27 | #include 28 | 29 | namespace TCLAP { 30 | 31 | /** 32 | * A Vistor that tells the CmdLine to begin ignoring arguments after 33 | * this one is parsed. 34 | */ 35 | class IgnoreRestVisitor: public Visitor 36 | { 37 | public: 38 | 39 | /** 40 | * Constructor. 41 | */ 42 | IgnoreRestVisitor() : Visitor() {} 43 | 44 | /** 45 | * Sets Arg::_ignoreRest. 46 | */ 47 | void visit() { Arg::beginIgnoring(); } 48 | }; 49 | 50 | } 51 | 52 | #endif 53 | -------------------------------------------------------------------------------- /ext/tclap/MultiSwitchArg.h: -------------------------------------------------------------------------------- 1 | 2 | /****************************************************************************** 3 | * 4 | * file: MultiSwitchArg.h 5 | * 6 | * Copyright (c) 2003, Michael E. Smoot . 7 | * Copyright (c) 2004, Michael E. Smoot, Daniel Aarno. 8 | * Copyright (c) 2005, Michael E. Smoot, Daniel Aarno, Erik Zeek. 9 | * All rights reverved. 10 | * 11 | * See the file COPYING in the top directory of this distribution for 12 | * more information. 13 | * 14 | * THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS 15 | * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 | * DEALINGS IN THE SOFTWARE. 21 | * 22 | *****************************************************************************/ 23 | 24 | 25 | #ifndef TCLAP_MULTI_SWITCH_ARG_H 26 | #define TCLAP_MULTI_SWITCH_ARG_H 27 | 28 | #include 29 | #include 30 | 31 | #include 32 | 33 | namespace TCLAP { 34 | 35 | /** 36 | * A multiple switch argument. If the switch is set on the command line, then 37 | * the getValue method will return the number of times the switch appears. 38 | */ 39 | class MultiSwitchArg : public SwitchArg 40 | { 41 | protected: 42 | 43 | /** 44 | * The value of the switch. 45 | */ 46 | int _value; 47 | 48 | /** 49 | * Used to support the reset() method so that ValueArg can be 50 | * reset to their constructed value. 51 | */ 52 | int _default; 53 | 54 | public: 55 | 56 | /** 57 | * MultiSwitchArg constructor. 58 | * \param flag - The one character flag that identifies this 59 | * argument on the command line. 60 | * \param name - A one word name for the argument. Can be 61 | * used as a long flag on the command line. 62 | * \param desc - A description of what the argument is for or 63 | * does. 64 | * \param init - Optional. The initial/default value of this Arg. 65 | * Defaults to 0. 66 | * \param v - An optional visitor. You probably should not 67 | * use this unless you have a very good reason. 68 | */ 69 | MultiSwitchArg(const std::string& flag, 70 | const std::string& name, 71 | const std::string& desc, 72 | int init = 0, 73 | Visitor* v = NULL); 74 | 75 | 76 | /** 77 | * MultiSwitchArg constructor. 78 | * \param flag - The one character flag that identifies this 79 | * argument on the command line. 80 | * \param name - A one word name for the argument. Can be 81 | * used as a long flag on the command line. 82 | * \param desc - A description of what the argument is for or 83 | * does. 84 | * \param parser - A CmdLine parser object to add this Arg to 85 | * \param init - Optional. The initial/default value of this Arg. 86 | * Defaults to 0. 87 | * \param v - An optional visitor. You probably should not 88 | * use this unless you have a very good reason. 89 | */ 90 | MultiSwitchArg(const std::string& flag, 91 | const std::string& name, 92 | const std::string& desc, 93 | CmdLineInterface& parser, 94 | int init = 0, 95 | Visitor* v = NULL); 96 | 97 | 98 | /** 99 | * Handles the processing of the argument. 100 | * This re-implements the SwitchArg version of this method to set the 101 | * _value of the argument appropriately. 102 | * \param i - Pointer the the current argument in the list. 103 | * \param args - Mutable list of strings. Passed 104 | * in from main(). 105 | */ 106 | virtual bool processArg(int* i, std::vector& args); 107 | 108 | /** 109 | * Returns int, the number of times the switch has been set. 110 | */ 111 | int getValue(); 112 | 113 | /** 114 | * Returns the shortID for this Arg. 115 | */ 116 | std::string shortID(const std::string& val) const; 117 | 118 | /** 119 | * Returns the longID for this Arg. 120 | */ 121 | std::string longID(const std::string& val) const; 122 | 123 | void reset(); 124 | 125 | }; 126 | 127 | ////////////////////////////////////////////////////////////////////// 128 | //BEGIN MultiSwitchArg.cpp 129 | ////////////////////////////////////////////////////////////////////// 130 | inline MultiSwitchArg::MultiSwitchArg(const std::string& flag, 131 | const std::string& name, 132 | const std::string& desc, 133 | int init, 134 | Visitor* v ) 135 | : SwitchArg(flag, name, desc, false, v), 136 | _value( init ), 137 | _default( init ) 138 | { } 139 | 140 | inline MultiSwitchArg::MultiSwitchArg(const std::string& flag, 141 | const std::string& name, 142 | const std::string& desc, 143 | CmdLineInterface& parser, 144 | int init, 145 | Visitor* v ) 146 | : SwitchArg(flag, name, desc, false, v), 147 | _value( init ), 148 | _default( init ) 149 | { 150 | parser.add( this ); 151 | } 152 | 153 | inline int MultiSwitchArg::getValue() { return _value; } 154 | 155 | inline bool MultiSwitchArg::processArg(int *i, std::vector& args) 156 | { 157 | if ( _ignoreable && Arg::ignoreRest() ) 158 | return false; 159 | 160 | if ( argMatches( args[*i] )) 161 | { 162 | // so the isSet() method will work 163 | _alreadySet = true; 164 | 165 | // Matched argument: increment value. 166 | ++_value; 167 | 168 | _checkWithVisitor(); 169 | 170 | return true; 171 | } 172 | else if ( combinedSwitchesMatch( args[*i] ) ) 173 | { 174 | // so the isSet() method will work 175 | _alreadySet = true; 176 | 177 | // Matched argument: increment value. 178 | ++_value; 179 | 180 | // Check for more in argument and increment value. 181 | while ( combinedSwitchesMatch( args[*i] ) ) 182 | ++_value; 183 | 184 | _checkWithVisitor(); 185 | 186 | return false; 187 | } 188 | else 189 | return false; 190 | } 191 | 192 | inline std::string 193 | MultiSwitchArg::shortID(const std::string& val) const 194 | { 195 | return Arg::shortID(val) + " ... "; 196 | } 197 | 198 | inline std::string 199 | MultiSwitchArg::longID(const std::string& val) const 200 | { 201 | return Arg::longID(val) + " (accepted multiple times)"; 202 | } 203 | 204 | inline void 205 | MultiSwitchArg::reset() 206 | { 207 | MultiSwitchArg::_value = MultiSwitchArg::_default; 208 | } 209 | 210 | ////////////////////////////////////////////////////////////////////// 211 | //END MultiSwitchArg.cpp 212 | ////////////////////////////////////////////////////////////////////// 213 | 214 | } //namespace TCLAP 215 | 216 | #endif 217 | -------------------------------------------------------------------------------- /ext/tclap/OptionalUnlabeledTracker.h: -------------------------------------------------------------------------------- 1 | 2 | 3 | /****************************************************************************** 4 | * 5 | * file: OptionalUnlabeledTracker.h 6 | * 7 | * Copyright (c) 2005, Michael E. Smoot . 8 | * All rights reverved. 9 | * 10 | * See the file COPYING in the top directory of this distribution for 11 | * more information. 12 | * 13 | * THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS 14 | * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 16 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 19 | * DEALINGS IN THE SOFTWARE. 20 | * 21 | *****************************************************************************/ 22 | 23 | 24 | #ifndef TCLAP_OPTIONAL_UNLABELED_TRACKER_H 25 | #define TCLAP_OPTIONAL_UNLABELED_TRACKER_H 26 | 27 | #include 28 | 29 | namespace TCLAP { 30 | 31 | class OptionalUnlabeledTracker 32 | { 33 | 34 | public: 35 | 36 | static void check( bool req, const std::string& argName ); 37 | 38 | static void gotOptional() { alreadyOptionalRef() = true; } 39 | 40 | static bool& alreadyOptional() { return alreadyOptionalRef(); } 41 | 42 | private: 43 | 44 | static bool& alreadyOptionalRef() { static bool ct = false; return ct; } 45 | }; 46 | 47 | 48 | inline void OptionalUnlabeledTracker::check( bool req, const std::string& argName ) 49 | { 50 | if ( OptionalUnlabeledTracker::alreadyOptional() ) 51 | throw( SpecificationException( 52 | "You can't specify ANY Unlabeled Arg following an optional Unlabeled Arg", 53 | argName ) ); 54 | 55 | if ( !req ) 56 | OptionalUnlabeledTracker::gotOptional(); 57 | } 58 | 59 | 60 | } // namespace TCLAP 61 | 62 | #endif 63 | -------------------------------------------------------------------------------- /ext/tclap/StandardTraits.h: -------------------------------------------------------------------------------- 1 | // -*- Mode: c++; c-basic-offset: 4; tab-width: 4; -*- 2 | 3 | /****************************************************************************** 4 | * 5 | * file: StandardTraits.h 6 | * 7 | * Copyright (c) 2007, Daniel Aarno, Michael E. Smoot . 8 | * All rights reverved. 9 | * 10 | * See the file COPYING in the top directory of this distribution for 11 | * more information. 12 | * 13 | * THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS 14 | * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 16 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 19 | * DEALINGS IN THE SOFTWARE. 20 | * 21 | *****************************************************************************/ 22 | 23 | // This is an internal tclap file, you should probably not have to 24 | // include this directly 25 | 26 | #ifndef TCLAP_STANDARD_TRAITS_H 27 | #define TCLAP_STANDARD_TRAITS_H 28 | 29 | #ifdef HAVE_CONFIG_H 30 | #include // To check for long long 31 | #endif 32 | 33 | namespace TCLAP { 34 | 35 | // ====================================================================== 36 | // Integer types 37 | // ====================================================================== 38 | 39 | /** 40 | * long longs have value-like semantics. 41 | */ 42 | template<> 43 | struct ArgTraits { 44 | typedef ValueLike ValueCategory; 45 | }; 46 | 47 | /** 48 | * longs have value-like semantics. 49 | */ 50 | template<> 51 | struct ArgTraits { 52 | typedef ValueLike ValueCategory; 53 | }; 54 | 55 | /** 56 | * ints have value-like semantics. 57 | */ 58 | template<> 59 | struct ArgTraits { 60 | typedef ValueLike ValueCategory; 61 | }; 62 | 63 | /** 64 | * shorts have value-like semantics. 65 | */ 66 | template<> 67 | struct ArgTraits { 68 | typedef ValueLike ValueCategory; 69 | }; 70 | 71 | /** 72 | * chars have value-like semantics. 73 | */ 74 | template<> 75 | struct ArgTraits { 76 | typedef ValueLike ValueCategory; 77 | }; 78 | 79 | #ifdef HAVE_LONG_LONG 80 | /** 81 | * long longs have value-like semantics. 82 | */ 83 | template<> 84 | struct ArgTraits { 85 | typedef ValueLike ValueCategory; 86 | }; 87 | #endif 88 | 89 | // ====================================================================== 90 | // Unsigned integer types 91 | // ====================================================================== 92 | 93 | /** 94 | * unsigned long longs have value-like semantics. 95 | */ 96 | template<> 97 | struct ArgTraits { 98 | typedef ValueLike ValueCategory; 99 | }; 100 | 101 | /** 102 | * unsigned longs have value-like semantics. 103 | */ 104 | template<> 105 | struct ArgTraits { 106 | typedef ValueLike ValueCategory; 107 | }; 108 | 109 | /** 110 | * unsigned ints have value-like semantics. 111 | */ 112 | template<> 113 | struct ArgTraits { 114 | typedef ValueLike ValueCategory; 115 | }; 116 | 117 | /** 118 | * unsigned shorts have value-like semantics. 119 | */ 120 | template<> 121 | struct ArgTraits { 122 | typedef ValueLike ValueCategory; 123 | }; 124 | 125 | /** 126 | * unsigned chars have value-like semantics. 127 | */ 128 | template<> 129 | struct ArgTraits { 130 | typedef ValueLike ValueCategory; 131 | }; 132 | 133 | #ifdef HAVE_LONG_LONG 134 | /** 135 | * unsigned long longs have value-like semantics. 136 | */ 137 | template<> 138 | struct ArgTraits { 139 | typedef ValueLike ValueCategory; 140 | }; 141 | #endif 142 | 143 | // ====================================================================== 144 | // Float types 145 | // ====================================================================== 146 | 147 | /** 148 | * floats have value-like semantics. 149 | */ 150 | template<> 151 | struct ArgTraits { 152 | typedef ValueLike ValueCategory; 153 | }; 154 | 155 | /** 156 | * doubles have value-like semantics. 157 | */ 158 | template<> 159 | struct ArgTraits { 160 | typedef ValueLike ValueCategory; 161 | }; 162 | 163 | // ====================================================================== 164 | // Other types 165 | // ====================================================================== 166 | 167 | /** 168 | * bools have value-like semantics. 169 | */ 170 | template<> 171 | struct ArgTraits { 172 | typedef ValueLike ValueCategory; 173 | }; 174 | 175 | /** 176 | * wchar_ts have value-like semantics. 177 | */ 178 | template<> 179 | struct ArgTraits { 180 | typedef ValueLike ValueCategory; 181 | }; 182 | 183 | /** 184 | * Strings have string like argument traits. 185 | */ 186 | template<> 187 | struct ArgTraits { 188 | typedef StringLike ValueCategory; 189 | }; 190 | 191 | template 192 | void SetString(T &dst, const std::string &src) 193 | { 194 | dst = src; 195 | } 196 | 197 | } // namespace 198 | 199 | #endif 200 | 201 | -------------------------------------------------------------------------------- /ext/tclap/StdOutput.h: -------------------------------------------------------------------------------- 1 | // -*- Mode: c++; c-basic-offset: 4; tab-width: 4; -*- 2 | 3 | /****************************************************************************** 4 | * 5 | * file: StdOutput.h 6 | * 7 | * Copyright (c) 2004, Michael E. Smoot 8 | * All rights reverved. 9 | * 10 | * See the file COPYING in the top directory of this distribution for 11 | * more information. 12 | * 13 | * THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS 14 | * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 16 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 19 | * DEALINGS IN THE SOFTWARE. 20 | * 21 | *****************************************************************************/ 22 | 23 | #ifndef TCLAP_STDCMDLINEOUTPUT_H 24 | #define TCLAP_STDCMDLINEOUTPUT_H 25 | 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | 32 | #include 33 | #include 34 | #include 35 | #include 36 | 37 | namespace TCLAP { 38 | 39 | /** 40 | * A class that isolates any output from the CmdLine object so that it 41 | * may be easily modified. 42 | */ 43 | class StdOutput : public CmdLineOutput 44 | { 45 | 46 | public: 47 | 48 | /** 49 | * Prints the usage to stdout. Can be overridden to 50 | * produce alternative behavior. 51 | * \param c - The CmdLine object the output is generated for. 52 | */ 53 | virtual void usage(CmdLineInterface& c); 54 | 55 | /** 56 | * Prints the version to stdout. Can be overridden 57 | * to produce alternative behavior. 58 | * \param c - The CmdLine object the output is generated for. 59 | */ 60 | virtual void version(CmdLineInterface& c); 61 | 62 | /** 63 | * Prints (to stderr) an error message, short usage 64 | * Can be overridden to produce alternative behavior. 65 | * \param c - The CmdLine object the output is generated for. 66 | * \param e - The ArgException that caused the failure. 67 | */ 68 | virtual void failure(CmdLineInterface& c, 69 | ArgException& e ); 70 | 71 | protected: 72 | 73 | /** 74 | * Writes a brief usage message with short args. 75 | * \param c - The CmdLine object the output is generated for. 76 | * \param os - The stream to write the message to. 77 | */ 78 | void _shortUsage( CmdLineInterface& c, std::ostream& os ) const; 79 | 80 | /** 81 | * Writes a longer usage message with long and short args, 82 | * provides descriptions and prints message. 83 | * \param c - The CmdLine object the output is generated for. 84 | * \param os - The stream to write the message to. 85 | */ 86 | void _longUsage( CmdLineInterface& c, std::ostream& os ) const; 87 | 88 | /** 89 | * This function inserts line breaks and indents long strings 90 | * according the params input. It will only break lines at spaces, 91 | * commas and pipes. 92 | * \param os - The stream to be printed to. 93 | * \param s - The string to be printed. 94 | * \param maxWidth - The maxWidth allowed for the output line. 95 | * \param indentSpaces - The number of spaces to indent the first line. 96 | * \param secondLineOffset - The number of spaces to indent the second 97 | * and all subsequent lines in addition to indentSpaces. 98 | */ 99 | void spacePrint( std::ostream& os, 100 | const std::string& s, 101 | int maxWidth, 102 | int indentSpaces, 103 | int secondLineOffset ) const; 104 | 105 | }; 106 | 107 | 108 | inline void StdOutput::version(CmdLineInterface& _cmd) 109 | { 110 | std::string progName = _cmd.getProgramName(); 111 | std::string version = _cmd.getVersion(); 112 | 113 | std::cout << std::endl << progName << " version: " 114 | << version << std::endl << std::endl; 115 | } 116 | 117 | inline void StdOutput::usage(CmdLineInterface& _cmd ) 118 | { 119 | std::cout << std::endl << "USAGE: " << std::endl << std::endl; 120 | 121 | _shortUsage( _cmd, std::cout ); 122 | 123 | std::cout << std::endl << std::endl << "Where: " << std::endl << std::endl; 124 | 125 | _longUsage( _cmd, std::cout ); 126 | 127 | std::cout << std::endl; 128 | 129 | } 130 | 131 | inline void StdOutput::failure( CmdLineInterface& _cmd, 132 | ArgException& e ) 133 | { 134 | std::string progName = _cmd.getProgramName(); 135 | 136 | std::cerr << "PARSE ERROR: " << e.argId() << std::endl 137 | << " " << e.error() << std::endl << std::endl; 138 | 139 | if ( _cmd.hasHelpAndVersion() ) 140 | { 141 | std::cerr << "Brief USAGE: " << std::endl; 142 | 143 | _shortUsage( _cmd, std::cerr ); 144 | 145 | std::cerr << std::endl << "For complete USAGE and HELP type: " 146 | << std::endl << " " << progName << " --help" 147 | << std::endl << std::endl; 148 | } 149 | else 150 | usage(_cmd); 151 | 152 | throw ExitException(1); 153 | } 154 | 155 | inline void 156 | StdOutput::_shortUsage( CmdLineInterface& _cmd, 157 | std::ostream& os ) const 158 | { 159 | std::list argList = _cmd.getArgList(); 160 | std::string progName = _cmd.getProgramName(); 161 | XorHandler xorHandler = _cmd.getXorHandler(); 162 | std::vector< std::vector > xorList = xorHandler.getXorList(); 163 | 164 | std::string s = progName + " "; 165 | 166 | // first the xor 167 | for ( int i = 0; static_cast(i) < xorList.size(); i++ ) 168 | { 169 | s += " {"; 170 | for ( ArgVectorIterator it = xorList[i].begin(); 171 | it != xorList[i].end(); it++ ) 172 | s += (*it)->shortID() + "|"; 173 | 174 | s[s.length()-1] = '}'; 175 | } 176 | 177 | // then the rest 178 | for (ArgListIterator it = argList.begin(); it != argList.end(); it++) 179 | if ( !xorHandler.contains( (*it) ) ) 180 | s += " " + (*it)->shortID(); 181 | 182 | // if the program name is too long, then adjust the second line offset 183 | int secondLineOffset = static_cast(progName.length()) + 2; 184 | if ( secondLineOffset > 75/2 ) 185 | secondLineOffset = static_cast(75/2); 186 | 187 | spacePrint( os, s, 75, 3, secondLineOffset ); 188 | } 189 | 190 | inline void 191 | StdOutput::_longUsage( CmdLineInterface& _cmd, 192 | std::ostream& os ) const 193 | { 194 | std::list argList = _cmd.getArgList(); 195 | std::string message = _cmd.getMessage(); 196 | XorHandler xorHandler = _cmd.getXorHandler(); 197 | std::vector< std::vector > xorList = xorHandler.getXorList(); 198 | 199 | // first the xor 200 | for ( int i = 0; static_cast(i) < xorList.size(); i++ ) 201 | { 202 | for ( ArgVectorIterator it = xorList[i].begin(); 203 | it != xorList[i].end(); 204 | it++ ) 205 | { 206 | spacePrint( os, (*it)->longID(), 75, 3, 3 ); 207 | spacePrint( os, (*it)->getDescription(), 75, 5, 0 ); 208 | 209 | if ( it+1 != xorList[i].end() ) 210 | spacePrint(os, "-- OR --", 75, 9, 0); 211 | } 212 | os << std::endl << std::endl; 213 | } 214 | 215 | // then the rest 216 | for (ArgListIterator it = argList.begin(); it != argList.end(); it++) 217 | if ( !xorHandler.contains( (*it) ) ) 218 | { 219 | spacePrint( os, (*it)->longID(), 75, 3, 3 ); 220 | spacePrint( os, (*it)->getDescription(), 75, 5, 0 ); 221 | os << std::endl; 222 | } 223 | 224 | os << std::endl; 225 | 226 | spacePrint( os, message, 75, 3, 0 ); 227 | } 228 | 229 | inline void StdOutput::spacePrint( std::ostream& os, 230 | const std::string& s, 231 | int maxWidth, 232 | int indentSpaces, 233 | int secondLineOffset ) const 234 | { 235 | int len = static_cast(s.length()); 236 | 237 | if ( (len + indentSpaces > maxWidth) && maxWidth > 0 ) 238 | { 239 | int allowedLen = maxWidth - indentSpaces; 240 | int start = 0; 241 | while ( start < len ) 242 | { 243 | // find the substring length 244 | // int stringLen = std::min( len - start, allowedLen ); 245 | // doing it this way to support a VisualC++ 2005 bug 246 | using namespace std; 247 | int stringLen = min( len - start, allowedLen ); 248 | 249 | // trim the length so it doesn't end in middle of a word 250 | if ( stringLen == allowedLen ) 251 | while ( stringLen >= 0 && 252 | s[stringLen+start] != ' ' && 253 | s[stringLen+start] != ',' && 254 | s[stringLen+start] != '|' ) 255 | stringLen--; 256 | 257 | // ok, the word is longer than the line, so just split 258 | // wherever the line ends 259 | if ( stringLen <= 0 ) 260 | stringLen = allowedLen; 261 | 262 | // check for newlines 263 | for ( int i = 0; i < stringLen; i++ ) 264 | if ( s[start+i] == '\n' ) 265 | stringLen = i+1; 266 | 267 | // print the indent 268 | for ( int i = 0; i < indentSpaces; i++ ) 269 | os << " "; 270 | 271 | if ( start == 0 ) 272 | { 273 | // handle second line offsets 274 | indentSpaces += secondLineOffset; 275 | 276 | // adjust allowed len 277 | allowedLen -= secondLineOffset; 278 | } 279 | 280 | os << s.substr(start,stringLen) << std::endl; 281 | 282 | // so we don't start a line with a space 283 | while ( s[stringLen+start] == ' ' && start < len ) 284 | start++; 285 | 286 | start += stringLen; 287 | } 288 | } 289 | else 290 | { 291 | for ( int i = 0; i < indentSpaces; i++ ) 292 | os << " "; 293 | os << s << std::endl; 294 | } 295 | } 296 | 297 | } //namespace TCLAP 298 | #endif 299 | -------------------------------------------------------------------------------- /ext/tclap/SwitchArg.h: -------------------------------------------------------------------------------- 1 | 2 | /****************************************************************************** 3 | * 4 | * file: SwitchArg.h 5 | * 6 | * Copyright (c) 2003, Michael E. Smoot . 7 | * Copyright (c) 2004, Michael E. Smoot, Daniel Aarno. 8 | * All rights reverved. 9 | * 10 | * See the file COPYING in the top directory of this distribution for 11 | * more information. 12 | * 13 | * THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS 14 | * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 16 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 19 | * DEALINGS IN THE SOFTWARE. 20 | * 21 | *****************************************************************************/ 22 | 23 | 24 | #ifndef TCLAP_SWITCH_ARG_H 25 | #define TCLAP_SWITCH_ARG_H 26 | 27 | #include 28 | #include 29 | 30 | #include 31 | 32 | namespace TCLAP { 33 | 34 | /** 35 | * A simple switch argument. If the switch is set on the command line, then 36 | * the getValue method will return the opposite of the default value for the 37 | * switch. 38 | */ 39 | class SwitchArg : public Arg 40 | { 41 | protected: 42 | 43 | /** 44 | * The value of the switch. 45 | */ 46 | bool _value; 47 | 48 | /** 49 | * Used to support the reset() method so that ValueArg can be 50 | * reset to their constructed value. 51 | */ 52 | bool _default; 53 | 54 | public: 55 | 56 | /** 57 | * SwitchArg constructor. 58 | * \param flag - The one character flag that identifies this 59 | * argument on the command line. 60 | * \param name - A one word name for the argument. Can be 61 | * used as a long flag on the command line. 62 | * \param desc - A description of what the argument is for or 63 | * does. 64 | * \param def - The default value for this Switch. 65 | * \param v - An optional visitor. You probably should not 66 | * use this unless you have a very good reason. 67 | */ 68 | SwitchArg(const std::string& flag, 69 | const std::string& name, 70 | const std::string& desc, 71 | bool def = false, 72 | Visitor* v = NULL); 73 | 74 | 75 | /** 76 | * SwitchArg constructor. 77 | * \param flag - The one character flag that identifies this 78 | * argument on the command line. 79 | * \param name - A one word name for the argument. Can be 80 | * used as a long flag on the command line. 81 | * \param desc - A description of what the argument is for or 82 | * does. 83 | * \param parser - A CmdLine parser object to add this Arg to 84 | * \param def - The default value for this Switch. 85 | * \param v - An optional visitor. You probably should not 86 | * use this unless you have a very good reason. 87 | */ 88 | SwitchArg(const std::string& flag, 89 | const std::string& name, 90 | const std::string& desc, 91 | CmdLineInterface& parser, 92 | bool def = false, 93 | Visitor* v = NULL); 94 | 95 | 96 | /** 97 | * Handles the processing of the argument. 98 | * This re-implements the Arg version of this method to set the 99 | * _value of the argument appropriately. 100 | * \param i - Pointer the the current argument in the list. 101 | * \param args - Mutable list of strings. Passed 102 | * in from main(). 103 | */ 104 | virtual bool processArg(int* i, std::vector& args); 105 | 106 | /** 107 | * Checks a string to see if any of the chars in the string 108 | * match the flag for this Switch. 109 | */ 110 | bool combinedSwitchesMatch(std::string& combined); 111 | 112 | /** 113 | * Returns bool, whether or not the switch has been set. 114 | */ 115 | bool getValue(); 116 | 117 | virtual void reset(); 118 | 119 | }; 120 | 121 | ////////////////////////////////////////////////////////////////////// 122 | //BEGIN SwitchArg.cpp 123 | ////////////////////////////////////////////////////////////////////// 124 | inline SwitchArg::SwitchArg(const std::string& flag, 125 | const std::string& name, 126 | const std::string& desc, 127 | bool default_val, 128 | Visitor* v ) 129 | : Arg(flag, name, desc, false, false, v), 130 | _value( default_val ), 131 | _default( default_val ) 132 | { } 133 | 134 | inline SwitchArg::SwitchArg(const std::string& flag, 135 | const std::string& name, 136 | const std::string& desc, 137 | CmdLineInterface& parser, 138 | bool default_val, 139 | Visitor* v ) 140 | : Arg(flag, name, desc, false, false, v), 141 | _value( default_val ), 142 | _default(default_val) 143 | { 144 | parser.add( this ); 145 | } 146 | 147 | inline bool SwitchArg::getValue() { return _value; } 148 | 149 | inline bool SwitchArg::combinedSwitchesMatch(std::string& combinedSwitches ) 150 | { 151 | // make sure this is actually a combined switch 152 | if ( combinedSwitches.length() > 0 && 153 | combinedSwitches[0] != Arg::flagStartString()[0] ) 154 | return false; 155 | 156 | // make sure it isn't a long name 157 | if ( combinedSwitches.substr( 0, Arg::nameStartString().length() ) == 158 | Arg::nameStartString() ) 159 | return false; 160 | 161 | // make sure the delimiter isn't in the string 162 | if ( combinedSwitches.find_first_of( Arg::delimiter() ) != std::string::npos ) 163 | return false; 164 | 165 | // ok, we're not specifying a ValueArg, so we know that we have 166 | // a combined switch list. 167 | for ( unsigned int i = 1; i < combinedSwitches.length(); i++ ) 168 | if ( _flag.length() > 0 && 169 | combinedSwitches[i] == _flag[0] && 170 | _flag[0] != Arg::flagStartString()[0] ) 171 | { 172 | // update the combined switches so this one is no longer present 173 | // this is necessary so that no unlabeled args are matched 174 | // later in the processing. 175 | //combinedSwitches.erase(i,1); 176 | combinedSwitches[i] = Arg::blankChar(); 177 | return true; 178 | } 179 | 180 | // none of the switches passed in the list match. 181 | return false; 182 | } 183 | 184 | 185 | inline bool SwitchArg::processArg(int *i, std::vector& args) 186 | { 187 | if ( _ignoreable && Arg::ignoreRest() ) 188 | return false; 189 | 190 | if ( argMatches( args[*i] ) || combinedSwitchesMatch( args[*i] ) ) 191 | { 192 | // If we match on a combined switch, then we want to return false 193 | // so that other switches in the combination will also have a 194 | // chance to match. 195 | bool ret = false; 196 | if ( argMatches( args[*i] ) ) 197 | ret = true; 198 | 199 | if ( _alreadySet || ( !ret && combinedSwitchesMatch( args[*i] ) ) ) 200 | throw(CmdLineParseException("Argument already set!", toString())); 201 | 202 | _alreadySet = true; 203 | 204 | if ( _value == true ) 205 | _value = false; 206 | else 207 | _value = true; 208 | 209 | _checkWithVisitor(); 210 | 211 | return ret; 212 | } 213 | else 214 | return false; 215 | } 216 | 217 | inline void SwitchArg::reset() 218 | { 219 | Arg::reset(); 220 | _value = _default; 221 | } 222 | ////////////////////////////////////////////////////////////////////// 223 | //End SwitchArg.cpp 224 | ////////////////////////////////////////////////////////////////////// 225 | 226 | } //namespace TCLAP 227 | 228 | #endif 229 | -------------------------------------------------------------------------------- /ext/tclap/ValuesConstraint.h: -------------------------------------------------------------------------------- 1 | 2 | 3 | /****************************************************************************** 4 | * 5 | * file: ValuesConstraint.h 6 | * 7 | * Copyright (c) 2005, Michael E. Smoot 8 | * All rights reverved. 9 | * 10 | * See the file COPYING in the top directory of this distribution for 11 | * more information. 12 | * 13 | * THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS 14 | * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 16 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 19 | * DEALINGS IN THE SOFTWARE. 20 | * 21 | *****************************************************************************/ 22 | 23 | #ifndef TCLAP_VALUESCONSTRAINT_H 24 | #define TCLAP_VALUESCONSTRAINT_H 25 | 26 | #include 27 | #include 28 | #include 29 | 30 | #ifdef HAVE_CONFIG_H 31 | #include 32 | #else 33 | #define HAVE_SSTREAM 34 | #endif 35 | 36 | #if defined(HAVE_SSTREAM) 37 | #include 38 | #elif defined(HAVE_STRSTREAM) 39 | #include 40 | #else 41 | #error "Need a stringstream (sstream or strstream) to compile!" 42 | #endif 43 | 44 | namespace TCLAP { 45 | 46 | /** 47 | * A Constraint that constrains the Arg to only those values specified 48 | * in the constraint. 49 | */ 50 | template 51 | class ValuesConstraint : public Constraint 52 | { 53 | 54 | public: 55 | 56 | /** 57 | * Constructor. 58 | * \param allowed - vector of allowed values. 59 | */ 60 | ValuesConstraint(std::vector& allowed); 61 | 62 | /** 63 | * Virtual destructor. 64 | */ 65 | virtual ~ValuesConstraint() {} 66 | 67 | /** 68 | * Returns a description of the Constraint. 69 | */ 70 | virtual std::string description() const; 71 | 72 | /** 73 | * Returns the short ID for the Constraint. 74 | */ 75 | virtual std::string shortID() const; 76 | 77 | /** 78 | * The method used to verify that the value parsed from the command 79 | * line meets the constraint. 80 | * \param value - The value that will be checked. 81 | */ 82 | virtual bool check(const T& value) const; 83 | 84 | protected: 85 | 86 | /** 87 | * The list of valid values. 88 | */ 89 | std::vector _allowed; 90 | 91 | /** 92 | * The string used to describe the allowed values of this constraint. 93 | */ 94 | std::string _typeDesc; 95 | 96 | }; 97 | 98 | template 99 | ValuesConstraint::ValuesConstraint(std::vector& allowed) 100 | : _allowed(allowed) 101 | { 102 | for ( unsigned int i = 0; i < _allowed.size(); i++ ) 103 | { 104 | 105 | #if defined(HAVE_SSTREAM) 106 | std::ostringstream os; 107 | #elif defined(HAVE_STRSTREAM) 108 | std::ostrstream os; 109 | #else 110 | #error "Need a stringstream (sstream or strstream) to compile!" 111 | #endif 112 | 113 | os << _allowed[i]; 114 | 115 | std::string temp( os.str() ); 116 | 117 | if ( i > 0 ) 118 | _typeDesc += "|"; 119 | _typeDesc += temp; 120 | } 121 | } 122 | 123 | template 124 | bool ValuesConstraint::check( const T& val ) const 125 | { 126 | if ( std::find(_allowed.begin(),_allowed.end(),val) == _allowed.end() ) 127 | return false; 128 | else 129 | return true; 130 | } 131 | 132 | template 133 | std::string ValuesConstraint::shortID() const 134 | { 135 | return _typeDesc; 136 | } 137 | 138 | template 139 | std::string ValuesConstraint::description() const 140 | { 141 | return _typeDesc; 142 | } 143 | 144 | 145 | } //namespace TCLAP 146 | #endif 147 | 148 | -------------------------------------------------------------------------------- /ext/tclap/VersionVisitor.h: -------------------------------------------------------------------------------- 1 | // -*- Mode: c++; c-basic-offset: 4; tab-width: 4; -*- 2 | 3 | /****************************************************************************** 4 | * 5 | * file: VersionVisitor.h 6 | * 7 | * Copyright (c) 2003, Michael E. Smoot . 8 | * All rights reverved. 9 | * 10 | * See the file COPYING in the top directory of this distribution for 11 | * more information. 12 | * 13 | * THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS 14 | * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 16 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 19 | * DEALINGS IN THE SOFTWARE. 20 | * 21 | *****************************************************************************/ 22 | 23 | 24 | #ifndef TCLAP_VERSION_VISITOR_H 25 | #define TCLAP_VERSION_VISITOR_H 26 | 27 | #include 28 | #include 29 | #include 30 | 31 | namespace TCLAP { 32 | 33 | /** 34 | * A Vistor that will call the version method of the given CmdLineOutput 35 | * for the specified CmdLine object and then exit. 36 | */ 37 | class VersionVisitor: public Visitor 38 | { 39 | protected: 40 | 41 | /** 42 | * The CmdLine of interest. 43 | */ 44 | CmdLineInterface* _cmd; 45 | 46 | /** 47 | * The output object. 48 | */ 49 | CmdLineOutput** _out; 50 | 51 | public: 52 | 53 | /** 54 | * Constructor. 55 | * \param cmd - The CmdLine the output is generated for. 56 | * \param out - The type of output. 57 | */ 58 | VersionVisitor( CmdLineInterface* cmd, CmdLineOutput** out ) 59 | : Visitor(), _cmd( cmd ), _out( out ) { } 60 | 61 | /** 62 | * Calls the version method of the output object using the 63 | * specified CmdLine. 64 | */ 65 | void visit() { 66 | (*_out)->version(*_cmd); 67 | throw ExitException(0); 68 | } 69 | 70 | }; 71 | 72 | } 73 | 74 | #endif 75 | -------------------------------------------------------------------------------- /ext/tclap/Visitor.h: -------------------------------------------------------------------------------- 1 | 2 | /****************************************************************************** 3 | * 4 | * file: Visitor.h 5 | * 6 | * Copyright (c) 2003, Michael E. Smoot . 7 | * All rights reverved. 8 | * 9 | * See the file COPYING in the top directory of this distribution for 10 | * more information. 11 | * 12 | * THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS 13 | * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 14 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 15 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 16 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 17 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 18 | * DEALINGS IN THE SOFTWARE. 19 | * 20 | *****************************************************************************/ 21 | 22 | 23 | #ifndef TCLAP_VISITOR_H 24 | #define TCLAP_VISITOR_H 25 | 26 | namespace TCLAP { 27 | 28 | /** 29 | * A base class that defines the interface for visitors. 30 | */ 31 | class Visitor 32 | { 33 | public: 34 | 35 | /** 36 | * Constructor. Does nothing. 37 | */ 38 | Visitor() { } 39 | 40 | /** 41 | * Destructor. Does nothing. 42 | */ 43 | virtual ~Visitor() { } 44 | 45 | /** 46 | * Does nothing. Should be overridden by child. 47 | */ 48 | virtual void visit() { } 49 | }; 50 | 51 | } 52 | 53 | #endif 54 | -------------------------------------------------------------------------------- /ext/tclap/XorHandler.h: -------------------------------------------------------------------------------- 1 | 2 | /****************************************************************************** 3 | * 4 | * file: XorHandler.h 5 | * 6 | * Copyright (c) 2003, Michael E. Smoot . 7 | * Copyright (c) 2004, Michael E. Smoot, Daniel Aarno. 8 | * All rights reverved. 9 | * 10 | * See the file COPYING in the top directory of this distribution for 11 | * more information. 12 | * 13 | * THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS 14 | * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 16 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 19 | * DEALINGS IN THE SOFTWARE. 20 | * 21 | *****************************************************************************/ 22 | 23 | #ifndef TCLAP_XORHANDLER_H 24 | #define TCLAP_XORHANDLER_H 25 | 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | 32 | namespace TCLAP { 33 | 34 | /** 35 | * This class handles lists of Arg's that are to be XOR'd on the command 36 | * line. This is used by CmdLine and you shouldn't ever use it. 37 | */ 38 | class XorHandler 39 | { 40 | protected: 41 | 42 | /** 43 | * The list of of lists of Arg's to be or'd together. 44 | */ 45 | std::vector< std::vector > _orList; 46 | 47 | public: 48 | 49 | /** 50 | * Constructor. Does nothing. 51 | */ 52 | XorHandler( ) {} 53 | 54 | /** 55 | * Add a list of Arg*'s that will be orred together. 56 | * \param ors - list of Arg* that will be xor'd. 57 | */ 58 | void add( std::vector& ors ); 59 | 60 | /** 61 | * Checks whether the specified Arg is in one of the xor lists and 62 | * if it does match one, returns the size of the xor list that the 63 | * Arg matched. If the Arg matches, then it also sets the rest of 64 | * the Arg's in the list. You shouldn't use this. 65 | * \param a - The Arg to be checked. 66 | */ 67 | int check( const Arg* a ); 68 | 69 | /** 70 | * Returns the XOR specific short usage. 71 | */ 72 | std::string shortUsage(); 73 | 74 | /** 75 | * Prints the XOR specific long usage. 76 | * \param os - Stream to print to. 77 | */ 78 | void printLongUsage(std::ostream& os); 79 | 80 | /** 81 | * Simply checks whether the Arg is contained in one of the arg 82 | * lists. 83 | * \param a - The Arg to be checked. 84 | */ 85 | bool contains( const Arg* a ); 86 | 87 | std::vector< std::vector >& getXorList(); 88 | 89 | }; 90 | 91 | 92 | ////////////////////////////////////////////////////////////////////// 93 | //BEGIN XOR.cpp 94 | ////////////////////////////////////////////////////////////////////// 95 | inline void XorHandler::add( std::vector& ors ) 96 | { 97 | _orList.push_back( ors ); 98 | } 99 | 100 | inline int XorHandler::check( const Arg* a ) 101 | { 102 | // iterate over each XOR list 103 | for ( int i = 0; static_cast(i) < _orList.size(); i++ ) 104 | { 105 | // if the XOR list contains the arg.. 106 | ArgVectorIterator ait = std::find( _orList[i].begin(), 107 | _orList[i].end(), a ); 108 | if ( ait != _orList[i].end() ) 109 | { 110 | // go through and set each arg that is not a 111 | for ( ArgVectorIterator it = _orList[i].begin(); 112 | it != _orList[i].end(); 113 | it++ ) 114 | if ( a != (*it) ) 115 | (*it)->xorSet(); 116 | 117 | // return the number of required args that have now been set 118 | if ( (*ait)->allowMore() ) 119 | return 0; 120 | else 121 | return static_cast(_orList[i].size()); 122 | } 123 | } 124 | 125 | if ( a->isRequired() ) 126 | return 1; 127 | else 128 | return 0; 129 | } 130 | 131 | inline bool XorHandler::contains( const Arg* a ) 132 | { 133 | for ( int i = 0; static_cast(i) < _orList.size(); i++ ) 134 | for ( ArgVectorIterator it = _orList[i].begin(); 135 | it != _orList[i].end(); 136 | it++ ) 137 | if ( a == (*it) ) 138 | return true; 139 | 140 | return false; 141 | } 142 | 143 | inline std::vector< std::vector >& XorHandler::getXorList() 144 | { 145 | return _orList; 146 | } 147 | 148 | 149 | 150 | ////////////////////////////////////////////////////////////////////// 151 | //END XOR.cpp 152 | ////////////////////////////////////////////////////////////////////// 153 | 154 | } //namespace TCLAP 155 | 156 | #endif 157 | -------------------------------------------------------------------------------- /ext/tclap/ZshCompletionOutput.h: -------------------------------------------------------------------------------- 1 | // -*- Mode: c++; c-basic-offset: 4; tab-width: 4; -*- 2 | 3 | /****************************************************************************** 4 | * 5 | * file: ZshCompletionOutput.h 6 | * 7 | * Copyright (c) 2006, Oliver Kiddle 8 | * All rights reverved. 9 | * 10 | * See the file COPYING in the top directory of this distribution for 11 | * more information. 12 | * 13 | * THE SOFTWARE IS PROVIDED _AS IS_, WITHOUT WARRANTY OF ANY KIND, EXPRESS 14 | * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 16 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 19 | * DEALINGS IN THE SOFTWARE. 20 | * 21 | *****************************************************************************/ 22 | 23 | #ifndef TCLAP_ZSHCOMPLETIONOUTPUT_H 24 | #define TCLAP_ZSHCOMPLETIONOUTPUT_H 25 | 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | 32 | #include 33 | #include 34 | #include 35 | #include 36 | 37 | namespace TCLAP { 38 | 39 | /** 40 | * A class that generates a Zsh completion function as output from the usage() 41 | * method for the given CmdLine and its Args. 42 | */ 43 | class ZshCompletionOutput : public CmdLineOutput 44 | { 45 | 46 | public: 47 | 48 | ZshCompletionOutput(); 49 | 50 | /** 51 | * Prints the usage to stdout. Can be overridden to 52 | * produce alternative behavior. 53 | * \param c - The CmdLine object the output is generated for. 54 | */ 55 | virtual void usage(CmdLineInterface& c); 56 | 57 | /** 58 | * Prints the version to stdout. Can be overridden 59 | * to produce alternative behavior. 60 | * \param c - The CmdLine object the output is generated for. 61 | */ 62 | virtual void version(CmdLineInterface& c); 63 | 64 | /** 65 | * Prints (to stderr) an error message, short usage 66 | * Can be overridden to produce alternative behavior. 67 | * \param c - The CmdLine object the output is generated for. 68 | * \param e - The ArgException that caused the failure. 69 | */ 70 | virtual void failure(CmdLineInterface& c, 71 | ArgException& e ); 72 | 73 | protected: 74 | 75 | void basename( std::string& s ); 76 | void quoteSpecialChars( std::string& s ); 77 | 78 | std::string getMutexList( CmdLineInterface& _cmd, Arg* a ); 79 | void printOption( Arg* it, std::string mutex ); 80 | void printArg( Arg* it ); 81 | 82 | std::map common; 83 | char theDelimiter; 84 | }; 85 | 86 | ZshCompletionOutput::ZshCompletionOutput() 87 | { 88 | common["host"] = "_hosts"; 89 | common["hostname"] = "_hosts"; 90 | common["file"] = "_files"; 91 | common["filename"] = "_files"; 92 | common["user"] = "_users"; 93 | common["username"] = "_users"; 94 | common["directory"] = "_directories"; 95 | common["path"] = "_directories"; 96 | common["url"] = "_urls"; 97 | } 98 | 99 | inline void ZshCompletionOutput::version(CmdLineInterface& _cmd) 100 | { 101 | std::cout << _cmd.getVersion() << std::endl; 102 | } 103 | 104 | inline void ZshCompletionOutput::usage(CmdLineInterface& _cmd ) 105 | { 106 | std::list argList = _cmd.getArgList(); 107 | std::string progName = _cmd.getProgramName(); 108 | std::string version = _cmd.getVersion(); 109 | theDelimiter = _cmd.getDelimiter(); 110 | basename(progName); 111 | 112 | std::cout << "#compdef " << progName << std::endl << std::endl << 113 | "# " << progName << " version " << _cmd.getVersion() << std::endl << std::endl << 114 | "_arguments -s -S"; 115 | 116 | for (ArgListIterator it = argList.begin(); it != argList.end(); it++) 117 | { 118 | if ( (*it)->shortID().at(0) == '<' ) 119 | printArg((*it)); 120 | else if ( (*it)->getFlag() != "-" ) 121 | printOption((*it), getMutexList(_cmd, *it)); 122 | } 123 | 124 | std::cout << std::endl; 125 | } 126 | 127 | inline void ZshCompletionOutput::failure( CmdLineInterface& _cmd, 128 | ArgException& e ) 129 | { 130 | static_cast(_cmd); // unused 131 | std::cout << e.what() << std::endl; 132 | } 133 | 134 | inline void ZshCompletionOutput::quoteSpecialChars( std::string& s ) 135 | { 136 | size_t idx = s.find_last_of(':'); 137 | while ( idx != std::string::npos ) 138 | { 139 | s.insert(idx, 1, '\\'); 140 | idx = s.find_last_of(':', idx); 141 | } 142 | idx = s.find_last_of('\''); 143 | while ( idx != std::string::npos ) 144 | { 145 | s.insert(idx, "'\\'"); 146 | if (idx == 0) 147 | idx = std::string::npos; 148 | else 149 | idx = s.find_last_of('\'', --idx); 150 | } 151 | } 152 | 153 | inline void ZshCompletionOutput::basename( std::string& s ) 154 | { 155 | size_t p = s.find_last_of('/'); 156 | if ( p != std::string::npos ) 157 | { 158 | s.erase(0, p + 1); 159 | } 160 | } 161 | 162 | inline void ZshCompletionOutput::printArg(Arg* a) 163 | { 164 | static int count = 1; 165 | 166 | std::cout << " \\" << std::endl << " '"; 167 | if ( a->acceptsMultipleValues() ) 168 | std::cout << '*'; 169 | else 170 | std::cout << count++; 171 | std::cout << ':'; 172 | if ( !a->isRequired() ) 173 | std::cout << ':'; 174 | 175 | std::cout << a->getName() << ':'; 176 | std::map::iterator compArg = common.find(a->getName()); 177 | if ( compArg != common.end() ) 178 | { 179 | std::cout << compArg->second; 180 | } 181 | else 182 | { 183 | std::cout << "_guard \"^-*\" " << a->getName(); 184 | } 185 | std::cout << '\''; 186 | } 187 | 188 | inline void ZshCompletionOutput::printOption(Arg* a, std::string mutex) 189 | { 190 | std::string flag = a->flagStartChar() + a->getFlag(); 191 | std::string name = a->nameStartString() + a->getName(); 192 | std::string desc = a->getDescription(); 193 | 194 | // remove full stop and capitalisation from description as 195 | // this is the convention for zsh function 196 | if (!desc.compare(0, 12, "(required) ")) 197 | { 198 | desc.erase(0, 12); 199 | } 200 | if (!desc.compare(0, 15, "(OR required) ")) 201 | { 202 | desc.erase(0, 15); 203 | } 204 | size_t len = desc.length(); 205 | if (len && desc.at(--len) == '.') 206 | { 207 | desc.erase(len); 208 | } 209 | if (len) 210 | { 211 | desc.replace(0, 1, 1, tolower(desc.at(0))); 212 | } 213 | 214 | std::cout << " \\" << std::endl << " '" << mutex; 215 | 216 | if ( a->getFlag().empty() ) 217 | { 218 | std::cout << name; 219 | } 220 | else 221 | { 222 | std::cout << "'{" << flag << ',' << name << "}'"; 223 | } 224 | if ( theDelimiter == '=' && a->isValueRequired() ) 225 | std::cout << "=-"; 226 | quoteSpecialChars(desc); 227 | std::cout << '[' << desc << ']'; 228 | 229 | if ( a->isValueRequired() ) 230 | { 231 | std::string arg = a->shortID(); 232 | arg.erase(0, arg.find_last_of(theDelimiter) + 1); 233 | if ( arg.at(arg.length()-1) == ']' ) 234 | arg.erase(arg.length()-1); 235 | if ( arg.at(arg.length()-1) == ']' ) 236 | { 237 | arg.erase(arg.length()-1); 238 | } 239 | if ( arg.at(0) == '<' ) 240 | { 241 | arg.erase(arg.length()-1); 242 | arg.erase(0, 1); 243 | } 244 | size_t p = arg.find('|'); 245 | if ( p != std::string::npos ) 246 | { 247 | do 248 | { 249 | arg.replace(p, 1, 1, ' '); 250 | } 251 | while ( (p = arg.find_first_of('|', p)) != std::string::npos ); 252 | quoteSpecialChars(arg); 253 | std::cout << ": :(" << arg << ')'; 254 | } 255 | else 256 | { 257 | std::cout << ':' << arg; 258 | std::map::iterator compArg = common.find(arg); 259 | if ( compArg != common.end() ) 260 | { 261 | std::cout << ':' << compArg->second; 262 | } 263 | } 264 | } 265 | 266 | std::cout << '\''; 267 | } 268 | 269 | inline std::string ZshCompletionOutput::getMutexList( CmdLineInterface& _cmd, Arg* a) 270 | { 271 | XorHandler xorHandler = _cmd.getXorHandler(); 272 | std::vector< std::vector > xorList = xorHandler.getXorList(); 273 | 274 | if (a->getName() == "help" || a->getName() == "version") 275 | { 276 | return "(-)"; 277 | } 278 | 279 | std::ostringstream list; 280 | if ( a->acceptsMultipleValues() ) 281 | { 282 | list << '*'; 283 | } 284 | 285 | for ( int i = 0; static_cast(i) < xorList.size(); i++ ) 286 | { 287 | for ( ArgVectorIterator it = xorList[i].begin(); 288 | it != xorList[i].end(); 289 | it++) 290 | if ( a == (*it) ) 291 | { 292 | list << '('; 293 | for ( ArgVectorIterator iu = xorList[i].begin(); 294 | iu != xorList[i].end(); 295 | iu++ ) 296 | { 297 | bool notCur = (*iu) != a; 298 | bool hasFlag = !(*iu)->getFlag().empty(); 299 | if ( iu != xorList[i].begin() && (notCur || hasFlag) ) 300 | list << ' '; 301 | if (hasFlag) 302 | list << (*iu)->flagStartChar() << (*iu)->getFlag() << ' '; 303 | if ( notCur || hasFlag ) 304 | list << (*iu)->nameStartString() << (*iu)->getName(); 305 | } 306 | list << ')'; 307 | return list.str(); 308 | } 309 | } 310 | 311 | // wasn't found in xor list 312 | if (!a->getFlag().empty()) { 313 | list << "(" << a->flagStartChar() << a->getFlag() << ' ' << 314 | a->nameStartString() << a->getName() << ')'; 315 | } 316 | 317 | return list.str(); 318 | } 319 | 320 | } //namespace TCLAP 321 | #endif 322 | -------------------------------------------------------------------------------- /scripts/.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | -------------------------------------------------------------------------------- /scripts/build-lto-cloudflare-zlib.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Builds zlib from source ready for LTO 4 | 5 | set -e 6 | ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." > /dev/null && pwd ) 7 | GCC_ROOT=$1 8 | 9 | if [[ $# -ne 1 ]]; then 10 | echo "Usage: build-lto-zlib.sh " 11 | exit 1 12 | fi 13 | 14 | mkdir -p ${ROOT_DIR}/build 15 | cd ${ROOT_DIR}/build 16 | rm -rf zlib-src 17 | git clone https://github.com/cloudflare/zlib zlib-src 18 | cd zlib-src 19 | 20 | export LIBRARY_PATH=/usr/lib/$(gcc -print-multiarch) 21 | export C_INCLUDE_PATH=/usr/include/$(gcc -print-multiarch) 22 | export CPLUS_INCLUDE_PATH=/usr/include/$(gcc -print-multiarch) 23 | export CXX=${GCC_ROOT}/bin/g++ 24 | export CC=${GCC_ROOT}/bin/gcc 25 | export AR=${GCC_ROOT}/bin/gcc-ar 26 | export RANLIB=${GCC_ROOT}/bin/gcc-ranlib 27 | export CFLAGS="-O3 -flto -fuse-linker-plugin -fuse-ld=gold -march=native" 28 | export LDFLAGS="-O3 -flto -fuse-linker-plugin -fuse-ld=gold -march=native" 29 | ./configure --static --prefix ${ROOT_DIR}/build/zlib --64 30 | make -j4 31 | make test 32 | make install 33 | -------------------------------------------------------------------------------- /scripts/build-lto-zlib.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Builds zlib from source ready for LTO 4 | 5 | set -e 6 | ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." > /dev/null && pwd ) 7 | GCC_ROOT=$1 8 | 9 | if [[ $# -ne 1 ]]; then 10 | echo "Usage: build-lto-zlib.sh " 11 | exit 1 12 | fi 13 | 14 | ZLIB_VERSION=1.2.8 15 | 16 | mkdir -p ${ROOT_DIR}/build 17 | cd ${ROOT_DIR}/build 18 | curl -s -L "https://github.com/madler/zlib/archive/v${ZLIB_VERSION}.tar.gz" | tar zxf - 19 | cd zlib-${ZLIB_VERSION} 20 | 21 | export LIBRARY_PATH=/usr/lib/$(gcc -print-multiarch) 22 | export C_INCLUDE_PATH=/usr/include/$(gcc -print-multiarch) 23 | export CPLUS_INCLUDE_PATH=/usr/include/$(gcc -print-multiarch) 24 | export CXX=${GCC_ROOT}/bin/g++ 25 | export CC=${GCC_ROOT}/bin/gcc 26 | export AR=${GCC_ROOT}/bin/gcc-ar 27 | export RANLIB=${GCC_ROOT}/bin/gcc-ranlib 28 | export CFLAGS="-O3 -flto -fuse-linker-plugin -fuse-ld=gold -march=native" 29 | export LDFLAGS="-O3 -flto -fuse-linker-plugin -fuse-ld=gold -march=native" 30 | ./configure --static --prefix ${ROOT_DIR}/build/zlib --64 31 | make -j4 32 | make test 33 | make install 34 | -------------------------------------------------------------------------------- /scripts/pgo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Somewhat hacky approach to try building a full PGO/LTO binary 4 | # Pass the root path to GCC as the first parameter 5 | 6 | set -e 7 | 8 | ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." > /dev/null && pwd ) 9 | 10 | if [[ $# -ne 1 ]] && [[ $# -ne 2 ]]; then 11 | echo "Usage: pgo.sh []" 12 | exit 1 13 | fi 14 | 15 | GCC_ROOT=$1 16 | EXERCISE_SCRIPT=$2 17 | 18 | export LIBRARY_PATH=/usr/lib/$(gcc -print-multiarch) 19 | export C_INCLUDE_PATH=/usr/include/$(gcc -print-multiarch) 20 | export CPLUS_INCLUDE_PATH=/usr/include/$(gcc -print-multiarch) 21 | export CXX=${GCC_ROOT}/bin/g++ 22 | export CC=${GCC_ROOT}/bin/gcc 23 | export CMAKE_AR=${GCC_ROOT}/bin/gcc-ar 24 | export CMAKE_RANLIB=${GCC_ROOT}/bin/gcc-ranlib 25 | 26 | cd ${ROOT_DIR} 27 | mkdir -p build 28 | cd build 29 | mkdir -p pgo-gen 30 | pushd pgo-gen 31 | PGO_GEN=$(pwd) 32 | cmake ${ROOT_DIR} -DStatic:BOOL=Yes -DUseLTO:BOOL=Yes -DCMAKE_AR=${CMAKE_AR} -DCMAKE_RANLIB=${CMAKE_RANLIB} -DArchNative:BOOL=yes -DCMAKE_BUILD_TYPE=Release -DPGO="-fprofile-generate=${PGO_GEN}" -DCMAKE_PREFIX_PATH=${ROOT_DIR}/build/zlib 33 | make zq zindex 34 | popd 35 | 36 | pushd pgo-gen 37 | if [[ -z "${EXERCISE_SCRIPT}" ]]; then 38 | echo "Using built-in exercise script" 39 | if [[ ! -e testfile.gz ]]; then 40 | echo "Creating a test file" 41 | seq 1000000 > testfile 42 | echo "Gzipping test file" 43 | gzip -f -9 testfile 44 | fi 45 | echo "Indexing test file" 46 | ./zindex --regex '([0-9]+)' testfile.gz 47 | echo "Querying test file" 48 | for i in $(seq 1 12345 1000000); do ./zq testfile.gz $i $((i + 100)) $((i + 200)); done 49 | else 50 | echo "Using external exercise script" 51 | ${EXERCISE_SCRIPT} 52 | fi 53 | popd 54 | 55 | echo "Making PGO executables" 56 | 57 | mkdir -p pgo 58 | cd pgo 59 | cmake ${ROOT_DIR} -DStatic:BOOL=Yes -DUseLTO:BOOL=Yes -DCMAKE_AR=${CMAKE_AR} -DCMAKE_RANLIB=${CMAKE_RANLIB} -DArchNative:BOOL=yes -DCMAKE_BUILD_TYPE=Release -DPGO="-fprofile-use=${PGO_GEN}" -DCMAKE_PREFIX_PATH=${ROOT_DIR}/build/zlib 60 | make zq zindex 61 | echo "Done!" 62 | -------------------------------------------------------------------------------- /src/ConsoleLog.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include "ConsoleLog.h" 3 | #include 4 | 5 | namespace { 6 | const char *prefix(Log::Severity severity) { 7 | switch (severity) { 8 | default: return ""; 9 | case Log::Severity::Debug: return "Debug: "; 10 | case Log::Severity::Warning: return "Warning: "; 11 | case Log::Severity::Error: return "ERROR: "; 12 | } 13 | } 14 | } 15 | 16 | ConsoleLog::ConsoleLog(Log::Severity logLevel, bool forceColour, bool logWarningsToInfo) 17 | : Log(logLevel) { 18 | ansiColour_ = forceColour || ::isatty(STDOUT_FILENO); 19 | logWarningsToInfo_ = logWarningsToInfo; 20 | } 21 | 22 | void ConsoleLog::log(Log::Severity severity, const std::string &message) { 23 | auto logToError = severity > Log::Severity::Warning || (severity == Log::Severity::Warning && !logWarningsToInfo_); 24 | auto &out = logToError ? std::cerr : std::cout; 25 | 26 | bool needReset = ansiColour_; 27 | if (ansiColour_) { 28 | switch (severity) { 29 | case Log::Severity::Debug: 30 | out << "\x1b[32m"; 31 | break; 32 | case Log::Severity::Info: 33 | needReset = false; 34 | break; 35 | case Log::Severity::Warning: 36 | out << "\x1b[33m"; 37 | break; 38 | case Log::Severity::Error: 39 | out << "\x1b[31;1m"; 40 | break; 41 | } 42 | } 43 | out << prefix(severity == Log::Severity::Warning && logWarningsToInfo_? Log::Severity::Info : severity) << message; 44 | if (needReset) { 45 | out << "\x1b[0;m"; 46 | } 47 | out << std::endl; 48 | } 49 | -------------------------------------------------------------------------------- /src/ConsoleLog.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "Log.h" 4 | 5 | // A Log implementation which prints to the console, in colour if forced or if 6 | // a TTY. 7 | class ConsoleLog : public Log { 8 | bool ansiColour_; 9 | bool logWarningsToInfo_; 10 | public: 11 | ConsoleLog(Log::Severity logLevel, bool forceColour, bool logWarningsToInfo); 12 | 13 | void log(Severity severity, const std::string &message) override; 14 | }; 15 | -------------------------------------------------------------------------------- /src/ExternalIndexer.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include "ExternalIndexer.h" 8 | #include "IndexSink.h" 9 | 10 | namespace { 11 | void X(int error) { 12 | // Prevent errors from shutting down the child "gracefully". 13 | if (error == -1) { 14 | std::cerr << "Error creating child process: " << errno << std::endl; 15 | exit(1); 16 | } 17 | } 18 | } 19 | 20 | void ExternalIndexer::index(IndexSink &sink, StringView line) { 21 | log_.debug("Writing to child..."); 22 | auto bytes = ::write(sendPipe_.writeFd(), line.begin(), line.length()); 23 | if (bytes != (ssize_t)line.length()) { 24 | log_.error("Failed to write to child process: ", errno); 25 | throw std::runtime_error("Unable to write to child process"); 26 | } 27 | bytes = ::write(sendPipe_.writeFd(), "\n", 1); 28 | if (bytes != 1) { 29 | log_.error("Failed to write to child process: ", errno); 30 | throw std::runtime_error("Unable to write to child process"); 31 | } 32 | log_.debug("Finished writing"); 33 | std::vector buf; 34 | for (; ;) { 35 | auto readPos = buf.size(); 36 | constexpr auto blockSize = 4096u; 37 | auto initSize = buf.size(); 38 | buf.resize(initSize + blockSize); 39 | bytes = ::read(receivePipe_.readFd(), &buf[readPos], blockSize); 40 | if (bytes < 0) 41 | throw std::runtime_error("Error reading from child process"); 42 | if (bytes == 0) 43 | throw std::runtime_error("Child process died"); 44 | buf.resize(initSize + bytes); 45 | if (!buf.empty() && buf.back() == '\n') 46 | break; 47 | if (memchr(&buf[0], '\n', buf.size())) { 48 | auto strBuf = std::string(&buf[0], buf.size()); 49 | throw std::runtime_error( 50 | "Child process emitted more than one line: '" + strBuf + 51 | "'"); 52 | } 53 | } 54 | auto ptr = &buf[0]; 55 | auto end = &buf[buf.size() - 1]; 56 | while (ptr < end) { 57 | auto nextSep = static_cast(memmem(ptr, end - ptr, separator_.c_str(), separator_.size())); 58 | if (!nextSep) nextSep = end; 59 | auto length = nextSep - ptr; 60 | if (length) sink.add(StringView(ptr, length), 0); // TODO: offset 61 | ptr = nextSep + separator_.size(); 62 | } 63 | } 64 | 65 | ExternalIndexer::ExternalIndexer(Log &log, const std::string &command, 66 | const std::string &separator) 67 | : log_(log), childPid_(0), separator_(separator) { 68 | auto forkResult = fork(); 69 | if (forkResult == -1) { 70 | log_.error("Unable to fork: ", errno); 71 | throw std::runtime_error("Unable to fork"); 72 | } else if (forkResult == 0) { 73 | runChild(command); // never returns 74 | } 75 | childPid_ = forkResult; 76 | log_.debug("Forked child process ", childPid_); 77 | sendPipe_.closeRead(); 78 | receivePipe_.closeWrite(); 79 | signal(SIGCHLD, SIG_IGN); 80 | signal(SIGPIPE, SIG_IGN); 81 | } 82 | 83 | ExternalIndexer::~ExternalIndexer() { 84 | if (childPid_ > 0) { 85 | log_.debug("Sending child process TERM"); 86 | auto result = kill(childPid_, SIGTERM); 87 | if (result == -1) { 88 | log_.error("Unable to send kill: ", errno); 89 | std::terminate(); 90 | } 91 | int status = 0; 92 | log_.debug("Waiting on child"); 93 | waitpid(childPid_, &status, 0); 94 | log_.debug("Child exited"); 95 | signal(SIGCHLD, SIG_DFL); 96 | signal(SIGPIPE, SIG_DFL); 97 | } 98 | } 99 | 100 | void ExternalIndexer::runChild(const std::string &command) { 101 | // Send and receive are from the point of view of the parent. 102 | sendPipe_.closeWrite(); 103 | receivePipe_.closeRead(); 104 | X(dup2(sendPipe_.readFd(), STDIN_FILENO)); 105 | X(dup2(receivePipe_.writeFd(), STDOUT_FILENO)); 106 | X(execl("/bin/sh", "/bin/sh", "-c", command.c_str(), nullptr)); 107 | } 108 | -------------------------------------------------------------------------------- /src/ExternalIndexer.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "LineIndexer.h" 4 | #include "Log.h" 5 | #include "Pipe.h" 6 | 7 | #include 8 | #include 9 | 10 | // A LineIndexer that runs an external command and pipes output to it, and 11 | // awaits its response. 12 | class ExternalIndexer : public LineIndexer { 13 | Log &log_; 14 | pid_t childPid_; 15 | Pipe sendPipe_; 16 | Pipe receivePipe_; 17 | std::string separator_; 18 | 19 | public: 20 | ExternalIndexer(Log &log, const std::string &command, 21 | const std::string &separator); 22 | ~ExternalIndexer(); 23 | 24 | ExternalIndexer(const ExternalIndexer &) = delete; 25 | ExternalIndexer &operator=(const ExternalIndexer &) = delete; 26 | 27 | void index(IndexSink &sink, StringView line) override; 28 | 29 | private: 30 | void runChild(const std::string &command); 31 | }; 32 | 33 | 34 | -------------------------------------------------------------------------------- /src/FieldIndexer.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include "FieldIndexer.h" 3 | #include "IndexSink.h" 4 | 5 | void FieldIndexer::index(IndexSink &sink, StringView line) { 6 | auto ptr = line.begin(); 7 | auto end = line.end(); 8 | for (auto i = 1; i < field_; ++i) { 9 | auto nextSep = static_cast( 10 | memmem(ptr, end - ptr, separator_.c_str(), separator_.size())); 11 | if (!nextSep) return; 12 | ptr = nextSep + separator_.size(); 13 | } 14 | auto lastSep = memmem(ptr, end - ptr, separator_.c_str(), separator_.size()); 15 | if (lastSep) end = static_cast(lastSep); 16 | if (ptr != end) 17 | sink.add(StringView(ptr, end - ptr), ptr - line.begin()); 18 | } 19 | -------------------------------------------------------------------------------- /src/FieldIndexer.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "LineIndexer.h" 4 | 5 | #include 6 | 7 | // A LineIndexer that indexes based on a separator and field number. 8 | class FieldIndexer : public LineIndexer { 9 | std::string separator_; 10 | int field_; 11 | public: 12 | FieldIndexer(std::string separator, int field) 13 | : separator_(move(separator)), 14 | field_(field) { } 15 | 16 | void index(IndexSink &sink, StringView line) override; 17 | }; 18 | -------------------------------------------------------------------------------- /src/File.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | namespace impl { 7 | struct Closer { 8 | void operator()(FILE *f) { 9 | if (f) ::fclose(f); 10 | } 11 | }; 12 | } 13 | 14 | // A File is a self-closing FILE *. 15 | using File = std::unique_ptr; 16 | -------------------------------------------------------------------------------- /src/Index.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "File.h" 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | class Log; 12 | 13 | class LineSink; 14 | 15 | class LineIndexer; 16 | 17 | class Sqlite; 18 | 19 | // An index wraps an existing index on a compressed file, or offers a way to 20 | // create an index on a compressed file. 21 | // Load an existing index with Index::load(...), and see the Index::Builder 22 | // class for details on building a new index. 23 | class Index { 24 | struct Impl; 25 | std::unique_ptr impl_; 26 | 27 | Index(); 28 | 29 | Index(std::unique_ptr &&imp); 30 | 31 | public: 32 | Index(Index &&other); 33 | ~Index(); 34 | 35 | struct IndexConfig { 36 | bool numeric; 37 | bool unique; 38 | bool sparse; 39 | bool indexLineOffsets; 40 | 41 | IndexConfig() : 42 | numeric{false}, unique{false}, sparse{false}, indexLineOffsets{false} {}; 43 | 44 | IndexConfig withNumeric(bool b) { numeric = b; return *this; }; 45 | IndexConfig withUnique(bool b) { unique = b; return *this; }; 46 | IndexConfig withSparse(bool b) { sparse = b; return *this; }; 47 | IndexConfig withIndexLineOffsets(bool b) { indexLineOffsets = b; return *this; }; 48 | }; 49 | 50 | // Retrieve a single line by line number, calling the supplied LineSink with 51 | // the line, if found. Returns true if a link was found, false otherwise. 52 | bool getLine(uint64_t line, LineSink &sink); 53 | // Retrieve multiple lines by line numbers, calling the supplied LinkSink 54 | // with each matching line, if found. Returns the number of lines matched. 55 | size_t getLines(const std::vector &lines, LineSink &sink); 56 | 57 | // A function type used to be given a series of matching line numbers. 58 | using LineFunction = std::function; 59 | 60 | // Return a LineFunction which fetches each line in turn and provides them 61 | // to the supplied LineSink. 62 | LineFunction sinkFetch(LineSink &sink); 63 | 64 | // Query the given sub-index with the supplied query. Each matching line 65 | // number is passed in turn to the supplied LineFunction. Returns the number 66 | // of index matches. 67 | size_t queryIndex(const std::string &index, const std::string &query, 68 | LineFunction lineFunction); 69 | 70 | // Query the given sub-index with the supplied query. Each matching line is 71 | // looked up and the line data passed to the supplied LineSink. Returns the 72 | // number of index matches. 73 | size_t queryIndex(const std::string &index, const std::string &query, 74 | LineSink &sink) { 75 | return queryIndex(index, query, sinkFetch(sink)); 76 | } 77 | 78 | // Query the given sub-index with the supplied array of queries. Each 79 | // matching line number is passed to the supplied lineFunction. Returns the 80 | // total number of matches. 81 | size_t queryIndexMulti(const std::string &index, 82 | const std::vector &queries, 83 | LineFunction lineFunction); 84 | 85 | // Query the given sub-index with the supplied array of queries. Each 86 | // matching line is looked up and the line data passed to the supplied 87 | // LineSink. Returns the total number of index matches. 88 | size_t queryIndexMulti(const std::string &index, 89 | const std::vector &queries, 90 | LineSink &sink) { 91 | return queryIndexMulti(index, queries, sinkFetch(sink)); 92 | } 93 | 94 | // Query all indexes with the supplied query. Each 95 | // matching line number is passed to the supplied lineFunction. Returns 96 | // the total number of index matches 97 | size_t queryCustom(const std::string &customQuery, LineFunction lineFunc); 98 | 99 | // Return the number of entries in a particular sub-index. 100 | size_t indexSize(const std::string &index) const; 101 | 102 | // Metadata is a blob of strings that describe aspects of the index. They're 103 | // pretty opaque and not designed to be used in user code. 104 | using Metadata = std::unordered_map; 105 | 106 | // Retrieve the metadata for this index. 107 | const Metadata &getMetadata() const; 108 | 109 | // A builder is used to construct a new index by passing it information it 110 | // needs; the file to index and some configuration options. Additionally 111 | // several indexers may be added to generate indices based on the contents 112 | // of each line. With no indexers, only a line number index is built. 113 | class Builder { 114 | struct Impl; 115 | std::unique_ptr impl_; 116 | public: 117 | // Construct a builder with the given file and index filename. 118 | Builder(Log &log, File &&from, const std::string &fromPath, 119 | const std::string &indexFilename); 120 | ~Builder(); 121 | // Modify the builder to checkpoint only every given number of bytes. 122 | Builder &indexEvery(uint64_t bytes); 123 | // Modify the builder to skip the first few lines. 124 | Builder &skipFirst(uint64_t skipFirst); 125 | 126 | // Add an indexer to the builder. The indexer will be given each line 127 | // in turn and asked to provide matches. The name is the name of the 128 | // sub-indexer to create, the 'creation' is a user-facing tag to give 129 | // the index. If the indexer generates numeric, or guaranteed unique 130 | // indices, then this can be noted here to make the underlying SQL index 131 | // more efficient. 132 | Builder &addIndexer(const std::string &name, 133 | const std::string &creation, 134 | IndexConfig config, 135 | std::unique_ptr indexer); 136 | 137 | // Build the index. Once this method's been called, the Builder should 138 | // not be used again. 139 | void build(); 140 | }; 141 | 142 | // Load an existing index. forceLoad will load the index even if it appears 143 | // not to match the underlying compressed file. 144 | static Index load(Log &log, File &&fromCompressed, 145 | const std::string &indexFilename, bool forceLoad); 146 | }; 147 | -------------------------------------------------------------------------------- /src/IndexParser.cpp: -------------------------------------------------------------------------------- 1 | #include "IndexParser.h" 2 | #include "RegExpIndexer.h" 3 | #include "FieldIndexer.h" 4 | #include "ExternalIndexer.h" 5 | #include 6 | #include 7 | #include 8 | 9 | void IndexParser::buildIndexes(Index::Builder *builder, ConsoleLog &log) { 10 | std::ifstream in(fileName_, std::ifstream::in); 11 | std::string contents((std::istreambuf_iterator(in)), 12 | std::istreambuf_iterator()); 13 | cJSON *root = cJSON_Parse(contents.c_str()); 14 | if (root == nullptr) { 15 | throw std::runtime_error("Could not parse the json config file. " 16 | "Careful, this is a strict parser."); 17 | } 18 | if (!cJSON_HasObjectItem(root, "indexes")) { 19 | throw std::runtime_error("No indexes to be found in the config file."); 20 | } 21 | cJSON *indexes = cJSON_GetObjectItem(root, "indexes"); 22 | auto arraySize = cJSON_GetArraySize(indexes); 23 | for (int i = 0; i < arraySize; i++) { 24 | parseIndex(cJSON_GetArrayItem(indexes, i), builder, log); 25 | } 26 | } 27 | 28 | void IndexParser::parseIndex(cJSON *index, Index::Builder *builder, 29 | ConsoleLog &log) { 30 | if (!cJSON_HasObjectItem(index, "type")) { 31 | throw std::runtime_error( 32 | "All indexes must have a type field, i.e. Regex, Field, "); 33 | } 34 | std::string indexName = "default"; 35 | if (cJSON_HasObjectItem(index, "name")) { 36 | indexName = cJSON_GetObjectItem(index, "name")->valuestring; 37 | } 38 | std::string type = cJSON_GetObjectItem(index, "type")->valuestring; 39 | Index::IndexConfig config{}; 40 | config.numeric = getBoolean(index, "numeric"); 41 | config.unique = getBoolean(index, "unique"); 42 | config.sparse = getBoolean(index, "sparse"); 43 | config.indexLineOffsets = getBoolean(index, "indexLineOffsets"); 44 | 45 | if (type == "regex") { 46 | auto regex = getOrThrowStr(index, "regex"); 47 | if (cJSON_HasObjectItem(index, "capture")) { 48 | uint capture = cJSON_GetObjectItem(index, "capture")->valueint; 49 | builder->addIndexer(indexName, regex, config, 50 | std::unique_ptr( 51 | new RegExpIndexer(regex, capture))); 52 | } else { 53 | builder->addIndexer(indexName, regex, config, 54 | std::unique_ptr( 55 | new RegExpIndexer(regex))); 56 | } 57 | } else if (type == "field") { 58 | auto delimiter = getOrThrowStr(index, "delimiter"); 59 | auto fieldNum = getOrThrowUint(index, "fieldNum"); 60 | std::ostringstream name; 61 | name << "Field " << fieldNum << " delimited by '" 62 | << delimiter << "'"; 63 | builder->addIndexer(indexName, name.str(), config, 64 | std::unique_ptr( 65 | new FieldIndexer(delimiter, fieldNum))); 66 | } else if (type == "pipe") { 67 | auto pipeCommand = getOrThrowStr(index, "command"); 68 | auto delimiter = getOrThrowStr(index, "delimiter"); 69 | 70 | builder->addIndexer( 71 | indexName, pipeCommand, config, 72 | std::unique_ptr( 73 | new ExternalIndexer(log, pipeCommand, delimiter))); 74 | } else { 75 | throw std::runtime_error("unknown index " + type); 76 | } 77 | } 78 | 79 | bool IndexParser::getBoolean(cJSON *index, const char *field) const { 80 | return cJSON_GetObjectItem(index, field) 81 | && cJSON_GetObjectItem(index, field)->type == cJSON_True; 82 | } 83 | 84 | std::string IndexParser::getOrThrowStr(cJSON *index, const char *field) const { 85 | auto *item = cJSON_GetObjectItem(index, field); 86 | if (!item) 87 | throw std::runtime_error("Could not parse the json config file. Field '" 88 | + std::string(field) + "' was missing"); 89 | return item->valuestring; 90 | } 91 | 92 | unsigned IndexParser::getOrThrowUint(cJSON *index, const char *field) const { 93 | auto *item = cJSON_GetObjectItem(index, field); 94 | if (!item) 95 | throw std::runtime_error("Could not parse the json config file. Field '" 96 | + std::string(field) + "' was missing"); 97 | if (item->valueint < 0) 98 | throw std::runtime_error("Could not parse the json config file. Field '" 99 | + std::string(field) + "' was negative"); 100 | return static_cast(item->valueint); 101 | } 102 | -------------------------------------------------------------------------------- /src/IndexParser.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "LineIndexer.h" 4 | #include "Index.h" 5 | #include "ConsoleLog.h" 6 | #include 7 | #include "cJSON/cJSON.h" 8 | 9 | // Parses multiple indexes from a json configuration. 10 | class IndexParser { 11 | std::string fileName_; 12 | public: 13 | IndexParser(std::string fileName) : 14 | fileName_{fileName} 15 | {} 16 | 17 | void buildIndexes(Index::Builder *builder, ConsoleLog& log); 18 | 19 | private: 20 | void parseIndex(cJSON *index, Index::Builder* builder, ConsoleLog& log); 21 | bool getBoolean(cJSON *index, const char *field) const; 22 | std::string getOrThrowStr(cJSON *index, const char *field) const; 23 | unsigned getOrThrowUint(cJSON *index, const char *field) const; 24 | }; 25 | -------------------------------------------------------------------------------- /src/IndexSink.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "StringView.h" 4 | 5 | #include 6 | 7 | // An IndexSink is a sink for a LineIndexer: each item to index should be given 8 | // to the IndexSink's add method. 9 | class IndexSink { 10 | public: 11 | virtual ~IndexSink() = default; 12 | 13 | // Add a key to be indexed. offset should be the offset within the line. 14 | virtual void add(StringView item, size_t offset) = 0; 15 | }; -------------------------------------------------------------------------------- /src/LineFinder.cpp: -------------------------------------------------------------------------------- 1 | #include "LineFinder.h" 2 | 3 | #include "LineSink.h" 4 | 5 | #include 6 | 7 | LineFinder::LineFinder(LineSink &sink) 8 | : sink_(sink), currentLineOffset_(0) { 9 | } 10 | 11 | void LineFinder::add(const uint8_t *data, uint64_t length, bool last) { 12 | auto endData = data + length; 13 | while (data < endData) { 14 | auto lineEnd = static_cast(memchr(data, '\n', 15 | endData - data)); 16 | if (lineEnd) { 17 | lineData(data, lineEnd); 18 | data = lineEnd + 1; 19 | } else { 20 | append(data, endData); 21 | break; 22 | } 23 | } 24 | if (last && !lineBuffer_.empty()) 25 | lineData(nullptr, nullptr); 26 | if (last) 27 | lineOffsets_.emplace_back(currentLineOffset_); 28 | } 29 | 30 | void LineFinder::lineData(const uint8_t *begin, const uint8_t *end) { 31 | uint64_t length; 32 | bool shouldAddLine; 33 | if (lineBuffer_.empty()) { 34 | shouldAddLine = sink_.onLine(lineOffsets_.size() + 1, currentLineOffset_, 35 | reinterpret_cast(begin), end - begin); 36 | length = (end - begin) + 1; 37 | } else { 38 | append(begin, end); 39 | shouldAddLine = sink_.onLine(lineOffsets_.size() + 1, currentLineOffset_, 40 | &lineBuffer_[0], lineBuffer_.size()); 41 | length = lineBuffer_.size() + 1; 42 | lineBuffer_.clear(); 43 | } 44 | if (shouldAddLine) 45 | lineOffsets_.emplace_back(currentLineOffset_); 46 | currentLineOffset_ += length; 47 | } 48 | 49 | void LineFinder::append(const uint8_t *begin, const uint8_t *end) { 50 | // Was: std::copy(begin, end, std::back_inserter(lineBuffer_)); 51 | // but it turns out that's pretty slow... so let's do this instead: 52 | auto offset = lineBuffer_.size(); 53 | auto size = end - begin; 54 | lineBuffer_.resize(offset + size); 55 | std::memcpy(lineBuffer_.data() + offset, begin, size); 56 | } -------------------------------------------------------------------------------- /src/LineFinder.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | class LineSink; 7 | 8 | // Finds line boundaries within a file. File data is presented to the LineFinder 9 | //in blocks. Offsets are stored within the LineFinder, which means memory usage 10 | //proportional to the number of lines in the file. Each line found is presented 11 | //to the supplied LineSink. 12 | class LineFinder { 13 | LineSink &sink_; 14 | std::vector lineBuffer_; 15 | std::vector lineOffsets_; 16 | uint64_t currentLineOffset_; 17 | public: 18 | explicit LineFinder(LineSink &sink); 19 | 20 | // Process the following data as further input. Calls back to the LineSink. 21 | void add(const uint8_t *data, uint64_t length, bool last); 22 | 23 | // Accesses the array of line offsets. Only valid after the last data has 24 | // been given to the finder. 25 | const std::vector &lineOffsets() const { return lineOffsets_; } 26 | 27 | private: 28 | void lineData(const uint8_t *begin, const uint8_t *end); 29 | void append(const uint8_t *begin, const uint8_t *end); 30 | }; 31 | -------------------------------------------------------------------------------- /src/LineIndexer.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include "StringView.h" 6 | 7 | class IndexSink; 8 | 9 | // A LineIndexer is given lines by the Indexer, and tells an IndexSink about 10 | // matches within it (based on whatever it indexes, e.g. a matching RegExp). 11 | class LineIndexer { 12 | public: 13 | virtual ~LineIndexer() = default; 14 | 15 | virtual void index(IndexSink &sink, StringView line) = 0; 16 | }; 17 | -------------------------------------------------------------------------------- /src/LineSink.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | // A sink to be called with each line in a file. 6 | class LineSink { 7 | public: 8 | virtual ~LineSink() = default; 9 | 10 | virtual bool onLine( 11 | size_t lineNumber, 12 | size_t fileOffset, 13 | const char *line, size_t length) = 0; 14 | }; 15 | -------------------------------------------------------------------------------- /src/Log.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | // A lightweight logging system. Concrete implementations include ConsoleLog and 7 | // CaptureLog. 8 | class Log { 9 | public: 10 | virtual ~Log() = default; 11 | 12 | enum class Severity { 13 | Debug, Info, Warning, Error 14 | }; 15 | 16 | static const char *name(Severity severity) { 17 | switch (severity) { 18 | case Severity::Debug: 19 | return "debug"; 20 | case Severity::Info: 21 | return "info"; 22 | case Severity::Warning: 23 | return "warn"; 24 | case Severity::Error: 25 | return "error"; 26 | } 27 | return "unknown"; 28 | } 29 | 30 | virtual void log(Severity severity, const std::string &message) = 0; 31 | 32 | #define LOG_IMPLEMENT(FUNC, SEV) \ 33 | void FUNC(const std::string &message) { \ 34 | if (minSeverity_ <= SEV) \ 35 | log(SEV, message); \ 36 | } \ 37 | template \ 38 | void FUNC(Args &&... args) { \ 39 | if (minSeverity_ <= SEV) \ 40 | log(SEV, format(std::forward(args)...)); \ 41 | } 42 | 43 | LOG_IMPLEMENT(debug, Severity::Debug) 44 | 45 | LOG_IMPLEMENT(info, Severity::Info) 46 | 47 | LOG_IMPLEMENT(warn, Severity::Warning) 48 | 49 | LOG_IMPLEMENT(error, Severity::Error) 50 | 51 | #undef LOG_IMPLEMENT 52 | 53 | template 54 | static std::string format(Args &&... args) { 55 | std::stringstream output; 56 | streamTo(output, std::forward(args)...); 57 | return output.str(); 58 | } 59 | 60 | protected: 61 | Severity minSeverity_; 62 | 63 | Log(Severity severity = Severity::Info) 64 | : minSeverity_(severity) { } 65 | 66 | private: 67 | static void streamTo(std::ostream &) { } 68 | 69 | template 70 | static void streamTo(std::ostream &o, const T &arg, Args &&... rest) { 71 | o << arg; 72 | streamTo(o, std::forward(rest)...); 73 | } 74 | }; 75 | -------------------------------------------------------------------------------- /src/Pipe.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include "Pipe.h" 7 | 8 | Pipe::Pipe() 9 | : pipeFds_{ -1, -1 } { 10 | if (pipe(pipeFds_) == -1) 11 | throw std::runtime_error("Unable to create a pipe: " 12 | + std::to_string(errno)); // TODO: errno 13 | } 14 | 15 | Pipe::~Pipe() { 16 | close(); 17 | } 18 | 19 | void Pipe::close() { 20 | closeRead(); 21 | closeWrite(); 22 | } 23 | 24 | void Pipe::closeRead() { 25 | if (pipeFds_[0] != -1) 26 | ::close(pipeFds_[0]); 27 | } 28 | 29 | void Pipe::closeWrite() { 30 | if (pipeFds_[1] != -1) 31 | ::close(pipeFds_[1]); 32 | } 33 | 34 | Pipe::Pipe(Pipe &&other) { 35 | for (auto i = 0; i < 2; ++i) 36 | pipeFds_[i] = other.pipeFds_[i]; 37 | other.destroy(); 38 | } 39 | 40 | void Pipe::destroy() { 41 | pipeFds_[0] = pipeFds_[1] = -1; 42 | } 43 | 44 | Pipe &Pipe::operator=(Pipe &&other) { 45 | if (this != &other) { 46 | close(); 47 | for (auto i = 0; i < 2; ++i) 48 | pipeFds_[i] = other.pipeFds_[i]; 49 | other.destroy(); 50 | } 51 | return *this; 52 | } 53 | -------------------------------------------------------------------------------- /src/Pipe.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | // RAII wrapper over an anonymous pipe. 4 | class Pipe { 5 | int pipeFds_[2]; 6 | public: 7 | // Constructs a new anonymous pipe. 8 | Pipe(); 9 | ~Pipe(); 10 | 11 | // Close the read side of the pipe. 12 | void closeRead(); 13 | // Close the write side of the pipe. 14 | void closeWrite(); 15 | // Close both sides of the pipe. 16 | void close(); 17 | 18 | Pipe(const Pipe &) = delete; 19 | Pipe &operator=(const Pipe &) = delete; 20 | Pipe(Pipe &&); 21 | Pipe &operator=(Pipe &&); 22 | 23 | // Get the file descriptor for the read side of the pipe. 24 | int readFd() const { return pipeFds_[0]; } 25 | // get the file descriptor for the write side of the pipe. 26 | int writeFd() const { return pipeFds_[1]; } 27 | 28 | private: 29 | void destroy(); 30 | }; 31 | 32 | 33 | -------------------------------------------------------------------------------- /src/PrettyBytes.cpp: -------------------------------------------------------------------------------- 1 | #include "PrettyBytes.h" 2 | 3 | #include 4 | #include 5 | 6 | using namespace std; 7 | 8 | ostream &operator<<(ostream &o, PrettyBytes pb) { 9 | if (pb.bytes_ == 1) return o << "1 byte"; 10 | if (pb.bytes_ < 1024) return o << pb.bytes_ << " bytes"; 11 | o << setprecision(2) << fixed; 12 | if (pb.bytes_ < 1024 * 1024) 13 | return o << (pb.bytes_ / (1024.0)) << " KiB"; 14 | if (pb.bytes_ < 1024 * 1024 * 1024) 15 | return o << (pb.bytes_ / (1024 * 1024.0)) << " MiB"; 16 | return o << (pb.bytes_ / (1024 * 1024 * 1024.0)) << " GiB"; 17 | } 18 | -------------------------------------------------------------------------------- /src/PrettyBytes.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | // Simple wrapper over a number which, when streamed out, displays the number as 7 | // a "pretty" number of bytes. For example: 8 | // cout << PrettyBytes(1); // yields "1 byte" 9 | // cout << PrettyBytes(1024); // yields "1 KiB" 10 | // etc 11 | class PrettyBytes { 12 | uint64_t bytes_; 13 | public: 14 | explicit PrettyBytes(uint64_t bytes) : bytes_(bytes) { } 15 | 16 | friend std::ostream &operator<<(std::ostream &, PrettyBytes pb); 17 | }; 18 | -------------------------------------------------------------------------------- /src/RangeFetcher.cpp: -------------------------------------------------------------------------------- 1 | #include "RangeFetcher.h" 2 | 3 | RangeFetcher::RangeFetcher(RangeFetcher::Handler &handler, 4 | uint64_t linesBefore, 5 | uint64_t linesAfter) 6 | : linesBefore_(linesBefore), linesAfter_(linesAfter), 7 | prevBegin_(0), prevEnd_(0), handler_(handler) { 8 | 9 | } 10 | 11 | void RangeFetcher::operator()(uint64_t line) { 12 | auto beginRange = line; 13 | if (line > linesBefore_) 14 | beginRange -= linesBefore_; 15 | else 16 | beginRange = 1; 17 | auto endRange = line + linesAfter_; 18 | auto currentLine = beginRange; 19 | if (beginRange >= prevBegin_ && beginRange <= prevEnd_) { 20 | currentLine = prevEnd_ + 1; 21 | } else { 22 | if (prevEnd_ && prevEnd_ + 1 != beginRange) 23 | handler_.onSeparator(); 24 | } 25 | for (; currentLine <= endRange; ++currentLine) 26 | handler_.onLine(currentLine); 27 | prevBegin_ = beginRange; 28 | prevEnd_ = endRange; 29 | } 30 | -------------------------------------------------------------------------------- /src/RangeFetcher.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | // A RangeFetcher is usable as an Index::LineFunction, and extends each matching 6 | // line number into a range from line-linesBefore to line+linesAfter, but with 7 | // overlapping regions removed. The Handler is called with each line and upon 8 | // separation of ranges. 9 | class RangeFetcher { 10 | const uint64_t linesBefore_; 11 | const uint64_t linesAfter_; 12 | uint64_t prevBegin_; 13 | uint64_t prevEnd_; 14 | 15 | public: 16 | // A handler for line ranges. 17 | class Handler { 18 | public: 19 | virtual ~Handler() = default; 20 | 21 | // Called for each line within a group. 22 | virtual void onLine(uint64_t line) = 0; 23 | // Called between each group. 24 | virtual void onSeparator() = 0; 25 | }; 26 | 27 | RangeFetcher(Handler &handler, uint64_t linesBefore, uint64_t linesAfter); 28 | 29 | void operator()(uint64_t line); 30 | 31 | private: 32 | Handler &handler_; 33 | }; 34 | -------------------------------------------------------------------------------- /src/RegExp.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "RegExp.h" 4 | 5 | RegExp::RegExp(const std::string ®ex) 6 | : RegExp(regex.c_str()) {} 7 | 8 | RegExp::RegExp(const char *regex) { 9 | R(regcomp(&re_, regex, REG_EXTENDED), regex); 10 | owned_ = true; 11 | } 12 | 13 | RegExp::~RegExp() { 14 | release(); 15 | } 16 | 17 | void RegExp::release() { 18 | if (owned_) 19 | regfree(&re_); 20 | owned_ = false; 21 | } 22 | 23 | bool RegExp::exec(const std::string &match, std::vector &result, 24 | size_t offset) { 25 | return exec(match.c_str() + offset, result, offset == 0); 26 | } 27 | 28 | bool RegExp::exec(const char *against, RegExp::Matches &result, bool bol) { 29 | constexpr auto MaxMatches = 20u; 30 | regmatch_t matches[MaxMatches]; 31 | auto res = regexec(&re_, against, MaxMatches, matches, 32 | bol ? 0 : REG_NOTBOL); 33 | if (res == REG_NOMATCH) return false; 34 | R(res); 35 | result.clear(); 36 | for (auto i = 0u; i < MaxMatches; ++i) { 37 | if (matches[i].rm_so == -1) break; 38 | result.emplace_back(matches[i].rm_so, matches[i].rm_eo); 39 | } 40 | return true; 41 | } 42 | 43 | void RegExp::R(int e) const { 44 | if (!e) return; 45 | char error[1024]; 46 | regerror(e, &re_, error, sizeof(error)); 47 | throw std::runtime_error(error); 48 | } 49 | 50 | void RegExp::R(int e, const char *context) const { 51 | if (!e) return; 52 | char error[1024]; 53 | regerror(e, &re_, error, sizeof(error)); 54 | throw std::runtime_error(error + std::string(" in '") + context + "'"); 55 | } 56 | 57 | RegExp::RegExp(RegExp &&exp) 58 | : re_(exp.re_) { 59 | exp.owned_ = false; 60 | owned_ = true; 61 | } 62 | 63 | RegExp &RegExp::operator=(RegExp &&exp) { 64 | release(); 65 | re_ = exp.re_; 66 | exp.owned_ = false; 67 | owned_ = true; 68 | return *this; 69 | } 70 | -------------------------------------------------------------------------------- /src/RegExp.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | // Wrapper over the POSIX regex library. 8 | // Ideally we'd use std::regex, but that's broken on GCC 4.8 (which is what 9 | // I'm targeting). 10 | class RegExp { 11 | regex_t re_; 12 | bool owned_; 13 | 14 | public: 15 | explicit RegExp(const char *regex); 16 | explicit RegExp(const std::string ®ex); 17 | ~RegExp(); 18 | 19 | RegExp(const RegExp &) = delete; 20 | RegExp &operator=(const RegExp &) = delete; 21 | RegExp(RegExp &&); 22 | RegExp &operator=(RegExp &&); 23 | 24 | using Match = std::pair; 25 | using Matches = std::vector; 26 | bool exec(const std::string &against, Matches &result, size_t offset = 0); 27 | bool exec(const char *against, Matches &result, bool bol=true); 28 | 29 | private: 30 | void release(); 31 | void R(int e) const; 32 | void R(int e, const char *context) const; 33 | }; 34 | -------------------------------------------------------------------------------- /src/RegExpIndexer.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "RegExpIndexer.h" 6 | #include "IndexSink.h" 7 | 8 | RegExpIndexer::RegExpIndexer(const std::string ®ex) 9 | : RegExpIndexer(regex, 0) { } 10 | 11 | RegExpIndexer::RegExpIndexer(const std::string ®ex, uint captureGroup) 12 | : re_(regex), 13 | captureGroup_(captureGroup) { } 14 | 15 | void RegExpIndexer::index(IndexSink &sink, StringView line) { 16 | RegExp::Matches result; 17 | size_t offset = 0; 18 | auto lineString = line.str(); 19 | while (offset < lineString.length()) { 20 | if (!re_.exec(lineString, result, offset)) return; 21 | 22 | // Magic number - indicates the default use case in which 23 | // a user has not specified the desired capture group 24 | if (captureGroup_ == 0) { 25 | if (result.size() == 1) 26 | onMatch(sink, lineString, offset, result[0]); 27 | else if (result.size() == 2) 28 | onMatch(sink, lineString, offset, result[1]); 29 | else 30 | throw std::runtime_error( 31 | "Expected exactly one match (or one capture group - " 32 | "use '--capture' option if you have multiple " 33 | "capture groups)"); 34 | } else { 35 | if (result.size() > captureGroup_) 36 | onMatch(sink, lineString, offset, result[captureGroup_]); 37 | else 38 | throw std::runtime_error( 39 | "Did not find a match in the given capture group"); 40 | } 41 | offset += result[0].second; 42 | } 43 | } 44 | 45 | void RegExpIndexer::onMatch(IndexSink &sink, const std::string &line, 46 | size_t offset, const RegExp::Match &match) { 47 | auto matchLen = match.second - match.first; 48 | try { 49 | sink.add(StringView(line.c_str() + offset + match.first, matchLen), 50 | offset + match.first); 51 | } catch (const std::exception &e) { 52 | throw std::runtime_error( 53 | "Error handling index match '" + 54 | line.substr(offset + match.first, matchLen) + 55 | "' - " + e.what()); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/RegExpIndexer.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "Sqlite.h" 4 | #include "RegExp.h" 5 | #include "LineIndexer.h" 6 | #include "IndexSink.h" 7 | 8 | // A LineIndexer that uses the capture group of a regular expression to provide 9 | // indices to an IndexSink. 10 | class RegExpIndexer : public LineIndexer { 11 | RegExp re_; 12 | unsigned int captureGroup_; 13 | 14 | public: 15 | explicit RegExpIndexer(const std::string ®ex); 16 | explicit RegExpIndexer(const std::string ®ex, unsigned int captureGroup); 17 | void index(IndexSink &sink, StringView line) override; 18 | 19 | private: 20 | void onMatch(IndexSink &sink, const std::string &line, size_t offset, 21 | const RegExp::Match &match); 22 | }; -------------------------------------------------------------------------------- /src/Sqlite.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "Sqlite.h" 5 | #include "SqliteError.h" 6 | #include 7 | #include "Log.h" 8 | #include "RegExp.h" 9 | 10 | Sqlite::Sqlite(Log &log) 11 | : log_(&log), sql_(nullptr) { 12 | } 13 | 14 | Sqlite::~Sqlite() { 15 | close(); 16 | } 17 | 18 | void Sqlite::open(const std::string &filename, bool readOnly) { 19 | close(); 20 | log_->info("Opening database ", filename, " in ", 21 | readOnly ? "read-only" : "read-write", " mode"); 22 | R(sqlite3_open_v2(filename.c_str(), &sql_, 23 | readOnly ? SQLITE_OPEN_READONLY : 24 | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_URI, nullptr)); 25 | R(sqlite3_extended_result_codes(sql_, true)); 26 | } 27 | 28 | void Sqlite::close() { 29 | if (sql_) { 30 | log_->info("Closing database"); 31 | sqlite3_close(sql_); 32 | } 33 | sql_ = nullptr; 34 | } 35 | 36 | void Sqlite::Statement::destroy() { 37 | if (statement_) 38 | sqlite3_finalize(statement_); 39 | statement_ = nullptr; 40 | } 41 | 42 | Sqlite::Statement Sqlite::prepare(const std::string &sql) const { 43 | Sqlite::Statement statement(*log_); 44 | log_->debug("Preparing statement ", sql); 45 | R(sqlite3_prepare_v2(sql_, sql.c_str(), sql.size(), 46 | &statement.statement_, nullptr), sql); 47 | return statement; 48 | } 49 | 50 | Sqlite::Statement::Statement(Statement &&other) { 51 | log_ = other.log_; 52 | statement_ = other.statement_; 53 | other.statement_ = nullptr; 54 | } 55 | 56 | Sqlite::Statement &Sqlite::Statement::operator=(Sqlite::Statement &&other) { 57 | if (this == &other) return *this; 58 | destroy(); 59 | log_ = other.log_; 60 | statement_ = other.statement_; 61 | other.statement_ = nullptr; 62 | return *this; 63 | } 64 | 65 | Sqlite::Statement &Sqlite::Statement::reset() { 66 | R(sqlite3_reset(statement_)); 67 | return *this; 68 | } 69 | 70 | bool Sqlite::Statement::step() { 71 | auto res = sqlite3_step(statement_); 72 | if (res == SQLITE_DONE) return true; 73 | if (res == SQLITE_ROW) return false; 74 | throw SqliteError(res); 75 | } 76 | 77 | Sqlite::Statement &Sqlite::Statement::bindBlob( 78 | StringView param, const void *data, size_t length) { 79 | R(sqlite3_bind_blob(statement_, P(param), data, length, SQLITE_TRANSIENT)); 80 | return *this; 81 | } 82 | 83 | Sqlite::Statement &Sqlite::Statement::bindInt64(StringView param, 84 | int64_t data) { 85 | R(sqlite3_bind_int64(statement_, P(param), data)); 86 | return *this; 87 | } 88 | 89 | Sqlite::Statement &Sqlite::Statement::bindString(StringView param, 90 | StringView data) { 91 | R(sqlite3_bind_text(statement_, P(param), data.begin(), data.length(), 92 | SQLITE_TRANSIENT)); 93 | return *this; 94 | } 95 | 96 | int64_t Sqlite::Statement::columnInt64(int index) const { 97 | return sqlite3_column_int64(statement_, index); 98 | } 99 | 100 | std::string Sqlite::Statement::columnString(int index) const { 101 | return (const char *)sqlite3_column_text(statement_, index); 102 | } 103 | 104 | int Sqlite::Statement::columnCount() const { 105 | return sqlite3_column_count(statement_); 106 | } 107 | 108 | std::string Sqlite::Statement::columnName(int index) const { 109 | return sqlite3_column_name(statement_, index); 110 | } 111 | 112 | int Sqlite::Statement::P(StringView param) const { 113 | std::string asStringStorage; 114 | auto index = sqlite3_bind_parameter_index(statement_, 115 | param.c_str(asStringStorage)); 116 | if (index == 0) 117 | throw std::runtime_error( 118 | "Unable to find bound parameter '" + param.str() + "'"); 119 | return index; 120 | } 121 | 122 | std::vector Sqlite::Statement::columnBlob(int index) const { 123 | auto ptr = sqlite3_column_blob(statement_, index); 124 | std::vector data(sqlite3_column_bytes(statement_, index)); 125 | std::memcpy(&data[0], ptr, data.size()); 126 | return data; 127 | } 128 | 129 | void Sqlite::exec(const std::string &sql) { 130 | log_->debug("Executing ", sql); 131 | R(sqlite3_exec(sql_, sql.c_str(), nullptr, nullptr, nullptr), sql); 132 | } 133 | 134 | void Sqlite::R(int result) const { 135 | if (result != SQLITE_OK) 136 | throw SqliteError(sqlite3_errmsg(sql_)); 137 | } 138 | 139 | void Sqlite::R(int result, const std::string &context) const { 140 | if (result != SQLITE_OK) 141 | throw SqliteError(sqlite3_errmsg(sql_), context); 142 | } 143 | 144 | void Sqlite::Statement::R(int result) const { 145 | if (result != SQLITE_OK) 146 | throw SqliteError(sqlite3_errmsg(sqlite3_db_handle(statement_))); 147 | } 148 | 149 | std::string Sqlite::toFile(const std::string &uri) { 150 | RegExp uriRegex("^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)"); 151 | RegExp::Matches matches; 152 | if (!uriRegex.exec(uri, matches)) { 153 | return uri; 154 | } 155 | auto protocol = matches[1]; 156 | auto result = std::string(uri.c_str() + protocol.first, protocol.second - protocol.first); 157 | if (result.find("file:") != 0) { 158 | throw std::runtime_error( 159 | "no uri support for '" + result+ "'"); 160 | } 161 | auto start = protocol.second; 162 | if (matches.size() > 3) { 163 | auto slashes = matches[3]; 164 | start = (slashes.second != 0) ? slashes.second : slashes.first; 165 | } 166 | auto path = std::string(uri.c_str() + start); 167 | auto paramsStart = path.rfind("?"); 168 | if (paramsStart != std::string::npos) { 169 | path.resize(paramsStart); 170 | } 171 | return path; 172 | } -------------------------------------------------------------------------------- /src/Sqlite.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "StringView.h" 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | class Log; 12 | 13 | // A fairly simple wrapper over sqlite. 14 | class Sqlite { 15 | Log *log_; 16 | sqlite3 *sql_; 17 | 18 | public: 19 | explicit Sqlite(Log &log); 20 | ~Sqlite(); 21 | Sqlite(const Sqlite &) = delete; 22 | Sqlite &operator=(const Sqlite &) = delete; 23 | 24 | Sqlite(Sqlite &&other) : log_(other.log_), sql_(other.sql_) 25 | { other.sql_ = nullptr; } 26 | 27 | Sqlite &operator=(Sqlite &&other) { 28 | if (this != &other) { 29 | close(); 30 | log_ = other.log_; 31 | sql_ = other.sql_; 32 | other.sql_ = nullptr; 33 | } 34 | return *this; 35 | } 36 | 37 | void open(const std::string &filename, bool readOnly); 38 | void close(); 39 | std::string toFile(const std::string &uri); 40 | 41 | class Statement { 42 | Log *log_; 43 | sqlite3_stmt *statement_; 44 | 45 | void destroy(); 46 | void R(int result) const; 47 | 48 | friend class Sqlite; 49 | 50 | public: 51 | Statement(Log &log) : log_(&log), statement_(nullptr) { } 52 | 53 | ~Statement() { 54 | destroy(); 55 | } 56 | 57 | Statement(const Statement &) = delete; 58 | Statement &operator=(const Statement &) = delete; 59 | Statement(Statement &&other); 60 | Statement &operator=(Statement &&other); 61 | 62 | Statement &reset(); 63 | Statement &bindInt64(StringView param, int64_t data); 64 | Statement &bindBlob(StringView param, const void *data, 65 | size_t length); 66 | Statement &bindString(StringView param, StringView string); 67 | 68 | bool step(); 69 | 70 | int columnCount() const; 71 | std::string columnName(int index) const; 72 | 73 | int64_t columnInt64(int index) const; 74 | std::string columnString(int index) const; 75 | std::vector columnBlob(int index) const; 76 | 77 | private: 78 | int P(StringView param) const; 79 | }; 80 | 81 | Statement prepare(const std::string &sql) const; 82 | void exec(const std::string &sql); 83 | 84 | private: 85 | void R(int result) const; 86 | void R(int result, const std::string &context) const; 87 | }; 88 | -------------------------------------------------------------------------------- /src/SqliteError.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | // An exception in sqlite. 7 | struct SqliteError : std::runtime_error { 8 | SqliteError(int result) 9 | : std::runtime_error(sqlite3_errstr(result)) { 10 | } 11 | 12 | SqliteError(int result, const std::string &context) 13 | : std::runtime_error( 14 | sqlite3_errstr(result) + std::string(" (") + context + ")") { 15 | } 16 | 17 | explicit SqliteError(const std::string &errMsg) 18 | : std::runtime_error(errMsg) { } 19 | 20 | SqliteError(const std::string &errMsg, const std::string &context) 21 | : std::runtime_error(errMsg + " (" + context + ")") { } 22 | }; 23 | -------------------------------------------------------------------------------- /src/StringView.cpp: -------------------------------------------------------------------------------- 1 | #include "StringView.h" 2 | 3 | #include 4 | 5 | std::ostream &operator<<(std::ostream &o, const StringView &sv) { 6 | for (auto i = 0u; i < sv.length_; ++i) 7 | o << sv.begin_[i]; 8 | return o; 9 | } -------------------------------------------------------------------------------- /src/StringView.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | // Very simple lightweight view over a un-owned character array. Designed to 9 | // mimic std::experimental::string_view. 10 | class StringView { 11 | const char *begin_; 12 | size_t length_; 13 | public: 14 | // Construct from a zero-terminated c-style string, which must outlive the 15 | // StringView. 16 | StringView(const char *cStr) 17 | : begin_(cStr), length_(strlen(cStr)) { } 18 | 19 | // Construct from a char array or the given length. The array must outlive 20 | // the StringView. 21 | StringView(const char *begin, size_t length) 22 | : begin_(begin), length_(length) { } 23 | 24 | // Construct from a std::string, which must outlive the StringView and not 25 | // be modified. 26 | StringView(const std::string &string) 27 | : begin_(string.empty() ? nullptr : &string[0]), 28 | length_(string.length()) { } 29 | 30 | const char *begin() const { return begin_; } 31 | 32 | const char *end() const { return begin_ + length_; } 33 | 34 | size_t length() const { return length_; } 35 | 36 | // Construct and return a std::string representation of the StringView. 37 | std::string str() const { 38 | return length_ ? std::string(begin_, length_) : std::string(); 39 | } 40 | 41 | // Return a zero-terminated c-style string, using the StringView storage 42 | // itself if possible, else placing the zero-terminated string in the 43 | // supplied std::string. Only works for strings without embedded nuls. 44 | const char *c_str(std::string &storage) const { 45 | if (length_ == 0) return ""; 46 | if (begin_[length_] == '\0') return begin_; // already zero-terminated 47 | storage.assign(begin_, length_); 48 | return storage.c_str(); 49 | } 50 | 51 | friend std::ostream &operator<<(std::ostream &o, const StringView &sv); 52 | }; -------------------------------------------------------------------------------- /src/zindex.cpp: -------------------------------------------------------------------------------- 1 | #include "File.h" 2 | #include "Index.h" 3 | #include "RegExpIndexer.h" 4 | #include "ConsoleLog.h" 5 | #include "FieldIndexer.h" 6 | #include "IndexParser.h" 7 | 8 | #include 9 | 10 | #include 11 | #include 12 | #include 13 | #include "ExternalIndexer.h" 14 | 15 | using namespace std; 16 | using namespace TCLAP; 17 | 18 | namespace { 19 | 20 | string getRealPath(const string &relPath) { 21 | char realPathBuf[PATH_MAX]; 22 | auto result = realpath(relPath.c_str(), realPathBuf); 23 | if (result == nullptr) return relPath; 24 | return string(relPath); 25 | } 26 | 27 | } 28 | 29 | int Main(int argc, const char *argv[]) { 30 | CmdLine cmd("Create indices in a compressed text file"); 31 | UnlabeledValueArg inputFile( 32 | "input-file", "Read input from ", true, "", "file", cmd); 33 | SwitchArg verbose("v", "verbose", "Be more verbose", cmd); 34 | SwitchArg debug("", "debug", "Be even more verbose", cmd); 35 | SwitchArg forceColour("", "colour", "Use colour even on non-TTY", cmd); 36 | SwitchArg forceColor("", "color", "Use color even on non-TTY", cmd); 37 | SwitchArg numeric("n", "numeric", "Assume the index is numeric", cmd); 38 | SwitchArg unique("u", "unique", "Assume each line's index is unique", cmd); 39 | SwitchArg sparse("s", "sparse", "Sparse - only save line offsets for rows" 40 | " that have ids. Merges all rows that have no id to the most recent" 41 | "id. Useful if your file is one id row followed by n data rows.", 42 | cmd); 43 | SwitchArg warnings("w", "warnings", "Log warnings at info level", cmd); 44 | ValueArg checkpointEvery( 45 | "", "checkpoint-every", 46 | "Create a compression checkpoint every ", false, 47 | 0, "bytes", cmd); 48 | ValueArg regex("", "regex", "Create an index using ", false, 49 | "", "regex", cmd); 50 | ValueArg capture("", "capture", 51 | "Determines which capture group in an regex to use", 52 | false, 0, "capture", cmd); 53 | ValueArg skipFirst("", "skip-first", "Skip the first lines", 54 | false, 0, "num", cmd); 55 | ValueArg field("f", "field", "Create an index using field " 56 | "(delimited by -d/--delimiter, 1-based)", 57 | false, 0, "num", cmd); 58 | ValueArg configFile("c", "config", "Create indexes using json " 59 | "config file ", false, "", "indexes", cmd); 60 | ValueArg delimiterArg( 61 | "d", "delimiter", "Use as the field delimiter", false, " ", 62 | "delim", cmd); 63 | SwitchArg tabDelimiterArg( 64 | "", "tab-delimiter", "Use a tab character as the field delimiter", 65 | cmd); 66 | ValueArg externalIndexer( 67 | "p", "pipe", 68 | "Create indices by piping output through which should output " 69 | "a single line for each input line. " 70 | "Multiple keys should be separated by the --delimiter " 71 | "character (which defaults to a space). " 72 | "The CMD should be unbuffered " 73 | "(man stdbuf(1) for one way of doing this).\n" 74 | "Example: --pipe 'jq --raw-output --unbuffered .eventId')", 75 | false, "", "CMD", cmd); 76 | ValueArg indexFilename("", "index-file", 77 | "Store index in " 78 | "(default .zindex)", false, "", 79 | "index-file", cmd); 80 | cmd.parse(argc, argv); 81 | 82 | ConsoleLog log( 83 | debug.isSet() ? Log::Severity::Debug : verbose.isSet() 84 | ? Log::Severity::Info 85 | : Log::Severity::Warning, 86 | forceColour.isSet() || forceColor.isSet(), warnings.isSet()); 87 | 88 | try { 89 | auto realPath = getRealPath(inputFile.getValue()); 90 | File in(fopen(realPath.c_str(), "rb")); 91 | if (in.get() == nullptr) { 92 | log.debug("Unable to open ", inputFile.getValue(), " (as ", 93 | realPath, 94 | ")"); 95 | log.error("Could not open ", inputFile.getValue(), " for reading"); 96 | return 1; 97 | } 98 | 99 | auto outputFile = indexFilename.isSet() ? indexFilename.getValue() : 100 | inputFile.getValue() + ".zindex"; 101 | Index::Builder builder(log, move(in), realPath, outputFile); 102 | if (skipFirst.isSet()) 103 | builder.skipFirst(skipFirst.getValue()); 104 | 105 | Index::IndexConfig config{}; 106 | config.numeric = numeric.isSet(); 107 | config.unique = unique.isSet(); 108 | config.sparse = sparse.isSet(); 109 | //config.indexLineOffsets = // TODO - add command line flag if desired 110 | 111 | auto delimiter = delimiterArg.getValue(); 112 | if (tabDelimiterArg.isSet() && delimiterArg.isSet()) { 113 | log.error("Cannot set both --delimiter and --tab-delimiter"); 114 | return 1; 115 | } 116 | if (tabDelimiterArg.isSet()) 117 | delimiter = "\t"; 118 | 119 | if (configFile.isSet()) { 120 | auto indexParser = IndexParser(configFile.getValue()); 121 | indexParser.buildIndexes(&builder, log); 122 | } else { 123 | if (regex.isSet() && field.isSet()) { 124 | throw std::runtime_error( 125 | "Sorry; multiple indices must be defined by an " 126 | "indexes file - see '-i' option"); 127 | } 128 | if (regex.isSet()) { 129 | auto regexIndexer = new RegExpIndexer(regex.getValue(), 130 | capture.getValue()); 131 | builder.addIndexer("default", regex.getValue(), config, 132 | std::unique_ptr(regexIndexer)); 133 | } 134 | if (field.isSet()) { 135 | ostringstream name; 136 | name << "Field " << field.getValue() << " delimited by '" 137 | << delimiter << "'"; 138 | builder.addIndexer("default", name.str(), config, 139 | std::unique_ptr( 140 | new FieldIndexer( 141 | delimiter, 142 | field.getValue()))); 143 | } 144 | if (externalIndexer.isSet()) { 145 | auto indexer = std::unique_ptr( 146 | new ExternalIndexer(log, 147 | externalIndexer.getValue(), 148 | delimiter)); 149 | builder.addIndexer("default", externalIndexer.getValue(), 150 | config, std::move(indexer)); 151 | } 152 | } 153 | if (checkpointEvery.isSet()) 154 | builder.indexEvery(checkpointEvery.getValue()); 155 | builder.build(); 156 | } catch (const exception &e) { 157 | log.error(e.what()); 158 | return 1; 159 | } 160 | return 0; 161 | } 162 | 163 | int main(int argc, const char *argv[]) { 164 | try { 165 | return Main(argc, argv); 166 | } catch (const exception &e) { 167 | cerr << e.what() << endl; 168 | return 1; 169 | } 170 | } 171 | -------------------------------------------------------------------------------- /src/zq.cpp: -------------------------------------------------------------------------------- 1 | #include "File.h" 2 | #include "Index.h" 3 | #include "LineSink.h" 4 | #include "ConsoleLog.h" 5 | 6 | #include 7 | 8 | #include 9 | #include 10 | #include 11 | #include "RangeFetcher.h" 12 | 13 | using namespace std; 14 | using namespace TCLAP; 15 | 16 | namespace { 17 | 18 | struct PrintSink : LineSink { 19 | bool printLineNum; 20 | 21 | explicit PrintSink(bool printLineNum) : printLineNum(printLineNum) { } 22 | 23 | bool onLine(size_t l, size_t, const char *line, size_t length) override { 24 | if (printLineNum) cout << l << ":"; 25 | cout << string(line, length) << endl; 26 | return true; 27 | } 28 | }; 29 | 30 | struct PrintHandler : RangeFetcher::Handler { 31 | Index &index; 32 | LineSink &sink; 33 | const bool printSep; 34 | const std::string sep; 35 | 36 | PrintHandler(Index &index, LineSink &sink, bool printSep, 37 | std::string sep) 38 | : index(index), sink(sink), printSep(printSep), 39 | sep(std::move(sep)) { } 40 | 41 | void onLine(uint64_t line) override { 42 | index.getLine(line, sink); 43 | } 44 | 45 | void onSeparator() override { 46 | if (printSep) 47 | cout << sep << endl; 48 | } 49 | }; 50 | 51 | uint64_t toInt(const string &s) { 52 | char *endP; 53 | auto res = strtoull(&s[0], &endP, 10); 54 | if (*endP != '\0') throw runtime_error("Non-numeric value: '" + s + "'"); 55 | return res; 56 | } 57 | 58 | } 59 | 60 | int Main(int argc, const char *argv[]) { 61 | CmdLine cmd("Lookup indices in a compressed text file"); 62 | UnlabeledValueArg inputFile( 63 | "input-file", "Read input from ", true, "", "file", cmd); 64 | UnlabeledMultiArg query( 65 | "query", "Query for ", false, "", cmd); 66 | SwitchArg lineMode("l", "line", 67 | "Treat query as a series of line numbers to print", cmd); 68 | SwitchArg verbose("v", "verbose", "Be more verbose", cmd); 69 | SwitchArg debug("", "debug", "Be even more verbose", cmd); 70 | SwitchArg forceColour("", "colour", "Use colour even on non-TTY", cmd); 71 | SwitchArg forceColor("", "color", "Use color even on non-TTY", cmd); 72 | SwitchArg forceLoad("", "force", "Load index even if it appears " 73 | "inconsistent with the index", cmd); 74 | SwitchArg lineNum("n", "line-number", 75 | "Prefix each line of output with its line number", cmd); 76 | SwitchArg warnings("w", "warnings", "Log warnings at info level", cmd); 77 | ValueArg afterArg("A", "after-context", 78 | "Print NUM lines of context after each match", 79 | false, 0, "NUM", cmd); 80 | ValueArg beforeArg("B", "before-context", 81 | "Print NUM lines of context before each match", 82 | false, 0, "NUM", cmd); 83 | ValueArg contextArg("C", "context", 84 | "Print NUM lines of context around each match", 85 | false, 0, "NUM", cmd); 86 | SwitchArg noSepArg("", "no-separator", 87 | "Don't print a separator between non-overlapping contexts", 88 | cmd); 89 | ValueArg sepArg("S", "separator", 90 | "Print SEPARATOR between non-overlapping contexts " 91 | "(if -A, -B or -C specified)", 92 | false, "--", "SEPARATOR", cmd); 93 | ValueArg indexArg("", "index-file", "Use index from " 94 | "(default .zindex)", false, "", "index", cmd); 95 | 96 | ValueArg queryIndexArg("i", "index", "Use specified index for searching" 97 | , false, "", "index", cmd); 98 | 99 | ValueArg rawSqlQueryArg("", "raw", "Expert Mode - Since zindex is " 100 | "a sqlite3 database under the covers, this flag lets you run a custom " 101 | "query for use cases not supported by command line args." 102 | " example: (--raw \"select a.line from index_default a, " 103 | "index_secondary b where a.line == b.line and a.key == '2' " 104 | "and b.key == 'KEY_2';\")" 105 | , false, "", "raw", cmd); 106 | 107 | cmd.parse(argc, argv); 108 | 109 | ConsoleLog log( 110 | debug.isSet() ? Log::Severity::Debug : verbose.isSet() 111 | ? Log::Severity::Info 112 | : Log::Severity::Warning, 113 | forceColour.isSet() || forceColor.isSet(), warnings.isSet()); 114 | 115 | try { 116 | auto compressedFile = inputFile.getValue(); 117 | File in(fopen(compressedFile.c_str(), "rb")); 118 | if (in.get() == nullptr) { 119 | log.error("Could not open ", compressedFile, " for reading"); 120 | return 1; 121 | } 122 | 123 | auto indexFile = indexArg.isSet() ? indexArg.getValue() : 124 | inputFile.getValue() + ".zindex"; 125 | auto index = Index::load(log, move(in), indexFile.c_str(), 126 | forceLoad.isSet()); 127 | auto queryIndex = queryIndexArg.isSet() ? queryIndexArg.getValue() : "default"; 128 | 129 | uint64_t before = 0u; 130 | uint64_t after = 0u; 131 | if (beforeArg.isSet()) before = beforeArg.getValue(); 132 | if (afterArg.isSet()) after = afterArg.getValue(); 133 | if (contextArg.isSet()) before = after = contextArg.getValue(); 134 | log.debug("Fetching context of ", before, " lines before and ", after, 135 | " lines after"); 136 | PrintSink sink(lineNum.isSet()); 137 | PrintHandler ph(index, sink, (before || after) && !noSepArg.isSet(), 138 | sepArg.getValue()); 139 | RangeFetcher rangeFetcher(ph, before, after); 140 | if (lineMode.isSet()) { 141 | for (auto &q : query.getValue()) 142 | rangeFetcher(toInt(q)); 143 | } else if (rawSqlQueryArg.isSet()) { 144 | index.queryCustom(rawSqlQueryArg.getValue(), rangeFetcher); 145 | } else { 146 | index.queryIndexMulti(queryIndex, query.getValue(), rangeFetcher); 147 | } 148 | } catch (const exception &e) { 149 | log.error(e.what()); 150 | return 1; 151 | } 152 | 153 | return 0; 154 | } 155 | 156 | int main(int argc, const char *argv[]) { 157 | try { 158 | return Main(argc, argv); 159 | } catch (const exception &e) { 160 | cerr << e.what() << endl; 161 | return 1; 162 | } 163 | } 164 | -------------------------------------------------------------------------------- /tests/.gitignore: -------------------------------------------------------------------------------- 1 | /check_zindex 2 | /*.log 3 | /*.trs 4 | -------------------------------------------------------------------------------- /tests/CaptureLog.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "Log.h" 4 | #include 5 | #include 6 | 7 | struct CaptureLog : Log { 8 | using Log::minSeverity_; 9 | 10 | struct Record { 11 | Log::Severity severity; 12 | std::string message; 13 | 14 | Record(Log::Severity severity, std::string message) 15 | : severity(severity), message(message) { } 16 | 17 | friend std::ostream &operator<<(std::ostream &o, const Record &r) { 18 | return o << Log::name(r.severity) << " - " << r.message; 19 | } 20 | 21 | friend bool operator==(const Record &lhs, const Record &rhs) { 22 | return lhs.severity == rhs.severity && lhs.message == rhs.message; 23 | } 24 | }; 25 | 26 | using Records = std::vector; 27 | Records records; 28 | 29 | void log(Severity severity, const std::string &message) override { 30 | records.emplace_back(severity, message); 31 | } 32 | }; 33 | 34 | inline CaptureLog::Record D(const std::string &s) { 35 | return CaptureLog::Record(Log::Severity::Debug, s); 36 | } 37 | 38 | inline CaptureLog::Record I(const std::string &s) { 39 | return CaptureLog::Record(Log::Severity::Info, s); 40 | } 41 | 42 | inline CaptureLog::Record W(const std::string &s) { 43 | return CaptureLog::Record(Log::Severity::Warning, s); 44 | } 45 | 46 | inline CaptureLog::Record E(const std::string &s) { 47 | return CaptureLog::Record(Log::Severity::Error, s); 48 | } 49 | -------------------------------------------------------------------------------- /tests/CaptureSink.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "IndexSink.h" 4 | 5 | struct CaptureSink : IndexSink { 6 | std::vector captured; 7 | 8 | virtual void add(StringView index, 9 | size_t /*offset*/) override { 10 | captured.emplace_back(index.str()); 11 | } 12 | 13 | void reset() { captured.clear(); } 14 | }; 15 | -------------------------------------------------------------------------------- /tests/ExternalIndexerTest.cpp: -------------------------------------------------------------------------------- 1 | #include "ExternalIndexer.h" 2 | 3 | #include "catch.hpp" 4 | #include "CaptureSink.h" 5 | #include "CaptureLog.h" 6 | 7 | using vs = std::vector; 8 | 9 | TEST_CASE("external indexes", "[ExternalIndexer]") { 10 | CaptureSink sink; 11 | CaptureLog log; 12 | SECTION("Simple test") { 13 | ExternalIndexer indexer(log, "cat", " "); 14 | indexer.index(sink, "these are words"); 15 | CHECK(sink.captured == vs({ "these", "are", "words" })); 16 | sink.reset(); 17 | indexer.index(sink, ""); 18 | CHECK(sink.captured == vs()); 19 | sink.reset(); 20 | indexer.index(sink, "Also multiple separators"); 21 | CHECK(sink.captured == vs({ "Also", "multiple", "separators" })); 22 | } 23 | SECTION("Transforming test") { 24 | ExternalIndexer indexer(log, "stdbuf -oL tr a-z A-Z", " "); 25 | indexer.index(sink, "these are words"); 26 | CHECK(sink.captured == vs({ "THESE", "ARE", "WORDS" })); 27 | } 28 | SECTION("Gigantic output") { 29 | ExternalIndexer indexer(log, "cat", " "); 30 | std::string giant(8192, 'A'); 31 | indexer.index(sink, giant); 32 | CHECK(sink.captured == vs({ giant })); 33 | } 34 | } -------------------------------------------------------------------------------- /tests/FieldIndexerTest.cpp: -------------------------------------------------------------------------------- 1 | #include "FieldIndexer.h" 2 | 3 | #include "catch.hpp" 4 | #include "CaptureSink.h" 5 | 6 | using vs = std::vector; 7 | 8 | TEST_CASE("field indexes", "[FieldIndexer]") { 9 | CaptureSink sink; 10 | SECTION("First") { 11 | FieldIndexer indexer(" ", 1); 12 | indexer.index(sink, "these are words"); 13 | CHECK(sink.captured == vs({ "these" })); 14 | sink.reset(); 15 | indexer.index(sink, ""); 16 | CHECK(sink.captured == vs()); 17 | } 18 | SECTION("Second") { 19 | FieldIndexer indexer(" ", 2); 20 | indexer.index(sink, "these are words"); 21 | CHECK(sink.captured == vs({ "are" })); 22 | } 23 | SECTION("Third") { 24 | FieldIndexer indexer(" ", 3); 25 | indexer.index(sink, "these are words"); 26 | CHECK(sink.captured == vs({ "words" })); 27 | } 28 | SECTION("Off the end") { 29 | FieldIndexer indexer(" ", 4); 30 | indexer.index(sink, "these are words"); 31 | CHECK(sink.captured == vs()); 32 | } 33 | SECTION("Works for tabs") { 34 | FieldIndexer indexer("\t", 2); 35 | indexer.index(sink, "yibble\tbibble\tboing"); 36 | CHECK(sink.captured == vs({"bibble"})); 37 | } 38 | SECTION("Works for strings") { 39 | FieldIndexer indexer(":sep:", 2); 40 | indexer.index(sink, "yibble:sep:bibble:sep:boing"); 41 | CHECK(sink.captured == vs({"bibble"})); 42 | } 43 | } -------------------------------------------------------------------------------- /tests/LineFinderTest.cpp: -------------------------------------------------------------------------------- 1 | #include "LineFinder.h" 2 | #include "LineSink.h" 3 | 4 | #include "catch.hpp" 5 | 6 | #include 7 | #include 8 | 9 | namespace { 10 | 11 | struct RecordingSink : LineSink { 12 | std::vector lines; 13 | std::vector fileOffsets; 14 | 15 | bool onLine(size_t, size_t offset, 16 | const char *line, size_t length) override { 17 | lines.emplace_back(line, length); 18 | fileOffsets.emplace_back(offset); 19 | return true; 20 | } 21 | 22 | bool empty() const { 23 | return lines.empty(); 24 | } 25 | }; 26 | 27 | } 28 | 29 | TEST_CASE("finds lines", "[LineFinder]") { 30 | RecordingSink sink; 31 | LineFinder finder(sink); 32 | 33 | REQUIRE(finder.lineOffsets().empty()); 34 | REQUIRE(sink.empty()); 35 | 36 | SECTION("empty input") { 37 | finder.add(nullptr, 0, true); 38 | REQUIRE(finder.lineOffsets().size() == 1); 39 | REQUIRE(finder.lineOffsets()[0] == uint64_t(0)); 40 | REQUIRE(sink.empty()); 41 | } 42 | 43 | SECTION("single line") { 44 | static const uint8_t one[] = "One\n"; 45 | finder.add(one, sizeof(one) - 1, true); 46 | const auto &lo = finder.lineOffsets(); 47 | REQUIRE(lo.size() == 2); 48 | REQUIRE(lo[0] == uint64_t(0)); 49 | REQUIRE(lo[1] == uint64_t(4)); 50 | REQUIRE(sink.lines.size() == 1); 51 | REQUIRE(sink.lines[0] == "One"); 52 | REQUIRE(sink.fileOffsets[0] == 0); 53 | } 54 | SECTION("two lines") { 55 | static const uint8_t oneTwo[] = "One\nTwo\n"; 56 | SECTION("in one go") { 57 | finder.add(oneTwo, sizeof(oneTwo) - 1, true); 58 | const auto &lo = finder.lineOffsets(); 59 | REQUIRE(lo.size() == 3); 60 | REQUIRE(lo[0] == uint64_t(0)); 61 | REQUIRE(lo[1] == uint64_t(4)); 62 | REQUIRE(lo[2] == uint64_t(8)); 63 | REQUIRE(sink.lines.size() == 2); 64 | REQUIRE(sink.lines[0] == "One"); 65 | REQUIRE(sink.lines[1] == "Two"); 66 | REQUIRE(sink.fileOffsets[0] == 0); 67 | REQUIRE(sink.fileOffsets[1] == 4); 68 | } 69 | SECTION("byte at a time") { 70 | for (auto i = 0u; i < sizeof(oneTwo) - 1; ++i) 71 | finder.add(oneTwo + i, 1, i == sizeof(oneTwo) - 2); 72 | const auto &lo = finder.lineOffsets(); 73 | REQUIRE(lo.size() == 3); 74 | REQUIRE(lo[0] == uint64_t(0)); 75 | REQUIRE(lo[1] == uint64_t(4)); 76 | REQUIRE(lo[2] == uint64_t(8)); 77 | REQUIRE(sink.lines.size() == 2); 78 | REQUIRE(sink.lines[0] == "One"); 79 | REQUIRE(sink.lines[1] == "Two"); 80 | REQUIRE(sink.fileOffsets[0] == 0); 81 | REQUIRE(sink.fileOffsets[1] == 4); 82 | } 83 | SECTION("3 bytes at a time") { 84 | for (auto i = 0u; i < sizeof(oneTwo) - 1; i += 3) { 85 | int remaining = sizeof(oneTwo) - 1 - i; 86 | auto length = remaining < 3 ? remaining : 3; 87 | finder.add(oneTwo + i, length, remaining <= 3); 88 | } 89 | const auto &lo = finder.lineOffsets(); 90 | REQUIRE(lo.size() == 3); 91 | REQUIRE(lo[0] == uint64_t(0)); 92 | REQUIRE(lo[1] == uint64_t(4)); 93 | REQUIRE(lo[2] == uint64_t(8)); 94 | REQUIRE(sink.lines.size() == 2); 95 | REQUIRE(sink.lines[0] == "One"); 96 | REQUIRE(sink.lines[1] == "Two"); 97 | REQUIRE(sink.fileOffsets[0] == 0); 98 | REQUIRE(sink.fileOffsets[1] == 4); 99 | } 100 | SECTION("in one go missing newline") { 101 | finder.add(oneTwo, sizeof(oneTwo) - 2, true); 102 | const auto &lo = finder.lineOffsets(); 103 | REQUIRE(lo.size() == 3); 104 | REQUIRE(lo[0] == uint64_t(0)); 105 | REQUIRE(lo[1] == uint64_t(4)); 106 | REQUIRE(lo[2] == uint64_t(8)); 107 | REQUIRE(sink.lines.size() == 2); 108 | REQUIRE(sink.lines[0] == "One"); 109 | REQUIRE(sink.lines[1] == "Two"); 110 | REQUIRE(sink.fileOffsets[0] == 0); 111 | REQUIRE(sink.fileOffsets[1] == 4); 112 | } 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /tests/LogTest.cpp: -------------------------------------------------------------------------------- 1 | #include "Log.h" 2 | 3 | #include "catch.hpp" 4 | 5 | #include 6 | #include 7 | #include 8 | #include "CaptureLog.h" 9 | 10 | namespace { 11 | 12 | struct StreamNoticer { 13 | mutable int numStreams = 0; 14 | 15 | friend std::ostream &operator<<(std::ostream &o, const StreamNoticer &s) { 16 | s.numStreams++; 17 | return o << "[stream]"; 18 | } 19 | }; 20 | 21 | } 22 | 23 | TEST_CASE("formats strings", "[Log]") { 24 | REQUIRE(Log::format() == ""); 25 | REQUIRE(Log::format("hello") == "hello"); 26 | REQUIRE(Log::format("hello", ' ', "world") == "hello world"); 27 | REQUIRE(Log::format("There are ", 26, " letters in the alphabet") == 28 | "There are 26 letters in the alphabet"); 29 | } 30 | 31 | TEST_CASE("captures logs", "[Log]") { 32 | CaptureLog log; 33 | SECTION("min level debug") { 34 | log.minSeverity_ = Log::Severity::Debug; 35 | log.debug("debug"); 36 | log.info("info"); 37 | log.warn("warn"); 38 | log.error("error"); 39 | CHECK(log.records == CaptureLog::Records({ D("debug"), I("info"), 40 | W("warn"), E("error") })); 41 | } 42 | SECTION("min level info") { 43 | log.minSeverity_ = Log::Severity::Info; 44 | log.debug("debug"); 45 | log.info("info"); 46 | log.warn("warn"); 47 | log.error("error"); 48 | CHECK(log.records == 49 | CaptureLog::Records({ I("info"), W("warn"), E("error") })); 50 | } 51 | SECTION("min level warn") { 52 | log.minSeverity_ = Log::Severity::Warning; 53 | log.debug("debug"); 54 | log.info("info"); 55 | log.warn("warn"); 56 | log.error("error"); 57 | CHECK(log.records == CaptureLog::Records({ W("warn"), E("error") })); 58 | } 59 | SECTION("min level error") { 60 | log.minSeverity_ = Log::Severity::Error; 61 | log.debug("debug"); 62 | log.info("info"); 63 | log.warn("warn"); 64 | log.error("error"); 65 | CHECK(log.records == CaptureLog::Records({ E("error") })); 66 | } 67 | } 68 | 69 | TEST_CASE("doesn't unnecessarily format if disabled", "[Log]") { 70 | CaptureLog log; 71 | log.minSeverity_ = Log::Severity::Warning; 72 | StreamNoticer debugNoticer, infoNoticer, warnNoticer, errNoticer; 73 | log.debug("this shouldn't get streamed: ", debugNoticer); 74 | log.info("This also shouldn't get streamed: ", infoNoticer); 75 | log.warn("This should: ", warnNoticer); 76 | log.error("As should this: ", errNoticer); 77 | CHECK(debugNoticer.numStreams == 0); 78 | CHECK(infoNoticer.numStreams == 0); 79 | CHECK(warnNoticer.numStreams == 1); 80 | CHECK(errNoticer.numStreams == 1); 81 | CHECK(log.records == CaptureLog::Records( 82 | { W("This should: [stream]"), E("As should this: [stream]") })); 83 | 84 | } 85 | -------------------------------------------------------------------------------- /tests/PrettyBytesTest.cpp: -------------------------------------------------------------------------------- 1 | #include "PrettyBytes.h" 2 | 3 | #include "catch.hpp" 4 | #include 5 | 6 | #define STR(XX) ([&]() ->std::string { \ 7 | std::stringstream STR__STREAM; \ 8 | STR__STREAM << XX; \ 9 | return STR__STREAM.str(); \ 10 | })() 11 | 12 | TEST_CASE("pretty bytes", "[PrettyBytes]") { 13 | CHECK(STR(PrettyBytes(0)) == "0 bytes"); 14 | CHECK(STR(PrettyBytes(1)) == "1 byte"); 15 | CHECK(STR(PrettyBytes(2)) == "2 bytes"); 16 | CHECK(STR(PrettyBytes(1023)) == "1023 bytes"); 17 | CHECK(STR(PrettyBytes(1024)) == "1.00 KiB"); 18 | CHECK(STR(PrettyBytes(1200)) == "1.17 KiB"); 19 | CHECK(STR(PrettyBytes(1200000)) == "1.14 MiB"); 20 | CHECK(STR(PrettyBytes(1200000000)) == "1.12 GiB"); 21 | CHECK(STR(PrettyBytes(std::numeric_limits::max())) == 22 | "17179869184.00 GiB"); 23 | } 24 | -------------------------------------------------------------------------------- /tests/RangeFetcherTest.cpp: -------------------------------------------------------------------------------- 1 | #include "RangeFetcher.h" 2 | #include "catch.hpp" 3 | #include 4 | 5 | namespace { 6 | using LL = std::vector; 7 | 8 | struct MockHandler : RangeFetcher::Handler { 9 | LL lines; 10 | 11 | void onLine(uint64_t line) override { 12 | lines.emplace_back(line); 13 | } 14 | 15 | void onSeparator() override { 16 | lines.emplace_back(0); 17 | } 18 | }; 19 | 20 | } 21 | 22 | TEST_CASE("fetches just the lines asked", "[RangeFetcher]") { 23 | MockHandler handler; 24 | RangeFetcher rf(handler, 0, 0); 25 | 26 | SECTION("single") { 27 | rf(1); 28 | CHECK(handler.lines == LL({ 1 })); 29 | } 30 | SECTION("several") { 31 | rf(1); 32 | rf(2); 33 | rf(3); 34 | CHECK(handler.lines == LL({ 1, 2, 3 })); 35 | } 36 | SECTION("several with gaps") { 37 | rf(10); 38 | rf(20); 39 | rf(30); 40 | CHECK(handler.lines == LL({ 10, 0, 20, 0, 30 })); 41 | } 42 | SECTION("going backwards") { 43 | rf(3); 44 | rf(2); 45 | rf(1); 46 | CHECK(handler.lines == LL({ 3, 0, 2, 0, 1 })); 47 | } 48 | SECTION("duplicate") { 49 | rf(3); 50 | rf(3); 51 | rf(3); 52 | CHECK(handler.lines == LL({ 3 })); 53 | } 54 | } 55 | 56 | TEST_CASE("handles before", "[RangeFetcher]") { 57 | MockHandler handler; 58 | RangeFetcher rf(handler, 2, 0); 59 | 60 | SECTION("single near start") { 61 | rf(1); 62 | CHECK(handler.lines == LL({ 1 })); 63 | } 64 | SECTION("single in middle") { 65 | rf(10); 66 | CHECK(handler.lines == LL({ 8, 9, 10 })); 67 | } 68 | SECTION("several") { 69 | rf(11); 70 | rf(12); 71 | rf(13); 72 | CHECK(handler.lines == LL({ 9, 10, 11, 12, 13 })); 73 | } 74 | SECTION("several with gaps") { 75 | rf(10); 76 | rf(20); 77 | rf(30); 78 | CHECK(handler.lines == LL({ 8, 9, 10, 0, 18, 19, 20, 0, 28, 29, 30 })); 79 | } 80 | SECTION("several with tiny gaps") { 81 | rf(10); 82 | rf(12); 83 | rf(14); 84 | CHECK(handler.lines == LL({ 8, 9, 10, 11, 12, 13, 14 })); 85 | } 86 | SECTION("going backwards") { 87 | rf(13); 88 | rf(12); 89 | rf(11); 90 | CHECK(handler.lines == LL({ 11, 12, 13, 0, 10, 11, 12, 0, 9, 10, 11 })); 91 | } 92 | SECTION("duplicate") { 93 | rf(5); 94 | rf(5); 95 | rf(5); 96 | CHECK(handler.lines == LL({ 3, 4, 5 })); 97 | } 98 | } 99 | 100 | TEST_CASE("handles after", "[RangeFetcher]") { 101 | MockHandler handler; 102 | RangeFetcher rf(handler, 0, 2); 103 | 104 | SECTION("single near start") { 105 | rf(1); 106 | CHECK(handler.lines == LL({ 1, 2, 3 })); 107 | } 108 | SECTION("single in middle") { 109 | rf(10); 110 | CHECK(handler.lines == LL({ 10, 11, 12 })); 111 | } 112 | SECTION("several") { 113 | rf(11); 114 | rf(12); 115 | rf(13); 116 | CHECK(handler.lines == LL({ 11, 12, 13, 14, 15 })); 117 | } 118 | SECTION("several with gaps") { 119 | rf(10); 120 | rf(20); 121 | rf(30); 122 | CHECK(handler.lines == 123 | LL({ 10, 11, 12, 0, 20, 21, 22, 0, 30, 31, 32 })); 124 | } 125 | SECTION("several with tiny gaps") { 126 | rf(10); 127 | rf(12); 128 | rf(14); 129 | CHECK(handler.lines == LL({ 10, 11, 12, 13, 14, 15, 16 })); 130 | } 131 | SECTION("going backwards") { 132 | rf(13); 133 | rf(12); 134 | rf(11); 135 | CHECK(handler.lines == 136 | LL({ 13, 14, 15, 0, 12, 13, 14, 0, 11, 12, 13 })); 137 | } 138 | SECTION("duplicate") { 139 | rf(5); 140 | rf(5); 141 | rf(5); 142 | CHECK(handler.lines == LL({ 5, 6, 7 })); 143 | } 144 | } 145 | 146 | TEST_CASE("handles both", "[RangeFetcher]") { 147 | MockHandler handler; 148 | RangeFetcher rf(handler, 2, 2); 149 | 150 | SECTION("single near start") { 151 | rf(1); 152 | CHECK(handler.lines == LL({ 1, 2, 3 })); 153 | } 154 | SECTION("single in middle") { 155 | rf(10); 156 | CHECK(handler.lines == LL({ 8, 9, 10, 11, 12 })); 157 | } 158 | SECTION("several") { 159 | rf(11); 160 | rf(12); 161 | rf(13); 162 | CHECK(handler.lines == LL({ 9, 10, 11, 12, 13, 14, 15 })); 163 | } 164 | SECTION("several with gaps") { 165 | rf(10); 166 | rf(20); 167 | rf(30); 168 | CHECK(handler.lines == 169 | LL({ 8, 9, 10, 11, 12, 0, 18, 19, 20, 21, 22, 0, 170 | 28, 29, 30, 31, 32 })); 171 | } 172 | SECTION("several with tiny gaps") { 173 | rf(10); 174 | rf(12); 175 | rf(14); 176 | CHECK(handler.lines == LL({ 8, 9, 10, 11, 12, 13, 14, 15, 16 })); 177 | } 178 | SECTION("going backwards") { 179 | rf(13); 180 | rf(12); 181 | rf(11); 182 | CHECK(handler.lines == 183 | LL({ 11, 12, 13, 14, 15, 0, 184 | 10, 11, 12, 13, 14, 0, 185 | 9, 10, 11, 12, 13 })); 186 | } 187 | SECTION("duplicate") { 188 | rf(5); 189 | rf(5); 190 | rf(5); 191 | CHECK(handler.lines == LL({ 3, 4, 5, 6, 7 })); 192 | } 193 | } 194 | -------------------------------------------------------------------------------- /tests/RegExpIndexerTest.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include "RegExpIndexer.h" 3 | 4 | #include "catch.hpp" 5 | #include "CaptureSink.h" 6 | 7 | using vs = std::vector; 8 | 9 | TEST_CASE("indexes", "[RegExpIndexer]") { 10 | SECTION("matches multiple on a line") { 11 | RegExpIndexer words("\\w+"); 12 | CaptureSink sink; 13 | words.index(sink, "these are words"); 14 | CHECK(sink.captured == vs({ "these", "are", "words" })); 15 | } 16 | SECTION("matches only one if anchored") { 17 | RegExpIndexer firstWord("^\\w+"); 18 | CaptureSink sink; 19 | firstWord.index(sink, "these are words"); 20 | REQUIRE(sink.captured.size() == 1); 21 | CHECK(sink.captured[0] == "these"); 22 | } 23 | SECTION("stupid index") { 24 | RegExpIndexer singleChar("."); 25 | CaptureSink sink; 26 | singleChar.index(sink, "0123456789"); 27 | CHECK(sink.captured == 28 | vs({ "0", "1", "2", "3", "4", "5", "6", "7", "8", "9" })); 29 | } 30 | SECTION("multiple capture groups - matches specified group") { 31 | RegExpIndexer multipleCaptureGroups("(New|Update)\\|([^|]+)", 2); 32 | CaptureSink sink; 33 | multipleCaptureGroups.index(sink, "New|2-id|0"); 34 | CHECK(sink.captured == vs({ "2-id" })); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /tests/RegExpTest.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "RegExp.h" 4 | 5 | #include "catch.hpp" 6 | 7 | TEST_CASE("constructs regexes", "[RegExp]") { 8 | RegExp r1(""); 9 | RegExp r2(".*"); 10 | RegExp r3("\"eventId\":([0-9]+)"); 11 | 12 | SECTION("throws on bad") { 13 | bool threw = false; 14 | try { 15 | RegExp r("("); 16 | } catch (const std::runtime_error &e) { 17 | threw = true; 18 | REQUIRE(strstr(e.what(), "Unmatched (") != nullptr); 19 | } 20 | REQUIRE(threw); 21 | } 22 | } 23 | 24 | namespace { 25 | std::string S(const char *s, RegExp::Match match) { 26 | return std::string(s + match.first, match.second - match.first); 27 | } 28 | } 29 | 30 | TEST_CASE("matches", "[RegExp]") { 31 | RegExp r("\"eventId\":([0-9]+)"); 32 | RegExp::Matches matches; 33 | REQUIRE(r.exec("not a match", matches) == false); 34 | char const *against = "\"eventId\":123234,moose"; 35 | REQUIRE(r.exec(against, matches) == true); 36 | REQUIRE(matches.size() == 2); 37 | REQUIRE(S(against, matches[0]) == "\"eventId\":123234"); 38 | REQUIRE(S(against, matches[1]) == "123234"); 39 | } 40 | 41 | TEST_CASE("multiple capture groups", "[RegExp]") { 42 | RegExp r("(New|Update)\\|([^|]+)"); 43 | RegExp::Matches matches; 44 | char const *against = "New|2-id|0"; 45 | REQUIRE(r.exec(against, matches) == true); 46 | REQUIRE(matches.size() == 3); 47 | REQUIRE(S(against, matches[0]) == "New|2-id"); 48 | REQUIRE(S(against, matches[1]) == "New"); 49 | REQUIRE(S(against, matches[2]) == "2-id"); 50 | } 51 | 52 | TEST_CASE("moves", "[RegExp]") { 53 | RegExp r("[a-z]+"); 54 | RegExp::Matches matches; 55 | REQUIRE(r.exec("moo", matches) == true); 56 | RegExp nR("1234"); 57 | nR = std::move(r); 58 | REQUIRE(r.exec("moo", matches) == true); 59 | } -------------------------------------------------------------------------------- /tests/SqliteTest.cpp: -------------------------------------------------------------------------------- 1 | #include "Sqlite.h" 2 | 3 | #include "catch.hpp" 4 | 5 | #include 6 | #include 7 | #include "SqliteError.h" 8 | #include "TempDir.h" 9 | #include "CaptureLog.h" 10 | 11 | TEST_CASE("opens databases", "[Sqlite]") { 12 | TempDir tempDir; 13 | CaptureLog log; 14 | Sqlite sqlite(log); 15 | auto dbPath = tempDir.path + "/db.sqlite"; 16 | 17 | SECTION("creates a new db") { 18 | sqlite.open(dbPath, false); 19 | } 20 | SECTION("fails to open a non-existent db") { 21 | REQUIRE_THROWS(sqlite.open(dbPath, true)); 22 | } 23 | 24 | SECTION("reopens a new db") { 25 | sqlite.open(dbPath, false); 26 | sqlite.close(); 27 | Sqlite anotherSqlite(log); 28 | anotherSqlite.open(dbPath, true); 29 | } 30 | } 31 | 32 | TEST_CASE("prepares statements", "[Sqlite]") { 33 | TempDir tempDir; 34 | CaptureLog log; 35 | Sqlite sqlite(log); 36 | auto dbPath = tempDir.path + "/db.sqlite"; 37 | sqlite.open(dbPath, false); 38 | 39 | auto s = sqlite.prepare("create table t(one varchar(10), two integer)"); 40 | 41 | SECTION("handles exceptions") { 42 | std::string threw; 43 | try { 44 | sqlite.prepare("SELECT * FROM foo"); 45 | } catch (const SqliteError &e) { 46 | threw = e.what(); 47 | } 48 | CHECK(threw == "no such table: foo (SELECT * FROM foo)"); 49 | } 50 | } 51 | 52 | TEST_CASE("executes statements", "[Sqlite]") { 53 | TempDir tempDir; 54 | CaptureLog log; 55 | Sqlite sqlite(log); 56 | auto dbPath = tempDir.path + "/db.sqlite"; 57 | sqlite.open(dbPath, false); 58 | 59 | REQUIRE(sqlite.prepare("create table t(one varchar(10), two integer)").step() == true); 60 | REQUIRE(sqlite.prepare("insert into t values('moo', 1)").step() == true); 61 | REQUIRE(sqlite.prepare("insert into t values('foo', 2)").step() == true); 62 | auto select = sqlite.prepare("select * from t order by two desc"); 63 | CHECK(select.columnCount() == 2); 64 | int num = 0; 65 | for (;;) { 66 | if (select.step()) break; 67 | CHECK(select.columnName(0) == "one"); 68 | CHECK(select.columnName(1) == "two"); 69 | if (num == 0) { 70 | CHECK(select.columnString(0) == "foo"); 71 | CHECK(select.columnInt64(1) == 2); 72 | } else if (num == 1) { 73 | CHECK(select.columnString(0) == "moo"); 74 | CHECK(select.columnInt64(1) == 1); 75 | } 76 | ++num; 77 | } 78 | CHECK(num == 2); 79 | } 80 | 81 | TEST_CASE("handles binds and blobs", "[Sqlite]") { 82 | TempDir tempDir; 83 | CaptureLog log; 84 | Sqlite sqlite(log); 85 | auto dbPath = tempDir.path + "/db.sqlite"; 86 | sqlite.open(dbPath, false); 87 | 88 | REQUIRE(sqlite.prepare("create table t(offset integer, data blob)").step() == true); 89 | auto inserter = sqlite.prepare("insert into t values(:id, :data)"); 90 | inserter.bindInt64(":id", 1234); 91 | constexpr auto byteLen = 1024; 92 | char bytes[byteLen]; 93 | for (int i = 0; i < byteLen; ++i) bytes[i] = i & 0xff; 94 | inserter.bindBlob(":data", bytes, byteLen); 95 | CHECK(inserter.step() == true); 96 | inserter.reset(); 97 | inserter.bindInt64(":id", 5678); 98 | for (int i = 0; i < byteLen; ++i) bytes[i] = (i & 0xff) ^ 0xff; 99 | inserter.bindBlob(":data", bytes, byteLen); 100 | CHECK(inserter.step() == true); 101 | 102 | auto select = sqlite.prepare("select * from t order by offset"); 103 | CHECK(select.columnCount() == 2); 104 | CHECK(select.step() == false); 105 | CHECK(select.columnName(0) == "offset"); 106 | CHECK(select.columnName(1) == "data"); 107 | CHECK(select.columnInt64(0) == 1234); 108 | auto blob1 = select.columnBlob(1); 109 | REQUIRE(blob1.size() == byteLen); 110 | for (int i = 0; i < byteLen; ++i) REQUIRE(blob1[i] == (i & 0xff)); 111 | CHECK(select.step() == false); 112 | CHECK(select.columnInt64(0) == 5678); 113 | auto blob2 = select.columnBlob(1); 114 | REQUIRE(blob2.size() == byteLen); 115 | for (int i = 0; i < byteLen; ++i) REQUIRE(blob2[i] == (0xff ^ (i & 0xff))); 116 | CHECK(select.step() == true); 117 | } 118 | 119 | 120 | TEST_CASE("handles binds and blobs uri", "[Sqlite]") { 121 | TempDir tempDir; 122 | CaptureLog log; 123 | Sqlite sqlite(log); 124 | auto dbPath = "file:" +tempDir.path + "/db.sqlite?foo=bar"; 125 | sqlite.open(dbPath, false); 126 | 127 | REQUIRE(sqlite.prepare("create table t(offset integer, data blob)").step() == true); 128 | auto inserter = sqlite.prepare("insert into t values(:id, :data)"); 129 | inserter.bindInt64(":id", 1234); 130 | constexpr auto byteLen = 1024; 131 | char bytes[byteLen]; 132 | for (int i = 0; i < byteLen; ++i) bytes[i] = i & 0xff; 133 | inserter.bindBlob(":data", bytes, byteLen); 134 | CHECK(inserter.step() == true); 135 | inserter.reset(); 136 | inserter.bindInt64(":id", 5678); 137 | for (int i = 0; i < byteLen; ++i) bytes[i] = (i & 0xff) ^ 0xff; 138 | inserter.bindBlob(":data", bytes, byteLen); 139 | CHECK(inserter.step() == true); 140 | 141 | auto select = sqlite.prepare("select * from t order by offset"); 142 | CHECK(select.columnCount() == 2); 143 | CHECK(select.step() == false); 144 | CHECK(select.columnName(0) == "offset"); 145 | CHECK(select.columnName(1) == "data"); 146 | CHECK(select.columnInt64(0) == 1234); 147 | auto blob1 = select.columnBlob(1); 148 | REQUIRE(blob1.size() == byteLen); 149 | for (int i = 0; i < byteLen; ++i) REQUIRE(blob1[i] == (i & 0xff)); 150 | CHECK(select.step() == false); 151 | CHECK(select.columnInt64(0) == 5678); 152 | auto blob2 = select.columnBlob(1); 153 | REQUIRE(blob2.size() == byteLen); 154 | for (int i = 0; i < byteLen; ++i) REQUIRE(blob2[i] == (0xff ^ (i & 0xff))); 155 | CHECK(select.step() == true); 156 | } 157 | 158 | TEST_CASE("uri convert to file", "[Sqlite]") { 159 | TempDir tempDir; 160 | CaptureLog log; 161 | Sqlite sqlite(log); 162 | 163 | // file:data.db Open the file "data.db" in the current directory. 164 | REQUIRE(sqlite.toFile("file:data.db") == "data.db"); 165 | 166 | // file:/home/fred/data.db 167 | // file:///home/fred/data.db 168 | // file://localhost/home/fred/data.db Open the database file "/home/fred/data.db". 169 | REQUIRE(sqlite.toFile("file:/home/fred/data.db") == "/home/fred/data.db"); 170 | REQUIRE(sqlite.toFile("file:///home/fred/data.db") == "/home/fred/data.db"); 171 | REQUIRE(sqlite.toFile("file://localhost/home/fred/data.db") == "/home/fred/data.db"); 172 | // 173 | // 174 | // file://darkstar/home/fred/data.db An error. "darkstar" is not a recognized authority. 175 | //let sqlite handle 176 | REQUIRE(sqlite.toFile("file://darkstar/home/fred/data.db") == "/home/fred/data.db"); 177 | // 178 | // file:///C:/Documents%20and%20Settings/fred/Desktop/data.db Windows only: Open the file "data.db" on fred's desktop on drive C:. Note that the %20 escaping in this example is not strictly necessary - space characters can be used literally in URI filenames. 179 | //REQUIRE(sqlite.toFile("file:///C:/Documents%20and%20Settings/fred/Desktop/data.db") == "/home/fred/data.db"); 180 | // 181 | // file:data.db?mode=ro&cache=private Open file "data.db" in the current directory for read-only access. Regardless of whether or not shared-cache mode is enabled by default, use a private cache. 182 | REQUIRE(sqlite.toFile("file:data.db?mode=ro&cache=private") == "data.db"); 183 | 184 | REQUIRE(sqlite.toFile("file:/home/fred/foo/bar/data.db?mode=ro&cache=private") == "/home/fred/foo/bar/data.db"); 185 | // 186 | // file:/home/fred/data.db?vfs=unix-dotfile Open file "/home/fred/data.db". Use the special VFS "unix-dotfile" that uses dot-files in place of posix advisory locking. 187 | REQUIRE(sqlite.toFile("file:/home/fred/data.db?vfs=unix-dotfile") == "/home/fred/data.db"); 188 | // file:data.db?mode=readonly An error. "readonly" is not a valid option for the "mode" parameter. Use "ro" instead: "file:data.db?mode=ro". 189 | REQUIRE(sqlite.toFile("file:data.db?mode=readonly") == "data.db"); 190 | 191 | 192 | REQUIRE_THROWS(sqlite.toFile("http://data.db?mode=readonly")); 193 | 194 | } 195 | -------------------------------------------------------------------------------- /tests/TempDir.cpp: -------------------------------------------------------------------------------- 1 | #include "TempDir.h" 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | TempDir::TempDir() { 11 | char templte[] = "/tmp/zindex_tmp_dir_XXXXXX"; 12 | auto tempDir = mkdtemp(templte); 13 | if (!tempDir) 14 | throw std::runtime_error("Unable to create a temp dir"); 15 | path = tempDir; 16 | } 17 | 18 | TempDir::~TempDir() { 19 | auto dir = opendir(path.c_str()); 20 | if (!dir) { 21 | std::cerr << "Unable to remove temp dir " << path << std::endl; 22 | std::terminate(); 23 | } 24 | for (;;) { 25 | auto ent = readdir(dir); 26 | if (!ent) break; 27 | auto tempPath = path + "/" + ent->d_name; 28 | if (!strcmp(ent->d_name, "..") || !strcmp(ent->d_name, ".")) continue; 29 | if (unlink(tempPath.c_str()) != 0) { 30 | closedir(dir); 31 | std::cerr << "Unable to remove temp file " << tempPath << std::endl; 32 | std::terminate(); 33 | } 34 | } 35 | closedir(dir); 36 | if (rmdir(path.c_str()) != 0) { 37 | std::cerr << "Unable to remove temp dir " << path << std::endl; 38 | std::terminate(); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /tests/TempDir.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | struct TempDir { 6 | std::string path; 7 | 8 | TempDir(); 9 | ~TempDir(); 10 | }; 11 | -------------------------------------------------------------------------------- /tests/files/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "indexes" : [ 3 | { 4 | "type" : "regex", 5 | "regex" : "(New|Update)\\|([^-]+)", 6 | "capture" : 2, 7 | "sparse" : true 8 | }, 9 | { 10 | "name" : "timestamp", 11 | "type" : "regex", 12 | "regex" : "^([^\\|]+)\\|.*(New|Update)", 13 | "sparse" : true, 14 | "capture" : 1 15 | } 16 | ] 17 | } 18 | -------------------------------------------------------------------------------- /tests/test_main.cpp: -------------------------------------------------------------------------------- 1 | #define CATCH_CONFIG_MAIN 2 | #include "catch.hpp" 3 | --------------------------------------------------------------------------------