├── .github └── workflows │ ├── doc.yml │ ├── kmindex-osx.yml │ └── kmindex.yml ├── .gitignore ├── .gitmodules ├── CMakeLists.txt ├── LICENSE ├── README.md ├── app ├── CMakeLists.txt ├── kmindex-server │ ├── CMakeLists.txt │ ├── compress.cpp │ ├── compress.hpp │ ├── main.cpp │ ├── request.hpp │ ├── server.cpp │ ├── server.hpp │ ├── utils.cpp │ └── utils.hpp └── kmindex │ ├── CMakeLists.txt │ ├── build.cpp │ ├── build.hpp │ ├── cli.cpp │ ├── cli.hpp │ ├── common.cpp │ ├── common.hpp │ ├── index_infos.cpp │ ├── index_infos.hpp │ ├── main.cpp │ ├── merge.cpp │ ├── merge.hpp │ ├── query.cpp │ ├── query.hpp │ ├── register.cpp │ ├── register.hpp │ ├── utils.cpp │ └── utils.hpp ├── cmake └── modules │ ├── CPackConfig.cmake │ └── FindXXHASH.cmake ├── conda └── kmindex │ ├── build.sh │ ├── conda_build_config.yaml │ └── meta.yaml ├── docker ├── Dockerfile └── README.md ├── docs ├── changelogs │ ├── v0.2.0.md │ ├── v0.3.0.md │ ├── v0.4.0.md │ ├── v0.5.0.md │ ├── v0.5.1.md │ └── v0.5.2.md └── kmindex │ ├── docs │ ├── construction.md │ ├── index.md │ ├── installation.md │ ├── merge.md │ ├── python-api.md │ ├── query.md │ ├── server-deploy.md │ ├── server-query.md │ └── stylesheets │ │ └── extra.css │ └── mkdocs.yml ├── examples ├── README.md └── data │ ├── 1.fasta │ ├── 1_build.sh │ ├── 2.fasta │ ├── 2_query.sh │ ├── 3.fasta │ ├── 3_run_server.sh │ ├── 4.fasta │ ├── 4_get_request.sh │ ├── 5_post_request.sh │ ├── fof1.txt │ ├── fof2.txt │ ├── query_1S1_1S2_in_index_D1_D2.json │ ├── query_1S1_in_index_D1.json │ └── query_1S1_in_index_D1_D2.json ├── flake.lock ├── flake.nix ├── install.sh ├── lib ├── CMakeLists.txt ├── config.hpp.in ├── include │ └── kmindex │ │ ├── exceptions.hpp │ │ ├── index │ │ ├── index.hpp │ │ ├── index_infos.hpp │ │ ├── kindex.hpp │ │ └── merge.hpp │ │ ├── mer.hpp │ │ ├── query │ │ ├── format.hpp │ │ ├── query.hpp │ │ └── query_results.hpp │ │ ├── spinlock.hpp │ │ ├── threadpool.hpp │ │ ├── utils.hpp │ │ └── version.hpp └── src │ ├── exceptions.cpp │ ├── index │ ├── index.cpp │ ├── index_infos.cpp │ ├── kindex.cpp │ └── merge.cpp │ ├── query │ ├── format.cpp │ ├── query.cpp │ └── query_results.cpp │ ├── threadpool.cpp │ └── utils.cpp ├── pykmindex ├── .gitignore ├── README.md ├── poetry.lock ├── pykmindex │ ├── __init__.py │ ├── query.py │ └── server.py └── pyproject.toml ├── scripts └── git-archive-all.sh ├── tests ├── CMakeLists.txt ├── app │ ├── CMakeLists.txt │ ├── run_test_abs.sh │ └── run_test_pa.sh ├── data │ ├── datasets │ │ ├── abs_dataset │ │ │ ├── 0.fasta │ │ │ ├── 1.fasta │ │ │ ├── 10.fasta │ │ │ ├── 11.fasta │ │ │ ├── 12.fasta │ │ │ ├── 13.fasta │ │ │ ├── 14.fasta │ │ │ ├── 15.fasta │ │ │ ├── 2.fasta │ │ │ ├── 3.fasta │ │ │ ├── 4.fasta │ │ │ ├── 5.fasta │ │ │ ├── 6.fasta │ │ │ ├── 7.fasta │ │ │ ├── 8.fasta │ │ │ ├── 9.fasta │ │ │ └── fof.txt │ │ └── pa_dataset │ │ │ ├── 0.fasta │ │ │ ├── 1.fasta │ │ │ ├── 10.fasta │ │ │ ├── 11.fasta │ │ │ ├── 12.fasta │ │ │ ├── 13.fasta │ │ │ ├── 14.fasta │ │ │ ├── 15.fasta │ │ │ ├── 2.fasta │ │ │ ├── 3.fasta │ │ │ ├── 4.fasta │ │ │ ├── 5.fasta │ │ │ ├── 6.fasta │ │ │ ├── 7.fasta │ │ │ ├── 8.fasta │ │ │ ├── 9.fasta │ │ │ └── fof.txt │ ├── indexes │ │ ├── abs_index │ │ │ ├── build_infos.txt │ │ │ ├── config_gatb │ │ │ │ └── gatb.config │ │ │ ├── hash.info │ │ │ ├── kmtricks.fof │ │ │ ├── matrices │ │ │ │ ├── matrix_0.cmbf │ │ │ │ ├── matrix_1.cmbf │ │ │ │ ├── matrix_2.cmbf │ │ │ │ └── matrix_3.cmbf │ │ │ ├── merge_infos │ │ │ │ ├── partition0.merge_info │ │ │ │ ├── partition1.merge_info │ │ │ │ ├── partition2.merge_info │ │ │ │ └── partition3.merge_info │ │ │ ├── options.txt │ │ │ ├── partition_infos │ │ │ │ ├── D0.pinfo │ │ │ │ ├── D1.pinfo │ │ │ │ ├── D10.pinfo │ │ │ │ ├── D11.pinfo │ │ │ │ ├── D12.pinfo │ │ │ │ ├── D13.pinfo │ │ │ │ ├── D14.pinfo │ │ │ │ ├── D15.pinfo │ │ │ │ ├── D2.pinfo │ │ │ │ ├── D3.pinfo │ │ │ │ ├── D4.pinfo │ │ │ │ ├── D5.pinfo │ │ │ │ ├── D6.pinfo │ │ │ │ ├── D7.pinfo │ │ │ │ ├── D8.pinfo │ │ │ │ └── D9.pinfo │ │ │ ├── repartition_gatb │ │ │ │ └── repartition.minimRepart │ │ │ └── run_infos.txt │ │ ├── index │ │ │ ├── abs │ │ │ ├── index.json │ │ │ └── pa │ │ └── pa_index │ │ │ ├── build_infos.txt │ │ │ ├── config_gatb │ │ │ └── gatb.config │ │ │ ├── fpr │ │ │ ├── partition_0.txt │ │ │ ├── partition_1.txt │ │ │ ├── partition_2.txt │ │ │ └── partition_3.txt │ │ │ ├── hash.info │ │ │ ├── kmtricks.fof │ │ │ ├── matrices │ │ │ ├── matrix_0.cmbf │ │ │ ├── matrix_1.cmbf │ │ │ ├── matrix_2.cmbf │ │ │ └── matrix_3.cmbf │ │ │ ├── merge_infos │ │ │ ├── partition0.merge_info │ │ │ ├── partition1.merge_info │ │ │ ├── partition2.merge_info │ │ │ └── partition3.merge_info │ │ │ ├── options.txt │ │ │ ├── partition_infos │ │ │ ├── D0.pinfo │ │ │ ├── D1.pinfo │ │ │ ├── D10.pinfo │ │ │ ├── D11.pinfo │ │ │ ├── D12.pinfo │ │ │ ├── D13.pinfo │ │ │ ├── D14.pinfo │ │ │ ├── D15.pinfo │ │ │ ├── D2.pinfo │ │ │ ├── D3.pinfo │ │ │ ├── D4.pinfo │ │ │ ├── D5.pinfo │ │ │ ├── D6.pinfo │ │ │ ├── D7.pinfo │ │ │ ├── D8.pinfo │ │ │ └── D9.pinfo │ │ │ ├── repartition_gatb │ │ │ └── repartition.minimRepart │ │ │ └── run_infos.txt │ └── outputs │ │ ├── q1_pa.json │ │ ├── q1_z4_abs.json │ │ ├── q1_z4_abs.tsv │ │ └── q1_z5_pa.tsv └── units │ └── kmindex-lib │ ├── CMakeLists.txt │ ├── main.cpp │ └── mer.cpp └── thirdparty ├── CMakeLists.txt ├── bitpacker └── include │ └── bitpacker │ └── bitpacker.hpp ├── fmt └── CMakeLists.txt ├── gtest └── CMakeLists.txt ├── nlohmann_json ├── CMakeLists.txt └── include │ ├── LICENSE.MIT │ ├── include │ └── nlohmann │ │ ├── adl_serializer.hpp │ │ ├── byte_container_with_subtype.hpp │ │ ├── detail │ │ ├── abi_macros.hpp │ │ ├── conversions │ │ │ ├── from_json.hpp │ │ │ ├── to_chars.hpp │ │ │ └── to_json.hpp │ │ ├── exceptions.hpp │ │ ├── hash.hpp │ │ ├── input │ │ │ ├── binary_reader.hpp │ │ │ ├── input_adapters.hpp │ │ │ ├── json_sax.hpp │ │ │ ├── lexer.hpp │ │ │ ├── parser.hpp │ │ │ └── position_t.hpp │ │ ├── iterators │ │ │ ├── internal_iterator.hpp │ │ │ ├── iter_impl.hpp │ │ │ ├── iteration_proxy.hpp │ │ │ ├── iterator_traits.hpp │ │ │ ├── json_reverse_iterator.hpp │ │ │ └── primitive_iterator.hpp │ │ ├── json_pointer.hpp │ │ ├── json_ref.hpp │ │ ├── macro_scope.hpp │ │ ├── macro_unscope.hpp │ │ ├── meta │ │ │ ├── call_std │ │ │ │ ├── begin.hpp │ │ │ │ └── end.hpp │ │ │ ├── cpp_future.hpp │ │ │ ├── detected.hpp │ │ │ ├── identity_tag.hpp │ │ │ ├── is_sax.hpp │ │ │ ├── std_fs.hpp │ │ │ ├── type_traits.hpp │ │ │ └── void_t.hpp │ │ ├── output │ │ │ ├── binary_writer.hpp │ │ │ ├── output_adapters.hpp │ │ │ └── serializer.hpp │ │ ├── string_concat.hpp │ │ ├── string_escape.hpp │ │ └── value_t.hpp │ │ ├── json.hpp │ │ ├── json_fwd.hpp │ │ ├── ordered_map.hpp │ │ └── thirdparty │ │ └── hedley │ │ ├── hedley.hpp │ │ └── hedley_undef.hpp │ ├── meson.build │ └── single_include │ └── nlohmann │ └── json.hpp ├── sha1 ├── README.md └── sha1.hpp ├── span-lite └── include │ └── nonstd │ └── span.hpp ├── spdlog └── CMakeLists.txt ├── xxhash └── CMakeLists.txt └── zlib └── CMakeLists.txt /.github/workflows/doc.yml: -------------------------------------------------------------------------------- 1 | name: kmindex-doc 2 | 3 | on: 4 | push: 5 | paths: ['docs/kmindex/**'] 6 | branches: ['main'] 7 | 8 | permissions: 9 | contents: write 10 | 11 | jobs: 12 | deploy: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v3 16 | - uses: actions/setup-python@v4 17 | with: 18 | python-version: 3.x 19 | - uses: actions/cache@v2 20 | with: 21 | key: ${{ github.ref }} 22 | path: .cache 23 | - run: pip install mkdocs-material 24 | - run: mkdocs gh-deploy -f ./docs/kmindex/mkdocs.yml --force 25 | 26 | 27 | -------------------------------------------------------------------------------- /.github/workflows/kmindex-osx.yml: -------------------------------------------------------------------------------- 1 | name: kmindex-osx 2 | 3 | on: 4 | push: 5 | paths-ignore: 6 | - '.github/workflows/doc.yml' 7 | - 'README.md' 8 | - 'docker/**' 9 | - 'conda/**' 10 | - 'scripts/**' 11 | - '.gitignore' 12 | - 'docs/kmindex/**' 13 | pull_request: 14 | paths-ignore: 15 | - '.github/workflows/doc.yml' 16 | - 'README.md' 17 | - 'docker/**' 18 | - 'conda/**' 19 | - 'scripts/**' 20 | - '.gitignore' 21 | - 'docs/kmindex/**' 22 | 23 | jobs: 24 | macOS: 25 | strategy: 26 | fail-fast: false 27 | matrix: 28 | distro: [macos-11,macos-12] 29 | build_type: [Release] 30 | 31 | runs-on: ${{ matrix.distro }} 32 | name: ${{ matrix.distro }} - ${{ matrix.build_type }} 33 | steps: 34 | - uses: actions/checkout@v3 35 | with: 36 | submodules: recursive 37 | 38 | - name: Dependencies 39 | run: | 40 | brew install cmake boost 41 | 42 | - name: Configure 43 | shell: bash 44 | run: | 45 | mkdir build 46 | cd build 47 | cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DWITH_TESTS=ON 48 | 49 | - name: Build 50 | shell: bash 51 | run: | 52 | cd build 53 | make 54 | 55 | - name: Test 56 | shell: bash 57 | run: | 58 | cd build 59 | ctest --verbose 60 | 61 | - name: Package 62 | if: matrix.distro == 'macos-11' && startsWith(github.event.ref, 'refs/tags/v') 63 | shell: bash 64 | run: cd build && make package 65 | 66 | - name: Get tag name 67 | if: matrix.distro == 'macos-11' && startsWith(github.event.ref, 'refs/tags/v') 68 | id: tag 69 | run: echo ::set-output name=TAG::${GITHUB_REF#refs/tags/} 70 | 71 | - name: Release archive 72 | if: matrix.distro == 'macos-11' && startsWith(github.event.ref, 'refs/tags/v') 73 | shell: bash 74 | run: | 75 | mkdir archive_files 76 | mv build/kmindex-${{ steps.tag.outputs.TAG }}-bin-Darwin.tar.gz archive_files 77 | 78 | - name: Release 79 | if: matrix.distro == 'macos-11' && startsWith(github.event.ref, 'refs/tags/v') 80 | uses: svenstaro/upload-release-action@v2 81 | with: 82 | release_name: Release ${{ steps.tag.outputs.TAG }} 83 | repo_token: ${{ secrets.GITHUB_TOKEN }} 84 | file: 'archive_files/*' 85 | file_glob: true 86 | tag: ${{ github.ref }} 87 | body: | 88 | kmindex v${{ steps.tags.outputs.TAG }} 89 | prerelease: true 90 | 91 | 92 | -------------------------------------------------------------------------------- /.github/workflows/kmindex.yml: -------------------------------------------------------------------------------- 1 | name: kmindex 2 | 3 | on: 4 | push: 5 | paths-ignore: 6 | - '.github/workflows/doc.yml' 7 | - 'README.md' 8 | - 'docker/**' 9 | - 'conda/**' 10 | - 'scripts/**' 11 | - '.gitignore' 12 | - 'docs/kmindex/**' 13 | pull_request: 14 | paths-ignore: 15 | - '.github/workflows/doc.yml' 16 | - 'README.md' 17 | - 'docker/**' 18 | - 'conda/**' 19 | - 'scripts/**' 20 | - '.gitignore' 21 | - 'docs/kmindex/**' 22 | 23 | jobs: 24 | Linux: 25 | runs-on: ubuntu-20.04 26 | strategy: 27 | fail-fast: false 28 | matrix: 29 | build_type: [Release, Debug] 30 | compiler: [g++-8, g++-9, g++-10] 31 | include: 32 | - compiler: g++-8 33 | cxx: g++-8 34 | cc: gcc-8 35 | - compiler: g++-9 36 | cxx: g++-9 37 | cc: gcc-9 38 | - compiler: g++-10 39 | cxx: g++-10 40 | cc: gcc-10 41 | 42 | name: Linux - ${{ matrix.build_type }} - ${{ matrix.compiler }} 43 | steps: 44 | - uses: actions/checkout@v3 45 | with: 46 | submodules: recursive 47 | 48 | - name: Dependencies 49 | run: | 50 | sudo apt-get install ${{ matrix.compiler }} 51 | sudo apt-get install zlib1g zlib1g-dev bzip2 libboost-all-dev 52 | 53 | - name: Configure 54 | shell: bash 55 | run: | 56 | mkdir build 57 | cd build 58 | export CC=$(which ${{ matrix.cc }}) 59 | export CXX=$(which ${{ matrix.cxx }}) 60 | cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DPORTABLE_BUILD=ON -DWITH_TESTS=ON -DFMT_HEADER_ONLY=ON -DSPDLOG_HEADER_ONLY=ON -DSTATIC_BUILD=ON 61 | 62 | - name: Build 63 | shell: bash 64 | run: | 65 | cd build 66 | export CC=$(which ${{ matrix.cc }}) 67 | export CXX=$(which ${{ matrix.cxx }}) 68 | make 69 | 70 | - name: Test 71 | shell: bash 72 | run: | 73 | cd build 74 | ctest --verbose 75 | 76 | - name: Package 77 | if: matrix.compiler == 'g++-10' && matrix.build_type == 'Release' && startsWith(github.event.ref, 'refs/tags/v') 78 | shell: bash 79 | run: cd build && make package 80 | 81 | - name: Get tag name 82 | if: matrix.compiler == 'g++-10' && matrix.build_type == 'Release' && startsWith(github.event.ref, 'refs/tags/v') 83 | id: tag 84 | run: echo ::set-output name=TAG::${GITHUB_REF#refs/tags/} 85 | 86 | - name: Release archive 87 | if: matrix.compiler == 'g++-10' && matrix.build_type == 'Release' && startsWith(github.event.ref, 'refs/tags/v') 88 | shell: bash 89 | run: | 90 | mkdir archive_files 91 | bash scripts/git-archive-all.sh --format tar.gz ./archive_files/kmindex-${{ steps.tag.outputs.TAG }}-sources.tar.gz 92 | mv build/kmindex-${{ steps.tag.outputs.TAG }}-bin-Linux.tar.gz archive_files 93 | 94 | - name: Release 95 | if: matrix.compiler == 'g++-10' && matrix.build_type == 'Release' && startsWith(github.event.ref, 'refs/tags/v') 96 | uses: svenstaro/upload-release-action@v2 97 | with: 98 | release_name: Release ${{ steps.tag.outputs.TAG }} 99 | repo_token: ${{ secrets.GITHUB_TOKEN }} 100 | file: 'archive_files/*' 101 | file_glob: true 102 | tag: ${{ github.ref }} 103 | body: | 104 | kmindex v${{ steps.tags.outputs.TAG }} 105 | prerelease: true 106 | 107 | 108 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Prerequisites 2 | *.d 3 | 4 | # Compiled Object files 5 | *.slo 6 | *.lo 7 | *.o 8 | *.obj 9 | 10 | # Precompiled Headers 11 | *.gch 12 | *.pch 13 | 14 | # Compiled Dynamic libraries 15 | *.so 16 | *.dylib 17 | *.dll 18 | 19 | # Fortran module files 20 | *.mod 21 | *.smod 22 | 23 | # Compiled Static libraries 24 | *.lai 25 | *.la 26 | *.a 27 | *.lib 28 | 29 | # Executables 30 | *.exe 31 | *.out 32 | *.app 33 | 34 | /tests/units/km_index 35 | tags 36 | /build/ 37 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "thirdparty/mio"] 2 | path = thirdparty/mio 3 | url = https://github.com/mandreyel/mio 4 | [submodule "thirdparty/kseqpp"] 5 | path = thirdparty/kseqpp 6 | url = https://github.com/cartoonist/kseqpp 7 | [submodule "thirdparty/bcli"] 8 | path = thirdparty/bcli 9 | url = https://github.com/tlemane/bcli 10 | [submodule "thirdparty/Simple-Web-Server"] 11 | path = thirdparty/Simple-Web-Server 12 | url = https://gitlab.com/eidheim/Simple-Web-Server 13 | [submodule "thirdparty/semver"] 14 | path = thirdparty/semver 15 | url = https://github.com/Neargye/semver 16 | [submodule "thirdparty/kmtricks"] 17 | path = thirdparty/kmtricks 18 | url = https://github.com/tlemane/kmtricks 19 | [submodule "thirdparty/atomic_queue"] 20 | path = thirdparty/atomic_queue 21 | url = https://github.com/max0x7ba/atomic_queue 22 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.24.0) 2 | project(kmindex VERSION 0.5.2 LANGUAGES C CXX) 3 | 4 | set(PROJECT_DESCRIPTION "kmindex") 5 | 6 | set(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake/modules) 7 | 8 | option(PORTABLE_BUILD "Generic x86_64 build" OFF) 9 | option(WITH_APP "Compile kmindex cli" ON) 10 | option(WITH_SERVER "Compile kmindex-server" ON) 11 | option(WITH_TESTS "Compile tests" OFF) 12 | option(STATIC_BUILD "Static build" OFF) 13 | 14 | if (NOT MAX_KMER_SIZE) 15 | set(MAX_KMER_SIZE 256) 16 | else() 17 | math(EXPR RESULT "(${MAX_KMER_SIZE} % 32)") 18 | if (${RESULT}) 19 | message(FATAL_ERROR "MAX_KMER_SIZE should be a multiple of 32.") 20 | endif() 21 | endif() 22 | 23 | add_compile_definitions(MAX_KMER_SIZE=${MAX_KMER_SIZE}) 24 | 25 | if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") 26 | set(FORCE_BUILD_FMT ON) 27 | set(FORCE_BUILD_SPDLOG ON) 28 | endif() 29 | 30 | if (NOT CMAKE_BUILD_TYPE) 31 | set(CMAKE_BUILD_TYPE="Release") 32 | endif() 33 | 34 | set(CMAKE_CXX_STANDARD 17) 35 | 36 | add_library(build_type_flags INTERFACE) 37 | target_compile_options(build_type_flags INTERFACE 38 | $<$:-O -g> 39 | $<$:-O3 -DNDEBUG> 40 | $<$:-O3 -ggdb3 -DNDEBUG -fno-inline> 41 | $<$:-O0 -g --coverage> 42 | $,-march=x86-64,-march=native> 43 | ) 44 | 45 | add_library(warning_flags INTERFACE) 46 | target_compile_options(warning_flags INTERFACE 47 | "-Wall" 48 | "-Wextra" 49 | "-pedantic" 50 | $<$: 51 | $<$:-Werror>> 52 | ) 53 | 54 | add_library(asan_flags INTERFACE) 55 | target_compile_options(asan_flags INTERFACE 56 | $<$:-fno-omit-frame-pointer -fsanitize=address> 57 | ) 58 | target_link_options(asan_flags INTERFACE 59 | $<$:-fno-omit-frame-pointer -fsanitize=address> 60 | ) 61 | target_link_libraries(asan_flags INTERFACE 62 | $<$:asan> 63 | ) 64 | 65 | add_subdirectory(thirdparty) 66 | add_subdirectory(lib) 67 | add_subdirectory(app) 68 | 69 | if (WITH_TESTS) 70 | enable_testing() 71 | add_subdirectory(tests) 72 | endif() 73 | 74 | include(CPackConfig) 75 | 76 | 77 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # kmindex 2 | 3 | [![License](http://img.shields.io/:license-affero-blue.svg)](http://www.gnu.org/licenses/agpl-3.0.en.html) 4 | [![kmindex](https://img.shields.io/github/actions/workflow/status/tlemane/kmindex/kmindex.yml?label=Linux)](https://github.com/tlemane/kmindex/actions/workflows/kmindex.yml) 5 | [![kmindex-osx](https://img.shields.io/github/actions/workflow/status/tlemane/kmindex/kmindex-osx.yml?label=macOS)](https://github.com/tlemane/kmindex/actions/workflows/kmindex-osx.yml) 6 | [![release](https://img.shields.io/github/v/release/tlemane/kmindex)](https://github.com/tlemane/kmindex/releases) 7 | [![dockerhub](https://img.shields.io/docker/v/tlemane/kmindex?label=tlemane/kmindex&logo=docker)](https://hub.docker.com/r/tlemane/kmindex/) 8 | [![anaconda](https://img.shields.io/conda/vn/tlemane/kmindex?color=green&label=tlemane%2Fkmindex&logo=anaconda)](https://anaconda.org/tlemane/kmindex) 9 | 10 | *kmindex* is a tool for indexing and querying sequencing samples. It is built on top of [kmtricks](https://github.com/tlemane/kmtricks). 11 | 12 | Given a databank $D = \{S_1, ..., S_n\}$, with each $S_i$ being any genomic dataset (genome or raw reads), *kmindex* allows to compute the percentage of shared k-mers between a query $Q$ and each $S \in D$. It supports multiple datasets and allows searching for each sub-index $D_i \in G = \{D_1,...,D_m\}$. Queries benefit from the [findere](https://github.com/lrobidou/findere) algorithm. In a few words, *findere* allows to reduce the false positive rate at query time by querying $(s+z)$-mers instead of $s$-mers, which are the indexed words, usually called $k$-mers. 13 | *kmindex* is a tool for querying sequencing samples indexed using [kmtricks](https://github.com/tlemane/kmtricks). 14 | 15 | 16 | 17 | 18 | **Indexing/Querying example** (can be tested in the `examples` directoy): 19 | 20 | 1. Index a dataset: 21 | ```bash 22 | kmindex build --fof fof1.txt --run-dir D1_index --index ./G --register-as D --hard-min 2 --kmer-size 25 --nb-cell 1000000 23 | ``` 24 | 25 | 2. Query the index: 26 | ```bash 27 | kmindex query --index ./G --fastx query.fasta --zvalue 3 28 | ``` 29 | 30 | **Full documentation** is available at [https://tlemane.github.io/kmindex](https://tlemane.github.io/kmindex) 31 | 32 | **Citation** Lemane, Téo, et al. "[Indexing and real-time user-friendly queries in terabyte-sized complex genomic datasets with kmindex and ORA](https://t.co/hH3GOY5YiP)" Nature Computational Science 4.2 (2024): 104-109. 33 | 34 | **Pre-print paper** is available on [bioRxiv](https://www.biorxiv.org/content/10.1101/2023.05.31.543043v3.abstract) 35 | -------------------------------------------------------------------------------- /app/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | include(CheckIPOSupported) 2 | check_ipo_supported(RESULT ipo_support OUTPUT error) 3 | 4 | if (ipo_support) 5 | message(STATUS "LTO enabled.") 6 | else() 7 | message(STATUS "LTO not supported: <${error}>.") 8 | endif() 9 | 10 | if (WITH_APP) 11 | add_subdirectory(kmindex) 12 | endif() 13 | 14 | if (WITH_SERVER) 15 | add_subdirectory(kmindex-server) 16 | endif() 17 | -------------------------------------------------------------------------------- /app/kmindex-server/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB_RECURSE KMINDEX_SERVER_SOURCES "*.cpp") 2 | 3 | if (STATIC_BUILD) 4 | set(Boost_USE_STATIC_LIBS ON) 5 | endif() 6 | 7 | find_package(Boost 1.70.0 REQUIRED COMPONENTS system) 8 | set(KMINDEX_SERVER "${PROJECT_NAME}-server") 9 | add_executable(${KMINDEX_SERVER} ${KMINDEX_SERVER_SOURCES}) 10 | 11 | target_compile_features(${KMINDEX_SERVER} PUBLIC cxx_std_17) 12 | target_link_libraries(${KMINDEX_SERVER} PUBLIC 13 | $ 14 | $ 15 | $ 16 | Boost::boost 17 | Boost::system 18 | bcli_lib 19 | kmindex-lib 20 | kseqpp 21 | zlib 22 | spdlog 23 | sws 24 | ) 25 | 26 | if (STATIC_BUILD) 27 | target_link_libraries(${KMINDEX_SERVER} PUBLIC 28 | -Wl,--whole-archive 29 | pthread 30 | -Wl,--no-whole-archive 31 | ) 32 | else() 33 | target_link_libraries(${KMINDEX_SERVER} PUBLIC pthread) 34 | endif() 35 | 36 | if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0.0) 37 | target_link_libraries(${KMINDEX_SERVER} PUBLIC stdc++fs) 38 | endif() 39 | 40 | if (ipo_supported) 41 | set_property(TARGET ${KMINDEX_SERVER} PROPERTY INTERPROCEDURAL_OPTIMIZATION TRUE) 42 | endif() 43 | 44 | if (STATIC_BUILD AND NOT ${CMAKE_BUILD_TYPE} MATCHES "Debug") 45 | target_link_options(${KMINDEX_SERVER} PRIVATE -static) 46 | endif() 47 | 48 | add_dependencies(${KMINDEX_SERVER} deps) 49 | 50 | install (TARGETS ${KMINDEX_SERVER} 51 | RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} 52 | COMPONENT ${KMINDEX_LIB}_Runtime 53 | ) 54 | 55 | -------------------------------------------------------------------------------- /app/kmindex-server/compress.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "compress.hpp" 5 | 6 | namespace kmq { 7 | 8 | // Source: https://panthema.net/2007/0328-ZLibString.html 9 | std::string compress_string(const std::string& str, 10 | int compressionlevel) 11 | { 12 | z_stream zs; // z_stream is zlib's control structure 13 | memset(&zs, 0, sizeof(zs)); 14 | 15 | if (deflateInit(&zs, compressionlevel) != Z_OK) 16 | throw(std::runtime_error("deflateInit failed while compressing.")); 17 | 18 | zs.next_in = (Bytef*)str.data(); 19 | zs.avail_in = str.size(); // set the z_stream's input 20 | 21 | int ret; 22 | char outbuffer[32768]; 23 | std::string outstring; 24 | 25 | // retrieve the compressed bytes blockwise 26 | do { 27 | zs.next_out = reinterpret_cast(outbuffer); 28 | zs.avail_out = sizeof(outbuffer); 29 | 30 | ret = deflate(&zs, Z_FINISH); 31 | 32 | if (outstring.size() < zs.total_out) { 33 | // append the block to the output string 34 | outstring.append(outbuffer, 35 | zs.total_out - outstring.size()); 36 | } 37 | } while (ret == Z_OK); 38 | 39 | deflateEnd(&zs); 40 | 41 | if (ret != Z_STREAM_END) { // an error occurred that was not EOF 42 | std::ostringstream oss; 43 | oss << "Exception during zlib compression: (" << ret << ") " << zs.msg; 44 | throw(std::runtime_error(oss.str())); 45 | } 46 | 47 | return outstring; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /app/kmindex-server/compress.hpp: -------------------------------------------------------------------------------- 1 | #ifndef COMPRESS_HPP_YEZUVILH 2 | #define COMPRESS_HPP_YEZUVILH 3 | 4 | #include 5 | #include 6 | 7 | namespace kmq { 8 | 9 | std::string compress_string(const std::string& s, int level = Z_BEST_COMPRESSION); 10 | 11 | } 12 | 13 | 14 | #endif /* end of include guard: COMPRESS_HPP_YEZUVILH */ 15 | -------------------------------------------------------------------------------- /app/kmindex-server/main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "server.hpp" 9 | 10 | int main(int argc, char* argv[]) 11 | { 12 | auto options = std::make_shared(); 13 | 14 | const std::string desc = 15 | "kmindex-server allows to perform queries via POST requests.\n\n" 16 | " The body is a json string with 4 entries:\n\n" 17 | " 'index': an array of strings corresponding to the indexes to query.\n\n" 18 | " 'id': a string used as query identifier.\n\n" 19 | " 'z': a integer which determine the k-mer size, (s+z)-mers. \n\n" 20 | " 'seq': an array of strings corresponding to the sequences to query, which\n" 21 | " are considered as a singe query.\n\n" 22 | " POST requests must be sent to /kmindex/query\n" 23 | " Index informations can be obtained via a GET request at /kmindex/infos\n\n" 24 | " Examples:\n\n" 25 | " curl -X POST http://127.0.0.1:8080/kmindex/query -H 'Content-type: application/json'\n" 26 | " -d '{\"index\":[\"index_1\"],\"seq\":[\"AGAGCCAGCAGCACCCCCAAAAAAAAA\"],\n" 27 | " \"id\":\"ID1\",\"z\":3}'\n\n" 28 | " curl -X GET http://127.0.0.1:8080/kmindex/infos"; 29 | 30 | auto cli_parser = std::make_shared>( 31 | "kmindex-server", desc, KMQ_PROJECT_TAG, ""); 32 | kmq::kmq_server_cli(cli_parser, options); 33 | 34 | try { 35 | cli_parser->parse(argc, argv); 36 | } 37 | catch (const bc::ex::BCliError& e) 38 | { 39 | bc::utils::exit_bcli(e); 40 | exit(EXIT_FAILURE); 41 | } 42 | 43 | if (options->verbosity == "debug") 44 | spdlog::set_level(spdlog::level::debug); 45 | else if (options->verbosity == "info") 46 | spdlog::set_level(spdlog::level::info); 47 | else if (options->verbosity == "warning") 48 | spdlog::set_level(spdlog::level::warn); 49 | else if (options->verbosity == "error") 50 | spdlog::set_level(spdlog::level::err); 51 | 52 | auto daily_sink = std::make_shared( 53 | fmt::format("{}/kmindex", options->log_directory), 0, 0); 54 | 55 | auto cerr_sink = std::make_shared(); 56 | auto dist_sink = std::make_shared(); 57 | 58 | if (!options->no_stderr) 59 | dist_sink->add_sink(cerr_sink); 60 | 61 | dist_sink->add_sink(daily_sink); 62 | 63 | spdlog::flush_every(std::chrono::seconds(5)); 64 | auto logger = std::make_shared("kmindex", dist_sink); 65 | logger->set_pattern("[kms:%t]-[%D %H:%M:%S] [%^%l%$] %v"); 66 | 67 | spdlog::set_default_logger(logger); 68 | 69 | try { 70 | kmq::main_server(options); 71 | } catch (const std::exception& e) { 72 | spdlog::error(e.what()); 73 | } 74 | 75 | return 0; 76 | } 77 | -------------------------------------------------------------------------------- /app/kmindex-server/server.hpp: -------------------------------------------------------------------------------- 1 | #ifndef SERVER_HPP_7KR2PWBZ 2 | #define SERVER_HPP_7KR2PWBZ 3 | 4 | #include 5 | #include 6 | 7 | namespace kmq { 8 | 9 | using parser_t = std::shared_ptr>; 10 | 11 | struct kmq_server_options 12 | { 13 | std::string verbosity; 14 | std::string index_path; 15 | std::size_t port; 16 | std::string address; 17 | std::string log_directory; 18 | std::size_t nb_threads; 19 | bool no_stderr {false}; 20 | }; 21 | 22 | using kmq_server_options_t = std::shared_ptr; 23 | 24 | kmq_server_options_t kmq_server_cli(parser_t parser, kmq_server_options_t options); 25 | 26 | void main_server(kmq_server_options_t opt); 27 | } 28 | 29 | #endif /* end of include guard: SERVER_HPP_7KR2PWBZ */ 30 | -------------------------------------------------------------------------------- /app/kmindex-server/utils.cpp: -------------------------------------------------------------------------------- 1 | #include "compress.hpp" 2 | #include "utils.hpp" 3 | 4 | namespace kmq { 5 | 6 | json json_error(const std::string& s) 7 | { 8 | json j; 9 | j["error"] = s; 10 | return j; 11 | } 12 | 13 | void send_response(response_t& response, const request_t& request, std::string&& msg) 14 | { 15 | SimpleWeb::CaseInsensitiveMultimap head; 16 | 17 | if (msg[0] == '{') 18 | { 19 | head = SimpleWeb::CaseInsensitiveMultimap( 20 | { { "Content-type", "application/json" } } ); 21 | } 22 | else 23 | { 24 | head = SimpleWeb::CaseInsensitiveMultimap( 25 | { { "Content-type", "text/csv" } } ); 26 | } 27 | 28 | auto ec = request->header.find("Accept-Encoding"); 29 | auto compress = 30 | ec != request->header.end() && ec->second.find("deflate") != std::string::npos; 31 | 32 | if (compress) 33 | { 34 | msg = compress_string(msg); 35 | head.insert(std::make_pair("Content-Encoding", "deflate")); 36 | head.insert(std::make_pair("Content-Length", std::to_string(msg.size()))); 37 | } 38 | 39 | response->write(SimpleWeb::StatusCode::success_ok, msg, head); 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /app/kmindex-server/utils.hpp: -------------------------------------------------------------------------------- 1 | #ifndef UTILS_HPP_OEH2LBRD 2 | #define UTILS_HPP_OEH2LBRD 3 | 4 | #include 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | using json = nlohmann::json; 11 | 12 | namespace kmq { 13 | 14 | struct string_iterator_wrapper 15 | { 16 | string_iterator_wrapper(const std::string& s) 17 | : beg(s.begin()), end(s.end()) {} 18 | std::string::const_iterator beg; 19 | std::string::const_iterator end; 20 | }; 21 | 22 | inline int wrapper_read(string_iterator_wrapper* it, char* buffer, unsigned int len) 23 | { 24 | auto& beg = it->beg; 25 | auto& end = it->end; 26 | 27 | std::size_t n = 0; 28 | 29 | if (beg >= end) 30 | return 0; 31 | 32 | for (std::size_t i = 0; i < len; ++i) 33 | { 34 | *buffer++ = *beg++; 35 | ++n; 36 | 37 | if (beg >= end) 38 | return n; 39 | } 40 | return n; 41 | } 42 | 43 | class SeqStreamString 44 | : public klibpp::KStreamIn 45 | { 46 | public: 47 | typedef klibpp::KStreamIn base_type; 48 | 49 | SeqStreamString(string_iterator_wrapper* itw) 50 | : base_type (itw, wrapper_read, nullptr) {} 51 | }; 52 | 53 | using http_server_t = SimpleWeb::Server; 54 | using response_t = std::shared_ptr; 55 | using request_t = std::shared_ptr; 56 | 57 | json json_error(const std::string& msg); 58 | 59 | void send_response(response_t& response, const request_t& request, std::string&& msg); 60 | } 61 | 62 | #endif /* end of include guard: UTILS_HPP_OEH2LBRD */ 63 | -------------------------------------------------------------------------------- /app/kmindex/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | file(GLOB_RECURSE KMINDEX_SOURCES "*.cpp") 2 | add_executable(${PROJECT_NAME} ${KMINDEX_SOURCES}) 3 | 4 | target_compile_features(${PROJECT_NAME} PUBLIC cxx_std_17) 5 | target_link_libraries(${PROJECT_NAME} PUBLIC 6 | $ 7 | $ 8 | $ 9 | bcli_lib 10 | kmindex-lib 11 | kseqpp 12 | zlib 13 | spdlog 14 | semver 15 | atomic_queue 16 | ) 17 | 18 | if (STATIC_BUILD) 19 | target_link_libraries(${PROJECT_NAME} PUBLIC 20 | -Wl,--whole-archive 21 | pthread 22 | -Wl,--no-whole-archive 23 | ) 24 | else() 25 | target_link_libraries(${PROJECT_NAME} PUBLIC pthread) 26 | endif() 27 | 28 | 29 | if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0.0) 30 | target_link_libraries(${PROJECT_NAME} PUBLIC stdc++fs) 31 | endif() 32 | 33 | if (ipo_supported) 34 | set_property(TARGET ${PROJECT_NAME} PROPERTY INTERPROCEDURAL_OPTIMIZATION TRUE) 35 | endif() 36 | 37 | if (STATIC_BUILD AND NOT ${CMAKE_BUILD_TYPE} MATCHES "Debug") 38 | target_link_options(${PROJECT_NAME} PRIVATE -static) 39 | endif() 40 | 41 | add_dependencies(${PROJECT_NAME} deps) 42 | 43 | install (TARGETS ${PROJECT_NAME} 44 | RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} 45 | COMPONENT ${KMINDEX_LIB}_Runtime 46 | ) 47 | 48 | -------------------------------------------------------------------------------- /app/kmindex/build.hpp: -------------------------------------------------------------------------------- 1 | #ifndef BUILD_HPP_4CDHAOMO 2 | #define BUILD_HPP_4CDHAOMO 3 | 4 | #include 5 | #include "common.hpp" 6 | 7 | namespace kmq { 8 | 9 | struct kmq_build_options : kmq_options 10 | { 11 | std::string fof; 12 | std::string directory; 13 | std::string from; 14 | std::string name; 15 | std::string km_path; 16 | 17 | std::size_t kmer_size {0}; 18 | std::size_t minim_size {0}; 19 | std::size_t hard_min {0}; 20 | std::size_t bloom_size {0}; 21 | std::size_t nb_cell {0}; 22 | std::size_t nb_partitions {0}; 23 | std::size_t bw {2}; 24 | bool no_check {false}; 25 | bool cpr {false}; 26 | }; 27 | 28 | using kmq_build_options_t = std::shared_ptr; 29 | 30 | kmq_options_t kmq_build_cli(parser_t parser, kmq_build_options_t options); 31 | 32 | void main_build(kmq_options_t opt); 33 | } 34 | 35 | #endif /* end of include guard: BUILD_HPP_4CDHAOMO */ 36 | -------------------------------------------------------------------------------- /app/kmindex/cli.cpp: -------------------------------------------------------------------------------- 1 | #include "cli.hpp" 2 | #include 3 | 4 | #include 5 | 6 | namespace kmq { 7 | 8 | kmq_cli::kmq_cli(const std::string& name, const std::string& desc, const std::string& version) 9 | { 10 | m_cli_parser = std::make_shared>(name, desc, version, std::string{}); 11 | m_kmq_register_opt = std::make_shared(kmq_register_options{}); 12 | m_kmq_query_opt = std::make_shared(kmq_query_options{}); 13 | m_kmq_build_opt = std::make_shared(kmq_build_options{}); 14 | m_kmq_infos_opt = std::make_shared(kmq_infos_options{}); 15 | m_kmq_merge_opt = std::make_shared(kmq_merge_options{}); 16 | 17 | kmq_build_cli(m_cli_parser, m_kmq_build_opt); 18 | kmq_register_cli(m_cli_parser, m_kmq_register_opt); 19 | kmq_query_cli(m_cli_parser, m_kmq_query_opt); 20 | kmq_merge_cli(m_cli_parser, m_kmq_merge_opt); 21 | kmq_infos_cli(m_cli_parser, m_kmq_infos_opt); 22 | } 23 | 24 | std::tuple kmq_cli::parse(int argc, char* argv[]) 25 | { 26 | if (argc > 1 && (std::string(argv[1]) == "--help" || std::string(argv[1]) == "-h")) 27 | { 28 | m_cli_parser->show_help(); 29 | exit(EXIT_SUCCESS); 30 | } 31 | else if (argc > 1 && (std::string(argv[1]) == "--version")) 32 | { 33 | std::cerr << fmt::format("kmindex {}", KMQ_PROJECT_TAG) << std::endl; 34 | exit(EXIT_SUCCESS); 35 | } 36 | 37 | m_cli_parser->parse(argc, argv); 38 | 39 | if (m_cli_parser->is("build")) 40 | return std::make_tuple(kmq_commands::kmq_build, m_kmq_build_opt); 41 | else if (m_cli_parser->is("register")) 42 | return std::make_tuple(kmq_commands::kmq_register, m_kmq_register_opt); 43 | else if (m_cli_parser->is("query")) 44 | return std::make_tuple(kmq_commands::kmq_query, m_kmq_query_opt); 45 | else if (m_cli_parser->is("index-infos")) 46 | return std::make_tuple(kmq_commands::kmq_index_infos, m_kmq_infos_opt); 47 | else if (m_cli_parser->is("merge")) 48 | return std::make_tuple(kmq_commands::kmq_merge, m_kmq_merge_opt); 49 | else 50 | exit(EXIT_FAILURE); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /app/kmindex/cli.hpp: -------------------------------------------------------------------------------- 1 | #ifndef CLI_HPP_RZKATMPJ 2 | #define CLI_HPP_RZKATMPJ 3 | 4 | #include "common.hpp" 5 | #include "register.hpp" 6 | #include "query.hpp" 7 | #include "build.hpp" 8 | #include "index_infos.hpp" 9 | #include "merge.hpp" 10 | 11 | namespace kmq { 12 | 13 | class kmq_cli 14 | { 15 | public: 16 | kmq_cli(const std::string& name, 17 | const std::string& desc, 18 | const std::string& version); 19 | 20 | std::tuple parse(int argc, char* argv[]); 21 | 22 | private: 23 | parser_t m_cli_parser {nullptr}; 24 | kmq_register_options_t m_kmq_register_opt {nullptr}; 25 | kmq_query_options_t m_kmq_query_opt {nullptr}; 26 | kmq_build_options_t m_kmq_build_opt {nullptr}; 27 | kmq_infos_options_t m_kmq_infos_opt {nullptr}; 28 | kmq_merge_options_t m_kmq_merge_opt {nullptr}; 29 | }; 30 | } 31 | 32 | #endif /* end of include guard: CLI_HPP_RZKATMPJ */ 33 | -------------------------------------------------------------------------------- /app/kmindex/common.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include "common.hpp" 3 | 4 | namespace kmq { 5 | 6 | void add_common_options(bc::cmd_t cmd, kmq_options_t options, bool with_threads, std::size_t n) 7 | { 8 | cmd->add_group("common", ""); 9 | 10 | if (with_threads) 11 | { 12 | cmd->add_param("-t/--threads", "Number of threads.") 13 | ->def(std::to_string(n)) 14 | ->meta("INT") 15 | ->setter(options->nb_threads) 16 | ->checker(bc::check::is_number); 17 | } 18 | 19 | cmd->add_param("-h/--help", "Show this message and exit.") 20 | ->as_flag() 21 | ->action(bc::Action::ShowHelp); 22 | cmd->add_param("--version", "Show version and exit.") 23 | ->as_flag() 24 | ->action(bc::Action::ShowVersion); 25 | cmd->add_param("-v/--verbose", "Verbosity level [debug|info|warning|error].") 26 | ->meta("STR") 27 | ->def("info") 28 | ->checker(bc::check::f::in("debug|info|warning|error")) 29 | ->setter(options->verbosity); 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /app/kmindex/common.hpp: -------------------------------------------------------------------------------- 1 | #ifndef COMMON_HPP_THWSML1X 2 | #define COMMON_HPP_THWSML1X 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include 10 | 11 | namespace kmq { 12 | 13 | enum class kmq_commands 14 | { 15 | kmq_build, 16 | kmq_query, 17 | kmq_register, 18 | kmq_merge, 19 | kmq_index_infos 20 | }; 21 | 22 | struct kmq_options 23 | { 24 | std::string verbosity; 25 | std::size_t nb_threads; 26 | std::string global_index_path; 27 | }; 28 | 29 | using kmq_options_t = std::shared_ptr; 30 | 31 | using parser_t = std::shared_ptr>; 32 | 33 | void add_common_options(bc::cmd_t cmd, 34 | kmq_options_t options, 35 | bool with_threads = false, 36 | std::size_t n = std::thread::hardware_concurrency()); 37 | 38 | } 39 | 40 | 41 | #endif /* end of include guard: COMMON_HPP_THWSML1X */ 42 | -------------------------------------------------------------------------------- /app/kmindex/index_infos.cpp: -------------------------------------------------------------------------------- 1 | #include "index_infos.hpp" 2 | 3 | #include 4 | #include 5 | 6 | #include 7 | #include 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | using json = nlohmann::json; 14 | 15 | namespace kmq { 16 | 17 | kmq_options_t kmq_infos_cli(parser_t parser, kmq_infos_options_t options) 18 | { 19 | auto cmd = parser->add_command("index-infos", "Print index informations."); 20 | 21 | cmd->add_param("-i/--index", "Global index path.") 22 | ->meta("STR") 23 | ->checker(bc::check::is_dir) 24 | ->setter(options->global_index_path); 25 | 26 | add_common_options(cmd, options, false); 27 | 28 | return options; 29 | } 30 | 31 | void main_infos(kmq_options_t opt) 32 | { 33 | kmq_infos_options_t o = std::static_pointer_cast(opt); 34 | 35 | auto mergeable = index(o->global_index_path).mergeable(); 36 | 37 | std::ifstream inf(fmt::format("{}/index.json", o->global_index_path), std::ios::in); 38 | check_fstream_good(o->global_index_path, inf); 39 | json data = json::parse(inf); 40 | 41 | for (auto& v : mergeable) 42 | spdlog::info("[{}] are mergeable.", fmt::join(v, ",")); 43 | 44 | std::cout << std::setw(4) << data << std::endl; 45 | } 46 | } 47 | 48 | -------------------------------------------------------------------------------- /app/kmindex/index_infos.hpp: -------------------------------------------------------------------------------- 1 | #ifndef INDEX_INFOS_HPP_15ZWXKGP 2 | #define INDEX_INFOS_HPP_15ZWXKGP 3 | 4 | #include 5 | #include "common.hpp" 6 | 7 | namespace kmq { 8 | 9 | struct kmq_infos_options : kmq_options 10 | { 11 | }; 12 | 13 | using kmq_infos_options_t = std::shared_ptr; 14 | 15 | kmq_options_t kmq_infos_cli(parser_t parser, kmq_infos_options_t options); 16 | 17 | void main_infos(kmq_options_t opt); 18 | } 19 | 20 | #endif /* end of include guard: INDEX_INFOS_HPP_15ZWXKGP */ 21 | -------------------------------------------------------------------------------- /app/kmindex/main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | #include 5 | 6 | #include "cli.hpp" 7 | 8 | int main(int argc, char* argv[]) 9 | { 10 | kmq::kmq_cli cli("kmindex", "Index/Query kmtricks bf/cbf matrices", KMQ_PROJECT_TAG); 11 | 12 | try { 13 | auto [cmd, options] = cli.parse(argc, argv); 14 | 15 | if (options->verbosity == "debug") 16 | spdlog::set_level(spdlog::level::debug); 17 | else if (options->verbosity == "info") 18 | spdlog::set_level(spdlog::level::info); 19 | else if (options->verbosity == "warning") 20 | spdlog::set_level(spdlog::level::warn); 21 | else if (options->verbosity == "error") 22 | spdlog::set_level(spdlog::level::err); 23 | 24 | auto cerr_logger = spdlog::stderr_color_mt("kmindex"); 25 | cerr_logger->set_pattern("[%C/%m/%d %H:%M:%S][%^%L%$:%t] %v"); 26 | spdlog::set_default_logger(cerr_logger); 27 | 28 | switch (cmd) 29 | { 30 | case kmq::kmq_commands::kmq_register: 31 | kmq::main_register(options); 32 | break; 33 | case kmq::kmq_commands::kmq_query: 34 | kmq::main_query(options); 35 | break; 36 | case kmq::kmq_commands::kmq_build: 37 | kmq::main_build(options); 38 | break; 39 | case kmq::kmq_commands::kmq_index_infos: 40 | kmq::main_infos(options); 41 | break; 42 | case kmq::kmq_commands::kmq_merge: 43 | kmq::main_merge(options); 44 | break; 45 | } 46 | 47 | } catch (const bc::ex::BCliError& e) { 48 | if (!e.get_name().empty()) 49 | spdlog::error("[{}] -> {}", e.get_name(), e.get_msg()); 50 | } catch (const std::exception& e) { 51 | spdlog::error(e.what()); 52 | exit(EXIT_FAILURE); 53 | } 54 | 55 | return 0; 56 | } 57 | -------------------------------------------------------------------------------- /app/kmindex/merge.hpp: -------------------------------------------------------------------------------- 1 | #ifndef MERGE_HPP_RTMU4VNU 2 | #define MERGE_HPP_RTMU4VNU 3 | 4 | #include 5 | #include 6 | 7 | #include 8 | 9 | #include "common.hpp" 10 | 11 | namespace kmq { 12 | 13 | struct kmq_merge_options : kmq_options 14 | { 15 | std::vector to_merge; 16 | std::string name; 17 | bool remove {false}; 18 | std::string new_path; 19 | 20 | std::string rename; 21 | rename_mode mode {rename_mode::none}; 22 | }; 23 | 24 | using kmq_merge_options_t = std::shared_ptr; 25 | 26 | kmq_options_t kmq_merge_cli(parser_t parser, kmq_merge_options_t options); 27 | 28 | void main_merge(kmq_options_t opt); 29 | } 30 | 31 | #endif /* end of include guard: MERGE_HPP_RTMU4VNU */ 32 | 33 | -------------------------------------------------------------------------------- /app/kmindex/query.hpp: -------------------------------------------------------------------------------- 1 | #ifndef QUERY_HPP_ZDULHSN1 2 | #define QUERY_HPP_ZDULHSN1 3 | 4 | #include 5 | #include 6 | #include "common.hpp" 7 | 8 | namespace kmq { 9 | 10 | struct kmq_query_options : kmq_options 11 | { 12 | std::string index; 13 | std::vector index_names; 14 | std::string output; 15 | std::string input; 16 | enum format format; 17 | std::string single; 18 | std::size_t z {0}; 19 | double sk_threshold {0}; 20 | std::size_t batch_size {0}; 21 | bool cache {false}; 22 | bool aggregate {false}; 23 | }; 24 | 25 | using kmq_query_options_t = std::shared_ptr; 26 | 27 | kmq_options_t kmq_query_cli(parser_t parser, kmq_query_options_t options); 28 | 29 | void main_query(kmq_options_t opt); 30 | } 31 | 32 | #endif /* end of include guard: QUERY_HPP_ZDULHSN1 */ 33 | -------------------------------------------------------------------------------- /app/kmindex/register.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "register.hpp" 4 | 5 | #include 6 | 7 | #include 8 | 9 | namespace kmq { 10 | 11 | kmq_options_t kmq_register_cli(parser_t parser, kmq_register_options_t options) 12 | { 13 | auto cmd = parser->add_command("register", "Register index."); 14 | 15 | cmd->add_param("-i/--global-index", "Global index path.") 16 | ->meta("STR") 17 | ->setter(options->global_index_path); 18 | 19 | cmd->add_param("-n/--name", "Index name.") 20 | ->meta("STR") 21 | ->setter(options->index_name); 22 | 23 | cmd->add_param("-p/--index-path", "Index path (a kmtricks run).") 24 | ->meta("STR") 25 | ->setter(options->index_path); 26 | 27 | add_common_options(cmd, options); 28 | 29 | return options; 30 | } 31 | 32 | void main_register(kmq_options_t opt) 33 | { 34 | kmq_register_options_t o = std::static_pointer_cast(opt); 35 | 36 | index i(o->global_index_path); 37 | i.add_index(o->index_name, o->index_path); 38 | i.save(); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /app/kmindex/register.hpp: -------------------------------------------------------------------------------- 1 | #ifndef REGISTER_HPP_X3UGAJR1 2 | #define REGISTER_HPP_X3UGAJR1 3 | 4 | #include "common.hpp" 5 | 6 | namespace kmq { 7 | 8 | struct kmq_register_options : kmq_options 9 | { 10 | std::string index_name; 11 | std::string index_path; 12 | }; 13 | 14 | using kmq_register_options_t = std::shared_ptr; 15 | 16 | kmq_options_t kmq_register_cli(parser_t parser, kmq_register_options_t); 17 | 18 | void main_register(kmq_options_t opt); 19 | } 20 | 21 | 22 | #endif /* end of include guard: REGISTER_HPP_X3UGAJR1 */ 23 | -------------------------------------------------------------------------------- /app/kmindex/utils.cpp: -------------------------------------------------------------------------------- 1 | #include "utils.hpp" 2 | 3 | #if __APPLE__ 4 | #include 5 | #include 6 | #endif 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | #include 13 | #include 14 | #include 15 | 16 | #include 17 | #include 18 | #include 19 | 20 | #include 21 | 22 | namespace kmq { 23 | 24 | namespace fs = std::filesystem; 25 | 26 | std::string get_binary_dir() 27 | { 28 | std::uint32_t size = 4096; 29 | char buffer[4096]; 30 | 31 | #if __APPLE__ 32 | _NSGetExecutablePath(buffer, &size); 33 | #else 34 | auto r = ::readlink("/proc/self/exe", buffer, size); 35 | if (r == -1) 36 | throw std::runtime_error("Unable to found kmindex binary path"); 37 | #endif 38 | return fs::path(buffer).parent_path(); 39 | } 40 | 41 | std::string cmd_exists(const std::string& dir, const std::string& cmd) 42 | { 43 | if (!std::system(fmt::format("which {} > /dev/null 2>&1", cmd).c_str())) 44 | return cmd; 45 | else if (!std::system(fmt::format("which {}/{} > /dev/null 2>&1", dir, cmd).c_str())) 46 | return fmt::format("{}/{}", dir, cmd); 47 | else 48 | throw std::runtime_error(fmt::format("{} not found.", cmd)); 49 | } 50 | 51 | int exec_cmd(const std::string& cmd, const std::string& args, 52 | const std::string& sout, const std::string& serr) 53 | { 54 | std::string bin_name = fs::path(cmd).filename().string(); 55 | std::vector argsv = bc::utils::split(args, ' '); 56 | pid_t pid; 57 | int p[2]; 58 | auto _i = pipe(p); unused(_i); 59 | int status; 60 | const char* cmd_name[] = {cmd.c_str()}; 61 | const char** argv = new const char*[argsv.size() + 2]; 62 | for (size_t i = 1; i < argsv.size() + 1; i++) 63 | { 64 | argv[i] = argsv[i - 1].c_str(); 65 | } 66 | 67 | spdlog::info(fmt::format("exec: {} {}", bin_name, args)); 68 | argv[0] = cmd_name[0]; 69 | argv[argsv.size() + 1] = NULL; 70 | 71 | pid = fork(); 72 | if (pid == 0) 73 | { 74 | int fd1 = -1, fd2 = -1; 75 | 76 | if (!sout.empty()) 77 | { 78 | fd1 = open(sout.c_str(), O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); 79 | dup2(fd1, STDOUT_FILENO); 80 | } 81 | 82 | if (!serr.empty()) 83 | { 84 | fd2 = open(serr.c_str(), O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); 85 | dup2(fd2, STDERR_FILENO); 86 | } 87 | 88 | if (bool failed = execvp(cmd_name[0], (char**)argv) == -1; failed) 89 | { 90 | delete[] argv; 91 | auto _w = write(p[1], &failed, sizeof(failed)); unused(_w); 92 | throw std::runtime_error(fmt::format("Failed to run {}.", cmd_name[0])); 93 | } 94 | 95 | close(fd1); 96 | close(fd2); 97 | 98 | delete[] argv; 99 | } 100 | else if (pid > 0) 101 | { 102 | if (wait(&status) == -1) 103 | { 104 | throw std::runtime_error("wait() failed."); 105 | } 106 | fcntl(p[0], F_SETFL, fcntl(p[0], F_GETFL) | O_NONBLOCK); 107 | bool failed = false; 108 | ssize_t r = read(p[0], &failed, sizeof(failed)); unused(r); 109 | 110 | if (WIFEXITED(status)) 111 | if (WEXITSTATUS(status) != 0) 112 | throw std::runtime_error(fmt::format("{} exit with {}.", cmd_name[0], WEXITSTATUS(status))); 113 | 114 | if (!failed) 115 | { 116 | spdlog::info("{} exit normally with ({}).", bin_name, WEXITSTATUS(status)); 117 | return 0; 118 | } 119 | else 120 | return 1; 121 | } 122 | else 123 | { 124 | throw std::runtime_error("fork() failed."); 125 | } 126 | return 1; 127 | } 128 | 129 | } 130 | -------------------------------------------------------------------------------- /app/kmindex/utils.hpp: -------------------------------------------------------------------------------- 1 | #ifndef UTILS_HPP_NLUODCAF 2 | #define UTILS_HPP_NLUODCAF 3 | 4 | #include 5 | 6 | namespace kmq { 7 | 8 | std::string get_binary_dir(); 9 | 10 | std::string cmd_exists(const std::string& dir, const std::string& cmd); 11 | 12 | int exec_cmd(const std::string& cmd, const std::string& args, 13 | const std::string& sout = "", const std::string& serr = ""); 14 | } 15 | 16 | #endif /* end of include guard: UTILS_HPP_NLUODCAF */ 17 | -------------------------------------------------------------------------------- /cmake/modules/CPackConfig.cmake: -------------------------------------------------------------------------------- 1 | SET (CPACK_PACKAGE_DESCRIPTION_SUMMARY ${PROJECT_NAME}) 2 | SET (CPACK_PACKAGE_VERSION_MAJOR "${CMAKE_PROJECT_VERSION_MAJOR}") 3 | SET (CPACK_PACKAGE_VERSION_MINOR "${CMAKE_PROJECT_VERSION_MINOR}") 4 | SET (CPACK_PACKAGE_VERSION_PATCH "${CMAKE_PROJECT_VERSION_PATCH}") 5 | SET (CPACK_PACKAGE_VERSION "${CMAKE_PROJECT_VERSION}") 6 | SET (CPACK_PACKAGE_FILE_NAME "${CMAKE_PROJECT_NAME}-v${CPACK_PACKAGE_VERSION}-bin-${CMAKE_SYSTEM_NAME}") 7 | SET (CPACK_GENERATOR "TGZ") 8 | SET (CPACK_SOURCE_GENERATOR "TGZ") 9 | SET (CPACK_SET_DESTDIR true) 10 | SET (CPACK_INSTALL_PREFIX /) 11 | 12 | INSTALL (TARGETS ${PROJECT_NAME} 13 | DESTINATION bin 14 | ) 15 | 16 | INSTALL (TARGETS ${KMINDEX_SERVER} 17 | DESTINATION bin 18 | ) 19 | 20 | include (CPack) 21 | -------------------------------------------------------------------------------- /cmake/modules/FindXXHASH.cmake: -------------------------------------------------------------------------------- 1 | include(FindPackageHandleStandardArgs) 2 | 3 | find_path(XXHASH_INCLUDE_DIR NAMES xxhash.h) 4 | find_library(XXHASH_LIBRARY NAMES xxhash) 5 | 6 | if (XXHASH_LIBRARY) 7 | file(READ "${XXHASH_INCLUDE_DIR}/xxhash.h" XXHASH_FILE) 8 | 9 | string(REGEX MATCH "XXH_VERSION_MAJOR *([0-9]*)" _ ${XXHASH_FILE}) 10 | set(XXHASH_VERSION_MAJOR ${CMAKE_MATCH_1}) 11 | string(REGEX MATCH "XXH_VERSION_MINOR *([0-9]*)" _ ${XXHASH_FILE}) 12 | set(XXHASH_VERSION_MINOR ${CMAKE_MATCH_1}) 13 | string(REGEX MATCH "XXH_VERSION_RELEASE *([0-9]*)" _ ${XXHASH_FILE}) 14 | set(XXHASH_VERSION_PATCH ${CMAKE_MATCH_1}) 15 | 16 | set(XXHASH_VERSION "${XXHASH_VERSION_MAJOR}.${XXHASH_VERSION_MINOR}.${XXHASH_VERSION_PATCH}") 17 | 18 | find_package_check_version(${XXHASH_VERSION} XXHASH_VERSION_OK 19 | HANDLE_VERSION_RANGE RESULT_MESSAGE_VARIABLE XXHASH_FAILED) 20 | 21 | include(CheckCSourceRuns) 22 | set(CMAKE_REQUIRED_INCLUDES ${XXHASH_INCLUDE_DIR}) 23 | set(CMAKE_REQUIRED_LIBRARIES ${XXHASH_LIBRARY}) 24 | check_c_source_runs(" 25 | #include 26 | int main() 27 | { 28 | return 0; 29 | } 30 | " XXHASH_COMPILE_OK) 31 | set(CMAKE_REQUIRED_INCLUDES) 32 | set(CMAKE_REQUIRED_LIBRARIES) 33 | endif() 34 | 35 | find_package_handle_standard_args( 36 | XXHASH 37 | REQUIRED_VARS XXHASH_LIBRARY XXHASH_INCLUDE_DIR XXHASH_VERSION_OK XXHASH_COMPILE_OK 38 | VERSION_VAR XXHASH_VERSION) 39 | 40 | mark_as_advanced( 41 | XXHASH_INCLUDE_DIR 42 | XXHASH_LIBRARY 43 | XXHASH_VERSION 44 | ) 45 | 46 | -------------------------------------------------------------------------------- /conda/kmindex/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | mkdir -p $PREFIX 4 | 5 | bash install.sh -n -p $PREFIX 6 | -------------------------------------------------------------------------------- /conda/kmindex/conda_build_config.yaml: -------------------------------------------------------------------------------- 1 | c_compiler_version: 2 | - 11.2.0 # [linux] 3 | - 11.1.0 # [osx] 4 | 5 | cxx_compiler_version: 6 | - 11.2.0 # [linux] 7 | - 11.1.0 # [osx] 8 | -------------------------------------------------------------------------------- /conda/kmindex/meta.yaml: -------------------------------------------------------------------------------- 1 | {% set version = "0.5.2" %} 2 | 3 | package: 4 | name: kmindex 5 | version: {{ version }} 6 | 7 | source: 8 | git_url: https://github.com/tlemane/kmindex 9 | git_rev: v{{ version }} 10 | 11 | requirements: 12 | build: 13 | - {{ compiler('cxx' )}} 14 | - make 15 | - cmake 16 | 17 | host: 18 | - zlib 19 | - xxhash 20 | - spdlog =1.10.0 21 | - fmt =8.1.1 22 | - boost-cpp 23 | - bzip2 24 | 25 | run: 26 | - kmtricks 27 | 28 | about: 29 | home: https://github.com/tlemane/kmindex 30 | license: AGPL-3.0; 31 | 32 | extra: 33 | recipe-maintainers: 34 | - tlemane 35 | 36 | 37 | -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:jammy 2 | 3 | ENV DEBIAN_FRONTEND=noninteractive 4 | 5 | ENV KMINDEX_VERSION v0.5.2 6 | 7 | ENV BUILD_INPUTS git cmake gcc g++ zlib1g zlib1g-dev libboost-all-dev make \ 8 | libspdlog-dev libspdlog1 bzip2 lbzip2 xxhash ca-certificates libbz2-dev 9 | 10 | WORKDIR /home 11 | 12 | RUN apt-get update \ 13 | && apt-get -y dist-upgrade \ 14 | && apt-get install -y --no-install-recommends wget ${BUILD_INPUTS} \ 15 | && apt-get clean 16 | 17 | RUN wget https://github.com/Kitware/CMake/releases/download/v3.26.4/cmake-3.26.4-linux-x86_64.tar.gz \ 18 | && tar -xzvf cmake-3.26.4-linux-x86_64.tar.gz 19 | 20 | RUN cd /home \ 21 | && mkdir kmindex \ 22 | && cd kmindex \ 23 | && wget https://github.com/tlemane/kmindex/releases/download/${KMINDEX_VERSION}/kmindex-${KMINDEX_VERSION}-sources.tar.gz \ 24 | && tar -xzvf kmindex-${KMINDEX_VERSION}-sources.tar.gz 25 | 26 | RUN cd /home \ 27 | && cd kmindex \ 28 | && rm -rf build \ 29 | && mkdir -p build \ 30 | && cd build \ 31 | && /home/cmake-3.26.4-linux-x86_64/bin/cmake .. -DPORTABLE_BUILD=ON \ 32 | && make -j8 && make install -p /bin 33 | 34 | RUN rm -rf /home/kmindex /home/cmake-3.26.4-linux-x86_64* 35 | 36 | 37 | WORKDIR /tmp 38 | 39 | ENTRYPOINT ["kmindex"] 40 | 41 | LABEL version="0.5.2" 42 | LABEL description="kmindex" 43 | LABEL author="Teo Lemane" 44 | 45 | -------------------------------------------------------------------------------- /docker/README.md: -------------------------------------------------------------------------------- 1 | # Build image 2 | 3 | ```bash 4 | git clone https://github.com/tlemane/kmindex.git 5 | cd kmindex/docker 6 | docker build -t kmindex-d . 7 | ``` 8 | 9 | # Run 10 | 11 | ```bash 12 | docker run --rm kmindex-d 13 | ``` 14 | 15 | The default entrypoint corresponds to `kmindex`. To run `kmindex-server`, use: 16 | 17 | ```bash 18 | docker run --rm --entrypoint /result/bin/kmindex-server kmindex-d 19 | ``` 20 | -------------------------------------------------------------------------------- /docs/changelogs/v0.2.0.md: -------------------------------------------------------------------------------- 1 | # kmindex v0.2.0 2 | 3 | ### Add 4 | 5 | * New commands: 6 | * kmindex build 7 | * kmindex index-infos 8 | * Abundance support 9 | 10 | ### Update 11 | 12 | * Faster queries 13 | 14 | See [documentation](https://tlemane.github.io/kmindex). 15 | -------------------------------------------------------------------------------- /docs/changelogs/v0.3.0.md: -------------------------------------------------------------------------------- 1 | # kmindex v0.3.0 2 | 3 | * `kmindex build`: add --cpr if kmtricks >= v1.4.0 4 | * `kmindex query`: add --fast 5 | * `kmindex server`: add fastx support 6 | * Various fixes 7 | 8 | -------------------------------------------------------------------------------- /docs/changelogs/v0.4.0.md: -------------------------------------------------------------------------------- 1 | # kmindex v0.4.0 2 | 3 | * `kmindex query`: add --format json_vec 4 | * `kmindex merge`: merge p/a and abs indexes 5 | 6 | -------------------------------------------------------------------------------- /docs/changelogs/v0.5.0.md: -------------------------------------------------------------------------------- 1 | # kmindex v0.5.0 2 | 3 | * Support for any k-mer size 4 | 5 | -------------------------------------------------------------------------------- /docs/changelogs/v0.5.1.md: -------------------------------------------------------------------------------- 1 | # kmindex v0.5.1 2 | 3 | * Fix sha1 in index_infos 4 | 5 | -------------------------------------------------------------------------------- /docs/changelogs/v0.5.2.md: -------------------------------------------------------------------------------- 1 | # kmindex v0.5.2 2 | 3 | * macOS support 4 | 5 | -------------------------------------------------------------------------------- /docs/kmindex/docs/index.md: -------------------------------------------------------------------------------- 1 | # Welcome to kmindex documentation 2 | 3 | [![License](http://img.shields.io/:license-affero-blue.svg)](http://www.gnu.org/licenses/agpl-3.0.en.html) 4 | [![kmindex](https://img.shields.io/github/actions/workflow/status/tlemane/kmindex/kmindex.yml?label=Linux)](https://github.com/tlemane/kmindex/actions/workflows/kmindex.yml) 5 | [![kmindex-osx](https://img.shields.io/github/actions/workflow/status/tlemane/kmindex/kmindex-osx.yml?label=macOS)](https://github.com/tlemane/kmindex/actions/workflows/kmindex-osx.yml) 6 | [![release](https://img.shields.io/github/v/release/tlemane/kmindex)](https://github.com/tlemane/kmindex/releases) 7 | [![dockerhub](https://img.shields.io/docker/v/tlemane/kmindex?label=tlemane/kmindex&logo=docker)](https://hub.docker.com/r/tlemane/kmindex/) 8 | [![anaconda](https://img.shields.io/conda/vn/tlemane/kmindex?color=green&label=tlemane%2Fkmindex&logo=anaconda)](https://anaconda.org/tlemane/kmindex) 9 | 10 | *kmindex* is a tool for indexing and querying sequencing samples. It is built on top of [kmtricks](https://github.com/tlemane/kmtricks). 11 | 12 | Given a databank $D = \{S_1, ..., S_n\}$, with each $S_i$ being any genomic dataset (genome or raw reads), *kmindex* allows to compute the percentage of shared k-mers between a query $Q$ and each $S \in D$. It supports multiple datasets and allows searching for each sub-index $D_i \in G = \{D_1,...,D_m\}$. Queries benefit from the [findere](https://github.com/lrobidou/findere) algorithm. In a few words, *findere* allows to reduce the false positive rate at query time by querying $(s+z)$-mers instead of $s$-mers, which are the indexed words, usually called $k$-mers. 13 | 14 | ## User guide 15 | 16 | * [Installation](installation.md) 17 | * [Index construction](construction.md) 18 | * [Index query](query.md) 19 | * [Index merge](merge.md) 20 | 21 | ## Server 22 | 23 | * [Deploy](server-deploy.md) 24 | * [Query](server-query.md) 25 | 26 | 27 | !!! info "Contact" 28 | Teo Lemane: teo[dot]lemane[at]genoscope[dot]cns[dot]fr 29 | 30 | -------------------------------------------------------------------------------- /docs/kmindex/docs/installation.md: -------------------------------------------------------------------------------- 1 | # kmindex installation 2 | 3 | !!! note 4 | **kmindex** supports GNU/Linux systems, and macOS since v0.5.2. 5 | 6 | ## **Conda** [![anaconda](https://img.shields.io/conda/vn/tlemane/kmindex?color=green&label=tlemane%2Fkmindex&logo=anaconda)](https://anaconda.org/tlemane/kmindex) ![anaconda-platform](https://img.shields.io/conda/pn/tlemane/kmindex?logo=anaconda) 7 | 8 | ```bash 9 | conda create -p kmindex_env 10 | conda activate ./kmindex_env 11 | conda install -c conda-forge -c tlemane kmindex 12 | ``` 13 | 14 | Note that the conda package includes [kmtricks](https://github.com/tlemane/kmtricks). 15 | 16 | ## **Docker** [![dockerhub](https://img.shields.io/docker/v/tlemane/kmindex?label=tlemane/kmindex&logo=docker)](https://hub.docker.com/r/tlemane/kmindex/) 17 | 18 | ```bash 19 | docker pull tlemane/kmindex 20 | ``` 21 | 22 | ## **Nix** 23 | 24 | ```bash 25 | nix --extra-experimental-features 'nix-command flakes' build github:tlemane/kmindex 26 | ``` 27 | 28 | ## **Release** [![release](https://img.shields.io/github/v/release/tlemane/kmindex)](https://github.com/tlemane/kmindex/releases) 29 | 30 | Portable binary releases are available at [kmindex/releases](https://github.com/tlemane/kmindex/releases). 31 | 32 | [:simple-linux:](https://github.com/tlemane/kmindex/releases/latest/download/kmindex-v0.5.2-bin-Linux.tar.gz){ .md-button .md-button--primary } [:simple-apple:](https://github.com/tlemane/kmindex/releases/latest/download/kmindex-v0.5.2-bin-Darwin.tar.gz){ .md-button .md-button--primary } 33 | 34 | 35 | ## **Install from sources** 36 | 37 | !!! info "**Requirements**" 38 | * cmake >= 3.13 39 | * gcc >= 8.1 40 | * Boost.Asio and Boost.System >= 1.70 41 | * bzip2 42 | 43 | All other dependencies are bundled with `kmindex`. 44 | 45 | **Installation** 46 | ``` bash 47 | git clone --recursive https://github.com/tlemane/kmindex # (1)! 48 | cd kmindex && ./install.sh # (2)! 49 | ``` 50 | 51 | 1. Cloning the repository 52 | 2. Build kmindex using `install.sh` 53 | 54 | !!! tip "`install.sh`" 55 | ``` 56 | kmindex build script - v0.5.0. 57 | Usage: 58 | ./install.sh [-r str] [-t int] [-j int] [-p str] [-k int] [-n] [-h] 59 | Options: 60 | -r -> build type {Release}. 61 | -t <0|1|2> -> tests: 0 = disabled, 1 = compile, 2 = compile and run {0}. 62 | -j -> nb threads {8}. 63 | -k -> Max k-mer size (should be a multiple of 32) {256} 64 | -n -> portable x86-64 build {disabled}. 65 | -p -> install path {./kmindex_install} 66 | -h -> show help. 67 | ``` 68 | -------------------------------------------------------------------------------- /docs/kmindex/docs/merge.md: -------------------------------------------------------------------------------- 1 | # Merge 2 | 3 | In addition to registering multiple indexes, compatible sub-indexes (i.e. sharing the parameters: k-mer size, Bloom filter size, number of partitions) can also be merged. This way, the indexes are physically extended resulting in optimal query times. 4 | 5 | !!! tip "Options" 6 | ``` 7 | kmindex merge v0.5.2 8 | 9 | DESCRIPTION 10 | Merge sub-indexes. 11 | 12 | USAGE 13 | kmindex merge -i/--index -n/--new-name -p/--new-path -m/--to-merge 14 | [-r/--rename ] [-t/--threads ] [-v/--verbose ] 15 | [-d/--delete-old] [-h/--help] [--version] 16 | 17 | OPTIONS 18 | [global] 19 | -i --index - Global index path. 20 | -n --new-name - Name of the new index. 21 | -p --new-path - Output path. 22 | -m --to-merge - Sub-indexes to merge, comma separated. 23 | -d --delete-old - Delete old sub-index files. [⚑] 24 | -r --rename - Rename sample ids. 25 | - A sub-index cannot contain samples with similar identifiers. 26 | Sub-indexes containing identical identifiers cannot be merged, the 27 | identifiers must be renamed. 28 | - Renaming can be done in three different ways: 29 | 30 | 1. Using identifier files (one per line). 31 | For example, if you want to merge three sub-indexes: 32 | '-r f:id1.txt,id2.txt,id3.txt' 33 | 34 | 2. Using a format string ('{}' is replaced by an integer in [0, nb_samples)). 35 | '-r "s:id_{}"' 36 | 37 | 3. Manually (not recommended). 38 | Identifiers can be changed in 'kmtricks.fof' files in sub-index directories. 39 | 40 | [common] 41 | -t --threads - Number of threads. {12} 42 | -h --help - Show this message and exit. [⚑] 43 | --version - Show version and exit. [⚑] 44 | -v --verbose - Verbosity level [debug|info|warning|error]. {info} 45 | ``` 46 | 47 | -------------------------------------------------------------------------------- /docs/kmindex/docs/python-api.md: -------------------------------------------------------------------------------- 1 | # Python API 2 | 3 | The python API allows to query a running kmindex server. In the following, we assume that a server is running at 127.0.0.1 on port 8080. 4 | 5 | ## **Installation** 6 | 7 | !!! info "Requirements" 8 | * python >= 3.10 9 | * pip (with [PEP-517 support](https://peps.python.org/pep-0517/) if installing from sources) 10 | 11 | ### Using pip [![pip](https://img.shields.io/pypi/v/pykmindex?logo=pypi)]() 12 | 13 | ```bash 14 | pip install pykmindex 15 | ``` 16 | 17 | ### From sources 18 | 19 | ```bash 20 | pip install git+https://github.com/tlemane/kmindex.git#subdirectory=pykmindex 21 | ``` 22 | 23 | ## **Usage** 24 | 25 | ### **Connection** 26 | 27 | ```py 28 | from pykmindex.server import connect 29 | 30 | km_serv = connect("127.0.0.1", 8080) 31 | 32 | infos = km_serv.infos() 33 | ``` 34 | 35 | ### **Query** 36 | 37 | #### **Synchronous** 38 | 39 | ```py 40 | from Bio import SeqIO 41 | 42 | from pykmindex.server import connect 43 | from pykmindex.query import Query 44 | 45 | km_serv = connect("127.0.0.1", 8080) 46 | 47 | for record in SeqIO.parse("query.fa", "fasta"): 48 | res = km_serv.submit(Query(record.id, 49 | record.seq, # Can be a list of sequences 50 | "index_id", # Can be a list of sub-indexes to query 51 | 0.0, # Min ratio to report a result (optional, default 0.0) 52 | 3)) # z value (optional, default 3) 53 | if res: 54 | print(res.id, res.result) 55 | else: 56 | print(res.err) 57 | ``` 58 | 59 | #### **Asynchronous** 60 | 61 | !!! note "Using QBatch" 62 | ```py 63 | from Bio import SeqIO 64 | from pykmindex.server import connect 65 | from pykmindex.query import QBatch 66 | 67 | km_serv = connect("127.0.0.1", 8080) 68 | 69 | batch = QBatch() 70 | 71 | for record in SeqIO.parse("query.fa", "fasta"): 72 | batch.add(record.id, record.seq, "index_id") 73 | 74 | responses = km_serv.submit_async(batch) 75 | 76 | for r in responses: 77 | if r: 78 | print(r.id, r.result) 79 | else: 80 | print(r.err) 81 | ``` 82 | 83 | !!! note "Using your own event loop" 84 | ```py 85 | import asyncio 86 | from Bio import SeqIO 87 | from pykmindex.server import connect 88 | from pykmindex.query import Query 89 | 90 | km_serv = connect("127.0.0.1", 8080) 91 | 92 | futures = [] 93 | for record in SeqIO.parse("query.fa", "fasta"): 94 | futures.append(km_serv.async_query(Query(record.id, record.seq, "index_id"))) 95 | 96 | loop = asyncio.new_event_loop() 97 | asyncio.set_event_loop(loop) 98 | responses = loop.run_until_complete(asyncio.gather(*futures)) 99 | 100 | for r in responses: 101 | if r: 102 | print(r.id, r.result) 103 | else: 104 | print(r.err) 105 | ``` 106 | 107 | -------------------------------------------------------------------------------- /docs/kmindex/docs/server-deploy.md: -------------------------------------------------------------------------------- 1 | # Server deploy 2 | 3 | ## kmindex-server 4 | 5 | For easy integration, *kmindex* is also composed of a server supporting http requests. 6 | The server can be started using the *kmindex-server* binary, see options below. 7 | 8 | !!! note "Options" 9 | 10 | ``` 11 | kmindex-server v0.5.2 12 | 13 | DESCRIPTION 14 | kmindex-server allows to perform queries via POST requests. 15 | 16 | USAGE 17 | kmindex-server -i/--index [-a/--address ] [-p/--port ] [-d/--log-directory ] 18 | [-t/--threads ] [--verbose ] [-s/--no-stderr] [-h/--help] 19 | [--version] 20 | 21 | OPTIONS 22 | [global] - global parameters 23 | -i --index - Index path. 24 | -a --address - Address to use (empty string to bind any address) 25 | IPv4: dotted decimal form 26 | IPv6: hexadimal form. 27 | default -> {127.0.0.1} 28 | -p --port - Port to use. {8080} 29 | -d --log-directory - Directory for daily logging. {kmindex_logs} 30 | -s --no-stderr - Disable stderr logging. [⚑] 31 | 32 | [common] 33 | -t --threads - Max number of parallel connections. {1} 34 | -h --help - Show this message and exit. [⚑] 35 | --version - Show version and exit. [⚑] 36 | --verbose - Verbosity level [debug|info|warning|error]. {info} 37 | ``` 38 | 39 | 40 | -------------------------------------------------------------------------------- /docs/kmindex/docs/server-query.md: -------------------------------------------------------------------------------- 1 | # Server query 2 | 3 | In the following, we assume that a *kmindex-server* is running at 127.0.0.1 on port 8080. 4 | 5 | 6 | | Method | Description | Path | 7 | | ------ | ----------- | --- | 8 | | `GET` | Index informations | /kmindex/infos | 9 | | `POST` | Index query | /kmindex/query | 10 | 11 | ## **Accessing index information** 12 | 13 | **Request** 14 | ```bash 15 | curl -X GET http://127.0.0.1:8080/kmindex/infos 16 | ``` 17 | 18 | **Response** 19 | ```json 20 | { 21 | "index": { 22 | "D1": { 23 | "bloom_size": 10000128, 24 | "index_size": 9, 25 | "minim_size": 10, 26 | "nb_partitions": 4, 27 | "nb_samples": 2, 28 | "samples": [ 29 | "S1", 30 | "S2" 31 | ], 32 | "smer_size": 25 33 | }, 34 | "D2": { 35 | "bloom_size": 10000128, 36 | "index_size": 9, 37 | "minim_size": 10, 38 | "nb_partitions": 4, 39 | "nb_samples": 2, 40 | "samples": [ 41 | "S3", 42 | "S4" 43 | ], 44 | "smer_size": 25 45 | } 46 | } 47 | ``` 48 | 49 | ## Query index 50 | 51 | !!! note "json body" 52 | * `index`: An array of strings corresponding to the indexes to query. 53 | * `id`: A string used as query identifier. 54 | * `seq`: An array of strings corresponding to the sequences to query. All sequences are considered as a unique query (same as `kmindex query --single-query `). 55 | * `fastx`: A string containing sequences in fasta/q format. 56 | * `format`: A string in ["json", "tsv"] corresponding to the output format. 57 | * `z`: The z parameter, see [z help](query.md#about-the-z-parameter) (default: 0). 58 | * `r`: A float in $[0,1]$, all ratios greater than `r` are reported (default: 0). 59 | 60 | `seq` and `fastx` are mutually exclusive. 61 | 62 | **Request** 63 | 64 | ```bash 65 | curl -X POST http://127.0.0.1:8080/kmindex/query \ 66 | -H 'Content-type: application/json' \ 67 | -d '{"index":["D1"],"seq":["ACGACGACGACGAGACGAGACGACAGCAGACAGAGACATAATATACT"], "id":"ID","z":3}' # (1)!. 68 | ``` 69 | 70 | 1. To avoid curl to change POST requests to GET on redirection, use `--post302`. 71 | 72 | **Response** 73 | 74 | ```json 75 | { 76 | "D1": { 77 | "ID": { 78 | "S1": 1.0, 79 | "S2": 0.0 80 | }, 81 | } 82 | } 83 | ``` 84 | 85 | 86 | -------------------------------------------------------------------------------- /docs/kmindex/docs/stylesheets/extra.css: -------------------------------------------------------------------------------- 1 | .md-grid { 2 | max-width: 100%; /* or 100%, if you want to stretch to full-width */ 3 | } 4 | -------------------------------------------------------------------------------- /docs/kmindex/mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: kmindex documentation 2 | 3 | nav: 4 | - Home: 'index.md' 5 | - 'User Guide': 6 | - 'Installation': 'installation.md' 7 | - 'Index construction': 'construction.md' 8 | - 'Index query': 'query.md' 9 | - 'Index merge': 'merge.md' 10 | - 'Server': 11 | - 'Deploy': 'server-deploy.md' 12 | - 'Query': 'server-query.md' 13 | - 'Python API': 'python-api.md' 14 | theme: 15 | name: material 16 | highlightjs: true 17 | palette: 18 | - scheme: default 19 | toggle: 20 | icon: material/brightness-7 21 | name: Switch to dark mode 22 | 23 | - scheme: slate 24 | toggle: 25 | icon: material/brightness-4 26 | name: Switch to light mode 27 | 28 | features: 29 | - navigation.tabs 30 | - navigation.tracking 31 | - content.code.copy 32 | - content.code.select 33 | - content.code.annotate 34 | 35 | markdown_extensions: 36 | - pymdownx.arithmatex: 37 | generic: true 38 | - admonition 39 | - pymdownx.highlight: 40 | anchor_linenums: true 41 | line_spans: __span 42 | pygments_lang_class: true 43 | - pymdownx.inlinehilite 44 | - pymdownx.snippets 45 | - pymdownx.superfences 46 | - attr_list 47 | - toc: 48 | permalink: true 49 | - tables 50 | - pymdownx.emoji: 51 | emoji_index: !!python/name:materialx.emoji.twemoji 52 | emoji_generator: !!python/name:materialx.emoji.to_svg 53 | 54 | extra_javascript: 55 | - javascripts/mathjax.js 56 | - https://polyfill.io/v3/polyfill.min.js?features=es6 57 | - https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js 58 | 59 | extra_css: 60 | - stylesheets/extra.css 61 | 62 | extra: 63 | version: 64 | provider: mike 65 | -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- 1 | Here is an example of how to use `kmindex`. We build two indexes, `D1` and `D2`, each containing 2 samples. These indexes are registered in a global index `G`. The `data` directory contains data and scripts to easily execute the commands presented here. 2 | 3 | ## CLI Examples 4 | 5 | ### Construction 6 | 7 | ```bash 8 | # Constructs D1 and registers it in G 9 | kmindex build --fof fof1.txt --run-dir ./D1 --index ./G --register-as D1 --hard-min 1 --kmer-size 25 --bloom-size 1000000 10 | # Constructs D2 and registers it in G 11 | kmindex build --fof fof2.txt --run-dir ./D2 --index ./G --register-as D2 --hard-min 1 --kmer-size 25 --bloom-size 1000000 12 | ``` 13 | 14 | ### Query 15 | 16 | ```bash 17 | # Query D1 and D2 in G, output in json format 18 | kmindex query --index ./G --names D1,D2 --output output_1 --format json --fastx 1.fasta 19 | kmindex query --index ./G --names D1,D2 --output output_2 --format json --fastx 2.fasta 20 | kmindex query --index ./G --names D1,D2 --output output_3 --format json --fastx 3.fasta 21 | kmindex query --index ./G --names D1,D2 --output output_4 --format json --fastx 4.fasta 22 | ``` 23 | 24 | ## Server Examples 25 | 26 | ### Start the server 27 | 28 | ```bash 29 | kmindex-server --index ./G 30 | ``` 31 | 32 | ### Get index informations 33 | ```bash 34 | curl -X GET http://127.0.0.1:8080/kmindex/infos 35 | ``` 36 | 37 | ### Query 38 | 39 | ```bash 40 | # Query seq 1_S1 from 1.fasta in D1" 41 | curl -X POST http://127.0.0.1:8080/kmindex/query -H 'Content-type: application/json' -d @./query_1S1_in_index_D1.json 42 | 43 | # Query seq 1_S1 from 1.fasta in D1 and D2" 44 | curl -X POST http://127.0.0.1:8080/kmindex/query -H 'Content-type: application/json' -d @./query_1S1_in_index_D1_D2.json 45 | 46 | # Query seq 1_S1 and 1_S2 from 1.fasta in D1 and D2" 47 | # 1_S1 and 1_S2 are considered as a single query 48 | curl -X POST http://127.0.0.1:8080/kmindex/query -H 'Content-type: application/json' -d @./query_1S1_1S2_in_index_D1_D2.json 49 | ``` 50 | -------------------------------------------------------------------------------- /examples/data/1.fasta: -------------------------------------------------------------------------------- 1 | >1_S1 2 | ACGACGACGACGAGACGAGACGACAGCAGACAGAGACATAATATACTATATAATATATATAGCGAGGGGGGGAGAGCCAGCAGCACCCCCAAAAAAAAA 3 | >1_S2 4 | AGGCGCGTTTGTGTCGCCGCGGCGCCTGCTCTTCTATATATAAAACGACGACCGCAACACGACGCCAGCAGAGAGACGCACACGAGACAGCGACGAGCG 5 | -------------------------------------------------------------------------------- /examples/data/1_build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | kmindex build --fof fof1.txt --run-dir ./D1 --index ./G --register-as D1 --hard-min 1 --kmer-size 25 --bloom-size 1000000 4 | kmindex build --fof fof2.txt --run-dir ./D2 --index ./G --register-as D2 --hard-min 1 --kmer-size 25 --bloom-size 1000000 5 | -------------------------------------------------------------------------------- /examples/data/2.fasta: -------------------------------------------------------------------------------- 1 | >2_S1 2 | AGAGGAAGGAATATTATATCTACTACCATCATCATCACTACTATCATACATACAGAGACAGCAGCAGAGCAGCACAAACGAGACACAAAAAAAGAGAGA 3 | >2_S2 4 | ACGACACTATATATTTTATATATTATATCTTCCTCTCTTCGGGGGGGGGGGGGGGAAAAAAAAAAAAAAAAAAAACGCGGCGAAAAAAAAAAAAAAAAA 5 | -------------------------------------------------------------------------------- /examples/data/2_query.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | kmindex query --index ./G --names D1,D2 --output output_1 --format json --fastx 1.fasta 4 | kmindex query --index ./G --names D1,D2 --output output_2 --format json --fastx 2.fasta 5 | kmindex query --index ./G --names D1,D2 --output output_3 --format json --fastx 3.fasta 6 | kmindex query --index ./G --names D1,D2 --output output_4 --format json --fastx 4.fasta 7 | -------------------------------------------------------------------------------- /examples/data/3.fasta: -------------------------------------------------------------------------------- 1 | >3_S1 2 | ACGACGACGACGAGAGCAGACTACACAGCAGACGACATTACACAGGACCGAGAGATAAACCAGACGAGCAGACGAAGCCAGCAGCACCCCCAAAAAAAAA 3 | >3_S2 4 | AGGCGCGTTTGTGTCGCCGCGGCGCCTGCTCTTCTATATATAAAAGACAGCAGGAGACTATCAGGCAGACGACGAGACGCACACGAGACAGCGACGAGCG 5 | -------------------------------------------------------------------------------- /examples/data/3_run_server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | kmindex-server --index ./G 4 | -------------------------------------------------------------------------------- /examples/data/4.fasta: -------------------------------------------------------------------------------- 1 | >4_S1 2 | GACGAATCTATCAAGACGACATCTATACTAAAAAGAGGAAGGAATATTATATCTACTACCATCATCATCACTACTATCATACATACAGAGACAGCAGAGA 3 | >4_S2 4 | AGCAGATACTACAGGCAAAAAAAAAAAATAACAGACGACAGCAGAGCATTACATCGAGCGAGCAGCAGGACAGACGCAGAGGACAGACAGCCCCACGACG 5 | -------------------------------------------------------------------------------- /examples/data/4_get_request.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | curl -X GET http://127.0.0.1:8080/kmindex/infos 4 | -------------------------------------------------------------------------------- /examples/data/5_post_request.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "Query seq 1_S1 from 1.fasta in D1" 4 | curl -X POST http://127.0.0.1:8080/kmindex/query -H 'Content-type: application/json' -d @./query_1S1_in_index_D1.json 5 | 6 | echo 7 | 8 | echo "Query seq 1_S1 from 1.fasta in D1 and D2" 9 | curl -X POST http://127.0.0.1:8080/kmindex/query -H 'Content-type: application/json' -d @./query_1S1_in_index_D1_D2.json 10 | 11 | echo 12 | 13 | echo "Query seq 1_S1 and 1_S2 from 1.fasta in D1 and D2" 14 | echo "1_S1 and 1_S2 are considered as a single query." 15 | curl -X POST http://127.0.0.1:8080/kmindex/query -H 'Content-type: application/json' -d @./query_1S1_1S2_in_index_D1_D2.json 16 | 17 | -------------------------------------------------------------------------------- /examples/data/fof1.txt: -------------------------------------------------------------------------------- 1 | S1: 1.fasta 2 | S2: 2.fasta 3 | -------------------------------------------------------------------------------- /examples/data/fof2.txt: -------------------------------------------------------------------------------- 1 | S3: 3.fasta 2 | S4: 4.fasta 3 | -------------------------------------------------------------------------------- /examples/data/query_1S1_1S2_in_index_D1_D2.json: -------------------------------------------------------------------------------- 1 | { 2 | "index":[ 3 | "D1","D2" 4 | ], 5 | "seq":[ 6 | "ACGACGACGACGAGACGAGACGACAGCAGACAGAGACATAATATACTATATAATATATATAGCGAGGGGGGGAGAGCCAGCAGCACCCCCAAAAAAAAA", 7 | "AGGCGCGTTTGTGTCGCCGCGGCGCCTGCTCTTCTATATATAAAACGACGACCGCAACACGACGCCAGCAGAGAGACGCACACGAGACAGCGACGAGCG" 8 | ], 9 | "id":"1_S1-1_S2-AsSingleQuery", 10 | "z":3 11 | } 12 | -------------------------------------------------------------------------------- /examples/data/query_1S1_in_index_D1.json: -------------------------------------------------------------------------------- 1 | { 2 | "index":[ 3 | "D1" 4 | ], 5 | "seq":[ 6 | "ACGACGACGACGAGACGAGACGACAGCAGACAGAGACATAATATACTATATAATATATATAGCGAGGGGGGGAGAGCCAGCAGCACCCCCAAAAAAAAA" 7 | ], 8 | "id":"1_S1", 9 | "z":3 10 | } 11 | -------------------------------------------------------------------------------- /examples/data/query_1S1_in_index_D1_D2.json: -------------------------------------------------------------------------------- 1 | { 2 | "index":[ 3 | "D1","D2" 4 | ], 5 | "seq":[ 6 | "ACGACGACGACGAGACGAGACGACAGCAGACAGAGACATAATATACTATATAATATATATAGCGAGGGGGGGAGAGCCAGCAGCACCCCCAAAAAAAAA" 7 | ], 8 | "id":"1_S1", 9 | "z":3 10 | } 11 | -------------------------------------------------------------------------------- /flake.lock: -------------------------------------------------------------------------------- 1 | { 2 | "nodes": { 3 | "flake-utils": { 4 | "locked": { 5 | "lastModified": 1676283394, 6 | "narHash": "sha256-XX2f9c3iySLCw54rJ/CZs+ZK6IQy7GXNY4nSOyu2QG4=", 7 | "owner": "numtide", 8 | "repo": "flake-utils", 9 | "rev": "3db36a8b464d0c4532ba1c7dda728f4576d6d073", 10 | "type": "github" 11 | }, 12 | "original": { 13 | "owner": "numtide", 14 | "repo": "flake-utils", 15 | "type": "github" 16 | } 17 | }, 18 | "nixpkgs": { 19 | "locked": { 20 | "lastModified": 1676481215, 21 | "narHash": "sha256-afma/1RU0EePRyrBPcjBdOt+dV8z1bJH9dtpTN/WXmY=", 22 | "owner": "nixos", 23 | "repo": "nixpkgs", 24 | "rev": "28319deb5ab05458d9cd5c7d99e1a24ec2e8fc4b", 25 | "type": "github" 26 | }, 27 | "original": { 28 | "owner": "nixos", 29 | "ref": "nixos-unstable", 30 | "repo": "nixpkgs", 31 | "type": "github" 32 | } 33 | }, 34 | "root": { 35 | "inputs": { 36 | "flake-utils": "flake-utils", 37 | "nixpkgs": "nixpkgs" 38 | } 39 | } 40 | }, 41 | "root": "root", 42 | "version": 7 43 | } 44 | -------------------------------------------------------------------------------- /flake.nix: -------------------------------------------------------------------------------- 1 | { 2 | inputs = { 3 | nixpkgs = { 4 | url = "github:nixos/nixpkgs/nixos-unstable"; 5 | }; 6 | flake-utils = { 7 | url = "github:numtide/flake-utils"; 8 | }; 9 | }; 10 | outputs = { self, nixpkgs, flake-utils, ... }: flake-utils.lib.eachSystem [ 11 | "x86_64-linux" 12 | ] (system: 13 | let 14 | pkgs = import nixpkgs { 15 | inherit system; 16 | }; 17 | 18 | kmindexBuildInputs = [ 19 | pkgs.gcc12 20 | pkgs.cmake 21 | pkgs.zlib 22 | pkgs.gbenchmark 23 | pkgs.boost 24 | pkgs.bzip2 25 | pkgs.xxHash 26 | pkgs.nlohmann_json 27 | pkgs.fmt_8 28 | pkgs.gtest 29 | pkgs.spdlog 30 | ]; 31 | 32 | kmindex = (with pkgs; stdenvNoCC.mkDerivation { 33 | pname = "kmindex"; 34 | version = "0.5.2"; 35 | src = builtins.fetchGit { 36 | url = "https://github.com/tlemane/kmindex"; 37 | rev = "0d3792fa5a242540582f16b4542311a473c0b7c3"; 38 | submodules = true; 39 | }; 40 | nativeBuildInputs = kmindexBuildInputs; 41 | 42 | configurePhase = '' 43 | cmake -S . -B build 44 | ''; 45 | 46 | buildPhase = '' 47 | cmake --build ./build 48 | ''; 49 | 50 | installPhase = '' 51 | mkdir -p $out/bin 52 | cp ./build/app/kmindex/kmindex $out/bin 53 | cp ./build/app/kmindex-server/kmindex-server $out/bin 54 | ''; 55 | } 56 | ); 57 | in rec { 58 | defaultApp = flake-utils.lib.mkApp { 59 | drv = defaultPackage; 60 | }; 61 | defaultPackage = kmindex; 62 | 63 | devShell = pkgs.mkShell { 64 | name = "kmindex_dev_env"; 65 | buildInputs = kmindexBuildInputs; 66 | }; 67 | 68 | apps.default = { 69 | type = "app"; 70 | program = "${self.defaultPackage.${system}}/bin/kmindex"; 71 | }; 72 | } 73 | ); 74 | } 75 | -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | function kmindex_build () 4 | { 5 | mkdir kmbuild 6 | cd kmbuild 7 | 8 | cmake .. -DCMAKE_BUILD_TYPE=${1} \ 9 | -DWITH_TESTS=${2} \ 10 | -DPORTABLE_BUILD=${4} \ 11 | -DCMAKE_INSTALL_PREFIX=$(realpath ${6}) \ 12 | -DMAX_KMER_SIZE=${7} 13 | 14 | make -j${5} 15 | 16 | if [[ ${3} == 1 ]]; then 17 | ctest --verbose 18 | fi 19 | 20 | make install 21 | } 22 | 23 | function usage () 24 | { 25 | echo "kmindex build script - v0.5.0." 26 | echo "Usage: " 27 | echo " ./install.sh [-r str] [-t int] [-j int] [-p str] [-k int] [-n] [-h]" 28 | echo "Options: " 29 | echo " -r -> build type {Release}." 30 | echo " -t <0|1|2> -> tests: 0 = disabled, 1 = compile, 2 = compile and run {0}." 31 | echo " -j -> nb threads {8}." 32 | echo " -k -> Max k-mer size (should be a multiple of 32) {256}" 33 | echo " -n -> portable x86-64 build {disabled}." 34 | echo " -p -> install path {./kmindex_install}" 35 | echo " -h -> show help." 36 | exit 1 37 | } 38 | 39 | rel="Release" 40 | deb="Debug" 41 | 42 | mode="Release" 43 | static="OFF" 44 | tests=0 45 | tests_str="OFF" 46 | tests_run=1 47 | jopt=8 48 | native="OFF" 49 | installp=$(realpath ./kmindex_install) 50 | max_ksize=256 51 | 52 | while getopts "r:t:j:k:p:nh" option; do 53 | case "$option" in 54 | r) 55 | mode=${OPTARG} 56 | [[ ${mode} != ${rel} && ${mode} != ${deb} ]] && usage 57 | ;; 58 | t) 59 | tests=${OPTARG} 60 | [[ ${tests} == 0 ]] || [[ ${tests} == 1 ]] || [[ ${tests} == 2 ]] || usage 61 | [[ ${tests} == 0 ]] && tests_run=0 62 | [[ ${tests} == 0 ]] && tests_str="OFF" 63 | [[ ${tests} == 1 ]] && tests_run=0 64 | [[ ${tests} == 1 ]] && tests_str="ON" 65 | [[ ${tests} == 2 ]] && tests_run=1 66 | [[ ${tests} == 2 ]] && tests_str="ON" 67 | ;; 68 | j) 69 | jopt=${OPTARG} 70 | ;; 71 | n) 72 | native="ON" 73 | ;; 74 | p) 75 | installp=${OPTARG} 76 | ;; 77 | k) 78 | max_ksize=${OPTARG} 79 | ;; 80 | h) 81 | usage 82 | ;; 83 | *) 84 | usage 85 | ;; 86 | esac 87 | done 88 | 89 | kmindex_build ${mode} ${tests_str} ${tests_run} ${native} ${jopt} ${installp} ${max_ksize} 90 | -------------------------------------------------------------------------------- /lib/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | configure_file("${CMAKE_SOURCE_DIR}/lib/config.hpp.in" "${CMAKE_BINARY_DIR}/include/kmindex/config.hpp") 2 | 3 | set(KMINDEX_LIB "kmindex-lib") 4 | if (NOT APPLE) 5 | set(CMAKE_INSTALL_RPATH $ORIGIN) 6 | endif() 7 | 8 | 9 | file(GLOB_RECURSE KMINDEX_SOURCES "src/*.cpp") 10 | add_library(${KMINDEX_LIB} ${KMINDEX_SOURCES}) 11 | 12 | set_target_properties(${KMINDEX_LIB} PROPERTIES OUTPUT_NAME "kmindex") 13 | target_compile_definitions(${KMINDEX_LIB} PUBLIC KMTRICKS_PUBLIC) 14 | 15 | add_library(${KMINDEX_LIB}::${KMINDEX_LIB} ALIAS ${KMINDEX_LIB}) 16 | 17 | target_compile_features(${KMINDEX_LIB} PUBLIC cxx_std_17) 18 | 19 | target_link_libraries(${KMINDEX_LIB} PUBLIC 20 | $ 21 | $ 22 | $ 23 | $ 24 | $ 25 | $ 26 | $ 27 | $ 28 | $ 29 | $ 30 | ) 31 | 32 | add_dependencies(${KMINDEX_LIB} deps) 33 | 34 | target_include_directories(${KMINDEX_LIB} PUBLIC 35 | $ 36 | $ 37 | $ 38 | ) 39 | 40 | set_target_properties(${KMINDEX_LIB} PROPERTIES 41 | SOVERSION ${CMAKE_PROJECT_VERSION_MAJOR} 42 | VERSION ${CMAKE_PROJECT_VERSION} 43 | ) 44 | 45 | -------------------------------------------------------------------------------- /lib/config.hpp.in: -------------------------------------------------------------------------------- 1 | #ifndef CONFIG_HPP_IN_EGRDPAWF 2 | #define CONFIG_HPP_IN_EGRDPAWF 3 | 4 | #define KMQ_PROJECT_NAME "@PROJECT_NAME@" 5 | #define KMQ_PROJECT_DESC "@PROJECT_DESCRIPTION@" 6 | 7 | #define KMQ_PROJECT_VER "@PROJECT_VERSION@" 8 | #define KMQ_PROJECT_TAG "v@PROJECT_VERSION@" 9 | #define KMQ_PROJECT_VER_MAJOR "@PROJECT_VERSION_MAJOR@" 10 | #define KMQ_PROJECT_VER_MINOR "@PROJECT_VERSION_MINOR@" 11 | #define KMQ_PROJECT_VER_PATCH "@PROJECT_VERSION_PATCH@" 12 | #define KMQ_VER_MAJOR @PROJECT_VERSION_MAJOR@ 13 | #define KMQ_VER_MINOR @PROJECT_VERSION_MINOR@ 14 | #define KMQ_VER_PATCH @PROJECT_VERSION_PATCH@ 15 | 16 | #define KMQ_HOST_SYSTEM "@CMAKE_HOST_SYSTEM@" 17 | #define KMQ_COMPILER_C "@CMAKE_C_COMPILER_ID@ @CMAKE_C_COMPILER_VERSION@" 18 | #define KMQ_COMPILER_CXX "@CMAKE_CXX_COMPILER_ID@ @CMAKE_CXX_COMPILER_VERSION@" 19 | 20 | 21 | #endif /* end of include guard: CONFIG_HPP_IN_EGRDPAWF */ 22 | -------------------------------------------------------------------------------- /lib/include/kmindex/exceptions.hpp: -------------------------------------------------------------------------------- 1 | #ifndef EXCEPTIONS_HPP_ZLDCNT1I 2 | #define EXCEPTIONS_HPP_ZLDCNT1I 3 | 4 | #include 5 | #include 6 | 7 | namespace kmq { 8 | 9 | class kmq_error : public std::exception 10 | { 11 | public: 12 | kmq_error(const std::string& msg) noexcept; 13 | const char* what() const noexcept override; 14 | virtual std::string name() const noexcept; 15 | private: 16 | std::string m_msg; 17 | }; 18 | 19 | class kmq_io_error : public kmq_error 20 | { 21 | using kmq_error::kmq_error; 22 | virtual std::string name() const noexcept override; 23 | }; 24 | 25 | class kmq_invalid_request : public kmq_error 26 | { 27 | using kmq_error::kmq_error; 28 | virtual std::string name() const noexcept override; 29 | }; 30 | 31 | class kmq_invalid_index : public kmq_error 32 | { 33 | using kmq_error::kmq_error; 34 | virtual std::string name() const noexcept override; 35 | }; 36 | } 37 | 38 | #endif /* end of include guard: EXCEPTIONS_HPP_ZLDCNT1I */ 39 | -------------------------------------------------------------------------------- /lib/include/kmindex/index/index.hpp: -------------------------------------------------------------------------------- 1 | #ifndef INDEX_HPP_W1NZBQZJ 2 | #define INDEX_HPP_W1NZBQZJ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | namespace kmq { 11 | 12 | constexpr semver::version min_kmv_required{1, 3, 0}; 13 | 14 | class index 15 | { 16 | using map_t = std::map; 17 | using iterator = map_t::iterator; 18 | 19 | public: 20 | index(const std::string& index_path); 21 | 22 | void add_index(const std::string& name, const std::string& km_path); 23 | void remove_index(const std::string& name); 24 | 25 | void save() const; 26 | 27 | iterator begin(); 28 | iterator end(); 29 | 30 | const index_infos& get(const std::string& name) const; 31 | index_infos& get(const std::string& name); 32 | 33 | std::vector all() const; 34 | 35 | std::vector> mergeable() const; 36 | 37 | const std::string& path() const { return m_index_path; } 38 | private: 39 | void init(const std::string& ipath); 40 | 41 | private: 42 | std::string m_index_path; 43 | map_t m_indexes; 44 | }; 45 | 46 | } 47 | 48 | 49 | #endif /* end of include guard: INDEX_HPP_W1NZBQZJ */ 50 | -------------------------------------------------------------------------------- /lib/include/kmindex/index/index_infos.hpp: -------------------------------------------------------------------------------- 1 | #ifndef INDEX_INFOS_HPP_IOEYNAWD 2 | #define INDEX_INFOS_HPP_IOEYNAWD 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #ifndef KMTRICKS_PUBLIC 9 | #define KMTRICKS_PUBLIC 10 | #endif 11 | 12 | #include 13 | #include 14 | #include 15 | 16 | #include 17 | #include 18 | 19 | using json = nlohmann::json; 20 | 21 | namespace kmq { 22 | 23 | class index_infos 24 | { 25 | private: 26 | std::string m_name; 27 | std::string m_path; 28 | std::size_t m_nb_partitions {0}; 29 | std::size_t m_bloom_size {0}; 30 | std::size_t m_nb_samples {0}; 31 | std::size_t m_smer_size {0}; 32 | std::size_t m_minim_size {0}; 33 | std::size_t m_index_size {0}; 34 | std::size_t m_bw {1}; 35 | std::vector m_samples; 36 | 37 | std::string m_sha1; 38 | 39 | std::shared_ptr m_hashw {nullptr}; 40 | std::shared_ptr m_repart {nullptr}; 41 | 42 | semver::version m_kmver; 43 | semver::version m_kmtver; 44 | 45 | public: 46 | 47 | index_infos() {} 48 | index_infos(const std::string& name, const std::string& km_dir); 49 | index_infos(const std::string& name, const json& jdata); 50 | 51 | 52 | std::shared_ptr get_hash_w() const; 53 | std::shared_ptr get_repartition() const; 54 | std::string get_partition(std::size_t partition) const; 55 | 56 | std::string name() const; 57 | std::size_t bloom_size() const; 58 | std::size_t nb_partitions() const; 59 | std::size_t nb_samples() const; 60 | std::size_t smer_size() const; 61 | std::size_t minim_size() const; 62 | std::size_t index_size() const; 63 | std::size_t bw() const; 64 | 65 | std::string path() const; 66 | std::string sha1() const; 67 | 68 | const std::vector& samples() const; 69 | 70 | std::size_t rename_samples(const std::string& format, std::size_t start_id); 71 | std::size_t rename_samples(const std::string& new_ids); 72 | 73 | std::string fof_str() const; 74 | 75 | const semver::version& km_version() const; 76 | const semver::version& kmt_version() const; 77 | 78 | private: 79 | void init(); 80 | void init(const json& data); 81 | void is_km_index() const; 82 | std::string km_sha1() const; 83 | }; 84 | 85 | } 86 | 87 | #endif /* end of include guard: INDEX_INFOS_HPP_IOEYNAWD */ 88 | -------------------------------------------------------------------------------- /lib/include/kmindex/index/kindex.hpp: -------------------------------------------------------------------------------- 1 | #ifndef INDEX_HPP_FJYOTLJN 2 | #define INDEX_HPP_FJYOTLJN 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | 11 | #include 12 | 13 | namespace kmq { 14 | 15 | class partition 16 | { 17 | public: 18 | partition(const std::string& matrix_path, std::size_t nb_samples, std::size_t width); 19 | 20 | ~partition(); 21 | 22 | void query(std::uint64_t pos, std::uint8_t* dest); 23 | 24 | private: 25 | int m_fd {0}; 26 | mio::mmap_source m_mapped; 27 | std::size_t m_nb_samples {0}; 28 | std::size_t m_bytes {0}; 29 | }; 30 | 31 | class kindex 32 | { 33 | public: 34 | 35 | kindex(); 36 | ~kindex(); 37 | kindex(const index_infos& i, bool cache = false); 38 | 39 | void init(std::size_t p); 40 | void unmap(std::size_t p); 41 | 42 | public: 43 | std::string name() const; 44 | std::string directory() const; 45 | 46 | void solve(batch_query& bq) 47 | { 48 | std::vector order; order.reserve(m_infos.nb_partitions()); 49 | for (std::size_t p = 0; p < m_infos.nb_partitions(); p++) 50 | order.push_back(p); 51 | #ifndef __APPLE__ 52 | std::random_shuffle(std::begin(order), std::end(order)); 53 | #endif 54 | for (auto const& p : order) 55 | solve_one(bq, p); 56 | } 57 | 58 | void solve_one(batch_query& bq, std::size_t p) 59 | { 60 | auto& smers = bq.partition(p); 61 | auto& responses = bq.response(); 62 | 63 | std::sort(std::begin(smers), std::end(smers)); 64 | 65 | std::unique_lock lock(m_mutexes[p]); 66 | init(p); 67 | for (auto& [mer, qid] : smers) 68 | { 69 | m_partitions[p]->query(mer.h, responses[qid]->get(mer.i)); 70 | } 71 | m_partitions[p] = nullptr; 72 | } 73 | 74 | void solve_cache(batch_query& bq) 75 | { 76 | std::vector order; order.reserve(m_infos.nb_partitions()); 77 | for (std::size_t p = 0; p < m_infos.nb_partitions(); p++) 78 | order.push_back(p); 79 | #ifndef __APPLE__ 80 | std::random_shuffle(std::begin(order), std::end(order)); 81 | #endif 82 | for (auto const& p : order) 83 | solve_one_cache(bq, p); 84 | } 85 | 86 | void solve_one_cache(batch_query& bq, std::size_t p) 87 | { 88 | auto& smers = bq.partition(p); 89 | auto& responses = bq.response(); 90 | 91 | std::sort(std::begin(smers), std::end(smers)); 92 | 93 | std::unique_lock lock(m_mutexes[p]); 94 | for (auto& [mer, qid] : smers) 95 | { 96 | m_partitions[p]->query(mer.h, responses[qid]->get(mer.i)); 97 | } 98 | } 99 | 100 | void solve_batch(batch_query& bq) 101 | { 102 | if (m_cache) 103 | solve_cache(bq); 104 | else 105 | solve(bq); 106 | } 107 | 108 | index_infos& infos(); 109 | private: 110 | index_infos m_infos; 111 | std::vector> m_partitions; 112 | std::vector m_mutexes; 113 | bool m_cache {false}; 114 | }; 115 | } 116 | 117 | 118 | 119 | #endif /* end of include guard: INDEX_HPP_FJYOTLJN */ 120 | -------------------------------------------------------------------------------- /lib/include/kmindex/index/merge.hpp: -------------------------------------------------------------------------------- 1 | #ifndef MERGE_HPP_BRXIFGDK 2 | #define MERGE_HPP_BRXIFGDK 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | namespace kmq { 11 | 12 | enum class rename_mode : std::uint8_t 13 | { 14 | file, 15 | format, 16 | none 17 | }; 18 | 19 | class index_merger 20 | { 21 | protected: 22 | index_merger(index* gindex, 23 | const std::vector& to_merge, 24 | const std::string& new_path, 25 | const std::string& new_name, 26 | rename_mode mode, 27 | std::size_t bw, 28 | bool rm); 29 | 30 | virtual ~index_merger() = default; 31 | 32 | void remove_old() const; 33 | void remove_old(std::size_t p) const; 34 | 35 | virtual void merge_one(std::size_t p) const = 0; 36 | 37 | void copy_tree(const std::string& path) const; 38 | void copy_trees() const; 39 | 40 | void zero(std::size_t p) const; 41 | 42 | public: 43 | void rename(const std::string& rename_string) const; 44 | virtual void merge(ThreadPool& pool) const = 0; 45 | protected: 46 | index* m_index; 47 | std::vector m_to_merge; 48 | std::string m_new_path; 49 | std::string m_new_name; 50 | rename_mode m_rename_mode {rename_mode::none}; 51 | std::size_t m_bw {0}; 52 | bool m_rm {false}; 53 | 54 | std::size_t m_psize {0}; 55 | std::size_t m_nb_samples {0}; 56 | std::size_t m_nb_parts {0}; 57 | mutable std::vector> m_buffer; 58 | }; 59 | 60 | using index_merger_t = std::shared_ptr; 61 | 62 | class index_merger_pa : public index_merger 63 | { 64 | public: 65 | index_merger_pa(index* gindex, 66 | const std::vector& to_merge, 67 | const std::string& new_path, 68 | const std::string& new_name, 69 | rename_mode mode, 70 | bool rm); 71 | 72 | void merge_one(std::size_t p) const override final; 73 | void merge(ThreadPool& pool) const override final; 74 | }; 75 | 76 | class index_merger_abs : public index_merger 77 | { 78 | public: 79 | index_merger_abs(index* gindex, 80 | const std::vector& to_merge, 81 | const std::string& new_path, 82 | const std::string& new_name, 83 | rename_mode mode, 84 | bool rm, 85 | std::size_t bw); 86 | 87 | void merge_one(std::size_t p) const override final; 88 | void merge(ThreadPool& pool) const override final; 89 | }; 90 | 91 | index_merger_t make_merger(index* gindex, 92 | const std::vector& to_merge, 93 | const std::string& new_path, 94 | const std::string& new_name, 95 | rename_mode mode, 96 | bool rm, 97 | std::size_t bw); 98 | 99 | 100 | 101 | } 102 | 103 | 104 | #endif /* end of include guard: MERGE_HPP_BRXIFGDK */ 105 | -------------------------------------------------------------------------------- /lib/include/kmindex/query/query_results.hpp: -------------------------------------------------------------------------------- 1 | #ifndef QUERY_RESULTS_HPP_VLPMAIEE 2 | #define QUERY_RESULTS_HPP_VLPMAIEE 3 | 4 | #include 5 | #include 6 | 7 | namespace kmq { 8 | 9 | class index_infos; 10 | 11 | enum class format; 12 | 13 | class query_result 14 | { 15 | public: 16 | 17 | query_result(query_response_t&& qr, std::size_t z, const index_infos& info, bool pos = false); 18 | 19 | public: 20 | 21 | void compute_ratios(); 22 | 23 | void compute_abs(); 24 | 25 | void compute_ratios_pos(); 26 | 27 | void compute_abs_pos(); 28 | 29 | std::size_t nbk() const; 30 | 31 | const std::vector& counts() const; 32 | 33 | const std::vector& ratios() const; 34 | 35 | const std::vector>& positions() const; 36 | 37 | const std::string& name() const; 38 | 39 | double threshold() const; 40 | 41 | private: 42 | std::vector m_ratios; 43 | std::vector m_counts; 44 | std::vector> m_positions; 45 | query_response_t m_qr; 46 | std::size_t m_z; 47 | const index_infos& m_infos; 48 | std::uint32_t m_nbk; 49 | }; 50 | 51 | class query_result_agg 52 | { 53 | using vec_t = std::vector; 54 | using iterator = vec_t::iterator; 55 | using const_iterator = vec_t::const_iterator; 56 | 57 | public: 58 | query_result_agg() = default; 59 | 60 | void add(query_result&& r); 61 | 62 | std::size_t size() const; 63 | 64 | const_iterator begin() const; 65 | 66 | const_iterator end() const; 67 | 68 | const vec_t& results() const; 69 | 70 | void output(const index_infos& infos, 71 | const std::string& output_dir, 72 | enum format f, 73 | const std::string& qname, 74 | double threshold); 75 | 76 | private: 77 | std::mutex m_mutex; 78 | std::vector m_results; 79 | }; 80 | 81 | } 82 | 83 | #endif /* end of include guard: QUERY_RESULTS_HPP_VLPMAIEE */ 84 | -------------------------------------------------------------------------------- /lib/include/kmindex/spinlock.hpp: -------------------------------------------------------------------------------- 1 | #ifndef SPINLOCK_HPP_OWQGS7C3 2 | #define SPINLOCK_HPP_OWQGS7C3 3 | 4 | #include 5 | #include 6 | 7 | namespace kmq { 8 | 9 | class spinlock 10 | { 11 | public: 12 | void lock() noexcept 13 | { 14 | static const std::size_t iters = 1024; 15 | 16 | while (true) 17 | { 18 | if (!m_lock.exchange(true, std::memory_order_acquire)) 19 | return; 20 | else 21 | { 22 | for (std::size_t i=0; i m_lock {false}; 55 | }; 56 | 57 | } 58 | 59 | #endif /* end of include guard: SPINLOCK_HPP_OWQGS7C3 */ 60 | -------------------------------------------------------------------------------- /lib/include/kmindex/threadpool.hpp: -------------------------------------------------------------------------------- 1 | #ifndef THREADPOOL_HPP_DMXPPU0Y 2 | #define THREADPOOL_HPP_DMXPPU0Y 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | namespace kmq { 13 | 14 | class ThreadPool 15 | { 16 | using size_type = std::result_of::type; 17 | 18 | public: 19 | ThreadPool(size_type threads); 20 | 21 | ThreadPool() = delete; 22 | ThreadPool(const ThreadPool&) = delete; 23 | ThreadPool& operator=(const ThreadPool&) = delete; 24 | ThreadPool(ThreadPool&&) = delete; 25 | ThreadPool& operator=(ThreadPool&&) = delete; 26 | 27 | ~ThreadPool(); 28 | 29 | void join_all(); 30 | 31 | void join(int i); 32 | void restart(size_type threads = 0); 33 | 34 | template 35 | void add_task(Callable&& f) 36 | { 37 | auto task = std::make_shared>(std::forward(f)); 38 | { 39 | std::unique_lock lock(_queue_mutex); 40 | if (_stop) throw std::runtime_error("Push on stopped Pool."); 41 | _queue.emplace([task](int thread_id) { (*task)(thread_id); }); 42 | } 43 | _condition.notify_one(); 44 | } 45 | 46 | private: 47 | void worker(int i); 48 | 49 | private: 50 | size_type _n{std::thread::hardware_concurrency()}; 51 | std::vector _pool; 52 | std::queue> _queue; 53 | std::mutex _queue_mutex; 54 | std::condition_variable _condition; 55 | bool _stop{false}; 56 | }; 57 | 58 | } 59 | 60 | #endif /* end of include guard: THREADPOOL_HPP_DMXPPU0Y */ 61 | -------------------------------------------------------------------------------- /lib/include/kmindex/utils.hpp: -------------------------------------------------------------------------------- 1 | #ifndef UTILS_HPP_TKVBI1VR 2 | #define UTILS_HPP_TKVBI1VR 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include 14 | namespace fs = std::filesystem; 15 | 16 | #define ROUND_UP(n, m) ((n + m - 1) / m) * m 17 | #define ROUND_UP(n, m) ((n + m - 1) / m) * m 18 | #define NBYTES(m) (m + 7) / 8 19 | #define BITMASK(b) (1 << ((b) % CHAR_BIT)) 20 | #define BITSLOT(b) ((b) / CHAR_BIT) 21 | #define BITSET(a, b) ((a)[BITSLOT(b)] |= BITMASK(b)) 22 | #define BITCHECK(a, b) ((a)[BITSLOT(b)] & BITMASK(b)) 23 | 24 | namespace kmq { 25 | 26 | template 27 | void unused(Args&&...) {} 28 | 29 | const std::string WHITECHAR = " \n\r\t\f\v"; 30 | 31 | template 32 | inline void check_fstream_good(const std::string& path, const T& stream) 33 | { 34 | if (!stream.good()) 35 | { 36 | if constexpr(std::is_same_v) 37 | throw std::runtime_error(fmt::format("Unable to write at {}.", path)); 38 | else 39 | throw std::runtime_error(fmt::format("Unable to read at {}.", path)); 40 | } 41 | } 42 | 43 | template> 44 | inline std::vector split(const std::string& s, char delim, Formatter format) 45 | { 46 | std::istringstream iss(s); 47 | std::vector ret; 48 | for (std::string tmp; std::getline(iss, tmp, delim);) 49 | ret.push_back(format(tmp)); 50 | return ret; 51 | } 52 | 53 | std::string rtrim(const std::string& s, const std::string& v = WHITECHAR); 54 | 55 | std::string ltrim(const std::string& s, const std::string& v = WHITECHAR); 56 | 57 | std::string trim(const std::string& s, const std::string& v = WHITECHAR); 58 | 59 | std::vector split(const std::string& s, char delim); 60 | 61 | template 62 | void free_container(C& c) 63 | { 64 | C{}.swap(c); 65 | } 66 | 67 | std::size_t directory_size(const std::string& p); 68 | 69 | class Timer 70 | { 71 | using time_point_t = std::chrono::time_point; 72 | using days = std::chrono::duration>; 73 | 74 | public: 75 | Timer(); 76 | 77 | template 78 | auto elapsed() 79 | { 80 | if (m_running) end(); 81 | return std::chrono::duration_cast(m_end_time - m_start_time); 82 | } 83 | 84 | std::string formatted(); 85 | 86 | template 87 | static auto time_it(std::function func) 88 | { 89 | auto start = std::chrono::steady_clock::now(); 90 | func(); 91 | auto end = std::chrono::steady_clock::now(); 92 | return std::chrono::duration_cast(end - start).count(); 93 | } 94 | 95 | void reset() 96 | { 97 | m_running = false; 98 | m_start_time = time_point_t{}; 99 | m_end_time = time_point_t{}; 100 | start(); 101 | } 102 | 103 | private: 104 | void start(); 105 | 106 | void end(); 107 | 108 | private: 109 | time_point_t m_start_time{}; 110 | time_point_t m_end_time{}; 111 | bool m_running{false}; 112 | }; 113 | 114 | template 115 | struct loop_executor 116 | { 117 | template typename Functor, typename... Args> 118 | static void exec(size_t kmer_size, Args&&... args) 119 | { 120 | if (kmer_size < SIZE) 121 | { 122 | Functor()(std::forward(args)...); 123 | return; 124 | } 125 | loop_executor::template exec( 126 | kmer_size, std::forward(args)...); 127 | } 128 | }; 129 | 130 | template 131 | struct loop_executor 132 | { 133 | template typename Functor, typename... Args> 134 | static void exec(size_t kmer_size, Args&&... args) 135 | { 136 | if (kmer_size < SIZE) 137 | Functor()(std::forward(args)...); 138 | else 139 | throw std::runtime_error("No implementation found for this k-mer size."); 140 | } 141 | }; 142 | 143 | } 144 | 145 | #endif /* end of include guard: UTILS_HPP_TKVBI1VR */ 146 | -------------------------------------------------------------------------------- /lib/include/kmindex/version.hpp: -------------------------------------------------------------------------------- 1 | #ifndef VERSION_HPP_KUCJIMKM 2 | #define VERSION_HPP_KUCJIMKM 3 | 4 | #include 5 | #include 6 | 7 | namespace kmq { 8 | 9 | constexpr semver::version kmindex_version { 10 | KMQ_VER_MAJOR, KMQ_VER_MINOR, KMQ_VER_PATCH 11 | }; 12 | 13 | } 14 | 15 | #endif /* end of include guard: VERSION_HPP_KUCJIMKM */ 16 | -------------------------------------------------------------------------------- /lib/src/exceptions.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | 5 | namespace kmq { 6 | 7 | kmq_error::kmq_error(const std::string& msg) noexcept 8 | : m_msg(fmt::format("[{}]: {}", name(), msg)) {} 9 | 10 | const char* kmq_error::what() const noexcept 11 | { 12 | return m_msg.c_str(); 13 | } 14 | 15 | std::string kmq_error::name() const noexcept 16 | { 17 | return "kmq_error"; 18 | } 19 | 20 | std::string kmq_io_error::name() const noexcept { return "kmq_io_error"; } 21 | std::string kmq_invalid_request::name() const noexcept { return "kmq_invalid_request"; } 22 | std::string kmq_invalid_index::name() const noexcept { return "kmq_invalid_index"; } 23 | } 24 | -------------------------------------------------------------------------------- /lib/src/index/index.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | using json = nlohmann::json; 11 | namespace kmq { 12 | 13 | index::index(const std::string& index_path) 14 | : m_index_path(fs::absolute(index_path).string()) 15 | { 16 | if (fs::exists(index_path)) 17 | init(index_path); 18 | else 19 | fs::create_directory(index_path); 20 | } 21 | 22 | void index::init(const std::string& index_path) 23 | { 24 | std::ifstream inf(fmt::format("{}/index.json", m_index_path), std::ios::in); 25 | check_fstream_good(index_path, inf); 26 | json data = json::parse(inf); 27 | 28 | for (auto& e : data["index"].items()) 29 | { 30 | m_indexes[e.key()] = index_infos(e.key(), data); 31 | } 32 | } 33 | 34 | void index::add_index(const std::string& name, const std::string& km_path) 35 | { 36 | if (m_indexes.count(name)) 37 | throw std::runtime_error(fmt::format("'{}' already exists.", name)); 38 | m_indexes[name] = index_infos(name, fs::absolute(km_path).string()); 39 | } 40 | 41 | void index::remove_index(const std::string& name) 42 | { 43 | if (m_indexes.count(name)) 44 | { 45 | m_indexes.erase(name); 46 | std::string p = fmt::format("{}/{}", m_index_path, name); 47 | ::unlink(p.c_str()); 48 | } 49 | } 50 | 51 | void index::save() const 52 | { 53 | nlohmann::json data; 54 | data["path"] = m_index_path; 55 | data["index"] = json({}); 56 | 57 | for (auto& [name, i] : m_indexes) 58 | { 59 | if (!fs::is_symlink(fmt::format("{}/{}", m_index_path, name))) 60 | fs::create_directory_symlink(i.path(), fmt::format("{}/{}", m_index_path, name)); 61 | 62 | data["index"][name]["nb_samples"] = i.nb_samples(); 63 | data["index"][name]["index_size"] = i.index_size(); 64 | data["index"][name]["bloom_size"] = i.bloom_size(); 65 | data["index"][name]["nb_partitions"] = i.nb_partitions(); 66 | data["index"][name]["smer_size"] = i.smer_size(); 67 | data["index"][name]["minim_size"] = i.minim_size(); 68 | data["index"][name]["samples"] = i.samples(); 69 | data["index"][name]["sha1"] = i.sha1(); 70 | data["index"][name]["bw"] = i.bw(); 71 | data["index"][name]["kmindex_version"] = i.km_version().to_string(); 72 | data["index"][name]["kmtricks_version"] = i.kmt_version().to_string(); 73 | } 74 | 75 | std::ofstream out(fmt::format("{}/index.json", m_index_path), std::ios::out); 76 | 77 | out << std::setw(4) << data << std::endl; 78 | } 79 | 80 | std::vector index::all() const 81 | { 82 | std::vector names; 83 | for (const auto& i : m_indexes) 84 | names.push_back(i.first); 85 | 86 | return names; 87 | } 88 | 89 | index::iterator index::begin() 90 | { 91 | return std::begin(m_indexes); 92 | } 93 | 94 | index::iterator index::end() 95 | { 96 | return std::end(m_indexes); 97 | } 98 | 99 | const index_infos& index::get(const std::string& name) const 100 | { 101 | if (m_indexes.count(name)) 102 | return m_indexes.at(name); 103 | throw kmq_invalid_index(fmt::format("'{}' is not registered by this instance", name)); 104 | } 105 | 106 | index_infos& index::get(const std::string& name) 107 | { 108 | if (m_indexes.count(name)) 109 | return m_indexes.at(name); 110 | throw kmq_invalid_index(fmt::format("'{}' is not registered by this instance", name)); 111 | } 112 | 113 | 114 | std::vector> index::mergeable() const 115 | { 116 | std::vector> vec; 117 | 118 | std::map> m; 119 | 120 | for (auto& [name, i] : m_indexes) 121 | m[i.sha1()].push_back(name); 122 | 123 | for (auto& [_, v] : m) 124 | { 125 | if (vec.size() > 1) 126 | vec.push_back(std::move(v)); 127 | } 128 | 129 | return vec; 130 | } 131 | 132 | } 133 | -------------------------------------------------------------------------------- /lib/src/index/kindex.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | namespace kmq { 9 | 10 | partition::partition(const std::string& matrix_path, std::size_t nb_samples, std::size_t width) 11 | : m_nb_samples(nb_samples), m_bytes(((nb_samples * width) + 7) / 8) 12 | { 13 | m_fd = open(matrix_path.c_str(), O_RDONLY); 14 | m_mapped = mio::mmap_source(m_fd, 0, mio::map_entire_file); 15 | posix_madvise(&m_mapped[0], m_mapped.length(), POSIX_MADV_SEQUENTIAL); 16 | } 17 | 18 | partition::~partition() 19 | { 20 | m_mapped.unmap(); 21 | close(m_fd); 22 | } 23 | 24 | void partition::query(std::uint64_t pos, std::uint8_t* dest) 25 | { 26 | std::memcpy(dest, m_mapped.begin() + (m_bytes * pos) + 49, m_bytes); 27 | } 28 | 29 | kindex::kindex() {} 30 | 31 | kindex::kindex(const index_infos& i, bool cache) 32 | : m_infos(i), m_mutexes(i.nb_partitions()), m_cache(cache) 33 | { 34 | m_partitions.resize(m_infos.nb_partitions()); 35 | 36 | if (m_cache) 37 | { 38 | for (std::size_t p = 0; p < m_infos.nb_partitions(); ++p) 39 | { 40 | init(p); 41 | } 42 | } 43 | } 44 | 45 | kindex::~kindex() 46 | { 47 | if (m_cache) 48 | { 49 | for (std::size_t p = 0; p < m_infos.nb_partitions(); ++p) 50 | { 51 | unmap(p); 52 | } 53 | } 54 | } 55 | 56 | void kindex::init(std::size_t p) 57 | { 58 | m_partitions[p] = std::make_unique(m_infos.get_partition(p), m_infos.nb_samples(), m_infos.bw()); 59 | } 60 | 61 | void kindex::unmap(std::size_t p) 62 | { 63 | m_partitions[p] = nullptr; 64 | } 65 | 66 | std::string kindex::name() const 67 | { 68 | return m_infos.name(); 69 | } 70 | 71 | std::string kindex::directory() const 72 | { 73 | return m_infos.path(); 74 | } 75 | 76 | index_infos& kindex::infos() 77 | { 78 | return m_infos; 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /lib/src/query/query.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | namespace kmq { 5 | 6 | query_response::query_response(const std::string& name, std::size_t n, std::size_t nbits, std::size_t width) 7 | : m_name(std::move(name)), m_block_size(((nbits * width) + 7) / 8) 8 | { 9 | m_responses.resize(n * m_block_size, 0); 10 | } 11 | 12 | std::size_t query_response::block_size() const 13 | { 14 | return m_block_size; 15 | } 16 | 17 | std::size_t query_response::nbk() const 18 | { 19 | return m_responses.size() / m_block_size; 20 | } 21 | 22 | const std::string& query_response::name() const 23 | { 24 | return m_name; 25 | } 26 | 27 | std::uint8_t* query_response::get(std::size_t mer_pos) 28 | { 29 | return m_responses.data() + (mer_pos * m_block_size); 30 | } 31 | 32 | void query_response::free() 33 | { 34 | free_container(m_responses); 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /lib/src/threadpool.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | namespace kmq { 4 | 5 | ThreadPool::ThreadPool(size_type threads) 6 | { 7 | if (threads < _n) _n = threads; 8 | for (size_t i = 0; i < _n; i++) 9 | { 10 | _pool.push_back(std::thread(&ThreadPool::worker, this, i)); 11 | } 12 | } 13 | 14 | void ThreadPool::restart(size_type threads) 15 | { 16 | _n = threads ? threads : _n; 17 | _stop = false; 18 | _pool.clear(); 19 | for (size_t i = 0; i < _n; i++) 20 | { 21 | _pool.push_back(std::thread(&ThreadPool::worker, this, i)); 22 | } 23 | } 24 | 25 | ThreadPool::~ThreadPool() 26 | { 27 | { 28 | std::unique_lock lock(_queue_mutex); 29 | _stop = true; 30 | } 31 | _condition.notify_all(); 32 | for (std::thread& t : _pool) 33 | if (t.joinable()) t.join(); 34 | } 35 | 36 | void ThreadPool::join_all() 37 | { 38 | { 39 | std::unique_lock lock(_queue_mutex); 40 | _stop = true; 41 | } 42 | _condition.notify_all(); 43 | for (std::thread& t : _pool) t.join(); 44 | } 45 | 46 | void ThreadPool::join(int i) 47 | { 48 | if (_pool[i].joinable()) _pool[i].join(); 49 | } 50 | 51 | void ThreadPool::worker(int i) 52 | { 53 | while (true) 54 | { 55 | std::function task; 56 | { 57 | std::unique_lock lock(this->_queue_mutex); 58 | this->_condition.wait(lock, [this] { return this->_stop || !this->_queue.empty(); }); 59 | if (this->_stop && this->_queue.empty()) return; 60 | task = std::move(this->_queue.front()); 61 | this->_queue.pop(); 62 | } 63 | task(i); 64 | } 65 | } 66 | 67 | } // end of namespace kmdiff 68 | 69 | -------------------------------------------------------------------------------- /lib/src/utils.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | namespace kmq { 4 | 5 | std::string rtrim(const std::string& s, const std::string& v) 6 | { 7 | size_t end = s.find_last_not_of(v); 8 | if (end == std::string::npos) 9 | return ""; 10 | return s.substr(0, end+1); 11 | } 12 | 13 | std::string ltrim(const std::string& s, const std::string& v) 14 | { 15 | size_t beg = s.find_first_not_of(v); 16 | if (beg == std::string::npos) 17 | return ""; 18 | return s.substr(beg); 19 | } 20 | 21 | std::string trim(const std::string& s, const std::string& v) 22 | { 23 | return ltrim(rtrim(s, v), v); 24 | } 25 | 26 | std::vector split(const std::string& s, char delim) 27 | { 28 | return split(s, delim, [](const std::string& s) -> std::string {return s;}); 29 | } 30 | 31 | std::size_t directory_size(const std::string& p) 32 | { 33 | std::size_t size = 0; 34 | for (auto& e : fs::directory_iterator(p)) 35 | { 36 | if (e.is_regular_file()) 37 | size += fs::file_size(e); 38 | } 39 | 40 | return size / 1024 / 1024; 41 | } 42 | 43 | Timer::Timer() 44 | { 45 | start(); 46 | } 47 | 48 | std::string Timer::formatted() 49 | { 50 | if (m_running) end(); 51 | std::chrono::seconds seconds(std::chrono::duration_cast(m_end_time - m_start_time)); 52 | auto d = std::chrono::duration_cast(seconds); seconds -= d; 53 | auto h = std::chrono::duration_cast(seconds); seconds -= h; 54 | auto m = std::chrono::duration_cast(seconds); seconds -= m; 55 | auto s = std::chrono::duration_cast(seconds); 56 | 57 | std::stringstream ss; ss.fill('0'); 58 | if (d.count()) 59 | ss << std::setw(2) << d.count() << "d"; 60 | if (h.count()) 61 | ss << std::setw(2) << h.count() << "h"; 62 | if (m.count()) 63 | ss << std::setw(2) << m.count() << "m"; 64 | ss << std::setw(2) << s.count() << "s"; 65 | return ss.str(); 66 | } 67 | 68 | void Timer::start() 69 | { 70 | m_running = true; 71 | m_start_time = std::chrono::steady_clock::now(); 72 | } 73 | void Timer::end() 74 | { 75 | m_running = false; 76 | m_end_time = std::chrono::steady_clock::now(); 77 | } 78 | } -------------------------------------------------------------------------------- /pykmindex/.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/#use-with-ide 110 | .pdm.toml 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | 131 | # Spyder project settings 132 | .spyderproject 133 | .spyproject 134 | 135 | # Rope project settings 136 | .ropeproject 137 | 138 | # mkdocs documentation 139 | /site 140 | 141 | # mypy 142 | .mypy_cache/ 143 | .dmypy.json 144 | dmypy.json 145 | 146 | # Pyre type checker 147 | .pyre/ 148 | 149 | # pytype static type analyzer 150 | .pytype/ 151 | 152 | # Cython debug symbols 153 | cython_debug/ 154 | 155 | # PyCharm 156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 158 | # and can be added to the global gitignore or merged into this file. For a more nuclear 159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 160 | #.idea/ 161 | -------------------------------------------------------------------------------- /pykmindex/README.md: -------------------------------------------------------------------------------- 1 | # pykmindex 2 | 3 | Python API to query kmindex-server. See [documentation](https://tlemane.github.io/kmindex/python-api/). 4 | -------------------------------------------------------------------------------- /pykmindex/pykmindex/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tlemane/kmindex/d60cbf4aa3b1cb6750c4f0090bc28c5458a6a348/pykmindex/pykmindex/__init__.py -------------------------------------------------------------------------------- /pykmindex/pykmindex/query.py: -------------------------------------------------------------------------------- 1 | from typing import List, Union, Iterator 2 | 3 | class Query: 4 | PAYLOAD = { 5 | "id": "", 6 | "seq": [""], 7 | "index": [""], 8 | "r": 0.0, 9 | "z": 3 10 | } 11 | 12 | def __init__(self, 13 | qid: str, 14 | sequence: Union[str, List[str]], 15 | index: Union[str, List[str]], 16 | r: float=0.0, 17 | z: int=3) -> None: 18 | self.qid = qid 19 | self.sequences = self._wrap_as_list(sequence) 20 | self.indexes = self._wrap_as_list(index) 21 | self.r = r 22 | self.z = z 23 | 24 | 25 | def _wrap_as_list(self, v: Union[str, List[str]]) -> List[str]: 26 | if isinstance(v, str): 27 | return [v] 28 | return v 29 | 30 | @property 31 | def payload(self): 32 | return { 33 | "id": self.qid, 34 | "seq": self.sequences, 35 | "index": self.indexes, 36 | "r": self.r, 37 | "z": self.z 38 | } 39 | 40 | class QBatch: 41 | def __init__(self) -> None: 42 | self.queries: List[Query] = [] 43 | 44 | def push(self, 45 | qid: str, 46 | sequence: Union[str, List[str]], 47 | index: Union[str, List[str]], 48 | r: float=0.0, 49 | z: int=3) -> None: 50 | self.queries.append(Query(qid, sequence, index, r, z)) 51 | 52 | def __iter__(self) -> Iterator[Query]: 53 | return iter(self.queries) 54 | 55 | class QResponse: 56 | def __init__(self, data) -> None: 57 | self.result = {} 58 | self.err = "" 59 | self._from_json(data) 60 | 61 | def _from_json(self, data) -> None: 62 | if "error" in data: 63 | self.err = data["error"] 64 | return 65 | 66 | self.id = list(data[list(data.keys())[0]].keys())[0] 67 | for k in data.keys(): 68 | self.result[k] = data[k][self.id] 69 | 70 | def __bool__(self) -> None: 71 | return not bool(self.err) 72 | 73 | -------------------------------------------------------------------------------- /pykmindex/pykmindex/server.py: -------------------------------------------------------------------------------- 1 | import uplink 2 | import asyncio 3 | 4 | from uplink import Consumer, Body, post, get, json 5 | from pykmindex.query import Query, QBatch, QResponse 6 | 7 | def connect(url: str, port: int): 8 | return KmIndexServer(url, port) 9 | 10 | class KmIndexAPI(Consumer): 11 | 12 | @get("kmindex/infos") 13 | def infos(self): 14 | "Fetches indexes informations." 15 | 16 | @json 17 | @post("kmindex/query") 18 | def query(self, data: Body): 19 | "Query index." 20 | 21 | 22 | async def query(async_client, payload): 23 | response = await async_client.query() 24 | 25 | class KmIndexServer: 26 | def __init__(self, url: str, port: int): 27 | self._url = url 28 | self._port = port 29 | self._api = KmIndexAPI(base_url = f"{self._url}:{self._port}") 30 | self._api_async = KmIndexAPI(base_url = f"{self._url}:{self._port}", client = uplink.AiohttpClient()) 31 | self.infos = dict(self._api.infos().json()) 32 | 33 | def submit(self, query: Query): 34 | return QResponse(self._api.query(query.payload).json()) 35 | 36 | async def async_query(self, query: Query): 37 | response = await self._api_async.query(query.payload) 38 | json = await response.json(content_type=None) 39 | return QResponse(json) 40 | 41 | def submit_async(self, queries: QBatch): 42 | futures = [self.async_query(q) for q in queries] 43 | loop = asyncio.new_event_loop() 44 | asyncio.set_event_loop(loop) 45 | return loop.run_until_complete(asyncio.gather(*futures)) 46 | 47 | 48 | -------------------------------------------------------------------------------- /pykmindex/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "pykmindex" 3 | version = "0.2.2" 4 | description = "Python API to query kmindex server." 5 | authors = [ "Teo Lemane " ] 6 | maintainers = [ "Teo Lemane " ] 7 | license = "MIT" 8 | classifiers = [ 9 | "License :: OSI Approved :: MIT License", 10 | "Programming Language :: Python :: 3" 11 | ] 12 | readme = "README.md" 13 | keywords = [ "k-mer", "indexing", "kmindex" ] 14 | homepage = "https://github/tlemane/kmindex" 15 | documentation = "https://github.io/tlemane/kmindex" 16 | 17 | [tool.poetry.dependencies] 18 | python = "^3.10" 19 | uplink = "^0.9.7" 20 | aiohttp = "^3.8.4" 21 | 22 | 23 | [build-system] 24 | requires = ["poetry-core"] 25 | build-backend = "poetry.core.masonry.api" 26 | -------------------------------------------------------------------------------- /tests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_subdirectory(units/kmindex-lib) 2 | add_subdirectory(app) 3 | -------------------------------------------------------------------------------- /tests/app/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_test( 2 | NAME kmindex:pa 3 | COMMAND ${CMAKE_SOURCE_DIR}/tests/app/run_test_pa.sh ${CMAKE_BINARY_DIR}/app/kmindex/kmindex . 4 | WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests/data 5 | ) 6 | 7 | add_test( 8 | NAME kmindex:abs 9 | COMMAND ${CMAKE_SOURCE_DIR}/tests/app/run_test_abs.sh ${CMAKE_BINARY_DIR}/app/kmindex/kmindex . 10 | WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests/data 11 | ) 12 | -------------------------------------------------------------------------------- /tests/app/run_test_abs.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | kmindex_bin=$1 4 | directory=$2 5 | 6 | cd ${directory} 7 | 8 | rm -rf out_tmp 9 | 10 | ${kmindex_bin} query -i indexes/index \ 11 | -n abs \ 12 | -q datasets/abs_dataset/1.fasta \ 13 | -z 4 \ 14 | -f matrix \ 15 | -o out_tmp -t 1 2> /dev/null 16 | 17 | diff out_tmp/abs.tsv outputs/q1_z4_abs.tsv || exit 1 18 | rm -rf out_tmp 19 | 20 | ${kmindex_bin} query -i indexes/index \ 21 | -n abs \ 22 | -q datasets/abs_dataset/1.fasta \ 23 | -z 4 \ 24 | -f json \ 25 | -o out_tmp -t 1 2> /dev/null 26 | 27 | diff out_tmp/abs.json outputs/q1_z4_abs.json || exit 1 28 | rm -rf out_tmp 29 | 30 | exit 0 31 | -------------------------------------------------------------------------------- /tests/app/run_test_pa.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | kmindex_bin=$1 4 | directory=$2 5 | 6 | cd ${directory} 7 | 8 | rm -rf out_tmp 9 | 10 | ${kmindex_bin} query -i indexes/index \ 11 | -n pa \ 12 | -q datasets/pa_dataset/1.fasta \ 13 | -z 4 \ 14 | -f matrix \ 15 | -z 5 \ 16 | -o out_tmp -t 1 2> /dev/null 17 | 18 | diff out_tmp/pa.tsv outputs/q1_z5_pa.tsv || exit 1 19 | rm -rf out_tmp 20 | 21 | ${kmindex_bin} query -i indexes/index \ 22 | -n pa \ 23 | -q datasets/pa_dataset/1.fasta \ 24 | -z 4 \ 25 | -f json \ 26 | -r 0.5 \ 27 | -o out_tmp -t 1 2> /dev/null 28 | 29 | diff out_tmp/pa.json outputs/q1_pa.json || exit 1 30 | rm -rf out_tmp 31 | 32 | exit 0 33 | -------------------------------------------------------------------------------- /tests/data/datasets/abs_dataset/fof.txt: -------------------------------------------------------------------------------- 1 | D0: /home/tlemane/work/dev/repo/tmp/kmindex/tests/units/data/datasets/abs_dataset/0.fasta 2 | D1: /home/tlemane/work/dev/repo/tmp/kmindex/tests/units/data/datasets/abs_dataset/1.fasta 3 | D2: /home/tlemane/work/dev/repo/tmp/kmindex/tests/units/data/datasets/abs_dataset/2.fasta 4 | D3: /home/tlemane/work/dev/repo/tmp/kmindex/tests/units/data/datasets/abs_dataset/3.fasta 5 | D4: /home/tlemane/work/dev/repo/tmp/kmindex/tests/units/data/datasets/abs_dataset/4.fasta 6 | D5: /home/tlemane/work/dev/repo/tmp/kmindex/tests/units/data/datasets/abs_dataset/5.fasta 7 | D6: /home/tlemane/work/dev/repo/tmp/kmindex/tests/units/data/datasets/abs_dataset/6.fasta 8 | D7: /home/tlemane/work/dev/repo/tmp/kmindex/tests/units/data/datasets/abs_dataset/7.fasta 9 | D8: /home/tlemane/work/dev/repo/tmp/kmindex/tests/units/data/datasets/abs_dataset/8.fasta 10 | D9: /home/tlemane/work/dev/repo/tmp/kmindex/tests/units/data/datasets/abs_dataset/9.fasta 11 | D10: /home/tlemane/work/dev/repo/tmp/kmindex/tests/units/data/datasets/abs_dataset/10.fasta 12 | D11: /home/tlemane/work/dev/repo/tmp/kmindex/tests/units/data/datasets/abs_dataset/11.fasta 13 | D12: /home/tlemane/work/dev/repo/tmp/kmindex/tests/units/data/datasets/abs_dataset/12.fasta 14 | D13: /home/tlemane/work/dev/repo/tmp/kmindex/tests/units/data/datasets/abs_dataset/13.fasta 15 | D14: /home/tlemane/work/dev/repo/tmp/kmindex/tests/units/data/datasets/abs_dataset/14.fasta 16 | D15: /home/tlemane/work/dev/repo/tmp/kmindex/tests/units/data/datasets/abs_dataset/15.fasta 17 | -------------------------------------------------------------------------------- /tests/data/datasets/pa_dataset/fof.txt: -------------------------------------------------------------------------------- 1 | D0: /home/tlemane/work/dev/repo/tmp/kmindex/tests/units/data/datasets/pa_dataset/0.fasta 2 | D1: /home/tlemane/work/dev/repo/tmp/kmindex/tests/units/data/datasets/pa_dataset/1.fasta 3 | D2: /home/tlemane/work/dev/repo/tmp/kmindex/tests/units/data/datasets/pa_dataset/2.fasta 4 | D3: /home/tlemane/work/dev/repo/tmp/kmindex/tests/units/data/datasets/pa_dataset/3.fasta 5 | D4: /home/tlemane/work/dev/repo/tmp/kmindex/tests/units/data/datasets/pa_dataset/4.fasta 6 | D5: /home/tlemane/work/dev/repo/tmp/kmindex/tests/units/data/datasets/pa_dataset/5.fasta 7 | D6: /home/tlemane/work/dev/repo/tmp/kmindex/tests/units/data/datasets/pa_dataset/6.fasta 8 | D7: /home/tlemane/work/dev/repo/tmp/kmindex/tests/units/data/datasets/pa_dataset/7.fasta 9 | D8: /home/tlemane/work/dev/repo/tmp/kmindex/tests/units/data/datasets/pa_dataset/8.fasta 10 | D9: /home/tlemane/work/dev/repo/tmp/kmindex/tests/units/data/datasets/pa_dataset/9.fasta 11 | D10: /home/tlemane/work/dev/repo/tmp/kmindex/tests/units/data/datasets/pa_dataset/10.fasta 12 | D11: /home/tlemane/work/dev/repo/tmp/kmindex/tests/units/data/datasets/pa_dataset/11.fasta 13 | D12: /home/tlemane/work/dev/repo/tmp/kmindex/tests/units/data/datasets/pa_dataset/12.fasta 14 | D13: /home/tlemane/work/dev/repo/tmp/kmindex/tests/units/data/datasets/pa_dataset/13.fasta 15 | D14: /home/tlemane/work/dev/repo/tmp/kmindex/tests/units/data/datasets/pa_dataset/14.fasta 16 | D15: /home/tlemane/work/dev/repo/tmp/kmindex/tests/units/data/datasets/pa_dataset/15.fasta 17 | -------------------------------------------------------------------------------- /tests/data/indexes/abs_index/build_infos.txt: -------------------------------------------------------------------------------- 1 | kmtricks v1.2.1 2 | 3 | - HOST - 4 | build host: Linux-6.1.3 5 | run host: Linux 6.1.3 6 | 7 | - BUILD - 8 | c compiler: GNU 11.3.0 9 | cxx compiler: GNU 11.3.0 10 | conda: OFF 11 | static: OFF 12 | native: ON 13 | modules: ON 14 | socks: OFF 15 | howde: OFF 16 | dev: OFF 17 | kmer: 32 18 | max_c: 4294967295 19 | 20 | - GIT SHA1 / VERSION - 21 | kmtricks: fa8c48d 22 | sdsl: c32874c 23 | bcli: 3e4f493 24 | fmt: 0544a227 25 | kff: 97d135e 26 | lz4: 4de56b3 27 | spdlog: v1.2.1-1811-g5b4c4f3f 28 | xxhash: 6853ddc 29 | gtest: release-1.8.0-2774-g96f4ce02 30 | croaring: v0.3.3-17-g2d5c927 31 | robin-hood-hasing: 24b3f50 32 | turbop: 4ab9f5b 33 | cfrcat: 2f9da97 34 | indicators: v1.9-36-gcdcff01 35 | 36 | Contact: teo.lemane@inria.fr 37 | -------------------------------------------------------------------------------- /tests/data/indexes/abs_index/config_gatb/gatb.config: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tlemane/kmindex/d60cbf4aa3b1cb6750c4f0090bc28c5458a6a348/tests/data/indexes/abs_index/config_gatb/gatb.config -------------------------------------------------------------------------------- /tests/data/indexes/abs_index/hash.info: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tlemane/kmindex/d60cbf4aa3b1cb6750c4f0090bc28c5458a6a348/tests/data/indexes/abs_index/hash.info -------------------------------------------------------------------------------- /tests/data/indexes/abs_index/kmtricks.fof: -------------------------------------------------------------------------------- 1 | D0: datasets/abs_dataset/0.fasta 2 | D1: datasets/abs_dataset/1.fasta 3 | D2: datasets/abs_dataset/2.fasta 4 | D3: datasets/abs_dataset/3.fasta 5 | D4: datasets/abs_dataset/4.fasta 6 | D5: datasets/abs_dataset/5.fasta 7 | D6: datasets/abs_dataset/6.fasta 8 | D7: datasets/abs_dataset/7.fasta 9 | D8: datasets/abs_dataset/8.fasta 10 | D9: datasets/abs_dataset/9.fasta 11 | D10: datasets/abs_dataset/10.fasta 12 | D11: datasets/abs_dataset/11.fasta 13 | D12: datasets/abs_dataset/12.fasta 14 | D13: datasets/abs_dataset/13.fasta 15 | D14: datasets/abs_dataset/14.fasta 16 | D15: datasets/abs_dataset/15.fasta 17 | -------------------------------------------------------------------------------- /tests/data/indexes/abs_index/matrices/matrix_0.cmbf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tlemane/kmindex/d60cbf4aa3b1cb6750c4f0090bc28c5458a6a348/tests/data/indexes/abs_index/matrices/matrix_0.cmbf -------------------------------------------------------------------------------- /tests/data/indexes/abs_index/matrices/matrix_1.cmbf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tlemane/kmindex/d60cbf4aa3b1cb6750c4f0090bc28c5458a6a348/tests/data/indexes/abs_index/matrices/matrix_1.cmbf -------------------------------------------------------------------------------- /tests/data/indexes/abs_index/matrices/matrix_2.cmbf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tlemane/kmindex/d60cbf4aa3b1cb6750c4f0090bc28c5458a6a348/tests/data/indexes/abs_index/matrices/matrix_2.cmbf -------------------------------------------------------------------------------- /tests/data/indexes/abs_index/matrices/matrix_3.cmbf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tlemane/kmindex/d60cbf4aa3b1cb6750c4f0090bc28c5458a6a348/tests/data/indexes/abs_index/matrices/matrix_3.cmbf -------------------------------------------------------------------------------- /tests/data/indexes/abs_index/merge_infos/partition0.merge_info: -------------------------------------------------------------------------------- 1 | NON_SOLID 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 | RESCUED 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 3 | UNIQUE_WO_RESCUE 3649 3701 3664 3876 3540 3605 3822 3649 3651 3874 3953 3862 3568 3709 4192 3874 4 | UNIQUE_W_RESCUE 3649 3701 3664 3876 3540 3605 3822 3649 3651 3874 3953 3862 3568 3709 4192 3874 5 | TOTAL_WO_RESCUE 9164 9606 8860 9869 9335 8650 9450 9118 9598 10122 9664 10291 8960 9274 11031 9519 6 | TOTAL_W_RESCUE 9164 9606 8860 9869 9335 8650 9450 9118 9598 10122 9664 10291 8960 9274 11031 9519 7 | -------------------------------------------------------------------------------- /tests/data/indexes/abs_index/merge_infos/partition1.merge_info: -------------------------------------------------------------------------------- 1 | NON_SOLID 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 | RESCUED 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 3 | UNIQUE_WO_RESCUE 3862 3798 3787 3764 3932 3709 3498 3922 3894 3456 3781 3926 3907 3876 3533 3560 4 | UNIQUE_W_RESCUE 3862 3798 3787 3764 3932 3709 3498 3922 3894 3456 3781 3926 3907 3876 3533 3560 5 | TOTAL_WO_RESCUE 9401 9906 9399 9314 10525 8871 8856 9648 10189 9302 9350 10192 9809 9794 9235 8619 6 | TOTAL_W_RESCUE 9401 9906 9399 9314 10525 8871 8856 9648 10189 9302 9350 10192 9809 9794 9235 8619 7 | -------------------------------------------------------------------------------- /tests/data/indexes/abs_index/merge_infos/partition2.merge_info: -------------------------------------------------------------------------------- 1 | NON_SOLID 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 | RESCUED 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 3 | UNIQUE_WO_RESCUE 3838 3832 3690 3820 3680 3867 3830 3761 3926 3724 3485 3621 3566 3752 3547 3836 4 | UNIQUE_W_RESCUE 3838 3832 3690 3820 3680 3867 3830 3761 3926 3724 3485 3621 3566 3752 3547 3836 5 | TOTAL_WO_RESCUE 9866 9686 9186 9386 9713 9077 9617 9599 10915 9957 8266 9503 8723 9285 9475 9602 6 | TOTAL_W_RESCUE 9866 9686 9186 9386 9713 9077 9617 9599 10915 9957 8266 9503 8723 9285 9475 9602 7 | -------------------------------------------------------------------------------- /tests/data/indexes/abs_index/merge_infos/partition3.merge_info: -------------------------------------------------------------------------------- 1 | NON_SOLID 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 | RESCUED 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 3 | UNIQUE_WO_RESCUE 3721 3732 3912 3603 3888 3875 3901 3740 3581 4008 3837 3650 4014 3718 3786 3784 4 | UNIQUE_W_RESCUE 3721 3732 3912 3603 3888 3875 3901 3740 3581 4008 3837 3650 4014 3718 3786 3784 5 | TOTAL_WO_RESCUE 9265 9866 9187 9127 9947 9806 10077 9255 9046 10747 9276 9610 9748 8659 10083 9348 6 | TOTAL_W_RESCUE 9265 9866 9187 9127 9947 9806 10077 9255 9046 10747 9276 9610 9748 8659 10083 9348 7 | -------------------------------------------------------------------------------- /tests/data/indexes/abs_index/options.txt: -------------------------------------------------------------------------------- 1 | Options: dir=indexes/abs_index, verbosity=info, nb_threads=12, fof=datasets/abs_dataset/fof.txt, kmer_size=25, c_ab_min=1, m_ab_min=1, r_min=1, m_ab_min_path=, m_ab_min_f=0, m_ab_float=0, save_if=0, minim_size=10, minim_type=0, repart_type=0, nb_parts=0, bloom_size=800000, keep_tmp=0, lz4=0, kff=0, skip_merge=0, hist=0, focus=0.5, restrict_to=1, bwidth=2, mode=bfc, format=bin, bf_format=howdesbt, count_format=hash, until=all -------------------------------------------------------------------------------- /tests/data/indexes/abs_index/partition_infos/D0.pinfo: -------------------------------------------------------------------------------- 1 | 9164 2 | 9401 3 | 9866 4 | 9265 5 | -------------------------------------------------------------------------------- /tests/data/indexes/abs_index/partition_infos/D1.pinfo: -------------------------------------------------------------------------------- 1 | 9606 2 | 9906 3 | 9686 4 | 9866 5 | -------------------------------------------------------------------------------- /tests/data/indexes/abs_index/partition_infos/D10.pinfo: -------------------------------------------------------------------------------- 1 | 9664 2 | 9350 3 | 8266 4 | 9276 5 | -------------------------------------------------------------------------------- /tests/data/indexes/abs_index/partition_infos/D11.pinfo: -------------------------------------------------------------------------------- 1 | 10291 2 | 10192 3 | 9503 4 | 9610 5 | -------------------------------------------------------------------------------- /tests/data/indexes/abs_index/partition_infos/D12.pinfo: -------------------------------------------------------------------------------- 1 | 8960 2 | 9809 3 | 8723 4 | 9748 5 | -------------------------------------------------------------------------------- /tests/data/indexes/abs_index/partition_infos/D13.pinfo: -------------------------------------------------------------------------------- 1 | 9274 2 | 9794 3 | 9285 4 | 8659 5 | -------------------------------------------------------------------------------- /tests/data/indexes/abs_index/partition_infos/D14.pinfo: -------------------------------------------------------------------------------- 1 | 11031 2 | 9235 3 | 9475 4 | 10083 5 | -------------------------------------------------------------------------------- /tests/data/indexes/abs_index/partition_infos/D15.pinfo: -------------------------------------------------------------------------------- 1 | 9519 2 | 8619 3 | 9602 4 | 9348 5 | -------------------------------------------------------------------------------- /tests/data/indexes/abs_index/partition_infos/D2.pinfo: -------------------------------------------------------------------------------- 1 | 8860 2 | 9399 3 | 9186 4 | 9187 5 | -------------------------------------------------------------------------------- /tests/data/indexes/abs_index/partition_infos/D3.pinfo: -------------------------------------------------------------------------------- 1 | 9869 2 | 9314 3 | 9386 4 | 9127 5 | -------------------------------------------------------------------------------- /tests/data/indexes/abs_index/partition_infos/D4.pinfo: -------------------------------------------------------------------------------- 1 | 9335 2 | 10525 3 | 9713 4 | 9947 5 | -------------------------------------------------------------------------------- /tests/data/indexes/abs_index/partition_infos/D5.pinfo: -------------------------------------------------------------------------------- 1 | 8650 2 | 8871 3 | 9077 4 | 9806 5 | -------------------------------------------------------------------------------- /tests/data/indexes/abs_index/partition_infos/D6.pinfo: -------------------------------------------------------------------------------- 1 | 9450 2 | 8856 3 | 9617 4 | 10077 5 | -------------------------------------------------------------------------------- /tests/data/indexes/abs_index/partition_infos/D7.pinfo: -------------------------------------------------------------------------------- 1 | 9118 2 | 9648 3 | 9599 4 | 9255 5 | -------------------------------------------------------------------------------- /tests/data/indexes/abs_index/partition_infos/D8.pinfo: -------------------------------------------------------------------------------- 1 | 9598 2 | 10189 3 | 10915 4 | 9046 5 | -------------------------------------------------------------------------------- /tests/data/indexes/abs_index/partition_infos/D9.pinfo: -------------------------------------------------------------------------------- 1 | 10122 2 | 9302 3 | 9957 4 | 10747 5 | -------------------------------------------------------------------------------- /tests/data/indexes/abs_index/run_infos.txt: -------------------------------------------------------------------------------- 1 | Time: 0 seconds 2 | Memory: 545.109375MB 3 | -------------------------------------------------------------------------------- /tests/data/indexes/index/abs: -------------------------------------------------------------------------------- 1 | ./indexes/abs_index -------------------------------------------------------------------------------- /tests/data/indexes/index/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "index": { 3 | "abs": { 4 | "bloom_size": 800000, 5 | "bw": 2, 6 | "index_size": 3, 7 | "kmindex_version": "0.2.0", 8 | "kmtricks_version": "1.2.1", 9 | "minim_size": 10, 10 | "nb_partitions": 4, 11 | "nb_samples": 16, 12 | "samples": [ 13 | "D0", 14 | "D1", 15 | "D2", 16 | "D3", 17 | "D4", 18 | "D5", 19 | "D6", 20 | "D7", 21 | "D8", 22 | "D9", 23 | "D10", 24 | "D11", 25 | "D12", 26 | "D13", 27 | "D14", 28 | "D15" 29 | ], 30 | "sha1": "6a0f7482d4533cf2fef2e44979a0d11912f18272", 31 | "smer_size": 25 32 | }, 33 | "pa": { 34 | "bloom_size": 800000, 35 | "bw": 1, 36 | "index_size": 1, 37 | "kmindex_version": "0.2.0", 38 | "kmtricks_version": "1.2.1", 39 | "minim_size": 10, 40 | "nb_partitions": 4, 41 | "nb_samples": 16, 42 | "samples": [ 43 | "D0", 44 | "D1", 45 | "D2", 46 | "D3", 47 | "D4", 48 | "D5", 49 | "D6", 50 | "D7", 51 | "D8", 52 | "D9", 53 | "D10", 54 | "D11", 55 | "D12", 56 | "D13", 57 | "D14", 58 | "D15" 59 | ], 60 | "sha1": "7a0284634c173063e0a136e8628249be6f68ef36", 61 | "smer_size": 25 62 | } 63 | }, 64 | "path": "./indexes/index" 65 | } 66 | -------------------------------------------------------------------------------- /tests/data/indexes/index/pa: -------------------------------------------------------------------------------- 1 | ./indexes/pa_index -------------------------------------------------------------------------------- /tests/data/indexes/pa_index/build_infos.txt: -------------------------------------------------------------------------------- 1 | kmtricks v1.2.1 2 | 3 | - HOST - 4 | build host: Linux-6.1.3 5 | run host: Linux 6.1.3 6 | 7 | - BUILD - 8 | c compiler: GNU 11.3.0 9 | cxx compiler: GNU 11.3.0 10 | conda: OFF 11 | static: OFF 12 | native: ON 13 | modules: ON 14 | socks: OFF 15 | howde: OFF 16 | dev: OFF 17 | kmer: 32 18 | max_c: 4294967295 19 | 20 | - GIT SHA1 / VERSION - 21 | kmtricks: fa8c48d 22 | sdsl: c32874c 23 | bcli: 3e4f493 24 | fmt: 0544a227 25 | kff: 97d135e 26 | lz4: 4de56b3 27 | spdlog: v1.2.1-1811-g5b4c4f3f 28 | xxhash: 6853ddc 29 | gtest: release-1.8.0-2774-g96f4ce02 30 | croaring: v0.3.3-17-g2d5c927 31 | robin-hood-hasing: 24b3f50 32 | turbop: 4ab9f5b 33 | cfrcat: 2f9da97 34 | indicators: v1.9-36-gcdcff01 35 | 36 | Contact: teo.lemane@inria.fr 37 | -------------------------------------------------------------------------------- /tests/data/indexes/pa_index/config_gatb/gatb.config: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tlemane/kmindex/d60cbf4aa3b1cb6750c4f0090bc28c5458a6a348/tests/data/indexes/pa_index/config_gatb/gatb.config -------------------------------------------------------------------------------- /tests/data/indexes/pa_index/fpr/partition_0.txt: -------------------------------------------------------------------------------- 1 | 0.086617 2 | 0.087014 3 | 0.085603 4 | 0.086274 5 | 0.083644 6 | 0.084372 7 | 0.088752 8 | 0.086051 9 | 0.085237 10 | 0.084999 11 | 0.088419 12 | 0.086247 13 | 0.087981 14 | 0.087881 15 | 0.086078 16 | 0.087621 17 | -------------------------------------------------------------------------------- /tests/data/indexes/pa_index/fpr/partition_1.txt: -------------------------------------------------------------------------------- 1 | 0.089294 2 | 0.087767 3 | 0.083969 4 | 0.086220 5 | 0.089057 6 | 0.087712 7 | 0.085859 8 | 0.087434 9 | 0.085177 10 | 0.084679 11 | 0.086411 12 | 0.087981 13 | 0.087703 14 | 0.088351 15 | 0.086786 16 | 0.084376 17 | -------------------------------------------------------------------------------- /tests/data/indexes/pa_index/fpr/partition_2.txt: -------------------------------------------------------------------------------- 1 | 0.085196 2 | 0.085918 3 | 0.089194 4 | 0.086727 5 | 0.086462 6 | 0.087553 7 | 0.084834 8 | 0.087279 9 | 0.088579 10 | 0.089175 11 | 0.084692 12 | 0.084981 13 | 0.084468 14 | 0.086622 15 | 0.084106 16 | 0.089294 17 | -------------------------------------------------------------------------------- /tests/data/indexes/pa_index/fpr/partition_3.txt: -------------------------------------------------------------------------------- 1 | 0.085552 2 | 0.085968 3 | 0.087968 4 | 0.087443 5 | 0.087804 6 | 0.086864 7 | 0.087315 8 | 0.085543 9 | 0.086932 10 | 0.087430 11 | 0.086695 12 | 0.087653 13 | 0.086073 14 | 0.083667 15 | 0.089043 16 | 0.085502 17 | -------------------------------------------------------------------------------- /tests/data/indexes/pa_index/hash.info: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tlemane/kmindex/d60cbf4aa3b1cb6750c4f0090bc28c5458a6a348/tests/data/indexes/pa_index/hash.info -------------------------------------------------------------------------------- /tests/data/indexes/pa_index/kmtricks.fof: -------------------------------------------------------------------------------- 1 | D0: datasets/pa_dataset/0.fasta 2 | D1: datasets/pa_dataset/1.fasta 3 | D2: datasets/pa_dataset/2.fasta 4 | D3: datasets/pa_dataset/3.fasta 5 | D4: datasets/pa_dataset/4.fasta 6 | D5: datasets/pa_dataset/5.fasta 7 | D6: datasets/pa_dataset/6.fasta 8 | D7: datasets/pa_dataset/7.fasta 9 | D8: datasets/pa_dataset/8.fasta 10 | D9: datasets/pa_dataset/9.fasta 11 | D10: datasets/pa_dataset/10.fasta 12 | D11: datasets/pa_dataset/11.fasta 13 | D12: datasets/pa_dataset/12.fasta 14 | D13: datasets/pa_dataset/13.fasta 15 | D14: datasets/pa_dataset/14.fasta 16 | D15: datasets/pa_dataset/15.fasta 17 | -------------------------------------------------------------------------------- /tests/data/indexes/pa_index/matrices/matrix_0.cmbf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tlemane/kmindex/d60cbf4aa3b1cb6750c4f0090bc28c5458a6a348/tests/data/indexes/pa_index/matrices/matrix_0.cmbf -------------------------------------------------------------------------------- /tests/data/indexes/pa_index/matrices/matrix_1.cmbf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tlemane/kmindex/d60cbf4aa3b1cb6750c4f0090bc28c5458a6a348/tests/data/indexes/pa_index/matrices/matrix_1.cmbf -------------------------------------------------------------------------------- /tests/data/indexes/pa_index/matrices/matrix_2.cmbf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tlemane/kmindex/d60cbf4aa3b1cb6750c4f0090bc28c5458a6a348/tests/data/indexes/pa_index/matrices/matrix_2.cmbf -------------------------------------------------------------------------------- /tests/data/indexes/pa_index/matrices/matrix_3.cmbf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tlemane/kmindex/d60cbf4aa3b1cb6750c4f0090bc28c5458a6a348/tests/data/indexes/pa_index/matrices/matrix_3.cmbf -------------------------------------------------------------------------------- /tests/data/indexes/pa_index/merge_infos/partition0.merge_info: -------------------------------------------------------------------------------- 1 | NON_SOLID 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 | RESCUED 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 3 | UNIQUE_WO_RESCUE 18120 18207 17898 18045 17470 17629 18588 17996 17818 17766 18515 18039 18419 18397 18002 18340 4 | UNIQUE_W_RESCUE 18120 18207 17898 18045 17470 17629 18588 17996 17818 17766 18515 18039 18419 18397 18002 18340 5 | TOTAL_WO_RESCUE 19020 19074 18780 18920 18294 18505 19502 18844 18729 18626 19481 18888 19351 19295 18855 19207 6 | TOTAL_W_RESCUE 19020 19074 18780 18920 18294 18505 19502 18844 18729 18626 19481 18888 19351 19295 18855 19207 7 | -------------------------------------------------------------------------------- /tests/data/indexes/pa_index/merge_infos/partition1.merge_info: -------------------------------------------------------------------------------- 1 | NON_SOLID 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 | RESCUED 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 3 | UNIQUE_WO_RESCUE 18707 18372 17541 18033 18655 18360 17954 18299 17805 17696 18075 18419 18358 18500 18157 17630 4 | UNIQUE_W_RESCUE 18707 18372 17541 18033 18655 18360 17954 18299 17805 17696 18075 18419 18358 18500 18157 17630 5 | TOTAL_WO_RESCUE 19641 19248 18319 18872 19554 19295 18795 19210 18702 18545 18945 19326 19286 19383 19019 18481 6 | TOTAL_W_RESCUE 19641 19248 18319 18872 19554 19295 18795 19210 18702 18545 18945 19326 19286 19383 19019 18481 7 | -------------------------------------------------------------------------------- /tests/data/indexes/pa_index/merge_infos/partition2.merge_info: -------------------------------------------------------------------------------- 1 | NON_SOLID 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 | RESCUED 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 3 | UNIQUE_WO_RESCUE 17809 17967 18685 18144 18086 18325 17730 18265 18550 18681 17699 17762 17650 18121 17571 18707 4 | UNIQUE_W_RESCUE 17809 17967 18685 18144 18086 18325 17730 18265 18550 18681 17699 17762 17650 18121 17571 18707 5 | TOTAL_WO_RESCUE 18616 18863 19588 19031 18910 19190 18588 19159 19503 19645 18548 18560 18520 19027 18471 19635 6 | TOTAL_W_RESCUE 18616 18863 19588 19031 18910 19190 18588 19159 19503 19645 18548 18560 18520 19027 18471 19635 7 | -------------------------------------------------------------------------------- /tests/data/indexes/pa_index/merge_infos/partition3.merge_info: -------------------------------------------------------------------------------- 1 | NON_SOLID 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 | RESCUED 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 3 | UNIQUE_WO_RESCUE 17887 17978 18416 18301 18380 18174 18273 17885 18189 18298 18137 18347 18001 17475 18652 17876 4 | UNIQUE_W_RESCUE 17887 17978 18416 18301 18380 18174 18273 17885 18189 18298 18137 18347 18001 17475 18652 17876 5 | TOTAL_WO_RESCUE 18723 18815 19313 19177 19242 19010 19115 18787 19066 19184 19026 19226 18843 18295 19655 18677 6 | TOTAL_W_RESCUE 18723 18815 19313 19177 19242 19010 19115 18787 19066 19184 19026 19226 18843 18295 19655 18677 7 | -------------------------------------------------------------------------------- /tests/data/indexes/pa_index/options.txt: -------------------------------------------------------------------------------- 1 | Options: dir=indexes/pa_index, verbosity=info, nb_threads=12, fof=datasets/pa_dataset/fof.txt, kmer_size=25, c_ab_min=1, m_ab_min=1, r_min=1, m_ab_min_path=, m_ab_min_f=0, m_ab_float=0, save_if=0, minim_size=10, minim_type=0, repart_type=0, nb_parts=0, bloom_size=800000, keep_tmp=0, lz4=0, kff=0, skip_merge=0, hist=0, focus=0.5, restrict_to=1, bwidth=2, mode=bf, format=bin, bf_format=howdesbt, count_format=hash, until=all -------------------------------------------------------------------------------- /tests/data/indexes/pa_index/partition_infos/D0.pinfo: -------------------------------------------------------------------------------- 1 | 19020 2 | 19641 3 | 18616 4 | 18723 5 | -------------------------------------------------------------------------------- /tests/data/indexes/pa_index/partition_infos/D1.pinfo: -------------------------------------------------------------------------------- 1 | 19074 2 | 19248 3 | 18863 4 | 18815 5 | -------------------------------------------------------------------------------- /tests/data/indexes/pa_index/partition_infos/D10.pinfo: -------------------------------------------------------------------------------- 1 | 19481 2 | 18945 3 | 18548 4 | 19026 5 | -------------------------------------------------------------------------------- /tests/data/indexes/pa_index/partition_infos/D11.pinfo: -------------------------------------------------------------------------------- 1 | 18888 2 | 19326 3 | 18560 4 | 19226 5 | -------------------------------------------------------------------------------- /tests/data/indexes/pa_index/partition_infos/D12.pinfo: -------------------------------------------------------------------------------- 1 | 19351 2 | 19286 3 | 18520 4 | 18843 5 | -------------------------------------------------------------------------------- /tests/data/indexes/pa_index/partition_infos/D13.pinfo: -------------------------------------------------------------------------------- 1 | 19295 2 | 19383 3 | 19027 4 | 18295 5 | -------------------------------------------------------------------------------- /tests/data/indexes/pa_index/partition_infos/D14.pinfo: -------------------------------------------------------------------------------- 1 | 18855 2 | 19019 3 | 18471 4 | 19655 5 | -------------------------------------------------------------------------------- /tests/data/indexes/pa_index/partition_infos/D15.pinfo: -------------------------------------------------------------------------------- 1 | 19207 2 | 18481 3 | 19635 4 | 18677 5 | -------------------------------------------------------------------------------- /tests/data/indexes/pa_index/partition_infos/D2.pinfo: -------------------------------------------------------------------------------- 1 | 18780 2 | 18319 3 | 19588 4 | 19313 5 | -------------------------------------------------------------------------------- /tests/data/indexes/pa_index/partition_infos/D3.pinfo: -------------------------------------------------------------------------------- 1 | 18920 2 | 18872 3 | 19031 4 | 19177 5 | -------------------------------------------------------------------------------- /tests/data/indexes/pa_index/partition_infos/D4.pinfo: -------------------------------------------------------------------------------- 1 | 18294 2 | 19554 3 | 18910 4 | 19242 5 | -------------------------------------------------------------------------------- /tests/data/indexes/pa_index/partition_infos/D5.pinfo: -------------------------------------------------------------------------------- 1 | 18505 2 | 19295 3 | 19190 4 | 19010 5 | -------------------------------------------------------------------------------- /tests/data/indexes/pa_index/partition_infos/D6.pinfo: -------------------------------------------------------------------------------- 1 | 19502 2 | 18795 3 | 18588 4 | 19115 5 | -------------------------------------------------------------------------------- /tests/data/indexes/pa_index/partition_infos/D7.pinfo: -------------------------------------------------------------------------------- 1 | 18844 2 | 19210 3 | 19159 4 | 18787 5 | -------------------------------------------------------------------------------- /tests/data/indexes/pa_index/partition_infos/D8.pinfo: -------------------------------------------------------------------------------- 1 | 18729 2 | 18702 3 | 19503 4 | 19066 5 | -------------------------------------------------------------------------------- /tests/data/indexes/pa_index/partition_infos/D9.pinfo: -------------------------------------------------------------------------------- 1 | 18626 2 | 18545 3 | 19645 4 | 19184 5 | -------------------------------------------------------------------------------- /tests/data/indexes/pa_index/run_infos.txt: -------------------------------------------------------------------------------- 1 | Time: 0 seconds 2 | Memory: 544.371094MB 3 | -------------------------------------------------------------------------------- /tests/units/kmindex-lib/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable(kmindex-lib-tests 2 | "main.cpp" 3 | "mer.cpp" 4 | ) 5 | 6 | target_link_libraries(kmindex-lib-tests PUBLIC gtest pthread kmindex-lib fmt) 7 | 8 | if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0.0) 9 | target_link_libraries(kmindex-lib-tests PUBLIC stdc++fs) 10 | endif() 11 | 12 | add_test( 13 | NAME kmindex:lib 14 | COMMAND kmindex-lib-tests 15 | ) 16 | set_property( 17 | TEST kmindex:lib 18 | PROPERTY ENVIRONMENT "KMINDEX_TEST_DATA=${CMAKE_SOURCE_DIR}/tests/data" 19 | ) 20 | -------------------------------------------------------------------------------- /tests/units/kmindex-lib/main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | namespace fs = std::filesystem; 5 | 6 | int main(int argc, char* argv[]) 7 | { 8 | ::testing::InitGoogleTest(&argc, argv); 9 | int r = RUN_ALL_TESTS(); 10 | return r; 11 | } 12 | -------------------------------------------------------------------------------- /tests/units/kmindex-lib/mer.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | static const std::string data_path(std::getenv("KMINDEX_TEST_DATA")); 9 | 10 | TEST(kmindex_lib_mer, smer_iterator) 11 | { 12 | kmq::index_infos infos("index", fmt::format("{}/indexes/pa_index", data_path)); 13 | 14 | auto r = infos.get_repartition(); 15 | auto h = infos.get_hash_w(); 16 | 17 | std::string seq("AGCAGATTAAGCATATATATCCGACGATAACACTA"); 18 | 19 | const std::vector parts { 20 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3 21 | }; 22 | 23 | const std::vector hashes { 24 | 102658, 25 | 123165, 26 | 161320, 27 | 193062, 28 | 164937, 29 | 169286, 30 | 60386, 31 | 144980, 32 | 49698, 33 | 4538, 34 | 189458 35 | }; 36 | 37 | EXPECT_EQ(infos.minim_size(), 10); 38 | EXPECT_EQ(infos.smer_size(), 25); 39 | EXPECT_EQ(infos.bw(), 1); 40 | 41 | kmq::smer_hasher<32> hh(r, h, infos.minim_size()); 42 | 43 | std::vector smers; 44 | for (auto& e : kmq::smer_iterator(seq, infos.smer_size(), hh)) 45 | smers.push_back(e); 46 | 47 | EXPECT_EQ(smers.size(), parts.size()); 48 | 49 | for (std::size_t i = 0; i < smers.size(); ++i) 50 | { 51 | EXPECT_EQ(smers[i].p, parts[i]); 52 | EXPECT_EQ(smers[i].h, hashes[i]); 53 | } 54 | } 55 | 56 | -------------------------------------------------------------------------------- /thirdparty/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_policy(SET CMP0077 NEW) 2 | 3 | if (STATIC_BUILD) 4 | set(ZLIB_USE_STATIC_LIBS ON) 5 | endif() 6 | 7 | set(THIRD_DIR ${CMAKE_SOURCE_DIR}/thirdparty) 8 | 9 | set(EXT_INSTALL_DIR ${CMAKE_BINARY_DIR}/ext-install) 10 | 11 | include(FetchContent) 12 | set(FETCHCONTENT_BASE_DIR "${CMAKE_BINARY_DIR}/ext-src") 13 | 14 | include(GNUInstallDirs) 15 | 16 | add_library(bcli_lib INTERFACE) 17 | target_include_directories(bcli_lib SYSTEM INTERFACE ${THIRD_DIR}/bcli/include) 18 | 19 | add_library(kmtricks INTERFACE) 20 | target_include_directories(kmtricks SYSTEM INTERFACE ${THIRD_DIR}/kmtricks/include/) 21 | 22 | add_subdirectory(fmt) 23 | add_subdirectory(nlohmann_json) 24 | 25 | add_library(mio_mmap INTERFACE) 26 | 27 | set(mio.installation OFF) 28 | set(INSTALL_SUBPROJECTS OFF) 29 | 30 | add_subdirectory(mio) 31 | 32 | target_link_libraries(mio_mmap INTERFACE mio::mio) 33 | 34 | if (WITH_TESTS) 35 | add_subdirectory(gtest) 36 | endif() 37 | add_subdirectory(xxhash) 38 | 39 | add_subdirectory(zlib) 40 | 41 | add_library(kseqpp INTERFACE) 42 | include(ExternalProject) 43 | 44 | ExternalProject_Add( 45 | KSEQPP 46 | PREFIX KSEQPP 47 | SOURCE_DIR ${CMAKE_SOURCE_DIR}/thirdparty/kseqpp 48 | INSTALL_COMMAND "" 49 | LOG_CONFIGURE ON 50 | LOG_BUILD ON 51 | ) 52 | 53 | target_include_directories(kseqpp INTERFACE ${THIRD_DIR}/kseqpp/include ${THIRD_DIR}/kseqpp/include/kseq++) 54 | 55 | add_subdirectory(spdlog) 56 | 57 | if (WITH_SERVER) 58 | add_library(sws INTERFACE) 59 | ExternalProject_Add( 60 | SWS 61 | PREFIX SWS 62 | SOURCE_DIR ${CMAKE_SOURCE_DIR}/thirdparty/Simple-Web-Server 63 | CMAKE_ARGS -DUSE_OPENSSL=OFF 64 | INSTALL_COMMAND "" 65 | LOG_CONFIGURE ON 66 | LOG_BUILD ON 67 | ) 68 | 69 | target_include_directories(sws SYSTEM INTERFACE ${THIRD_DIR}/Simple-Web-Server/) 70 | endif() 71 | 72 | add_library(sha1h INTERFACE) 73 | target_include_directories(sha1h INTERFACE ${THIRD_DIR}/sha1) 74 | 75 | add_library(semver INTERFACE) 76 | target_include_directories(semver INTERFACE ${THIRD_DIR}/semver/include) 77 | 78 | add_library(bitpack INTERFACE) 79 | target_include_directories(bitpack SYSTEM INTERFACE ${THIRD_DIR}/bitpacker/include) 80 | target_include_directories(bitpack SYSTEM INTERFACE ${THIRD_DIR}/span-lite/include) 81 | 82 | add_library(atomic_queue INTERFACE) 83 | target_include_directories(atomic_queue SYSTEM INTERFACE ${THIRD_DIR}/atomic_queue/include) 84 | 85 | add_library(deps INTERFACE) 86 | add_dependencies(deps KSEQPP SWS mio_mmap xxhash zlib bcli_lib fmt kmtricks) 87 | 88 | 89 | -------------------------------------------------------------------------------- /thirdparty/fmt/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(fmt INTERFACE) 2 | 3 | option(FORCE_BUILD_FMT "Always build, instead of using the system version." OFF) 4 | option(FMT_HEADER_ONLY "Use header only version of fmt." OFF) 5 | set(FMT_MIN_REQUIRED_VERSION "8.1.0...<9.0.0") 6 | 7 | if (NOT FORCE_BUILD_FMT AND NOT FORCE_BUILD_ALL) 8 | 9 | find_package(fmt ${FMT_MIN_REQUIRED_VERSION} CONFIG QUIET) 10 | 11 | if (TARGET fmt::fmt) 12 | 13 | get_target_property(FMT_INCLUDE_DIRS fmt::fmt INTERFACE_INCLUDE_DIRECTORIES) 14 | 15 | find_library(FMT_LIBRARY fmt) 16 | 17 | file(READ ${FMT_INCLUDE_DIRS}/fmt/core.h fmt_core) 18 | 19 | if (NOT fmt_core MATCHES "FMT_VERSION ([0-9]+)([0-9][0-9])([0-9][0-9])") 20 | set(FMT_VERSION "Unknown version") 21 | endif() 22 | 23 | set(FMT_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}") 24 | 25 | if (FMT_HEADER_ONLY) 26 | target_link_libraries(fmt INTERFACE fmt::fmt-header-only) 27 | message(STATUS "Found FMT: header-only (found suitable version \"${FMT_VERSION}\", minimum required is \"${FMT_MIN_REQUIRED_VERSION}\")") 28 | else() 29 | target_link_libraries(fmt INTERFACE fmt::fmt) 30 | message(STATUS "Found FMT: ${FMT_LIBRARY} (found suitable version \"${FMT_VERSION}\", minimum required is \"${FMT_MIN_REQUIRED_VERSION}\")") 31 | endif() 32 | 33 | return() 34 | 35 | else() 36 | message(STATUS "FMT not found: use bundled version.") 37 | endif() 38 | 39 | endif() 40 | 41 | set(FMT_URL "https://github.com/fmtlib/fmt/archive/refs/tags/9.1.0.tar.gz") 42 | set(FMT_HASH "21fac48cae8f3b4a5783ae06b443973a") 43 | set(FMT_DIR ${EXT_INSTALL_DIR}/fmt) 44 | 45 | include(ExternalProject) 46 | ExternalProject_Add( 47 | FMT-external 48 | PREFIX FMT-external 49 | URL ${FMT_URL} 50 | URL_MD5 ${FMT_HASH} 51 | EXCLUDE_FROM_ALL 52 | LOG_DOWNLOAD ON 53 | LOG_CONFIGURE ON 54 | LOG_BUILD ON 55 | LOG_INSTALL ON 56 | BUILD_IN_SOURCE true 57 | CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${FMT_DIR} 58 | ) 59 | 60 | target_include_directories(fmt SYSTEM INTERFACE 61 | "${FMT_DIR}/include" 62 | ) 63 | 64 | if (NOT FMT_HEADER_ONLY) 65 | target_link_libraries(fmt INTERFACE 66 | "${FMT_DIR}/${CMAKE_INSTALL_LIBDIR}/${CMAKE_STATIC_LIBRARY_PREFIX}fmt${CMAKE_STATIC_LIBRARY_SUFFIX}" 67 | ) 68 | else() 69 | target_compile_definitions(fmt INTERFACE FMT_HEADER_ONLY) 70 | endif() 71 | 72 | add_dependencies(fmt FMT-external) 73 | 74 | -------------------------------------------------------------------------------- /thirdparty/gtest/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(gtest INTERFACE) 2 | add_library(gmock INTERFACE) 3 | 4 | option(FORCE_BUILD_GTEST "Always build, instead of using the system version." OFF) 5 | set(GTEST_MIN_REQUIRED_VERSION "1.0") 6 | 7 | if (NOT FORCE_BUILD_GTEST AND NOT FORCE_BUILD_ALL) 8 | 9 | enable_testing() 10 | find_package(GTest ${FMT_MIN_REQUIRED_VERSION} CONFIG QUIET) 11 | 12 | if (TARGET GTest::gtest AND TARGET GTest::gmock) 13 | find_library(GTEST_LIBRARY gtest) 14 | target_link_libraries(gtest INTERFACE GTest::gtest GTest::gtest_main GTest::gmock GTest::gmock_main) 15 | message(STATUS "Found GTEST: ${GTEST_LIBRARY} (minimum required is \"${GTEST_MIN_REQUIRED_VERSION}\")") 16 | return() 17 | else() 18 | message(STATUS "GTEST not found: use bundled version.") 19 | endif() 20 | 21 | endif() 22 | 23 | set(GTEST_URL "https://github.com/google/googletest/archive/refs/tags/release-1.11.0.tar.gz") 24 | set(GTEST_HASH "e8a8df240b6938bb6384155d4c37d937") 25 | set(GTEST_DIR ${EXT_INSTALL_DIR}/gtest) 26 | 27 | include(ExternalProject) 28 | ExternalProject_Add( 29 | GTEST-external 30 | PREFIX GTEST-external 31 | URL ${GTEST_URL} 32 | URL_MD5 ${GTEST_HASH} 33 | EXCLUDE_FROM_ALL 34 | LOG_DOWNLOAD ON 35 | LOG_CONFIGURE ON 36 | LOG_BUILD ON 37 | LOG_INSTALL ON 38 | BUILD_IN_SOURCE true 39 | CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${GTEST_DIR} 40 | ) 41 | 42 | target_include_directories(gtest SYSTEM INTERFACE 43 | "${GTEST_DIR}/include" 44 | ) 45 | 46 | target_link_libraries(gtest INTERFACE 47 | "${GTEST_DIR}/${CMAKE_INSTALL_LIBDIR}/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}" 48 | "${GTEST_DIR}/${CMAKE_INSTALL_LIBDIR}/${CMAKE_STATIC_LIBRARY_PREFIX}gtest_main${CMAKE_STATIC_LIBRARY_SUFFIX}" 49 | pthread 50 | ) 51 | 52 | add_dependencies(gtest GTEST-external) 53 | -------------------------------------------------------------------------------- /thirdparty/nlohmann_json/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(json INTERFACE) 2 | target_include_directories(json INTERFACE ${THIRD_DIR}/nlohmann_json/include/single_include) 3 | -------------------------------------------------------------------------------- /thirdparty/nlohmann_json/include/LICENSE.MIT: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2013-2022 Niels Lohmann 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /thirdparty/nlohmann_json/include/include/nlohmann/adl_serializer.hpp: -------------------------------------------------------------------------------- 1 | // __ _____ _____ _____ 2 | // __| | __| | | | JSON for Modern C++ 3 | // | | |__ | | | | | | version 3.11.2 4 | // |_____|_____|_____|_|___| https://github.com/nlohmann/json 5 | // 6 | // SPDX-FileCopyrightText: 2013-2022 Niels Lohmann 7 | // SPDX-License-Identifier: MIT 8 | 9 | #pragma once 10 | 11 | #include 12 | 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | NLOHMANN_JSON_NAMESPACE_BEGIN 19 | 20 | /// @sa https://json.nlohmann.me/api/adl_serializer/ 21 | template 22 | struct adl_serializer 23 | { 24 | /// @brief convert a JSON value to any value type 25 | /// @sa https://json.nlohmann.me/api/adl_serializer/from_json/ 26 | template 27 | static auto from_json(BasicJsonType && j, TargetType& val) noexcept( 28 | noexcept(::nlohmann::from_json(std::forward(j), val))) 29 | -> decltype(::nlohmann::from_json(std::forward(j), val), void()) 30 | { 31 | ::nlohmann::from_json(std::forward(j), val); 32 | } 33 | 34 | /// @brief convert a JSON value to any value type 35 | /// @sa https://json.nlohmann.me/api/adl_serializer/from_json/ 36 | template 37 | static auto from_json(BasicJsonType && j) noexcept( 38 | noexcept(::nlohmann::from_json(std::forward(j), detail::identity_tag {}))) 39 | -> decltype(::nlohmann::from_json(std::forward(j), detail::identity_tag {})) 40 | { 41 | return ::nlohmann::from_json(std::forward(j), detail::identity_tag {}); 42 | } 43 | 44 | /// @brief convert any value type to a JSON value 45 | /// @sa https://json.nlohmann.me/api/adl_serializer/to_json/ 46 | template 47 | static auto to_json(BasicJsonType& j, TargetType && val) noexcept( 48 | noexcept(::nlohmann::to_json(j, std::forward(val)))) 49 | -> decltype(::nlohmann::to_json(j, std::forward(val)), void()) 50 | { 51 | ::nlohmann::to_json(j, std::forward(val)); 52 | } 53 | }; 54 | 55 | NLOHMANN_JSON_NAMESPACE_END 56 | -------------------------------------------------------------------------------- /thirdparty/nlohmann_json/include/include/nlohmann/byte_container_with_subtype.hpp: -------------------------------------------------------------------------------- 1 | // __ _____ _____ _____ 2 | // __| | __| | | | JSON for Modern C++ 3 | // | | |__ | | | | | | version 3.11.2 4 | // |_____|_____|_____|_|___| https://github.com/nlohmann/json 5 | // 6 | // SPDX-FileCopyrightText: 2013-2022 Niels Lohmann 7 | // SPDX-License-Identifier: MIT 8 | 9 | #pragma once 10 | 11 | #include // uint8_t, uint64_t 12 | #include // tie 13 | #include // move 14 | 15 | #include 16 | 17 | NLOHMANN_JSON_NAMESPACE_BEGIN 18 | 19 | /// @brief an internal type for a backed binary type 20 | /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/ 21 | template 22 | class byte_container_with_subtype : public BinaryType 23 | { 24 | public: 25 | using container_type = BinaryType; 26 | using subtype_type = std::uint64_t; 27 | 28 | /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/byte_container_with_subtype/ 29 | byte_container_with_subtype() noexcept(noexcept(container_type())) 30 | : container_type() 31 | {} 32 | 33 | /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/byte_container_with_subtype/ 34 | byte_container_with_subtype(const container_type& b) noexcept(noexcept(container_type(b))) 35 | : container_type(b) 36 | {} 37 | 38 | /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/byte_container_with_subtype/ 39 | byte_container_with_subtype(container_type&& b) noexcept(noexcept(container_type(std::move(b)))) 40 | : container_type(std::move(b)) 41 | {} 42 | 43 | /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/byte_container_with_subtype/ 44 | byte_container_with_subtype(const container_type& b, subtype_type subtype_) noexcept(noexcept(container_type(b))) 45 | : container_type(b) 46 | , m_subtype(subtype_) 47 | , m_has_subtype(true) 48 | {} 49 | 50 | /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/byte_container_with_subtype/ 51 | byte_container_with_subtype(container_type&& b, subtype_type subtype_) noexcept(noexcept(container_type(std::move(b)))) 52 | : container_type(std::move(b)) 53 | , m_subtype(subtype_) 54 | , m_has_subtype(true) 55 | {} 56 | 57 | bool operator==(const byte_container_with_subtype& rhs) const 58 | { 59 | return std::tie(static_cast(*this), m_subtype, m_has_subtype) == 60 | std::tie(static_cast(rhs), rhs.m_subtype, rhs.m_has_subtype); 61 | } 62 | 63 | bool operator!=(const byte_container_with_subtype& rhs) const 64 | { 65 | return !(rhs == *this); 66 | } 67 | 68 | /// @brief sets the binary subtype 69 | /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/set_subtype/ 70 | void set_subtype(subtype_type subtype_) noexcept 71 | { 72 | m_subtype = subtype_; 73 | m_has_subtype = true; 74 | } 75 | 76 | /// @brief return the binary subtype 77 | /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/subtype/ 78 | constexpr subtype_type subtype() const noexcept 79 | { 80 | return m_has_subtype ? m_subtype : static_cast(-1); 81 | } 82 | 83 | /// @brief return whether the value has a subtype 84 | /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/has_subtype/ 85 | constexpr bool has_subtype() const noexcept 86 | { 87 | return m_has_subtype; 88 | } 89 | 90 | /// @brief clears the binary subtype 91 | /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/clear_subtype/ 92 | void clear_subtype() noexcept 93 | { 94 | m_subtype = 0; 95 | m_has_subtype = false; 96 | } 97 | 98 | private: 99 | subtype_type m_subtype = 0; 100 | bool m_has_subtype = false; 101 | }; 102 | 103 | NLOHMANN_JSON_NAMESPACE_END 104 | -------------------------------------------------------------------------------- /thirdparty/nlohmann_json/include/include/nlohmann/detail/abi_macros.hpp: -------------------------------------------------------------------------------- 1 | // __ _____ _____ _____ 2 | // __| | __| | | | JSON for Modern C++ 3 | // | | |__ | | | | | | version 3.11.2 4 | // |_____|_____|_____|_|___| https://github.com/nlohmann/json 5 | // 6 | // SPDX-FileCopyrightText: 2013-2022 Niels Lohmann 7 | // SPDX-License-Identifier: MIT 8 | 9 | #pragma once 10 | 11 | // This file contains all macro definitions affecting or depending on the ABI 12 | 13 | #ifndef JSON_SKIP_LIBRARY_VERSION_CHECK 14 | #if defined(NLOHMANN_JSON_VERSION_MAJOR) && defined(NLOHMANN_JSON_VERSION_MINOR) && defined(NLOHMANN_JSON_VERSION_PATCH) 15 | #if NLOHMANN_JSON_VERSION_MAJOR != 3 || NLOHMANN_JSON_VERSION_MINOR != 11 || NLOHMANN_JSON_VERSION_PATCH != 2 16 | #warning "Already included a different version of the library!" 17 | #endif 18 | #endif 19 | #endif 20 | 21 | #define NLOHMANN_JSON_VERSION_MAJOR 3 // NOLINT(modernize-macro-to-enum) 22 | #define NLOHMANN_JSON_VERSION_MINOR 11 // NOLINT(modernize-macro-to-enum) 23 | #define NLOHMANN_JSON_VERSION_PATCH 2 // NOLINT(modernize-macro-to-enum) 24 | 25 | #ifndef JSON_DIAGNOSTICS 26 | #define JSON_DIAGNOSTICS 0 27 | #endif 28 | 29 | #ifndef JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON 30 | #define JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON 0 31 | #endif 32 | 33 | #if JSON_DIAGNOSTICS 34 | #define NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS _diag 35 | #else 36 | #define NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS 37 | #endif 38 | 39 | #if JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON 40 | #define NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON _ldvcmp 41 | #else 42 | #define NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON 43 | #endif 44 | 45 | #ifndef NLOHMANN_JSON_NAMESPACE_NO_VERSION 46 | #define NLOHMANN_JSON_NAMESPACE_NO_VERSION 0 47 | #endif 48 | 49 | // Construct the namespace ABI tags component 50 | #define NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b) json_abi ## a ## b 51 | #define NLOHMANN_JSON_ABI_TAGS_CONCAT(a, b) \ 52 | NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b) 53 | 54 | #define NLOHMANN_JSON_ABI_TAGS \ 55 | NLOHMANN_JSON_ABI_TAGS_CONCAT( \ 56 | NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS, \ 57 | NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON) 58 | 59 | // Construct the namespace version component 60 | #define NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT_EX(major, minor, patch) \ 61 | _v ## major ## _ ## minor ## _ ## patch 62 | #define NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT(major, minor, patch) \ 63 | NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT_EX(major, minor, patch) 64 | 65 | #if NLOHMANN_JSON_NAMESPACE_NO_VERSION 66 | #define NLOHMANN_JSON_NAMESPACE_VERSION 67 | #else 68 | #define NLOHMANN_JSON_NAMESPACE_VERSION \ 69 | NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT(NLOHMANN_JSON_VERSION_MAJOR, \ 70 | NLOHMANN_JSON_VERSION_MINOR, \ 71 | NLOHMANN_JSON_VERSION_PATCH) 72 | #endif 73 | 74 | // Combine namespace components 75 | #define NLOHMANN_JSON_NAMESPACE_CONCAT_EX(a, b) a ## b 76 | #define NLOHMANN_JSON_NAMESPACE_CONCAT(a, b) \ 77 | NLOHMANN_JSON_NAMESPACE_CONCAT_EX(a, b) 78 | 79 | #ifndef NLOHMANN_JSON_NAMESPACE 80 | #define NLOHMANN_JSON_NAMESPACE \ 81 | nlohmann::NLOHMANN_JSON_NAMESPACE_CONCAT( \ 82 | NLOHMANN_JSON_ABI_TAGS, \ 83 | NLOHMANN_JSON_NAMESPACE_VERSION) 84 | #endif 85 | 86 | #ifndef NLOHMANN_JSON_NAMESPACE_BEGIN 87 | #define NLOHMANN_JSON_NAMESPACE_BEGIN \ 88 | namespace nlohmann \ 89 | { \ 90 | inline namespace NLOHMANN_JSON_NAMESPACE_CONCAT( \ 91 | NLOHMANN_JSON_ABI_TAGS, \ 92 | NLOHMANN_JSON_NAMESPACE_VERSION) \ 93 | { 94 | #endif 95 | 96 | #ifndef NLOHMANN_JSON_NAMESPACE_END 97 | #define NLOHMANN_JSON_NAMESPACE_END \ 98 | } /* namespace (inline namespace) NOLINT(readability/namespace) */ \ 99 | } // namespace nlohmann 100 | #endif 101 | -------------------------------------------------------------------------------- /thirdparty/nlohmann_json/include/include/nlohmann/detail/hash.hpp: -------------------------------------------------------------------------------- 1 | // __ _____ _____ _____ 2 | // __| | __| | | | JSON for Modern C++ 3 | // | | |__ | | | | | | version 3.11.2 4 | // |_____|_____|_____|_|___| https://github.com/nlohmann/json 5 | // 6 | // SPDX-FileCopyrightText: 2013-2022 Niels Lohmann 7 | // SPDX-License-Identifier: MIT 8 | 9 | #pragma once 10 | 11 | #include // uint8_t 12 | #include // size_t 13 | #include // hash 14 | 15 | #include 16 | #include 17 | 18 | NLOHMANN_JSON_NAMESPACE_BEGIN 19 | namespace detail 20 | { 21 | 22 | // boost::hash_combine 23 | inline std::size_t combine(std::size_t seed, std::size_t h) noexcept 24 | { 25 | seed ^= h + 0x9e3779b9 + (seed << 6U) + (seed >> 2U); 26 | return seed; 27 | } 28 | 29 | /*! 30 | @brief hash a JSON value 31 | 32 | The hash function tries to rely on std::hash where possible. Furthermore, the 33 | type of the JSON value is taken into account to have different hash values for 34 | null, 0, 0U, and false, etc. 35 | 36 | @tparam BasicJsonType basic_json specialization 37 | @param j JSON value to hash 38 | @return hash value of j 39 | */ 40 | template 41 | std::size_t hash(const BasicJsonType& j) 42 | { 43 | using string_t = typename BasicJsonType::string_t; 44 | using number_integer_t = typename BasicJsonType::number_integer_t; 45 | using number_unsigned_t = typename BasicJsonType::number_unsigned_t; 46 | using number_float_t = typename BasicJsonType::number_float_t; 47 | 48 | const auto type = static_cast(j.type()); 49 | switch (j.type()) 50 | { 51 | case BasicJsonType::value_t::null: 52 | case BasicJsonType::value_t::discarded: 53 | { 54 | return combine(type, 0); 55 | } 56 | 57 | case BasicJsonType::value_t::object: 58 | { 59 | auto seed = combine(type, j.size()); 60 | for (const auto& element : j.items()) 61 | { 62 | const auto h = std::hash {}(element.key()); 63 | seed = combine(seed, h); 64 | seed = combine(seed, hash(element.value())); 65 | } 66 | return seed; 67 | } 68 | 69 | case BasicJsonType::value_t::array: 70 | { 71 | auto seed = combine(type, j.size()); 72 | for (const auto& element : j) 73 | { 74 | seed = combine(seed, hash(element)); 75 | } 76 | return seed; 77 | } 78 | 79 | case BasicJsonType::value_t::string: 80 | { 81 | const auto h = std::hash {}(j.template get_ref()); 82 | return combine(type, h); 83 | } 84 | 85 | case BasicJsonType::value_t::boolean: 86 | { 87 | const auto h = std::hash {}(j.template get()); 88 | return combine(type, h); 89 | } 90 | 91 | case BasicJsonType::value_t::number_integer: 92 | { 93 | const auto h = std::hash {}(j.template get()); 94 | return combine(type, h); 95 | } 96 | 97 | case BasicJsonType::value_t::number_unsigned: 98 | { 99 | const auto h = std::hash {}(j.template get()); 100 | return combine(type, h); 101 | } 102 | 103 | case BasicJsonType::value_t::number_float: 104 | { 105 | const auto h = std::hash {}(j.template get()); 106 | return combine(type, h); 107 | } 108 | 109 | case BasicJsonType::value_t::binary: 110 | { 111 | auto seed = combine(type, j.get_binary().size()); 112 | const auto h = std::hash {}(j.get_binary().has_subtype()); 113 | seed = combine(seed, h); 114 | seed = combine(seed, static_cast(j.get_binary().subtype())); 115 | for (const auto byte : j.get_binary()) 116 | { 117 | seed = combine(seed, std::hash {}(byte)); 118 | } 119 | return seed; 120 | } 121 | 122 | default: // LCOV_EXCL_LINE 123 | JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE 124 | return 0; // LCOV_EXCL_LINE 125 | } 126 | } 127 | 128 | } // namespace detail 129 | NLOHMANN_JSON_NAMESPACE_END 130 | -------------------------------------------------------------------------------- /thirdparty/nlohmann_json/include/include/nlohmann/detail/input/position_t.hpp: -------------------------------------------------------------------------------- 1 | // __ _____ _____ _____ 2 | // __| | __| | | | JSON for Modern C++ 3 | // | | |__ | | | | | | version 3.11.2 4 | // |_____|_____|_____|_|___| https://github.com/nlohmann/json 5 | // 6 | // SPDX-FileCopyrightText: 2013-2022 Niels Lohmann 7 | // SPDX-License-Identifier: MIT 8 | 9 | #pragma once 10 | 11 | #include // size_t 12 | 13 | #include 14 | 15 | NLOHMANN_JSON_NAMESPACE_BEGIN 16 | namespace detail 17 | { 18 | 19 | /// struct to capture the start position of the current token 20 | struct position_t 21 | { 22 | /// the total number of characters read 23 | std::size_t chars_read_total = 0; 24 | /// the number of characters read in the current line 25 | std::size_t chars_read_current_line = 0; 26 | /// the number of lines read 27 | std::size_t lines_read = 0; 28 | 29 | /// conversion to size_t to preserve SAX interface 30 | constexpr operator size_t() const 31 | { 32 | return chars_read_total; 33 | } 34 | }; 35 | 36 | } // namespace detail 37 | NLOHMANN_JSON_NAMESPACE_END 38 | -------------------------------------------------------------------------------- /thirdparty/nlohmann_json/include/include/nlohmann/detail/iterators/internal_iterator.hpp: -------------------------------------------------------------------------------- 1 | // __ _____ _____ _____ 2 | // __| | __| | | | JSON for Modern C++ 3 | // | | |__ | | | | | | version 3.11.2 4 | // |_____|_____|_____|_|___| https://github.com/nlohmann/json 5 | // 6 | // SPDX-FileCopyrightText: 2013-2022 Niels Lohmann 7 | // SPDX-License-Identifier: MIT 8 | 9 | #pragma once 10 | 11 | #include 12 | #include 13 | 14 | NLOHMANN_JSON_NAMESPACE_BEGIN 15 | namespace detail 16 | { 17 | 18 | /*! 19 | @brief an iterator value 20 | 21 | @note This structure could easily be a union, but MSVC currently does not allow 22 | unions members with complex constructors, see https://github.com/nlohmann/json/pull/105. 23 | */ 24 | template struct internal_iterator 25 | { 26 | /// iterator for JSON objects 27 | typename BasicJsonType::object_t::iterator object_iterator {}; 28 | /// iterator for JSON arrays 29 | typename BasicJsonType::array_t::iterator array_iterator {}; 30 | /// generic iterator for all other types 31 | primitive_iterator_t primitive_iterator {}; 32 | }; 33 | 34 | } // namespace detail 35 | NLOHMANN_JSON_NAMESPACE_END 36 | -------------------------------------------------------------------------------- /thirdparty/nlohmann_json/include/include/nlohmann/detail/iterators/iterator_traits.hpp: -------------------------------------------------------------------------------- 1 | // __ _____ _____ _____ 2 | // __| | __| | | | JSON for Modern C++ 3 | // | | |__ | | | | | | version 3.11.2 4 | // |_____|_____|_____|_|___| https://github.com/nlohmann/json 5 | // 6 | // SPDX-FileCopyrightText: 2013-2022 Niels Lohmann 7 | // SPDX-License-Identifier: MIT 8 | 9 | #pragma once 10 | 11 | #include // random_access_iterator_tag 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | NLOHMANN_JSON_NAMESPACE_BEGIN 18 | namespace detail 19 | { 20 | 21 | template 22 | struct iterator_types {}; 23 | 24 | template 25 | struct iterator_types < 26 | It, 27 | void_t> 29 | { 30 | using difference_type = typename It::difference_type; 31 | using value_type = typename It::value_type; 32 | using pointer = typename It::pointer; 33 | using reference = typename It::reference; 34 | using iterator_category = typename It::iterator_category; 35 | }; 36 | 37 | // This is required as some compilers implement std::iterator_traits in a way that 38 | // doesn't work with SFINAE. See https://github.com/nlohmann/json/issues/1341. 39 | template 40 | struct iterator_traits 41 | { 42 | }; 43 | 44 | template 45 | struct iterator_traits < T, enable_if_t < !std::is_pointer::value >> 46 | : iterator_types 47 | { 48 | }; 49 | 50 | template 51 | struct iterator_traits::value>> 52 | { 53 | using iterator_category = std::random_access_iterator_tag; 54 | using value_type = T; 55 | using difference_type = ptrdiff_t; 56 | using pointer = T*; 57 | using reference = T&; 58 | }; 59 | 60 | } // namespace detail 61 | NLOHMANN_JSON_NAMESPACE_END 62 | -------------------------------------------------------------------------------- /thirdparty/nlohmann_json/include/include/nlohmann/detail/iterators/json_reverse_iterator.hpp: -------------------------------------------------------------------------------- 1 | // __ _____ _____ _____ 2 | // __| | __| | | | JSON for Modern C++ 3 | // | | |__ | | | | | | version 3.11.2 4 | // |_____|_____|_____|_|___| https://github.com/nlohmann/json 5 | // 6 | // SPDX-FileCopyrightText: 2013-2022 Niels Lohmann 7 | // SPDX-License-Identifier: MIT 8 | 9 | #pragma once 10 | 11 | #include // ptrdiff_t 12 | #include // reverse_iterator 13 | #include // declval 14 | 15 | #include 16 | 17 | NLOHMANN_JSON_NAMESPACE_BEGIN 18 | namespace detail 19 | { 20 | 21 | ////////////////////// 22 | // reverse_iterator // 23 | ////////////////////// 24 | 25 | /*! 26 | @brief a template for a reverse iterator class 27 | 28 | @tparam Base the base iterator type to reverse. Valid types are @ref 29 | iterator (to create @ref reverse_iterator) and @ref const_iterator (to 30 | create @ref const_reverse_iterator). 31 | 32 | @requirement The class satisfies the following concept requirements: 33 | - 34 | [BidirectionalIterator](https://en.cppreference.com/w/cpp/named_req/BidirectionalIterator): 35 | The iterator that can be moved can be moved in both directions (i.e. 36 | incremented and decremented). 37 | - [OutputIterator](https://en.cppreference.com/w/cpp/named_req/OutputIterator): 38 | It is possible to write to the pointed-to element (only if @a Base is 39 | @ref iterator). 40 | 41 | @since version 1.0.0 42 | */ 43 | template 44 | class json_reverse_iterator : public std::reverse_iterator 45 | { 46 | public: 47 | using difference_type = std::ptrdiff_t; 48 | /// shortcut to the reverse iterator adapter 49 | using base_iterator = std::reverse_iterator; 50 | /// the reference type for the pointed-to element 51 | using reference = typename Base::reference; 52 | 53 | /// create reverse iterator from iterator 54 | explicit json_reverse_iterator(const typename base_iterator::iterator_type& it) noexcept 55 | : base_iterator(it) {} 56 | 57 | /// create reverse iterator from base class 58 | explicit json_reverse_iterator(const base_iterator& it) noexcept : base_iterator(it) {} 59 | 60 | /// post-increment (it++) 61 | json_reverse_iterator operator++(int)& // NOLINT(cert-dcl21-cpp) 62 | { 63 | return static_cast(base_iterator::operator++(1)); 64 | } 65 | 66 | /// pre-increment (++it) 67 | json_reverse_iterator& operator++() 68 | { 69 | return static_cast(base_iterator::operator++()); 70 | } 71 | 72 | /// post-decrement (it--) 73 | json_reverse_iterator operator--(int)& // NOLINT(cert-dcl21-cpp) 74 | { 75 | return static_cast(base_iterator::operator--(1)); 76 | } 77 | 78 | /// pre-decrement (--it) 79 | json_reverse_iterator& operator--() 80 | { 81 | return static_cast(base_iterator::operator--()); 82 | } 83 | 84 | /// add to iterator 85 | json_reverse_iterator& operator+=(difference_type i) 86 | { 87 | return static_cast(base_iterator::operator+=(i)); 88 | } 89 | 90 | /// add to iterator 91 | json_reverse_iterator operator+(difference_type i) const 92 | { 93 | return static_cast(base_iterator::operator+(i)); 94 | } 95 | 96 | /// subtract from iterator 97 | json_reverse_iterator operator-(difference_type i) const 98 | { 99 | return static_cast(base_iterator::operator-(i)); 100 | } 101 | 102 | /// return difference 103 | difference_type operator-(const json_reverse_iterator& other) const 104 | { 105 | return base_iterator(*this) - base_iterator(other); 106 | } 107 | 108 | /// access to successor 109 | reference operator[](difference_type n) const 110 | { 111 | return *(this->operator+(n)); 112 | } 113 | 114 | /// return the key of an object iterator 115 | auto key() const -> decltype(std::declval().key()) 116 | { 117 | auto it = --this->base(); 118 | return it.key(); 119 | } 120 | 121 | /// return the value of an iterator 122 | reference value() const 123 | { 124 | auto it = --this->base(); 125 | return it.operator * (); 126 | } 127 | }; 128 | 129 | } // namespace detail 130 | NLOHMANN_JSON_NAMESPACE_END 131 | -------------------------------------------------------------------------------- /thirdparty/nlohmann_json/include/include/nlohmann/detail/iterators/primitive_iterator.hpp: -------------------------------------------------------------------------------- 1 | // __ _____ _____ _____ 2 | // __| | __| | | | JSON for Modern C++ 3 | // | | |__ | | | | | | version 3.11.2 4 | // |_____|_____|_____|_|___| https://github.com/nlohmann/json 5 | // 6 | // SPDX-FileCopyrightText: 2013-2022 Niels Lohmann 7 | // SPDX-License-Identifier: MIT 8 | 9 | #pragma once 10 | 11 | #include // ptrdiff_t 12 | #include // numeric_limits 13 | 14 | #include 15 | 16 | NLOHMANN_JSON_NAMESPACE_BEGIN 17 | namespace detail 18 | { 19 | 20 | /* 21 | @brief an iterator for primitive JSON types 22 | 23 | This class models an iterator for primitive JSON types (boolean, number, 24 | string). It's only purpose is to allow the iterator/const_iterator classes 25 | to "iterate" over primitive values. Internally, the iterator is modeled by 26 | a `difference_type` variable. Value begin_value (`0`) models the begin, 27 | end_value (`1`) models past the end. 28 | */ 29 | class primitive_iterator_t 30 | { 31 | private: 32 | using difference_type = std::ptrdiff_t; 33 | static constexpr difference_type begin_value = 0; 34 | static constexpr difference_type end_value = begin_value + 1; 35 | 36 | JSON_PRIVATE_UNLESS_TESTED: 37 | /// iterator as signed integer type 38 | difference_type m_it = (std::numeric_limits::min)(); 39 | 40 | public: 41 | constexpr difference_type get_value() const noexcept 42 | { 43 | return m_it; 44 | } 45 | 46 | /// set iterator to a defined beginning 47 | void set_begin() noexcept 48 | { 49 | m_it = begin_value; 50 | } 51 | 52 | /// set iterator to a defined past the end 53 | void set_end() noexcept 54 | { 55 | m_it = end_value; 56 | } 57 | 58 | /// return whether the iterator can be dereferenced 59 | constexpr bool is_begin() const noexcept 60 | { 61 | return m_it == begin_value; 62 | } 63 | 64 | /// return whether the iterator is at end 65 | constexpr bool is_end() const noexcept 66 | { 67 | return m_it == end_value; 68 | } 69 | 70 | friend constexpr bool operator==(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept 71 | { 72 | return lhs.m_it == rhs.m_it; 73 | } 74 | 75 | friend constexpr bool operator<(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept 76 | { 77 | return lhs.m_it < rhs.m_it; 78 | } 79 | 80 | primitive_iterator_t operator+(difference_type n) noexcept 81 | { 82 | auto result = *this; 83 | result += n; 84 | return result; 85 | } 86 | 87 | friend constexpr difference_type operator-(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept 88 | { 89 | return lhs.m_it - rhs.m_it; 90 | } 91 | 92 | primitive_iterator_t& operator++() noexcept 93 | { 94 | ++m_it; 95 | return *this; 96 | } 97 | 98 | primitive_iterator_t operator++(int)& noexcept // NOLINT(cert-dcl21-cpp) 99 | { 100 | auto result = *this; 101 | ++m_it; 102 | return result; 103 | } 104 | 105 | primitive_iterator_t& operator--() noexcept 106 | { 107 | --m_it; 108 | return *this; 109 | } 110 | 111 | primitive_iterator_t operator--(int)& noexcept // NOLINT(cert-dcl21-cpp) 112 | { 113 | auto result = *this; 114 | --m_it; 115 | return result; 116 | } 117 | 118 | primitive_iterator_t& operator+=(difference_type n) noexcept 119 | { 120 | m_it += n; 121 | return *this; 122 | } 123 | 124 | primitive_iterator_t& operator-=(difference_type n) noexcept 125 | { 126 | m_it -= n; 127 | return *this; 128 | } 129 | }; 130 | 131 | } // namespace detail 132 | NLOHMANN_JSON_NAMESPACE_END 133 | -------------------------------------------------------------------------------- /thirdparty/nlohmann_json/include/include/nlohmann/detail/json_ref.hpp: -------------------------------------------------------------------------------- 1 | // __ _____ _____ _____ 2 | // __| | __| | | | JSON for Modern C++ 3 | // | | |__ | | | | | | version 3.11.2 4 | // |_____|_____|_____|_|___| https://github.com/nlohmann/json 5 | // 6 | // SPDX-FileCopyrightText: 2013-2022 Niels Lohmann 7 | // SPDX-License-Identifier: MIT 8 | 9 | #pragma once 10 | 11 | #include 12 | #include 13 | 14 | #include 15 | #include 16 | 17 | NLOHMANN_JSON_NAMESPACE_BEGIN 18 | namespace detail 19 | { 20 | 21 | template 22 | class json_ref 23 | { 24 | public: 25 | using value_type = BasicJsonType; 26 | 27 | json_ref(value_type&& value) 28 | : owned_value(std::move(value)) 29 | {} 30 | 31 | json_ref(const value_type& value) 32 | : value_ref(&value) 33 | {} 34 | 35 | json_ref(std::initializer_list init) 36 | : owned_value(init) 37 | {} 38 | 39 | template < 40 | class... Args, 41 | enable_if_t::value, int> = 0 > 42 | json_ref(Args && ... args) 43 | : owned_value(std::forward(args)...) 44 | {} 45 | 46 | // class should be movable only 47 | json_ref(json_ref&&) noexcept = default; 48 | json_ref(const json_ref&) = delete; 49 | json_ref& operator=(const json_ref&) = delete; 50 | json_ref& operator=(json_ref&&) = delete; 51 | ~json_ref() = default; 52 | 53 | value_type moved_or_copied() const 54 | { 55 | if (value_ref == nullptr) 56 | { 57 | return std::move(owned_value); 58 | } 59 | return *value_ref; 60 | } 61 | 62 | value_type const& operator*() const 63 | { 64 | return value_ref ? *value_ref : owned_value; 65 | } 66 | 67 | value_type const* operator->() const 68 | { 69 | return &** this; 70 | } 71 | 72 | private: 73 | mutable value_type owned_value = nullptr; 74 | value_type const* value_ref = nullptr; 75 | }; 76 | 77 | } // namespace detail 78 | NLOHMANN_JSON_NAMESPACE_END 79 | -------------------------------------------------------------------------------- /thirdparty/nlohmann_json/include/include/nlohmann/detail/macro_unscope.hpp: -------------------------------------------------------------------------------- 1 | // __ _____ _____ _____ 2 | // __| | __| | | | JSON for Modern C++ 3 | // | | |__ | | | | | | version 3.11.2 4 | // |_____|_____|_____|_|___| https://github.com/nlohmann/json 5 | // 6 | // SPDX-FileCopyrightText: 2013-2022 Niels Lohmann 7 | // SPDX-License-Identifier: MIT 8 | 9 | #pragma once 10 | 11 | // restore clang diagnostic settings 12 | #if defined(__clang__) 13 | #pragma clang diagnostic pop 14 | #endif 15 | 16 | // clean up 17 | #undef JSON_ASSERT 18 | #undef JSON_INTERNAL_CATCH 19 | #undef JSON_THROW 20 | #undef JSON_PRIVATE_UNLESS_TESTED 21 | #undef NLOHMANN_BASIC_JSON_TPL_DECLARATION 22 | #undef NLOHMANN_BASIC_JSON_TPL 23 | #undef JSON_EXPLICIT 24 | #undef NLOHMANN_CAN_CALL_STD_FUNC_IMPL 25 | #undef JSON_INLINE_VARIABLE 26 | #undef JSON_NO_UNIQUE_ADDRESS 27 | #undef JSON_DISABLE_ENUM_SERIALIZATION 28 | #undef JSON_USE_GLOBAL_UDLS 29 | 30 | #ifndef JSON_TEST_KEEP_MACROS 31 | #undef JSON_CATCH 32 | #undef JSON_TRY 33 | #undef JSON_HAS_CPP_11 34 | #undef JSON_HAS_CPP_14 35 | #undef JSON_HAS_CPP_17 36 | #undef JSON_HAS_CPP_20 37 | #undef JSON_HAS_FILESYSTEM 38 | #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM 39 | #undef JSON_HAS_THREE_WAY_COMPARISON 40 | #undef JSON_HAS_RANGES 41 | #undef JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON 42 | #endif 43 | 44 | #include 45 | -------------------------------------------------------------------------------- /thirdparty/nlohmann_json/include/include/nlohmann/detail/meta/call_std/begin.hpp: -------------------------------------------------------------------------------- 1 | // __ _____ _____ _____ 2 | // __| | __| | | | JSON for Modern C++ 3 | // | | |__ | | | | | | version 3.11.2 4 | // |_____|_____|_____|_|___| https://github.com/nlohmann/json 5 | // 6 | // SPDX-FileCopyrightText: 2013-2022 Niels Lohmann 7 | // SPDX-License-Identifier: MIT 8 | 9 | #pragma once 10 | 11 | #include 12 | 13 | NLOHMANN_JSON_NAMESPACE_BEGIN 14 | 15 | NLOHMANN_CAN_CALL_STD_FUNC_IMPL(begin); 16 | 17 | NLOHMANN_JSON_NAMESPACE_END 18 | -------------------------------------------------------------------------------- /thirdparty/nlohmann_json/include/include/nlohmann/detail/meta/call_std/end.hpp: -------------------------------------------------------------------------------- 1 | // __ _____ _____ _____ 2 | // __| | __| | | | JSON for Modern C++ 3 | // | | |__ | | | | | | version 3.11.2 4 | // |_____|_____|_____|_|___| https://github.com/nlohmann/json 5 | // 6 | // SPDX-FileCopyrightText: 2013-2022 Niels Lohmann 7 | // SPDX-License-Identifier: MIT 8 | 9 | #pragma once 10 | 11 | #include 12 | 13 | NLOHMANN_JSON_NAMESPACE_BEGIN 14 | 15 | NLOHMANN_CAN_CALL_STD_FUNC_IMPL(end); 16 | 17 | NLOHMANN_JSON_NAMESPACE_END 18 | -------------------------------------------------------------------------------- /thirdparty/nlohmann_json/include/include/nlohmann/detail/meta/detected.hpp: -------------------------------------------------------------------------------- 1 | // __ _____ _____ _____ 2 | // __| | __| | | | JSON for Modern C++ 3 | // | | |__ | | | | | | version 3.11.2 4 | // |_____|_____|_____|_|___| https://github.com/nlohmann/json 5 | // 6 | // SPDX-FileCopyrightText: 2013-2022 Niels Lohmann 7 | // SPDX-License-Identifier: MIT 8 | 9 | #pragma once 10 | 11 | #include 12 | 13 | #include 14 | 15 | NLOHMANN_JSON_NAMESPACE_BEGIN 16 | namespace detail 17 | { 18 | 19 | // https://en.cppreference.com/w/cpp/experimental/is_detected 20 | struct nonesuch 21 | { 22 | nonesuch() = delete; 23 | ~nonesuch() = delete; 24 | nonesuch(nonesuch const&) = delete; 25 | nonesuch(nonesuch const&&) = delete; 26 | void operator=(nonesuch const&) = delete; 27 | void operator=(nonesuch&&) = delete; 28 | }; 29 | 30 | template class Op, 33 | class... Args> 34 | struct detector 35 | { 36 | using value_t = std::false_type; 37 | using type = Default; 38 | }; 39 | 40 | template class Op, class... Args> 41 | struct detector>, Op, Args...> 42 | { 43 | using value_t = std::true_type; 44 | using type = Op; 45 | }; 46 | 47 | template class Op, class... Args> 48 | using is_detected = typename detector::value_t; 49 | 50 | template class Op, class... Args> 51 | struct is_detected_lazy : is_detected { }; 52 | 53 | template class Op, class... Args> 54 | using detected_t = typename detector::type; 55 | 56 | template class Op, class... Args> 57 | using detected_or = detector; 58 | 59 | template class Op, class... Args> 60 | using detected_or_t = typename detected_or::type; 61 | 62 | template class Op, class... Args> 63 | using is_detected_exact = std::is_same>; 64 | 65 | template class Op, class... Args> 66 | using is_detected_convertible = 67 | std::is_convertible, To>; 68 | 69 | } // namespace detail 70 | NLOHMANN_JSON_NAMESPACE_END 71 | -------------------------------------------------------------------------------- /thirdparty/nlohmann_json/include/include/nlohmann/detail/meta/identity_tag.hpp: -------------------------------------------------------------------------------- 1 | // __ _____ _____ _____ 2 | // __| | __| | | | JSON for Modern C++ 3 | // | | |__ | | | | | | version 3.11.2 4 | // |_____|_____|_____|_|___| https://github.com/nlohmann/json 5 | // 6 | // SPDX-FileCopyrightText: 2013-2022 Niels Lohmann 7 | // SPDX-License-Identifier: MIT 8 | 9 | #pragma once 10 | 11 | #include 12 | 13 | NLOHMANN_JSON_NAMESPACE_BEGIN 14 | namespace detail 15 | { 16 | 17 | // dispatching helper struct 18 | template struct identity_tag {}; 19 | 20 | } // namespace detail 21 | NLOHMANN_JSON_NAMESPACE_END 22 | -------------------------------------------------------------------------------- /thirdparty/nlohmann_json/include/include/nlohmann/detail/meta/std_fs.hpp: -------------------------------------------------------------------------------- 1 | // __ _____ _____ _____ 2 | // __| | __| | | | JSON for Modern C++ 3 | // | | |__ | | | | | | version 3.11.2 4 | // |_____|_____|_____|_|___| https://github.com/nlohmann/json 5 | // 6 | // SPDX-FileCopyrightText: 2013-2022 Niels Lohmann 7 | // SPDX-License-Identifier: MIT 8 | 9 | #pragma once 10 | 11 | #include 12 | 13 | #if JSON_HAS_EXPERIMENTAL_FILESYSTEM 14 | #include 15 | NLOHMANN_JSON_NAMESPACE_BEGIN 16 | namespace detail 17 | { 18 | namespace std_fs = std::experimental::filesystem; 19 | } // namespace detail 20 | NLOHMANN_JSON_NAMESPACE_END 21 | #elif JSON_HAS_FILESYSTEM 22 | #include 23 | NLOHMANN_JSON_NAMESPACE_BEGIN 24 | namespace detail 25 | { 26 | namespace std_fs = std::filesystem; 27 | } // namespace detail 28 | NLOHMANN_JSON_NAMESPACE_END 29 | #endif 30 | -------------------------------------------------------------------------------- /thirdparty/nlohmann_json/include/include/nlohmann/detail/meta/void_t.hpp: -------------------------------------------------------------------------------- 1 | // __ _____ _____ _____ 2 | // __| | __| | | | JSON for Modern C++ 3 | // | | |__ | | | | | | version 3.11.2 4 | // |_____|_____|_____|_|___| https://github.com/nlohmann/json 5 | // 6 | // SPDX-FileCopyrightText: 2013-2022 Niels Lohmann 7 | // SPDX-License-Identifier: MIT 8 | 9 | #pragma once 10 | 11 | #include 12 | 13 | NLOHMANN_JSON_NAMESPACE_BEGIN 14 | namespace detail 15 | { 16 | 17 | template struct make_void 18 | { 19 | using type = void; 20 | }; 21 | template using void_t = typename make_void::type; 22 | 23 | } // namespace detail 24 | NLOHMANN_JSON_NAMESPACE_END 25 | -------------------------------------------------------------------------------- /thirdparty/nlohmann_json/include/include/nlohmann/detail/string_escape.hpp: -------------------------------------------------------------------------------- 1 | // __ _____ _____ _____ 2 | // __| | __| | | | JSON for Modern C++ 3 | // | | |__ | | | | | | version 3.11.2 4 | // |_____|_____|_____|_|___| https://github.com/nlohmann/json 5 | // 6 | // SPDX-FileCopyrightText: 2013-2022 Niels Lohmann 7 | // SPDX-License-Identifier: MIT 8 | 9 | #pragma once 10 | 11 | #include 12 | 13 | NLOHMANN_JSON_NAMESPACE_BEGIN 14 | namespace detail 15 | { 16 | 17 | /*! 18 | @brief replace all occurrences of a substring by another string 19 | 20 | @param[in,out] s the string to manipulate; changed so that all 21 | occurrences of @a f are replaced with @a t 22 | @param[in] f the substring to replace with @a t 23 | @param[in] t the string to replace @a f 24 | 25 | @pre The search string @a f must not be empty. **This precondition is 26 | enforced with an assertion.** 27 | 28 | @since version 2.0.0 29 | */ 30 | template 31 | inline void replace_substring(StringType& s, const StringType& f, 32 | const StringType& t) 33 | { 34 | JSON_ASSERT(!f.empty()); 35 | for (auto pos = s.find(f); // find first occurrence of f 36 | pos != StringType::npos; // make sure f was found 37 | s.replace(pos, f.size(), t), // replace with t, and 38 | pos = s.find(f, pos + t.size())) // find next occurrence of f 39 | {} 40 | } 41 | 42 | /*! 43 | * @brief string escaping as described in RFC 6901 (Sect. 4) 44 | * @param[in] s string to escape 45 | * @return escaped string 46 | * 47 | * Note the order of escaping "~" to "~0" and "/" to "~1" is important. 48 | */ 49 | template 50 | inline StringType escape(StringType s) 51 | { 52 | replace_substring(s, StringType{"~"}, StringType{"~0"}); 53 | replace_substring(s, StringType{"/"}, StringType{"~1"}); 54 | return s; 55 | } 56 | 57 | /*! 58 | * @brief string unescaping as described in RFC 6901 (Sect. 4) 59 | * @param[in] s string to unescape 60 | * @return unescaped string 61 | * 62 | * Note the order of escaping "~1" to "/" and "~0" to "~" is important. 63 | */ 64 | template 65 | static void unescape(StringType& s) 66 | { 67 | replace_substring(s, StringType{"~1"}, StringType{"/"}); 68 | replace_substring(s, StringType{"~0"}, StringType{"~"}); 69 | } 70 | 71 | } // namespace detail 72 | NLOHMANN_JSON_NAMESPACE_END 73 | -------------------------------------------------------------------------------- /thirdparty/nlohmann_json/include/include/nlohmann/json_fwd.hpp: -------------------------------------------------------------------------------- 1 | // __ _____ _____ _____ 2 | // __| | __| | | | JSON for Modern C++ 3 | // | | |__ | | | | | | version 3.11.2 4 | // |_____|_____|_____|_|___| https://github.com/nlohmann/json 5 | // 6 | // SPDX-FileCopyrightText: 2013-2022 Niels Lohmann 7 | // SPDX-License-Identifier: MIT 8 | 9 | #ifndef INCLUDE_NLOHMANN_JSON_FWD_HPP_ 10 | #define INCLUDE_NLOHMANN_JSON_FWD_HPP_ 11 | 12 | #include // int64_t, uint64_t 13 | #include // map 14 | #include // allocator 15 | #include // string 16 | #include // vector 17 | 18 | #include 19 | 20 | /*! 21 | @brief namespace for Niels Lohmann 22 | @see https://github.com/nlohmann 23 | @since version 1.0.0 24 | */ 25 | NLOHMANN_JSON_NAMESPACE_BEGIN 26 | 27 | /*! 28 | @brief default JSONSerializer template argument 29 | 30 | This serializer ignores the template arguments and uses ADL 31 | ([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl)) 32 | for serialization. 33 | */ 34 | template 35 | struct adl_serializer; 36 | 37 | /// a class to store JSON values 38 | /// @sa https://json.nlohmann.me/api/basic_json/ 39 | template class ObjectType = 40 | std::map, 41 | template class ArrayType = std::vector, 42 | class StringType = std::string, class BooleanType = bool, 43 | class NumberIntegerType = std::int64_t, 44 | class NumberUnsignedType = std::uint64_t, 45 | class NumberFloatType = double, 46 | template class AllocatorType = std::allocator, 47 | template class JSONSerializer = 48 | adl_serializer, 49 | class BinaryType = std::vector> 50 | class basic_json; 51 | 52 | /// @brief JSON Pointer defines a string syntax for identifying a specific value within a JSON document 53 | /// @sa https://json.nlohmann.me/api/json_pointer/ 54 | template 55 | class json_pointer; 56 | 57 | /*! 58 | @brief default specialization 59 | @sa https://json.nlohmann.me/api/json/ 60 | */ 61 | using json = basic_json<>; 62 | 63 | /// @brief a minimal map-like container that preserves insertion order 64 | /// @sa https://json.nlohmann.me/api/ordered_map/ 65 | template 66 | struct ordered_map; 67 | 68 | /// @brief specialization that maintains the insertion order of object keys 69 | /// @sa https://json.nlohmann.me/api/ordered_json/ 70 | using ordered_json = basic_json; 71 | 72 | NLOHMANN_JSON_NAMESPACE_END 73 | 74 | #endif // INCLUDE_NLOHMANN_JSON_FWD_HPP_ 75 | -------------------------------------------------------------------------------- /thirdparty/nlohmann_json/include/meson.build: -------------------------------------------------------------------------------- 1 | project('nlohmann_json', 2 | 'cpp', 3 | version : '3.11.2', 4 | license : 'MIT', 5 | ) 6 | 7 | nlohmann_json_dep = declare_dependency( 8 | include_directories: include_directories('single_include') 9 | ) 10 | 11 | nlohmann_json_multiple_headers = declare_dependency( 12 | include_directories: include_directories('include') 13 | ) 14 | 15 | if not meson.is_subproject() 16 | install_headers('single_include/nlohmann/json.hpp', subdir: 'nlohmann') 17 | install_headers('single_include/nlohmann/json_fwd.hpp', subdir: 'nlohmann') 18 | 19 | pkgc = import('pkgconfig') 20 | pkgc.generate(name: 'nlohmann_json', 21 | version: meson.project_version(), 22 | description: 'JSON for Modern C++' 23 | ) 24 | endif 25 | -------------------------------------------------------------------------------- /thirdparty/sha1/README.md: -------------------------------------------------------------------------------- 1 | From [https://github.com/983/SHA1](https://github.com/983/SHA1) 2 | -------------------------------------------------------------------------------- /thirdparty/spdlog/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(spdlog INTERFACE) 2 | 3 | option(FORCE_BUILD_SPDLOG "Always build, instead of using the system version." OFF) 4 | option(SPDLOG_HEADER_ONLY "Use header only version of spdlog." OFF) 5 | 6 | set(SPDLOG_MIN_REQUIRED_VERSION "1.8.0") 7 | 8 | if (NOT FORCE_BUILD_SPDLOG AND NOT FORCE_BUILD_ALL) 9 | 10 | find_package(spdlog ${SPDLOG_MIN_REQUIRED_VERSION} CONFIG QUIET) 11 | 12 | if (TARGET spdlog::spdlog) 13 | 14 | get_target_property(SPDLOG_INCLUDE_DIRS spdlog::spdlog INTERFACE_INCLUDE_DIRECTORIES) 15 | 16 | find_library(SPDLOG_LIBRARY spdlog) 17 | 18 | file(READ "${SPDLOG_INCLUDE_DIRS}/spdlog/version.h" SPD_FILE) 19 | string(REGEX MATCH "SPDLOG_VER_MAJOR ([0-9]*)" _ ${SPD_FILE}) 20 | set(SPDLOG_VERSION_MAJOR ${CMAKE_MATCH_1}) 21 | string(REGEX MATCH "SPDLOG_VER_MINOR ([0-9]*)" _ ${SPD_FILE}) 22 | set(SPDLOG_VERSION_MINOR ${CMAKE_MATCH_1}) 23 | string(REGEX MATCH "SPDLOG_VER_PATCH ([0-9]*)" _ ${SPD_FILE}) 24 | set(SPDLOG_VERSION_PATCH ${CMAKE_MATCH_1}) 25 | 26 | set(SPDLOG_VERSION "${SPDLOG_VERSION_MAJOR}.${SPDLOG_VERSION_MINOR}.${SPDLOG_VERSION_PATCH}") 27 | 28 | if (SPDLOG_HEADER_ONLY) 29 | target_link_libraries(spdlog INTERFACE spdlog::spdlog_header_only) 30 | message(STATUS "Found SPDLOG: header-only (found suitable version \"${SPDLOG_VERSION}\", minimum required is \"${SPDLOG_MIN_REQUIRED_VERSION}\")") 31 | else() 32 | target_link_libraries(spdlog INTERFACE spdlog::spdlog) 33 | message(STATUS "Found SPDLOG: ${SPDLOG_LIBRARY} (found suitable version \"${SPDLOG_VERSION}\", minimum required is \"${SPDLOG_MIN_REQUIRED_VERSION}\")") 34 | endif() 35 | 36 | return() 37 | 38 | else() 39 | message(STATUS "SPDLOG not found: use bundled version.") 40 | endif() 41 | 42 | endif() 43 | 44 | set(SPDLOG_URL "https://github.com/gabime/spdlog/archive/refs/tags/v1.11.0.tar.gz") 45 | set(SPDLOG_HASH "287c6492c25044fd2da9947ab120b2bd") 46 | set(SPDLOG_DIR ${EXT_INSTALL_DIR}/spdlog) 47 | 48 | include(ExternalProject) 49 | ExternalProject_Add( 50 | SPDLOG-external 51 | PREFIX SPDLOG-external 52 | URL ${SPDLOG_URL} 53 | URL_MD5 ${SPDLOG_HASH} 54 | EXCLUDE_FROM_ALL 55 | LOG_DOWNLOAD ON 56 | LOG_CONFIGURE ON 57 | LOG_BUILD ON 58 | LOG_INSTALL ON 59 | CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${SPDLOG_DIR} 60 | ) 61 | 62 | target_include_directories(spdlog INTERFACE 63 | "${SPDLOG_DIR}/include" 64 | ) 65 | 66 | if (SPDLOG_HEADER_ONLY) 67 | #target_compile_definitions(spdlog INTERFACE SPDLOG_HEADER_ONLY) 68 | else() 69 | target_link_libraries(spdlog INTERFACE 70 | "${SPDLOG_DIR}/${CMAKE_INSTALL_LIBDIR}/${CMAKE_STATIC_LIBRARY_PREFIX}spdlog${CMAKE_STATIC_LIBRARY_SUFFIX}" 71 | ) 72 | endif() 73 | 74 | add_dependencies(spdlog SPDLOG-external) 75 | 76 | -------------------------------------------------------------------------------- /thirdparty/xxhash/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(xxhash INTERFACE) 2 | 3 | option(FORCE_BUILD_XXHASH "Always build, instead of using the system version." OFF) 4 | set(XXHASH_MIN_REQUIRED_VERSION "0.8.0") 5 | 6 | if (NOT FORCE_BUILD_XXHASH AND NOT FORCE_BUILD_ALL) 7 | find_package(XXHASH ${XXHASH_MIN_REQUIRED_VERSION}) 8 | 9 | if (XXHASH_FOUND) 10 | target_include_directories(xxhash INTERFACE ${XXHASH_INCLUDE_DIR}) 11 | target_link_libraries(xxhash INTERFACE ${XXHASH_LIBRARY}) 12 | return() 13 | else() 14 | message(STATUS "XXHASH not found: use bundled version.") 15 | endif() 16 | 17 | endif() 18 | 19 | set(XXHASH_URL "https://github.com/Cyan4973/xxHash/archive/refs/tags/v0.8.1.tar.gz") 20 | set(XXHASH_HASH "b67c587f5ff4894253da0095ba7ea393") 21 | set(XXHASH_DIR ${EXT_INSTALL_DIR}/xxhash) 22 | 23 | include(ExternalProject) 24 | ExternalProject_Add( 25 | XXHASH-external 26 | PREFIX XXHASH-external 27 | #URL ${XXHASH_URL} 28 | #URL_MD5 ${XXHASH_HASH} 29 | GIT_REPOSITORY "https://github.com/Cyan4973/xxHash" 30 | GIT_TAG dev 31 | EXCLUDE_FROM_ALL 32 | LOG_DOWNLOAD ON 33 | LOG_CONFIGURE ON 34 | LOG_BUILD ON 35 | LOG_INSTALL ON 36 | BUILD_IN_SOURCE true 37 | CONFIGURE_COMMAND "" 38 | BUILD_COMMAND make 39 | INSTALL_COMMAND make PREFIX=${XXHASH_DIR} install 40 | ) 41 | 42 | target_include_directories(xxhash SYSTEM INTERFACE 43 | "${XXHASH_DIR}/include" 44 | ) 45 | 46 | target_link_libraries(xxhash INTERFACE 47 | "${XXHASH_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}xxhash${CMAKE_STATIC_LIBRARY_SUFFIX}" 48 | ) 49 | 50 | add_dependencies(xxhash XXHASH-external) 51 | -------------------------------------------------------------------------------- /thirdparty/zlib/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(zlib INTERFACE) 2 | 3 | option(FORCE_BUILD_ZLIB "Always build, instead of using the system version." OFF) 4 | set(ZLIB_MIN_REQUIRED_VERSION "1.2.1") 5 | 6 | if (NOT FORCE_BUILD_ZLIB AND NOT FORCE_BUILD_ALL) 7 | find_package(ZLIB ${ZLIB_MIN_REQUIRED_VERSION}) 8 | 9 | if (ZLIB_FOUND) 10 | target_link_libraries(zlib INTERFACE ZLIB::ZLIB) 11 | return() 12 | else() 13 | message(STATUS "ZLIB not found: use bundled version.") 14 | endif() 15 | 16 | endif() 17 | 18 | set(ZLIB_URL "https://github.com/madler/zlib/archive/refs/tags/v1.2.12.tar.gz") 19 | set(ZLIB_HASH "db5b7326d4e0dbcbd1981b640d495c9b") 20 | set(ZLIB_DIR ${EXT_INSTALL_DIR}/zlib) 21 | 22 | FetchContent_Declare( 23 | ZLIB-fc 24 | URL ${ZLIB_URL} 25 | URL_HASH MD5=${ZLIB_HASH} 26 | ) 27 | FetchContent_MakeAvailable(ZLIB-fc) 28 | 29 | include(ExternalProject) 30 | ExternalProject_Add( 31 | ZLIB-external 32 | PREFIX ZLIB-external 33 | SOURCE_DIR ${CMAKE_BINARY_DIR}/ext-src/zlib-fc-src 34 | EXCLUDE_FROM_ALL 35 | LOG_DOWNLOAD ON 36 | LOG_CONFIGURE ON 37 | LOG_BUILD ON 38 | LOG_INSTALL ON 39 | BUILD_IN_SOURCE true 40 | CONFIGURE_COMMAND ./configure 41 | BUILD_COMMAND make 42 | INSTALL_COMMAND make prefix=${ZLIB_DIR} install 43 | ) 44 | 45 | target_include_directories(zlib INTERFACE 46 | "${ZLIB_DIR}/include" 47 | ) 48 | 49 | target_link_libraries(zlib INTERFACE 50 | "${ZLIB_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}z${CMAKE_STATIC_LIBRARY_SUFFIX}" 51 | ) 52 | 53 | add_dependencies(zlib ZLIB-external) 54 | 55 | --------------------------------------------------------------------------------