├── include └── rabitqlib │ ├── third │ ├── Eigen │ │ ├── Eigen │ │ ├── src │ │ │ ├── Core │ │ │ │ ├── util │ │ │ │ │ ├── NonMPL2.h │ │ │ │ │ ├── ReenableStupidWarnings.h │ │ │ │ │ └── ReshapedHelper.h │ │ │ │ ├── functors │ │ │ │ │ └── TernaryFunctors.h │ │ │ │ ├── arch │ │ │ │ │ ├── HIP │ │ │ │ │ │ └── hcc │ │ │ │ │ │ │ └── math_constants.h │ │ │ │ │ ├── SVE │ │ │ │ │ │ ├── MathFunctions.h │ │ │ │ │ │ └── TypeCasting.h │ │ │ │ │ ├── Default │ │ │ │ │ │ └── Settings.h │ │ │ │ │ ├── AVX512 │ │ │ │ │ │ └── TypeCasting.h │ │ │ │ │ ├── GPU │ │ │ │ │ │ ├── TypeCasting.h │ │ │ │ │ │ └── MathFunctions.h │ │ │ │ │ ├── AltiVec │ │ │ │ │ │ └── MathFunctions.h │ │ │ │ │ ├── SYCL │ │ │ │ │ │ └── TypeCasting.h │ │ │ │ │ ├── AVX │ │ │ │ │ │ └── TypeCasting.h │ │ │ │ │ └── NEON │ │ │ │ │ │ └── MathFunctions.h │ │ │ │ ├── DiagonalProduct.h │ │ │ │ ├── SelfCwiseBinaryOp.h │ │ │ │ ├── NestByValue.h │ │ │ │ ├── Swap.h │ │ │ │ ├── Assign.h │ │ │ │ ├── VectorBlock.h │ │ │ │ ├── Inverse.h │ │ │ │ └── NoAlias.h │ │ │ ├── misc │ │ │ │ ├── lapacke_mangling.h │ │ │ │ ├── RealSvd2x2.h │ │ │ │ ├── Kernel.h │ │ │ │ └── Image.h │ │ │ ├── SparseCore │ │ │ │ ├── SparseFuzzy.h │ │ │ │ ├── SparseRedux.h │ │ │ │ ├── MappedSparseMatrix.h │ │ │ │ ├── SparseDot.h │ │ │ │ └── SparseTranspose.h │ │ │ ├── SparseLU │ │ │ │ ├── SparseLU_Utils.h │ │ │ │ ├── SparseLU_relax_snode.h │ │ │ │ └── SparseLU_copy_to_ucol.h │ │ │ ├── StlSupport │ │ │ │ └── details.h │ │ │ ├── QR │ │ │ │ └── HouseholderQR_LAPACKE.h │ │ │ ├── plugins │ │ │ │ └── MatrixCwiseUnaryOps.h │ │ │ ├── LU │ │ │ │ ├── PartialPivLU_LAPACKE.h │ │ │ │ └── Determinant.h │ │ │ ├── Eigenvalues │ │ │ │ └── RealSchur_LAPACKE.h │ │ │ └── Geometry │ │ │ │ └── EulerAngles.h │ │ ├── Dense │ │ ├── StdList │ │ ├── StdDeque │ │ ├── StdVector │ │ ├── Householder │ │ ├── Jacobi │ │ ├── Sparse │ │ ├── QtAlignedMalloc │ │ ├── MetisSupport │ │ ├── PardisoSupport │ │ ├── SPQRSupport │ │ ├── SparseQR │ │ ├── Cholesky │ │ ├── SparseCholesky │ │ ├── LU │ │ ├── QR │ │ ├── UmfPackSupport │ │ ├── KLUSupport │ │ ├── SVD │ │ ├── PaStiXSupport │ │ ├── SparseLU │ │ ├── Eigenvalues │ │ ├── CholmodSupport │ │ ├── Geometry │ │ ├── IterativeLinearSolvers │ │ ├── SuperLUSupport │ │ ├── SparseCore │ │ └── OrderingMethods │ └── hnswlib │ │ └── visited_list_pool.h │ ├── utils │ ├── stopw.hpp │ ├── visited_pool.hpp │ ├── io.hpp │ ├── hashset.hpp │ ├── buffer.hpp │ └── tools.hpp │ ├── index │ ├── ivf │ │ └── cluster.hpp │ └── lut.hpp │ └── defines.hpp ├── docs ├── docs │ ├── assets │ │ └── img │ │ │ ├── SingleData.png │ │ │ └── compact │ │ │ ├── 2bit.png │ │ │ ├── 4bit.png │ │ │ └── 6bit.png │ ├── rabitq │ │ ├── reranking.md │ │ ├── rabitq.md │ │ └── rotator.md │ ├── kernel_ip.md │ ├── compact_code.md │ └── index │ │ └── qg.md └── mkdocs.yml ├── sample ├── CMakeLists.txt ├── quantizer.cpp ├── symqg_indexing.cpp ├── ivf_rabitq_indexing.cpp ├── symqg_querying.cpp └── hnsw_rabitq_indexing.cpp ├── .gitignore ├── CMakeLists.txt ├── example.sh └── python ├── utils └── io.py └── ivf.py /include/rabitqlib/third/Eigen/Eigen: -------------------------------------------------------------------------------- 1 | #include "Dense" 2 | #include "Sparse" 3 | -------------------------------------------------------------------------------- /docs/docs/assets/img/SingleData.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VectorDB-NTU/RaBitQ-Library/HEAD/docs/docs/assets/img/SingleData.png -------------------------------------------------------------------------------- /docs/docs/assets/img/compact/2bit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VectorDB-NTU/RaBitQ-Library/HEAD/docs/docs/assets/img/compact/2bit.png -------------------------------------------------------------------------------- /docs/docs/assets/img/compact/4bit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VectorDB-NTU/RaBitQ-Library/HEAD/docs/docs/assets/img/compact/4bit.png -------------------------------------------------------------------------------- /docs/docs/assets/img/compact/6bit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VectorDB-NTU/RaBitQ-Library/HEAD/docs/docs/assets/img/compact/6bit.png -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/Core/util/NonMPL2.h: -------------------------------------------------------------------------------- 1 | #ifdef EIGEN_MPL2_ONLY 2 | #error Including non-MPL2 code in EIGEN_MPL2_ONLY mode 3 | #endif 4 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/Dense: -------------------------------------------------------------------------------- 1 | #include "Core" 2 | #include "LU" 3 | #include "Cholesky" 4 | #include "QR" 5 | #include "SVD" 6 | #include "Geometry" 7 | #include "Eigenvalues" 8 | -------------------------------------------------------------------------------- /sample/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set (CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/bin) 2 | 3 | add_executable(symqg_indexing symqg_indexing.cpp) 4 | add_executable(symqg_querying symqg_querying.cpp) 5 | 6 | add_executable(ivf_rabitq_indexing ivf_rabitq_indexing.cpp) 7 | add_executable(ivf_rabitq_querying ivf_rabitq_querying.cpp) 8 | 9 | add_executable(hnsw_rabitq_indexing hnsw_rabitq_indexing.cpp) 10 | add_executable(hnsw_rabitq_querying hnsw_rabitq_querying.cpp) 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Prerequisites 2 | *.d 3 | 4 | # Compiled Object files 5 | *.slo 6 | *.lo 7 | *.o 8 | *.obj 9 | 10 | # Precompiled Headers 11 | *.gch 12 | *.pch 13 | 14 | # Compiled Dynamic libraries 15 | *.so 16 | *.dylib 17 | *.dll 18 | 19 | # Fortran module files 20 | *.mod 21 | *.smod 22 | 23 | # Compiled Static libraries 24 | *.lai 25 | *.la 26 | *.a 27 | *.lib 28 | 29 | # Executables 30 | *.exe 31 | *.out 32 | *.app 33 | 34 | 35 | .cache/ 36 | bin/ 37 | build/ 38 | data/ 39 | 40 | *.pyc 41 | .clang-* 42 | 43 | # macOS garbages 44 | .DS_Store 45 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 3.10) 2 | 3 | project(RaBitQLib LANGUAGES CXX) 4 | 5 | set(CMAKE_CXX_STANDARD 17) 6 | set(CMAKE_CXX_STANDARD_REQUIRED ON) 7 | 8 | 9 | include_directories(${PROJECT_SOURCE_DIR}/include) 10 | 11 | set(CMAKE_EXPORT_COMPILE_COMMANDS ON) 12 | 13 | SET(CMAKE_CXX_FLAGS "-Wall -Ofast -Wextra -lrt -march=native -fpic -fopenmp -ftree-vectorize -fexceptions") 14 | 15 | add_subdirectory(sample) 16 | 17 | add_library(rabitq_headers INTERFACE) 18 | 19 | target_include_directories(rabitq_headers INTERFACE ${PROJECT_SOURCE_DIR}/include) -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/misc/lapacke_mangling.h: -------------------------------------------------------------------------------- 1 | #ifndef LAPACK_HEADER_INCLUDED 2 | #define LAPACK_HEADER_INCLUDED 3 | 4 | #ifndef LAPACK_GLOBAL 5 | #if defined(LAPACK_GLOBAL_PATTERN_LC) || defined(ADD_) 6 | #define LAPACK_GLOBAL(lcname,UCNAME) lcname##_ 7 | #elif defined(LAPACK_GLOBAL_PATTERN_UC) || defined(UPPER) 8 | #define LAPACK_GLOBAL(lcname,UCNAME) UCNAME 9 | #elif defined(LAPACK_GLOBAL_PATTERN_MC) || defined(NOCHANGE) 10 | #define LAPACK_GLOBAL(lcname,UCNAME) lcname 11 | #else 12 | #define LAPACK_GLOBAL(lcname,UCNAME) lcname##_ 13 | #endif 14 | #endif 15 | 16 | #endif 17 | 18 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/Core/functors/TernaryFunctors.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2016 Eugene Brevdo 5 | // 6 | // This Source Code Form is subject to the terms of the Mozilla 7 | // Public License v. 2.0. If a copy of the MPL was not distributed 8 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | 10 | #ifndef EIGEN_TERNARY_FUNCTORS_H 11 | #define EIGEN_TERNARY_FUNCTORS_H 12 | 13 | namespace Eigen { 14 | 15 | namespace internal { 16 | 17 | //---------- associative ternary functors ---------- 18 | 19 | 20 | 21 | } // end namespace internal 22 | 23 | } // end namespace Eigen 24 | 25 | #endif // EIGEN_TERNARY_FUNCTORS_H 26 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/Core/arch/HIP/hcc/math_constants.h: -------------------------------------------------------------------------------- 1 | /* 2 | * math_constants.h - 3 | * HIP equivalent of the CUDA header of the same name 4 | */ 5 | 6 | #ifndef __MATH_CONSTANTS_H__ 7 | #define __MATH_CONSTANTS_H__ 8 | 9 | /* single precision constants */ 10 | 11 | #define HIPRT_INF_F __int_as_float(0x7f800000) 12 | #define HIPRT_NAN_F __int_as_float(0x7fffffff) 13 | #define HIPRT_MIN_DENORM_F __int_as_float(0x00000001) 14 | #define HIPRT_MAX_NORMAL_F __int_as_float(0x7f7fffff) 15 | #define HIPRT_NEG_ZERO_F __int_as_float(0x80000000) 16 | #define HIPRT_ZERO_F 0.0f 17 | #define HIPRT_ONE_F 1.0f 18 | 19 | /* double precision constants */ 20 | #define HIPRT_INF __hiloint2double(0x7ff00000, 0x00000000) 21 | #define HIPRT_NAN __hiloint2double(0xfff80000, 0x00000000) 22 | 23 | #endif 24 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/StdList: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2009 Hauke Heibel 5 | // 6 | // This Source Code Form is subject to the terms of the Mozilla 7 | // Public License v. 2.0. If a copy of the MPL was not distributed 8 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | 10 | #ifndef EIGEN_STDLIST_MODULE_H 11 | #define EIGEN_STDLIST_MODULE_H 12 | 13 | #include "Core" 14 | #include 15 | 16 | #if EIGEN_COMP_MSVC && EIGEN_OS_WIN64 && (EIGEN_MAX_STATIC_ALIGN_BYTES<=16) /* MSVC auto aligns up to 16 bytes in 64 bit builds */ 17 | 18 | #define EIGEN_DEFINE_STL_LIST_SPECIALIZATION(...) 19 | 20 | #else 21 | 22 | #include "src/StlSupport/StdList.h" 23 | 24 | #endif 25 | 26 | #endif // EIGEN_STDLIST_MODULE_H 27 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/StdDeque: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2009 Gael Guennebaud 5 | // Copyright (C) 2009 Hauke Heibel 6 | // 7 | // This Source Code Form is subject to the terms of the Mozilla 8 | // Public License v. 2.0. If a copy of the MPL was not distributed 9 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 10 | 11 | #ifndef EIGEN_STDDEQUE_MODULE_H 12 | #define EIGEN_STDDEQUE_MODULE_H 13 | 14 | #include "Core" 15 | #include 16 | 17 | #if EIGEN_COMP_MSVC && EIGEN_OS_WIN64 && (EIGEN_MAX_STATIC_ALIGN_BYTES<=16) /* MSVC auto aligns up to 16 bytes in 64 bit builds */ 18 | 19 | #define EIGEN_DEFINE_STL_DEQUE_SPECIALIZATION(...) 20 | 21 | #else 22 | 23 | #include "src/StlSupport/StdDeque.h" 24 | 25 | #endif 26 | 27 | #endif // EIGEN_STDDEQUE_MODULE_H 28 | -------------------------------------------------------------------------------- /docs/mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: RaBitQ Library 2 | 3 | nav: 4 | - Introduction: index.md 5 | - Quick Start: quick_start.md 6 | - RaBitQ: 7 | - Introduction: rabitq/rabitq.md 8 | - Rotator: rabitq/rotator.md 9 | - Quantizer: rabitq/quantizer.md 10 | - Estimator: rabitq/estimator.md 11 | - Reranking: rabitq/reranking.md 12 | - Kernel: 13 | - Compact Code Storage: compact_code.md 14 | - Inner Product: kernel_ip.md 15 | - RaBitQ for Vector Search: 16 | - IVF + RaBitQ: index/ivf.md 17 | - HNSW + RaBitQ: index/hnsw.md 18 | - QG + RaBitQ (SymphonyQG): index/qg.md 19 | 20 | 21 | markdown_extensions: 22 | - pymdownx.arithmatex 23 | - pymdownx.details 24 | - pymdownx.superfences 25 | - pymdownx.highlight 26 | - attr_list 27 | - footnotes 28 | 29 | extra_javascript: 30 | - https://cdn.jsdelivr.net/npm/mathjax@2/MathJax.js?config=TeX-AMS-MML_HTMLorMML 31 | 32 | theme: 33 | name: material 34 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/StdVector: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2009 Gael Guennebaud 5 | // Copyright (C) 2009 Hauke Heibel 6 | // 7 | // This Source Code Form is subject to the terms of the Mozilla 8 | // Public License v. 2.0. If a copy of the MPL was not distributed 9 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 10 | 11 | #ifndef EIGEN_STDVECTOR_MODULE_H 12 | #define EIGEN_STDVECTOR_MODULE_H 13 | 14 | #include "Core" 15 | #include 16 | 17 | #if EIGEN_COMP_MSVC && EIGEN_OS_WIN64 && (EIGEN_MAX_STATIC_ALIGN_BYTES<=16) /* MSVC auto aligns up to 16 bytes in 64 bit builds */ 18 | 19 | #define EIGEN_DEFINE_STL_VECTOR_SPECIALIZATION(...) 20 | 21 | #else 22 | 23 | #include "src/StlSupport/StdVector.h" 24 | 25 | #endif 26 | 27 | #endif // EIGEN_STDVECTOR_MODULE_H 28 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/Householder: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // This Source Code Form is subject to the terms of the Mozilla 5 | // Public License v. 2.0. If a copy of the MPL was not distributed 6 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 7 | 8 | #ifndef EIGEN_HOUSEHOLDER_MODULE_H 9 | #define EIGEN_HOUSEHOLDER_MODULE_H 10 | 11 | #include "Core" 12 | 13 | #include "src/Core/util/DisableStupidWarnings.h" 14 | 15 | /** \defgroup Householder_Module Householder module 16 | * This module provides Householder transformations. 17 | * 18 | * \code 19 | * #include 20 | * \endcode 21 | */ 22 | 23 | #include "src/Householder/Householder.h" 24 | #include "src/Householder/HouseholderSequence.h" 25 | #include "src/Householder/BlockHouseholder.h" 26 | 27 | #include "src/Core/util/ReenableStupidWarnings.h" 28 | 29 | #endif // EIGEN_HOUSEHOLDER_MODULE_H 30 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/Jacobi: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // This Source Code Form is subject to the terms of the Mozilla 5 | // Public License v. 2.0. If a copy of the MPL was not distributed 6 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 7 | 8 | #ifndef EIGEN_JACOBI_MODULE_H 9 | #define EIGEN_JACOBI_MODULE_H 10 | 11 | #include "Core" 12 | 13 | #include "src/Core/util/DisableStupidWarnings.h" 14 | 15 | /** \defgroup Jacobi_Module Jacobi module 16 | * This module provides Jacobi and Givens rotations. 17 | * 18 | * \code 19 | * #include 20 | * \endcode 21 | * 22 | * In addition to listed classes, it defines the two following MatrixBase methods to apply a Jacobi or Givens rotation: 23 | * - MatrixBase::applyOnTheLeft() 24 | * - MatrixBase::applyOnTheRight(). 25 | */ 26 | 27 | #include "src/Jacobi/Jacobi.h" 28 | 29 | #include "src/Core/util/ReenableStupidWarnings.h" 30 | 31 | #endif // EIGEN_JACOBI_MODULE_H 32 | 33 | -------------------------------------------------------------------------------- /include/rabitqlib/utils/stopw.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | namespace rabitqlib { 6 | class StopW { 7 | using clock = std::chrono::steady_clock; 8 | clock::time_point time_begin_; 9 | 10 | [[nodiscard]] clock::duration elapsed() const { return clock::now() - time_begin_; } 11 | 12 | public: 13 | StopW() : time_begin_(clock::now()) {} 14 | 15 | void reset() { time_begin_ = clock::now(); } 16 | 17 | [[nodiscard]] float get_elapsed_sec() const { 18 | return std::chrono::duration(elapsed()).count(); 19 | } 20 | 21 | [[nodiscard]] float get_elapsed_mili() const { 22 | return std::chrono::duration(elapsed()).count(); 23 | } 24 | 25 | [[nodiscard]] float get_elapsed_micro() const { 26 | return std::chrono::duration(elapsed()).count(); 27 | } 28 | 29 | [[nodiscard]] float get_elapsed_nano() const { 30 | return std::chrono::duration(elapsed()).count(); 31 | } 32 | }; 33 | } // namespace rabitqlib -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/Sparse: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // This Source Code Form is subject to the terms of the Mozilla 5 | // Public License v. 2.0. If a copy of the MPL was not distributed 6 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 7 | 8 | #ifndef EIGEN_SPARSE_MODULE_H 9 | #define EIGEN_SPARSE_MODULE_H 10 | 11 | /** \defgroup Sparse_Module Sparse meta-module 12 | * 13 | * Meta-module including all related modules: 14 | * - \ref SparseCore_Module 15 | * - \ref OrderingMethods_Module 16 | * - \ref SparseCholesky_Module 17 | * - \ref SparseLU_Module 18 | * - \ref SparseQR_Module 19 | * - \ref IterativeLinearSolvers_Module 20 | * 21 | \code 22 | #include 23 | \endcode 24 | */ 25 | 26 | #include "SparseCore" 27 | #include "OrderingMethods" 28 | #include "SparseCholesky" 29 | #include "SparseLU" 30 | #include "SparseQR" 31 | #include "IterativeLinearSolvers" 32 | 33 | #endif // EIGEN_SPARSE_MODULE_H 34 | 35 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/QtAlignedMalloc: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // This Source Code Form is subject to the terms of the Mozilla 5 | // Public License v. 2.0. If a copy of the MPL was not distributed 6 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 7 | 8 | #ifndef EIGEN_QTMALLOC_MODULE_H 9 | #define EIGEN_QTMALLOC_MODULE_H 10 | 11 | #include "Core" 12 | 13 | #if (!EIGEN_MALLOC_ALREADY_ALIGNED) 14 | 15 | #include "src/Core/util/DisableStupidWarnings.h" 16 | 17 | void *qMalloc(std::size_t size) 18 | { 19 | return Eigen::internal::aligned_malloc(size); 20 | } 21 | 22 | void qFree(void *ptr) 23 | { 24 | Eigen::internal::aligned_free(ptr); 25 | } 26 | 27 | void *qRealloc(void *ptr, std::size_t size) 28 | { 29 | void* newPtr = Eigen::internal::aligned_malloc(size); 30 | std::memcpy(newPtr, ptr, size); 31 | Eigen::internal::aligned_free(ptr); 32 | return newPtr; 33 | } 34 | 35 | #include "src/Core/util/ReenableStupidWarnings.h" 36 | 37 | #endif 38 | 39 | #endif // EIGEN_QTMALLOC_MODULE_H 40 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/Core/DiagonalProduct.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2008 Gael Guennebaud 5 | // Copyright (C) 2007-2009 Benoit Jacob 6 | // 7 | // This Source Code Form is subject to the terms of the Mozilla 8 | // Public License v. 2.0. If a copy of the MPL was not distributed 9 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 10 | 11 | #ifndef EIGEN_DIAGONALPRODUCT_H 12 | #define EIGEN_DIAGONALPRODUCT_H 13 | 14 | namespace Eigen { 15 | 16 | /** \returns the diagonal matrix product of \c *this by the diagonal matrix \a diagonal. 17 | */ 18 | template 19 | template 20 | EIGEN_DEVICE_FUNC inline const Product 21 | MatrixBase::operator*(const DiagonalBase &a_diagonal) const 22 | { 23 | return Product(derived(),a_diagonal.derived()); 24 | } 25 | 26 | } // end namespace Eigen 27 | 28 | #endif // EIGEN_DIAGONALPRODUCT_H 29 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/MetisSupport: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // This Source Code Form is subject to the terms of the Mozilla 5 | // Public License v. 2.0. If a copy of the MPL was not distributed 6 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 7 | 8 | #ifndef EIGEN_METISSUPPORT_MODULE_H 9 | #define EIGEN_METISSUPPORT_MODULE_H 10 | 11 | #include "SparseCore" 12 | 13 | #include "src/Core/util/DisableStupidWarnings.h" 14 | 15 | extern "C" { 16 | #include 17 | } 18 | 19 | 20 | /** \ingroup Support_modules 21 | * \defgroup MetisSupport_Module MetisSupport module 22 | * 23 | * \code 24 | * #include 25 | * \endcode 26 | * This module defines an interface to the METIS reordering package (http://glaros.dtc.umn.edu/gkhome/views/metis). 27 | * It can be used just as any other built-in method as explained in \link OrderingMethods_Module here. \endlink 28 | */ 29 | 30 | 31 | #include "src/MetisSupport/MetisSupport.h" 32 | 33 | #include "src/Core/util/ReenableStupidWarnings.h" 34 | 35 | #endif // EIGEN_METISSUPPORT_MODULE_H 36 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/Core/util/ReenableStupidWarnings.h: -------------------------------------------------------------------------------- 1 | #ifdef EIGEN_WARNINGS_DISABLED_2 2 | // "DisableStupidWarnings.h" was included twice recursively: Do not reenable warnings yet! 3 | # undef EIGEN_WARNINGS_DISABLED_2 4 | 5 | #elif defined(EIGEN_WARNINGS_DISABLED) 6 | #undef EIGEN_WARNINGS_DISABLED 7 | 8 | #ifndef EIGEN_PERMANENTLY_DISABLE_STUPID_WARNINGS 9 | #ifdef _MSC_VER 10 | #pragma warning( pop ) 11 | #elif defined __INTEL_COMPILER 12 | #pragma warning pop 13 | #elif defined __clang__ 14 | #pragma clang diagnostic pop 15 | #elif defined __GNUC__ && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) 16 | #pragma GCC diagnostic pop 17 | #endif 18 | 19 | #if defined __NVCC__ 20 | // Don't reenable the diagnostic messages, as it turns out these messages need 21 | // to be disabled at the point of the template instantiation (i.e the user code) 22 | // otherwise they'll be triggered by nvcc. 23 | // #pragma diag_default code_is_unreachable 24 | // #pragma diag_default initialization_not_reachable 25 | // #pragma diag_default 2651 26 | // #pragma diag_default 2653 27 | #endif 28 | 29 | #endif 30 | 31 | #endif // EIGEN_WARNINGS_DISABLED 32 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/SparseCore/SparseFuzzy.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2008-2014 Gael Guennebaud 5 | // 6 | // This Source Code Form is subject to the terms of the Mozilla 7 | // Public License v. 2.0. If a copy of the MPL was not distributed 8 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | 10 | #ifndef EIGEN_SPARSE_FUZZY_H 11 | #define EIGEN_SPARSE_FUZZY_H 12 | 13 | namespace Eigen { 14 | 15 | template 16 | template 17 | bool SparseMatrixBase::isApprox(const SparseMatrixBase& other, const RealScalar &prec) const 18 | { 19 | const typename internal::nested_eval::type actualA(derived()); 20 | typename internal::conditional::type, 22 | const PlainObject>::type actualB(other.derived()); 23 | 24 | return (actualA - actualB).squaredNorm() <= prec * prec * numext::mini(actualA.squaredNorm(), actualB.squaredNorm()); 25 | } 26 | 27 | } // end namespace Eigen 28 | 29 | #endif // EIGEN_SPARSE_FUZZY_H 30 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/PardisoSupport: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // This Source Code Form is subject to the terms of the Mozilla 5 | // Public License v. 2.0. If a copy of the MPL was not distributed 6 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 7 | 8 | #ifndef EIGEN_PARDISOSUPPORT_MODULE_H 9 | #define EIGEN_PARDISOSUPPORT_MODULE_H 10 | 11 | #include "SparseCore" 12 | 13 | #include "src/Core/util/DisableStupidWarnings.h" 14 | 15 | #include 16 | 17 | /** \ingroup Support_modules 18 | * \defgroup PardisoSupport_Module PardisoSupport module 19 | * 20 | * This module brings support for the Intel(R) MKL PARDISO direct sparse solvers. 21 | * 22 | * \code 23 | * #include 24 | * \endcode 25 | * 26 | * In order to use this module, the MKL headers must be accessible from the include paths, and your binary must be linked to the MKL library and its dependencies. 27 | * See this \ref TopicUsingIntelMKL "page" for more information on MKL-Eigen integration. 28 | * 29 | */ 30 | 31 | #include "src/PardisoSupport/PardisoSupport.h" 32 | 33 | #include "src/Core/util/ReenableStupidWarnings.h" 34 | 35 | #endif // EIGEN_PARDISOSUPPORT_MODULE_H 36 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/SPQRSupport: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // This Source Code Form is subject to the terms of the Mozilla 5 | // Public License v. 2.0. If a copy of the MPL was not distributed 6 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 7 | 8 | #ifndef EIGEN_SPQRSUPPORT_MODULE_H 9 | #define EIGEN_SPQRSUPPORT_MODULE_H 10 | 11 | #include "SparseCore" 12 | 13 | #include "src/Core/util/DisableStupidWarnings.h" 14 | 15 | #include "SuiteSparseQR.hpp" 16 | 17 | /** \ingroup Support_modules 18 | * \defgroup SPQRSupport_Module SuiteSparseQR module 19 | * 20 | * This module provides an interface to the SPQR library, which is part of the suitesparse package. 21 | * 22 | * \code 23 | * #include 24 | * \endcode 25 | * 26 | * In order to use this module, the SPQR headers must be accessible from the include paths, and your binary must be linked to the SPQR library and its dependencies (Cholmod, AMD, COLAMD,...). 27 | * For a cmake based project, you can use our FindSPQR.cmake and FindCholmod.Cmake modules 28 | * 29 | */ 30 | 31 | #include "src/CholmodSupport/CholmodSupport.h" 32 | #include "src/SPQRSupport/SuiteSparseQRSupport.h" 33 | 34 | #endif 35 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/SparseQR: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // This Source Code Form is subject to the terms of the Mozilla 5 | // Public License v. 2.0. If a copy of the MPL was not distributed 6 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 7 | 8 | #ifndef EIGEN_SPARSEQR_MODULE_H 9 | #define EIGEN_SPARSEQR_MODULE_H 10 | 11 | #include "SparseCore" 12 | #include "OrderingMethods" 13 | #include "src/Core/util/DisableStupidWarnings.h" 14 | 15 | /** \defgroup SparseQR_Module SparseQR module 16 | * \brief Provides QR decomposition for sparse matrices 17 | * 18 | * This module provides a simplicial version of the left-looking Sparse QR decomposition. 19 | * The columns of the input matrix should be reordered to limit the fill-in during the 20 | * decomposition. Built-in methods (COLAMD, AMD) or external methods (METIS) can be used to this end. 21 | * See the \link OrderingMethods_Module OrderingMethods\endlink module for the list 22 | * of built-in and external ordering methods. 23 | * 24 | * \code 25 | * #include 26 | * \endcode 27 | * 28 | * 29 | */ 30 | 31 | #include "src/SparseCore/SparseColEtree.h" 32 | #include "src/SparseQR/SparseQR.h" 33 | 34 | #include "src/Core/util/ReenableStupidWarnings.h" 35 | 36 | #endif 37 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/Cholesky: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // This Source Code Form is subject to the terms of the Mozilla 5 | // Public License v. 2.0. If a copy of the MPL was not distributed 6 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 7 | 8 | #ifndef EIGEN_CHOLESKY_MODULE_H 9 | #define EIGEN_CHOLESKY_MODULE_H 10 | 11 | #include "Core" 12 | #include "Jacobi" 13 | 14 | #include "src/Core/util/DisableStupidWarnings.h" 15 | 16 | /** \defgroup Cholesky_Module Cholesky module 17 | * 18 | * 19 | * 20 | * This module provides two variants of the Cholesky decomposition for selfadjoint (hermitian) matrices. 21 | * Those decompositions are also accessible via the following methods: 22 | * - MatrixBase::llt() 23 | * - MatrixBase::ldlt() 24 | * - SelfAdjointView::llt() 25 | * - SelfAdjointView::ldlt() 26 | * 27 | * \code 28 | * #include 29 | * \endcode 30 | */ 31 | 32 | #include "src/Cholesky/LLT.h" 33 | #include "src/Cholesky/LDLT.h" 34 | #ifdef EIGEN_USE_LAPACKE 35 | #ifdef EIGEN_USE_MKL 36 | #include "mkl_lapacke.h" 37 | #else 38 | #include "src/misc/lapacke.h" 39 | #endif 40 | #include "src/Cholesky/LLT_LAPACKE.h" 41 | #endif 42 | 43 | #include "src/Core/util/ReenableStupidWarnings.h" 44 | 45 | #endif // EIGEN_CHOLESKY_MODULE_H 46 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/SparseCholesky: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2008-2013 Gael Guennebaud 5 | // 6 | // This Source Code Form is subject to the terms of the Mozilla 7 | // Public License v. 2.0. If a copy of the MPL was not distributed 8 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | 10 | #ifndef EIGEN_SPARSECHOLESKY_MODULE_H 11 | #define EIGEN_SPARSECHOLESKY_MODULE_H 12 | 13 | #include "SparseCore" 14 | #include "OrderingMethods" 15 | 16 | #include "src/Core/util/DisableStupidWarnings.h" 17 | 18 | /** 19 | * \defgroup SparseCholesky_Module SparseCholesky module 20 | * 21 | * This module currently provides two variants of the direct sparse Cholesky decomposition for selfadjoint (hermitian) matrices. 22 | * Those decompositions are accessible via the following classes: 23 | * - SimplicialLLt, 24 | * - SimplicialLDLt 25 | * 26 | * Such problems can also be solved using the ConjugateGradient solver from the IterativeLinearSolvers module. 27 | * 28 | * \code 29 | * #include 30 | * \endcode 31 | */ 32 | 33 | #include "src/SparseCholesky/SimplicialCholesky.h" 34 | #include "src/SparseCholesky/SimplicialCholesky_impl.h" 35 | #include "src/Core/util/ReenableStupidWarnings.h" 36 | 37 | #endif // EIGEN_SPARSECHOLESKY_MODULE_H 38 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/Core/arch/SVE/MathFunctions.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2020, Arm Limited and Contributors 5 | // 6 | // This Source Code Form is subject to the terms of the Mozilla 7 | // Public License v. 2.0. If a copy of the MPL was not distributed 8 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | 10 | #ifndef EIGEN_MATH_FUNCTIONS_SVE_H 11 | #define EIGEN_MATH_FUNCTIONS_SVE_H 12 | 13 | namespace Eigen { 14 | namespace internal { 15 | 16 | template <> 17 | EIGEN_STRONG_INLINE EIGEN_UNUSED PacketXf pexp(const PacketXf& x) { 18 | return pexp_float(x); 19 | } 20 | 21 | template <> 22 | EIGEN_STRONG_INLINE EIGEN_UNUSED PacketXf plog(const PacketXf& x) { 23 | return plog_float(x); 24 | } 25 | 26 | template <> 27 | EIGEN_STRONG_INLINE EIGEN_UNUSED PacketXf psin(const PacketXf& x) { 28 | return psin_float(x); 29 | } 30 | 31 | template <> 32 | EIGEN_STRONG_INLINE EIGEN_UNUSED PacketXf pcos(const PacketXf& x) { 33 | return pcos_float(x); 34 | } 35 | 36 | // Hyperbolic Tangent function. 37 | template <> 38 | EIGEN_STRONG_INLINE EIGEN_UNUSED PacketXf ptanh(const PacketXf& x) { 39 | return internal::generic_fast_tanh_float(x); 40 | } 41 | } // end namespace internal 42 | } // end namespace Eigen 43 | 44 | #endif // EIGEN_MATH_FUNCTIONS_SVE_H 45 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/LU: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // This Source Code Form is subject to the terms of the Mozilla 5 | // Public License v. 2.0. If a copy of the MPL was not distributed 6 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 7 | 8 | #ifndef EIGEN_LU_MODULE_H 9 | #define EIGEN_LU_MODULE_H 10 | 11 | #include "Core" 12 | 13 | #include "src/Core/util/DisableStupidWarnings.h" 14 | 15 | /** \defgroup LU_Module LU module 16 | * This module includes %LU decomposition and related notions such as matrix inversion and determinant. 17 | * This module defines the following MatrixBase methods: 18 | * - MatrixBase::inverse() 19 | * - MatrixBase::determinant() 20 | * 21 | * \code 22 | * #include 23 | * \endcode 24 | */ 25 | 26 | #include "src/misc/Kernel.h" 27 | #include "src/misc/Image.h" 28 | #include "src/LU/FullPivLU.h" 29 | #include "src/LU/PartialPivLU.h" 30 | #ifdef EIGEN_USE_LAPACKE 31 | #ifdef EIGEN_USE_MKL 32 | #include "mkl_lapacke.h" 33 | #else 34 | #include "src/misc/lapacke.h" 35 | #endif 36 | #include "src/LU/PartialPivLU_LAPACKE.h" 37 | #endif 38 | #include "src/LU/Determinant.h" 39 | #include "src/LU/InverseImpl.h" 40 | 41 | #if defined EIGEN_VECTORIZE_SSE || defined EIGEN_VECTORIZE_NEON 42 | #include "src/LU/arch/InverseSize4.h" 43 | #endif 44 | 45 | #include "src/Core/util/ReenableStupidWarnings.h" 46 | 47 | #endif // EIGEN_LU_MODULE_H 48 | -------------------------------------------------------------------------------- /include/rabitqlib/utils/visited_pool.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | 5 | #include "rabitqlib/utils/hashset.hpp" 6 | 7 | namespace rabitqlib { 8 | class VisitedListPool { 9 | std::deque pool_; 10 | std::mutex poolguard_; 11 | size_t numelements_; 12 | 13 | public: 14 | VisitedListPool(size_t initpoolsize, size_t max_elements) { 15 | numelements_ = max_elements / 10; 16 | for (size_t i = 0; i < initpoolsize; i++) { 17 | pool_.push_front(new HashBasedBooleanSet(numelements_)); 18 | } 19 | } 20 | 21 | HashBasedBooleanSet* get_free_vislist() { 22 | HashBasedBooleanSet* rez; 23 | { 24 | std::unique_lock lock(poolguard_); 25 | if (pool_.size() > 0) { 26 | rez = pool_.front(); 27 | pool_.pop_front(); 28 | } else { 29 | rez = new HashBasedBooleanSet(numelements_); 30 | } 31 | } 32 | rez->clear(); 33 | return rez; 34 | } 35 | 36 | void release_vis_list(HashBasedBooleanSet* vl) { 37 | std::unique_lock lock(poolguard_); 38 | pool_.push_front(vl); 39 | } 40 | 41 | ~VisitedListPool() { 42 | while (pool_.size() > 0) { 43 | HashBasedBooleanSet* rez = pool_.front(); 44 | pool_.pop_front(); 45 | ::delete rez; 46 | } 47 | } 48 | }; 49 | } // namespace rabitqlib -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/QR: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // This Source Code Form is subject to the terms of the Mozilla 5 | // Public License v. 2.0. If a copy of the MPL was not distributed 6 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 7 | 8 | #ifndef EIGEN_QR_MODULE_H 9 | #define EIGEN_QR_MODULE_H 10 | 11 | #include "Core" 12 | 13 | #include "Cholesky" 14 | #include "Jacobi" 15 | #include "Householder" 16 | 17 | #include "src/Core/util/DisableStupidWarnings.h" 18 | 19 | /** \defgroup QR_Module QR module 20 | * 21 | * 22 | * 23 | * This module provides various QR decompositions 24 | * This module also provides some MatrixBase methods, including: 25 | * - MatrixBase::householderQr() 26 | * - MatrixBase::colPivHouseholderQr() 27 | * - MatrixBase::fullPivHouseholderQr() 28 | * 29 | * \code 30 | * #include 31 | * \endcode 32 | */ 33 | 34 | #include "src/QR/HouseholderQR.h" 35 | #include "src/QR/FullPivHouseholderQR.h" 36 | #include "src/QR/ColPivHouseholderQR.h" 37 | #include "src/QR/CompleteOrthogonalDecomposition.h" 38 | #ifdef EIGEN_USE_LAPACKE 39 | #ifdef EIGEN_USE_MKL 40 | #include "mkl_lapacke.h" 41 | #else 42 | #include "src/misc/lapacke.h" 43 | #endif 44 | #include "src/QR/HouseholderQR_LAPACKE.h" 45 | #include "src/QR/ColPivHouseholderQR_LAPACKE.h" 46 | #endif 47 | 48 | #include "src/Core/util/ReenableStupidWarnings.h" 49 | 50 | #endif // EIGEN_QR_MODULE_H 51 | -------------------------------------------------------------------------------- /example.sh: -------------------------------------------------------------------------------- 1 | # compiling 2 | mkdir build bin 3 | cd build 4 | cmake .. 5 | make 6 | 7 | # Download the dataset 8 | wget -P ./data/gist ftp://ftp.irisa.fr/local/texmex/corpus/gist.tar.gz 9 | tar -xzvf ./data/gist/gist.tar.gz -C ./data/gist 10 | 11 | # indexing and querying for symqg 12 | ./bin/symqg_indexing ./data/gist/gist_base.fvecs 32 400 ./data/gist/symqg_32.index 13 | 14 | ./bin/symqg_querying ./data/gist/symqg_32.index ./data/gist/gist_query.fvecs ./data/gist/gist_groundtruth.ivecs 15 | 16 | # indexing and querying for RabitQ+ with ivf, please refer to python/ivf.py for more information about clustering 17 | python ./python/ivf.py ./data/gist/gist_base.fvecs 4096 ./data/gist/gist_centroids_4096.fvecs ./data/gist/gist_clusterids_4096.ivecs 18 | 19 | ./bin/ivf_rabitq_indexing ./data/gist/gist_base.fvecs ./data/gist/gist_centroids_4096.fvecs ./data/gist/gist_clusterids_4096.ivecs 3 ./data/gist/ivf_4096_3.index 20 | 21 | ./bin/ivf_rabitq_querying ./data/gist/ivf_4096_3.index ./data/gist/gist_query.fvecs ./data/gist/gist_groundtruth.ivecs 22 | 23 | # indexing and querying for RabitQ+ with hnsw, do clustering first 24 | python ./python/ivf.py ./data/gist/gist_base.fvecs 16 ./data/gist/gist_centroids_16.fvecs ./data/gist/gist_clusterids_16.ivecs 25 | 26 | ./bin/hnsw_rabitq_indexing ./data/gist/gist_base.fvecs ./data/gist/gist_centroids_16.fvecs ./data/gist/gist_clusterids_16.ivecs 16 200 5 ./data/gist/hnsw_5.index 27 | 28 | ./bin/hnsw_rabitq_querying ./data/gist/hnsw_5.index ./data/gist/gist_query.fvecs ./data/gist/gist_groundtruth.ivecs 29 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/UmfPackSupport: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // This Source Code Form is subject to the terms of the Mozilla 5 | // Public License v. 2.0. If a copy of the MPL was not distributed 6 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 7 | 8 | #ifndef EIGEN_UMFPACKSUPPORT_MODULE_H 9 | #define EIGEN_UMFPACKSUPPORT_MODULE_H 10 | 11 | #include "SparseCore" 12 | 13 | #include "src/Core/util/DisableStupidWarnings.h" 14 | 15 | extern "C" { 16 | #include 17 | } 18 | 19 | /** \ingroup Support_modules 20 | * \defgroup UmfPackSupport_Module UmfPackSupport module 21 | * 22 | * This module provides an interface to the UmfPack library which is part of the suitesparse package. 23 | * It provides the following factorization class: 24 | * - class UmfPackLU: a multifrontal sequential LU factorization. 25 | * 26 | * \code 27 | * #include 28 | * \endcode 29 | * 30 | * In order to use this module, the umfpack headers must be accessible from the include paths, and your binary must be linked to the umfpack library and its dependencies. 31 | * The dependencies depend on how umfpack has been compiled. 32 | * For a cmake based project, you can use our FindUmfPack.cmake module to help you in this task. 33 | * 34 | */ 35 | 36 | #include "src/UmfPackSupport/UmfPackSupport.h" 37 | 38 | #include "src/Core/util/ReenableStupidWarnings.h" 39 | 40 | #endif // EIGEN_UMFPACKSUPPORT_MODULE_H 41 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/KLUSupport: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // This Source Code Form is subject to the terms of the Mozilla 5 | // Public License v. 2.0. If a copy of the MPL was not distributed 6 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 7 | 8 | #ifndef EIGEN_KLUSUPPORT_MODULE_H 9 | #define EIGEN_KLUSUPPORT_MODULE_H 10 | 11 | #include 12 | 13 | #include 14 | 15 | extern "C" { 16 | #include 17 | #include 18 | } 19 | 20 | /** \ingroup Support_modules 21 | * \defgroup KLUSupport_Module KLUSupport module 22 | * 23 | * This module provides an interface to the KLU library which is part of the suitesparse package. 24 | * It provides the following factorization class: 25 | * - class KLU: a sparse LU factorization, well-suited for circuit simulation. 26 | * 27 | * \code 28 | * #include 29 | * \endcode 30 | * 31 | * In order to use this module, the klu and btf headers must be accessible from the include paths, and your binary must be linked to the klu library and its dependencies. 32 | * The dependencies depend on how umfpack has been compiled. 33 | * For a cmake based project, you can use our FindKLU.cmake module to help you in this task. 34 | * 35 | */ 36 | 37 | #include "src/KLUSupport/KLUSupport.h" 38 | 39 | #include 40 | 41 | #endif // EIGEN_KLUSUPPORT_MODULE_H 42 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/Core/arch/SVE/TypeCasting.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2020, Arm Limited and Contributors 5 | // 6 | // This Source Code Form is subject to the terms of the Mozilla 7 | // Public License v. 2.0. If a copy of the MPL was not distributed 8 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | 10 | #ifndef EIGEN_TYPE_CASTING_SVE_H 11 | #define EIGEN_TYPE_CASTING_SVE_H 12 | 13 | namespace Eigen { 14 | namespace internal { 15 | 16 | template <> 17 | struct type_casting_traits { 18 | enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; 19 | }; 20 | 21 | template <> 22 | struct type_casting_traits { 23 | enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; 24 | }; 25 | 26 | template <> 27 | EIGEN_STRONG_INLINE PacketXf pcast(const PacketXi& a) { 28 | return svcvt_f32_s32_z(svptrue_b32(), a); 29 | } 30 | 31 | template <> 32 | EIGEN_STRONG_INLINE PacketXi pcast(const PacketXf& a) { 33 | return svcvt_s32_f32_z(svptrue_b32(), a); 34 | } 35 | 36 | template <> 37 | EIGEN_STRONG_INLINE PacketXf preinterpret(const PacketXi& a) { 38 | return svreinterpret_f32_s32(a); 39 | } 40 | 41 | template <> 42 | EIGEN_STRONG_INLINE PacketXi preinterpret(const PacketXf& a) { 43 | return svreinterpret_s32_f32(a); 44 | } 45 | 46 | } // namespace internal 47 | } // namespace Eigen 48 | 49 | #endif // EIGEN_TYPE_CASTING_SVE_H 50 | -------------------------------------------------------------------------------- /sample/quantizer.cpp: -------------------------------------------------------------------------------- 1 | #include "rabitqlib/quantization/rabitq.hpp" 2 | #include "rabitqlib/utils/rotator.hpp" 3 | 4 | int main() { 5 | // generate random data 6 | size_t dim = 128; 7 | size_t bit = 4; 8 | 9 | float* data = new float[dim]; 10 | for (size_t i = 0; i < dim; i++) { 11 | data[i] = static_cast(rand()) / static_cast(RAND_MAX); 12 | } 13 | 14 | // choose rotator 15 | rabitqlib::Rotator* rotator = rabitqlib::choose_rotator(dim); 16 | 17 | size_t padded_dim = rotator->size(); 18 | float* rotated_data = new float[padded_dim]; 19 | rotator->rotate(data, rotated_data); 20 | 21 | // print rotated_data 22 | for (size_t i = 0; i < padded_dim; i++) { 23 | std::cout << rotated_data[i] << " "; 24 | } 25 | std::cout << '\n'; 26 | 27 | // quantize 28 | uint8_t* code = new uint8_t[padded_dim]; 29 | float delta = 0; 30 | float vl = 0; 31 | rabitqlib::quant::quantize_scalar(rotated_data, padded_dim, bit, code, delta, vl); 32 | 33 | // [Note: we don't need to store vl as vl = - delta * (2^bit - 1) / 2] 34 | 35 | // reconstruct 36 | float* reconstructed_data = new float[padded_dim]; 37 | rabitqlib::quant::reconstruct_vec(code, delta, vl, padded_dim, reconstructed_data); 38 | 39 | // print reconstructed_data 40 | for (size_t i = 0; i < padded_dim; i++) { 41 | std::cout << reconstructed_data[i] << " "; 42 | } 43 | std::cout << '\n'; 44 | 45 | delete rotator; 46 | delete[] data; 47 | delete[] rotated_data; 48 | delete[] code; 49 | delete[] reconstructed_data; 50 | } 51 | -------------------------------------------------------------------------------- /python/utils/io.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import struct 3 | 4 | 5 | def read_ivecs(filename): 6 | print(f"Reading File - {filename}") 7 | a = np.fromfile(filename, dtype="int32") 8 | d = a[0] 9 | print(f"\t{filename} readed") 10 | return a.reshape(-1, d + 1)[:, 1:] 11 | 12 | 13 | def read_fvecs(filename): 14 | return read_ivecs(filename).view("float32") 15 | 16 | 17 | def write_ivecs(filename, m): 18 | print(f"Writing File - {filename}") 19 | n, d = m.shape 20 | myimt = "i" * d 21 | with open(filename, "wb") as f: 22 | for i in range(n): 23 | f.write(struct.pack("i", d)) 24 | bin = struct.pack(myimt, *m[i]) 25 | f.write(bin) 26 | print(f"\t{filename} wrote") 27 | 28 | 29 | def write_fvecs(filename, m): 30 | m = m.astype("float32") 31 | write_ivecs(filename, m.view("int32")) 32 | 33 | 34 | def read_ibin(filename): 35 | n, d = np.fromfile(filename, count=2, dtype="int32") 36 | a = np.fromfile(filename, dtype="int32") 37 | print(f"\t{filename} readed") 38 | return a[2:].reshape(n, d) 39 | 40 | 41 | def read_fbin(filename): 42 | return read_ibin(filename).view("float32") 43 | 44 | 45 | def write_sampled_data(filename, m, num_sampled, seed=42): 46 | num_points = m.shape[0] 47 | d = m.shape[1] 48 | m = m.view("int32") 49 | 50 | np.random.seed(seed) 51 | sequence = np.random.permutation(num_points) 52 | 53 | with open(filename, "wb") as f: 54 | for i in range(num_sampled): 55 | f.write(struct.pack("i", d)) 56 | bin = struct.pack(f"{d}i", *m[sequence[i]]) 57 | f.write(bin) 58 | -------------------------------------------------------------------------------- /include/rabitqlib/index/ivf/cluster.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include "rabitqlib/defines.hpp" 6 | 7 | namespace rabitqlib::ivf { 8 | 9 | /** 10 | * @brief Cluster is used for ivf index with rabitq+. Components are only used for record 11 | * the addresses for different part of data. 12 | * 13 | */ 14 | class Cluster { 15 | private: 16 | size_t num_; // Num of vectors in this cluster 17 | char* batch_data_ = nullptr; // RaBitQ code and factors 18 | char* ex_data_ = nullptr; // Ex code and factors 19 | PID* ids_ = nullptr; // PID of vectors 20 | 21 | public: 22 | explicit Cluster(size_t, char*, char*, PID*); 23 | Cluster(const Cluster& other); 24 | Cluster(Cluster&& other) noexcept; 25 | ~Cluster() {} 26 | 27 | [[nodiscard]] char* batch_data() const { return this->batch_data_; } 28 | 29 | [[nodiscard]] char* ex_data() const { return ex_data_; } 30 | 31 | [[nodiscard]] PID* ids() const { return this->ids_; } 32 | 33 | [[nodiscard]] size_t num() const { return num_; } 34 | }; 35 | 36 | inline Cluster::Cluster(size_t num, char* batch_data, char* ex_data, PID* ids) 37 | : num_(num), batch_data_(batch_data), ex_data_(ex_data), ids_(ids) {} 38 | 39 | inline Cluster::Cluster(const Cluster& other) 40 | : num_(other.num_) 41 | , batch_data_(other.batch_data_) 42 | , ex_data_(other.ex_data_) 43 | , ids_(other.ids_) {} 44 | 45 | inline Cluster::Cluster(Cluster&& other) noexcept 46 | : num_(other.num_) 47 | , batch_data_(other.batch_data_) 48 | , ex_data_(other.ex_data_) 49 | , ids_(other.ids_) {} 50 | } // namespace rabitqlib::ivf -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/Core/util/ReshapedHelper.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2017 Gael Guennebaud 5 | // 6 | // This Source Code Form is subject to the terms of the Mozilla 7 | // Public License v. 2.0. If a copy of the MPL was not distributed 8 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | 10 | 11 | #ifndef EIGEN_RESHAPED_HELPER_H 12 | #define EIGEN_RESHAPED_HELPER_H 13 | 14 | namespace Eigen { 15 | 16 | enum AutoSize_t { AutoSize }; 17 | const int AutoOrder = 2; 18 | 19 | namespace internal { 20 | 21 | template 22 | struct get_compiletime_reshape_size { 23 | enum { value = get_fixed_value::value }; 24 | }; 25 | 26 | template 27 | Index get_runtime_reshape_size(SizeType size, Index /*other*/, Index /*total*/) { 28 | return internal::get_runtime_value(size); 29 | } 30 | 31 | template 32 | struct get_compiletime_reshape_size { 33 | enum { 34 | other_size = get_fixed_value::value, 35 | value = (TotalSize==Dynamic || other_size==Dynamic) ? Dynamic : TotalSize / other_size }; 36 | }; 37 | 38 | inline Index get_runtime_reshape_size(AutoSize_t /*size*/, Index other, Index total) { 39 | return total/other; 40 | } 41 | 42 | template 43 | struct get_compiletime_reshape_order { 44 | enum { value = Order == AutoOrder ? Flags & RowMajorBit : Order }; 45 | }; 46 | 47 | } 48 | 49 | } // end namespace Eigen 50 | 51 | #endif // EIGEN_RESHAPED_HELPER_H 52 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/SVD: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // This Source Code Form is subject to the terms of the Mozilla 5 | // Public License v. 2.0. If a copy of the MPL was not distributed 6 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 7 | 8 | #ifndef EIGEN_SVD_MODULE_H 9 | #define EIGEN_SVD_MODULE_H 10 | 11 | #include "QR" 12 | #include "Householder" 13 | #include "Jacobi" 14 | 15 | #include "src/Core/util/DisableStupidWarnings.h" 16 | 17 | /** \defgroup SVD_Module SVD module 18 | * 19 | * 20 | * 21 | * This module provides SVD decomposition for matrices (both real and complex). 22 | * Two decomposition algorithms are provided: 23 | * - JacobiSVD implementing two-sided Jacobi iterations is numerically very accurate, fast for small matrices, but very slow for larger ones. 24 | * - BDCSVD implementing a recursive divide & conquer strategy on top of an upper-bidiagonalization which remains fast for large problems. 25 | * These decompositions are accessible via the respective classes and following MatrixBase methods: 26 | * - MatrixBase::jacobiSvd() 27 | * - MatrixBase::bdcSvd() 28 | * 29 | * \code 30 | * #include 31 | * \endcode 32 | */ 33 | 34 | #include "src/misc/RealSvd2x2.h" 35 | #include "src/SVD/UpperBidiagonalization.h" 36 | #include "src/SVD/SVDBase.h" 37 | #include "src/SVD/JacobiSVD.h" 38 | #include "src/SVD/BDCSVD.h" 39 | #if defined(EIGEN_USE_LAPACKE) && !defined(EIGEN_USE_LAPACKE_STRICT) 40 | #ifdef EIGEN_USE_MKL 41 | #include "mkl_lapacke.h" 42 | #else 43 | #include "src/misc/lapacke.h" 44 | #endif 45 | #include "src/SVD/JacobiSVD_LAPACKE.h" 46 | #endif 47 | 48 | #include "src/Core/util/ReenableStupidWarnings.h" 49 | 50 | #endif // EIGEN_SVD_MODULE_H 51 | -------------------------------------------------------------------------------- /python/ivf.py: -------------------------------------------------------------------------------- 1 | import faiss 2 | import sys 3 | from time import time 4 | from utils.io import read_fvecs, write_fvecs, write_ivecs, read_fbin 5 | 6 | if __name__ == "__main__": 7 | if len(sys.argv) < 5: 8 | print(f"Usage: {sys.argv[0]} ") 9 | print("arg1: path for data file, format .fvecs") 10 | print("arg2: number of clusters") 11 | print("arg3: path for centroid vectors") 12 | print("arg4: path for cluster ids") 13 | print("arg5: distance metric") 14 | exit(1) 15 | 16 | 17 | # path 18 | data_path = sys.argv[1] 19 | K = int(sys.argv[2]) 20 | centroids_path = sys.argv[3] 21 | cluster_id_path = sys.argv[4] 22 | if len(sys.argv) == 6: 23 | distance_metric = sys.argv[5].lower() 24 | if distance_metric == "l2": 25 | metric = faiss.METRIC_L2 26 | print("Using L2 metric") 27 | elif distance_metric in ["ip", "innerproduct"]: 28 | metric = faiss.METRIC_INNER_PRODUCT 29 | print("Using InnerProduct metric") 30 | else: 31 | raise ValueError("Unsupported distance metric. Use 'l2' or 'ip'.") 32 | else: 33 | metric = faiss.METRIC_L2 # by default, L2 metric 34 | print("Using L2 metric by default") 35 | 36 | X = read_fvecs(data_path) 37 | 38 | dim = X.shape[1] 39 | 40 | t1 = time() 41 | 42 | # cluster data vectors 43 | index = faiss.index_factory(dim, f"IVF{K},Flat", metric) 44 | index.verbose = True 45 | index.train(X) 46 | 47 | t2 = time() 48 | print(f"Time for training ivf {t2-t1} secs") 49 | 50 | centroids = index.quantizer.reconstruct_n(0, index.nlist) 51 | _, cluster_id = index.quantizer.search(X, 1) 52 | 53 | write_ivecs(cluster_id_path, cluster_id) 54 | write_fvecs(centroids_path, centroids) 55 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/Core/SelfCwiseBinaryOp.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2009-2010 Gael Guennebaud 5 | // 6 | // This Source Code Form is subject to the terms of the Mozilla 7 | // Public License v. 2.0. If a copy of the MPL was not distributed 8 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | 10 | #ifndef EIGEN_SELFCWISEBINARYOP_H 11 | #define EIGEN_SELFCWISEBINARYOP_H 12 | 13 | namespace Eigen { 14 | 15 | // TODO generalize the scalar type of 'other' 16 | 17 | template 18 | EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& DenseBase::operator*=(const Scalar& other) 19 | { 20 | internal::call_assignment(this->derived(), PlainObject::Constant(rows(),cols(),other), internal::mul_assign_op()); 21 | return derived(); 22 | } 23 | 24 | template 25 | EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& ArrayBase::operator+=(const Scalar& other) 26 | { 27 | internal::call_assignment(this->derived(), PlainObject::Constant(rows(),cols(),other), internal::add_assign_op()); 28 | return derived(); 29 | } 30 | 31 | template 32 | EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& ArrayBase::operator-=(const Scalar& other) 33 | { 34 | internal::call_assignment(this->derived(), PlainObject::Constant(rows(),cols(),other), internal::sub_assign_op()); 35 | return derived(); 36 | } 37 | 38 | template 39 | EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& DenseBase::operator/=(const Scalar& other) 40 | { 41 | internal::call_assignment(this->derived(), PlainObject::Constant(rows(),cols(),other), internal::div_assign_op()); 42 | return derived(); 43 | } 44 | 45 | } // end namespace Eigen 46 | 47 | #endif // EIGEN_SELFCWISEBINARYOP_H 48 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/SparseCore/SparseRedux.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2008-2014 Gael Guennebaud 5 | // 6 | // This Source Code Form is subject to the terms of the Mozilla 7 | // Public License v. 2.0. If a copy of the MPL was not distributed 8 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | 10 | #ifndef EIGEN_SPARSEREDUX_H 11 | #define EIGEN_SPARSEREDUX_H 12 | 13 | namespace Eigen { 14 | 15 | template 16 | typename internal::traits::Scalar 17 | SparseMatrixBase::sum() const 18 | { 19 | eigen_assert(rows()>0 && cols()>0 && "you are using a non initialized matrix"); 20 | Scalar res(0); 21 | internal::evaluator thisEval(derived()); 22 | for (Index j=0; j::InnerIterator iter(thisEval,j); iter; ++iter) 24 | res += iter.value(); 25 | return res; 26 | } 27 | 28 | template 29 | typename internal::traits >::Scalar 30 | SparseMatrix<_Scalar,_Options,_Index>::sum() const 31 | { 32 | eigen_assert(rows()>0 && cols()>0 && "you are using a non initialized matrix"); 33 | if(this->isCompressed()) 34 | return Matrix::Map(m_data.valuePtr(), m_data.size()).sum(); 35 | else 36 | return Base::sum(); 37 | } 38 | 39 | template 40 | typename internal::traits >::Scalar 41 | SparseVector<_Scalar,_Options,_Index>::sum() const 42 | { 43 | eigen_assert(rows()>0 && cols()>0 && "you are using a non initialized matrix"); 44 | return Matrix::Map(m_data.valuePtr(), m_data.size()).sum(); 45 | } 46 | 47 | } // end namespace Eigen 48 | 49 | #endif // EIGEN_SPARSEREDUX_H 50 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/PaStiXSupport: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // This Source Code Form is subject to the terms of the Mozilla 5 | // Public License v. 2.0. If a copy of the MPL was not distributed 6 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 7 | 8 | #ifndef EIGEN_PASTIXSUPPORT_MODULE_H 9 | #define EIGEN_PASTIXSUPPORT_MODULE_H 10 | 11 | #include "SparseCore" 12 | 13 | #include "src/Core/util/DisableStupidWarnings.h" 14 | 15 | extern "C" { 16 | #include 17 | #include 18 | } 19 | 20 | #ifdef complex 21 | #undef complex 22 | #endif 23 | 24 | /** \ingroup Support_modules 25 | * \defgroup PaStiXSupport_Module PaStiXSupport module 26 | * 27 | * This module provides an interface to the PaSTiX library. 28 | * PaSTiX is a general \b supernodal, \b parallel and \b opensource sparse solver. 29 | * It provides the two following main factorization classes: 30 | * - class PastixLLT : a supernodal, parallel LLt Cholesky factorization. 31 | * - class PastixLDLT: a supernodal, parallel LDLt Cholesky factorization. 32 | * - class PastixLU : a supernodal, parallel LU factorization (optimized for a symmetric pattern). 33 | * 34 | * \code 35 | * #include 36 | * \endcode 37 | * 38 | * In order to use this module, the PaSTiX headers must be accessible from the include paths, and your binary must be linked to the PaSTiX library and its dependencies. 39 | * This wrapper resuires PaStiX version 5.x compiled without MPI support. 40 | * The dependencies depend on how PaSTiX has been compiled. 41 | * For a cmake based project, you can use our FindPaSTiX.cmake module to help you in this task. 42 | * 43 | */ 44 | 45 | #include "src/PaStiXSupport/PaStiXSupport.h" 46 | 47 | #include "src/Core/util/ReenableStupidWarnings.h" 48 | 49 | #endif // EIGEN_PASTIXSUPPORT_MODULE_H 50 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/Core/arch/Default/Settings.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2008-2010 Gael Guennebaud 5 | // Copyright (C) 2006-2008 Benoit Jacob 6 | // 7 | // This Source Code Form is subject to the terms of the Mozilla 8 | // Public License v. 2.0. If a copy of the MPL was not distributed 9 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 10 | 11 | 12 | /* All the parameters defined in this file can be specialized in the 13 | * architecture specific files, and/or by the user. 14 | * More to come... */ 15 | 16 | #ifndef EIGEN_DEFAULT_SETTINGS_H 17 | #define EIGEN_DEFAULT_SETTINGS_H 18 | 19 | /** Defines the maximal loop size to enable meta unrolling of loops. 20 | * Note that the value here is expressed in Eigen's own notion of "number of FLOPS", 21 | * it does not correspond to the number of iterations or the number of instructions 22 | */ 23 | #ifndef EIGEN_UNROLLING_LIMIT 24 | #define EIGEN_UNROLLING_LIMIT 110 25 | #endif 26 | 27 | /** Defines the threshold between a "small" and a "large" matrix. 28 | * This threshold is mainly used to select the proper product implementation. 29 | */ 30 | #ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 31 | #define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8 32 | #endif 33 | 34 | /** Defines the maximal width of the blocks used in the triangular product and solver 35 | * for vectors (level 2 blas xTRMV and xTRSV). The default is 8. 36 | */ 37 | #ifndef EIGEN_TUNE_TRIANGULAR_PANEL_WIDTH 38 | #define EIGEN_TUNE_TRIANGULAR_PANEL_WIDTH 8 39 | #endif 40 | 41 | 42 | /** Defines the default number of registers available for that architecture. 43 | * Currently it must be 8 or 16. Other values will fail. 44 | */ 45 | #ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 46 | #define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 8 47 | #endif 48 | 49 | #endif // EIGEN_DEFAULT_SETTINGS_H 50 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/misc/RealSvd2x2.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2009-2010 Benoit Jacob 5 | // Copyright (C) 2013-2016 Gael Guennebaud 6 | // 7 | // This Source Code Form is subject to the terms of the Mozilla 8 | // Public License v. 2.0. If a copy of the MPL was not distributed 9 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 10 | 11 | #ifndef EIGEN_REALSVD2X2_H 12 | #define EIGEN_REALSVD2X2_H 13 | 14 | namespace Eigen { 15 | 16 | namespace internal { 17 | 18 | template 19 | void real_2x2_jacobi_svd(const MatrixType& matrix, Index p, Index q, 20 | JacobiRotation *j_left, 21 | JacobiRotation *j_right) 22 | { 23 | using std::sqrt; 24 | using std::abs; 25 | Matrix m; 26 | m << numext::real(matrix.coeff(p,p)), numext::real(matrix.coeff(p,q)), 27 | numext::real(matrix.coeff(q,p)), numext::real(matrix.coeff(q,q)); 28 | JacobiRotation rot1; 29 | RealScalar t = m.coeff(0,0) + m.coeff(1,1); 30 | RealScalar d = m.coeff(1,0) - m.coeff(0,1); 31 | 32 | if(abs(d) < (std::numeric_limits::min)()) 33 | { 34 | rot1.s() = RealScalar(0); 35 | rot1.c() = RealScalar(1); 36 | } 37 | else 38 | { 39 | // If d!=0, then t/d cannot overflow because the magnitude of the 40 | // entries forming d are not too small compared to the ones forming t. 41 | RealScalar u = t / d; 42 | RealScalar tmp = sqrt(RealScalar(1) + numext::abs2(u)); 43 | rot1.s() = RealScalar(1) / tmp; 44 | rot1.c() = u / tmp; 45 | } 46 | m.applyOnTheLeft(0,1,rot1); 47 | j_right->makeJacobi(m,0,1); 48 | *j_left = rot1 * j_right->transpose(); 49 | } 50 | 51 | } // end namespace internal 52 | 53 | } // end namespace Eigen 54 | 55 | #endif // EIGEN_REALSVD2X2_H 56 | -------------------------------------------------------------------------------- /docs/docs/rabitq/reranking.md: -------------------------------------------------------------------------------- 1 | # Reranking 2 | Reranking is a technique widely adopted for improving recall of vector search. During searching, ANN algorithms usually 3 | 4 | 1. shortlist a set of candidates based on an index and their quantization codes (e.g., **in memory**); 5 | 2. retrieve the raw vectors (e.g., **from disks**) for those candidates which have the smallest estimated distances; 6 | 3. computes the exact distances for the candidates to find the nearest neighbors. 7 | 8 | RaBitQ has a unique advantage in reranking due to its theoretical error bound. It can skip reranking a candidate if the lower bound of its estimated distance is larger than the upper bound of the distance of the nearest neighbors. 9 | 10 | Based on error bounds, it is possible to rerank **fewer than $K$ vectors** to achieve nearly perfect recall - it only reranks the vectors on the boundaries of KNNs. 11 | 12 | ## Algorithm Description 13 | Let $K$ be the number of nearest neighbors we target. After receiving the candidates and their estimated distances from an index, e.g., HNSW + RaBitQ, we perform the following strategy of reranking to minimize the number of retrieved raw vectors from disks. 14 | 15 | 1. Sort the candidates with respect to their estimated distances. 16 | 2. Initialize a max-heap (sorted by **upper bounds of distances**) `KNNs` with the top-$K$ candidates which have the smallest estimated distances. 17 | 3. Enumerate the remaining candidates. For a new candidate A, 18 | 1. [Condition 1 - A's lower bound $>$ the maximum upper bound in `KNNs`.] 19 | 1. Drop the new candidate and move on. 20 | 2. [Condition 2 - A's lower bound $\le$ the maximum upper bound in `KNNs`, and the top candidate in `KNNs` has been reranked.] 21 | 1. Rerank the new candidate, update `KNNs` and move on. 22 | 3. [Condition 3 - A's lower bound $\le$ the maximum upper bound in `KNNs`, and the top candidate in `KNNs` has NOT been reranked.] 23 | 1. Rerank the top candidate in `KNNs`, update `KNNs` and repeat the procedure for candidate A. 24 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/SparseLU: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2012 Désiré Nuentsa-Wakam 5 | // Copyright (C) 2012 Gael Guennebaud 6 | // 7 | // This Source Code Form is subject to the terms of the Mozilla 8 | // Public License v. 2.0. If a copy of the MPL was not distributed 9 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 10 | 11 | #ifndef EIGEN_SPARSELU_MODULE_H 12 | #define EIGEN_SPARSELU_MODULE_H 13 | 14 | #include "SparseCore" 15 | 16 | /** 17 | * \defgroup SparseLU_Module SparseLU module 18 | * This module defines a supernodal factorization of general sparse matrices. 19 | * The code is fully optimized for supernode-panel updates with specialized kernels. 20 | * Please, see the documentation of the SparseLU class for more details. 21 | */ 22 | 23 | // Ordering interface 24 | #include "OrderingMethods" 25 | 26 | #include "src/Core/util/DisableStupidWarnings.h" 27 | 28 | #include "src/SparseLU/SparseLU_gemm_kernel.h" 29 | 30 | #include "src/SparseLU/SparseLU_Structs.h" 31 | #include "src/SparseLU/SparseLU_SupernodalMatrix.h" 32 | #include "src/SparseLU/SparseLUImpl.h" 33 | #include "src/SparseCore/SparseColEtree.h" 34 | #include "src/SparseLU/SparseLU_Memory.h" 35 | #include "src/SparseLU/SparseLU_heap_relax_snode.h" 36 | #include "src/SparseLU/SparseLU_relax_snode.h" 37 | #include "src/SparseLU/SparseLU_pivotL.h" 38 | #include "src/SparseLU/SparseLU_panel_dfs.h" 39 | #include "src/SparseLU/SparseLU_kernel_bmod.h" 40 | #include "src/SparseLU/SparseLU_panel_bmod.h" 41 | #include "src/SparseLU/SparseLU_column_dfs.h" 42 | #include "src/SparseLU/SparseLU_column_bmod.h" 43 | #include "src/SparseLU/SparseLU_copy_to_ucol.h" 44 | #include "src/SparseLU/SparseLU_pruneL.h" 45 | #include "src/SparseLU/SparseLU_Utils.h" 46 | #include "src/SparseLU/SparseLU.h" 47 | 48 | #include "src/Core/util/ReenableStupidWarnings.h" 49 | 50 | #endif // EIGEN_SPARSELU_MODULE_H 51 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/Eigenvalues: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // This Source Code Form is subject to the terms of the Mozilla 5 | // Public License v. 2.0. If a copy of the MPL was not distributed 6 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 7 | 8 | #ifndef EIGEN_EIGENVALUES_MODULE_H 9 | #define EIGEN_EIGENVALUES_MODULE_H 10 | 11 | #include "Core" 12 | 13 | #include "Cholesky" 14 | #include "Jacobi" 15 | #include "Householder" 16 | #include "LU" 17 | #include "Geometry" 18 | 19 | #include "src/Core/util/DisableStupidWarnings.h" 20 | 21 | /** \defgroup Eigenvalues_Module Eigenvalues module 22 | * 23 | * 24 | * 25 | * This module mainly provides various eigenvalue solvers. 26 | * This module also provides some MatrixBase methods, including: 27 | * - MatrixBase::eigenvalues(), 28 | * - MatrixBase::operatorNorm() 29 | * 30 | * \code 31 | * #include 32 | * \endcode 33 | */ 34 | 35 | #include "src/misc/RealSvd2x2.h" 36 | #include "src/Eigenvalues/Tridiagonalization.h" 37 | #include "src/Eigenvalues/RealSchur.h" 38 | #include "src/Eigenvalues/EigenSolver.h" 39 | #include "src/Eigenvalues/SelfAdjointEigenSolver.h" 40 | #include "src/Eigenvalues/GeneralizedSelfAdjointEigenSolver.h" 41 | #include "src/Eigenvalues/HessenbergDecomposition.h" 42 | #include "src/Eigenvalues/ComplexSchur.h" 43 | #include "src/Eigenvalues/ComplexEigenSolver.h" 44 | #include "src/Eigenvalues/RealQZ.h" 45 | #include "src/Eigenvalues/GeneralizedEigenSolver.h" 46 | #include "src/Eigenvalues/MatrixBaseEigenvalues.h" 47 | #ifdef EIGEN_USE_LAPACKE 48 | #ifdef EIGEN_USE_MKL 49 | #include "mkl_lapacke.h" 50 | #else 51 | #include "src/misc/lapacke.h" 52 | #endif 53 | #include "src/Eigenvalues/RealSchur_LAPACKE.h" 54 | #include "src/Eigenvalues/ComplexSchur_LAPACKE.h" 55 | #include "src/Eigenvalues/SelfAdjointEigenSolver_LAPACKE.h" 56 | #endif 57 | 58 | #include "src/Core/util/ReenableStupidWarnings.h" 59 | 60 | #endif // EIGEN_EIGENVALUES_MODULE_H 61 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/CholmodSupport: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // This Source Code Form is subject to the terms of the Mozilla 5 | // Public License v. 2.0. If a copy of the MPL was not distributed 6 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 7 | 8 | #ifndef EIGEN_CHOLMODSUPPORT_MODULE_H 9 | #define EIGEN_CHOLMODSUPPORT_MODULE_H 10 | 11 | #include "SparseCore" 12 | 13 | #include "src/Core/util/DisableStupidWarnings.h" 14 | 15 | extern "C" { 16 | #include 17 | } 18 | 19 | /** \ingroup Support_modules 20 | * \defgroup CholmodSupport_Module CholmodSupport module 21 | * 22 | * This module provides an interface to the Cholmod library which is part of the suitesparse package. 23 | * It provides the two following main factorization classes: 24 | * - class CholmodSupernodalLLT: a supernodal LLT Cholesky factorization. 25 | * - class CholmodDecomposiiton: a general L(D)LT Cholesky factorization with automatic or explicit runtime selection of the underlying factorization method (supernodal or simplicial). 26 | * 27 | * For the sake of completeness, this module also propose the two following classes: 28 | * - class CholmodSimplicialLLT 29 | * - class CholmodSimplicialLDLT 30 | * Note that these classes does not bring any particular advantage compared to the built-in 31 | * SimplicialLLT and SimplicialLDLT factorization classes. 32 | * 33 | * \code 34 | * #include 35 | * \endcode 36 | * 37 | * In order to use this module, the cholmod headers must be accessible from the include paths, and your binary must be linked to the cholmod library and its dependencies. 38 | * The dependencies depend on how cholmod has been compiled. 39 | * For a cmake based project, you can use our FindCholmod.cmake module to help you in this task. 40 | * 41 | */ 42 | 43 | #include "src/CholmodSupport/CholmodSupport.h" 44 | 45 | #include "src/Core/util/ReenableStupidWarnings.h" 46 | 47 | #endif // EIGEN_CHOLMODSUPPORT_MODULE_H 48 | 49 | -------------------------------------------------------------------------------- /include/rabitqlib/third/hnswlib/visited_list_pool.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | namespace hnswlib { 8 | typedef unsigned short int vl_type; 9 | 10 | class VisitedList { 11 | public: 12 | vl_type curV; 13 | vl_type *mass; 14 | unsigned int numelements; 15 | 16 | VisitedList(int numelements1) { 17 | curV = -1; 18 | numelements = numelements1; 19 | mass = new vl_type[numelements]; 20 | } 21 | 22 | void reset() { 23 | curV++; 24 | if (curV == 0) { 25 | memset(mass, 0, sizeof(vl_type) * numelements); 26 | curV++; 27 | } 28 | } 29 | 30 | ~VisitedList() { delete[] mass; } 31 | }; 32 | /////////////////////////////////////////////////////////// 33 | // 34 | // Class for multi-threaded pool-management of VisitedLists 35 | // 36 | ///////////////////////////////////////////////////////// 37 | 38 | class VisitedListPool { 39 | std::deque pool; 40 | std::mutex poolguard; 41 | int numelements; 42 | 43 | public: 44 | VisitedListPool(int initmaxpools, int numelements1) { 45 | numelements = numelements1; 46 | for (int i = 0; i < initmaxpools; i++) 47 | pool.push_front(new VisitedList(numelements)); 48 | } 49 | 50 | VisitedList *getFreeVisitedList() { 51 | VisitedList *rez; 52 | { 53 | std::unique_lock lock(poolguard); 54 | if (pool.size() > 0) { 55 | rez = pool.front(); 56 | pool.pop_front(); 57 | } else { 58 | rez = new VisitedList(numelements); 59 | } 60 | } 61 | rez->reset(); 62 | return rez; 63 | } 64 | 65 | void releaseVisitedList(VisitedList *vl) { 66 | std::unique_lock lock(poolguard); 67 | pool.push_front(vl); 68 | } 69 | 70 | ~VisitedListPool() { 71 | while (pool.size()) { 72 | VisitedList *rez = pool.front(); 73 | pool.pop_front(); 74 | delete rez; 75 | } 76 | } 77 | }; 78 | } // namespace hnswlib 79 | -------------------------------------------------------------------------------- /sample/symqg_indexing.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "rabitqlib/defines.hpp" 4 | #include "rabitqlib/index/symqg/qg.hpp" 5 | #include "rabitqlib/index/symqg/qg_builder.hpp" 6 | #include "rabitqlib/utils/io.hpp" 7 | #include "rabitqlib/utils/stopw.hpp" 8 | 9 | using PID = rabitqlib::PID; 10 | using index_type = rabitqlib::symqg::QuantizedGraph; 11 | using data_type = rabitqlib::RowMajorArray; 12 | using gt_type = rabitqlib::RowMajorArray; 13 | 14 | int main(int argc, char** argv) { 15 | if (argc < 5) { 16 | std::cerr << "Usage: " << argv[0] << " \n" 17 | << "arg1: path for data file, format .fvecs\n" 18 | << "arg2: degree bound for symqg, must be a multiple of 32\n" 19 | << "arg3: ef for indexing \n" 20 | << "arg4: path for saving index\n" 21 | << "arg5: metric type (\"l2\" or \"ip\"), l2 by default\n"; 22 | exit(1); 23 | } 24 | 25 | char* data_file = argv[1]; 26 | size_t degree = atoi(argv[2]); 27 | size_t ef = atoi(argv[3]); 28 | char* index_file = argv[4]; 29 | 30 | rabitqlib::MetricType metric_type = rabitqlib::METRIC_L2; 31 | if (argc > 5) { 32 | std::string metric_str(argv[5]); 33 | if (metric_str == "ip" || metric_str == "IP") { 34 | metric_type = rabitqlib::METRIC_IP; 35 | } 36 | } 37 | if (metric_type == rabitqlib::METRIC_IP) { 38 | std::cout << "Metric Type: IP\n"; 39 | } else if (metric_type == rabitqlib::METRIC_L2) { 40 | std::cout << "Metric Type: L2\n"; 41 | } 42 | 43 | data_type data; 44 | 45 | rabitqlib::load_vecs(data_file, data); 46 | 47 | rabitqlib::StopW stopw; 48 | 49 | index_type qg(data.rows(), data.cols(), degree, metric_type); 50 | 51 | rabitqlib::symqg::QGBuilder builder(qg, ef, data.data()); 52 | 53 | // 3 iters, refine at last iter 54 | builder.build(); 55 | 56 | auto milisecs = stopw.get_elapsed_mili(); 57 | 58 | std::cout << "Indexing time " << milisecs / 1000.F << " secs\n"; 59 | 60 | qg.save(index_file); 61 | 62 | return 0; 63 | } -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/Geometry: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // This Source Code Form is subject to the terms of the Mozilla 5 | // Public License v. 2.0. If a copy of the MPL was not distributed 6 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 7 | 8 | #ifndef EIGEN_GEOMETRY_MODULE_H 9 | #define EIGEN_GEOMETRY_MODULE_H 10 | 11 | #include "Core" 12 | 13 | #include "SVD" 14 | #include "LU" 15 | #include 16 | 17 | #include "src/Core/util/DisableStupidWarnings.h" 18 | 19 | /** \defgroup Geometry_Module Geometry module 20 | * 21 | * This module provides support for: 22 | * - fixed-size homogeneous transformations 23 | * - translation, scaling, 2D and 3D rotations 24 | * - \link Quaternion quaternions \endlink 25 | * - cross products (\ref MatrixBase::cross, \ref MatrixBase::cross3) 26 | * - orthognal vector generation (\ref MatrixBase::unitOrthogonal) 27 | * - some linear components: \link ParametrizedLine parametrized-lines \endlink and \link Hyperplane hyperplanes \endlink 28 | * - \link AlignedBox axis aligned bounding boxes \endlink 29 | * - \link umeyama least-square transformation fitting \endlink 30 | * 31 | * \code 32 | * #include 33 | * \endcode 34 | */ 35 | 36 | #include "src/Geometry/OrthoMethods.h" 37 | #include "src/Geometry/EulerAngles.h" 38 | 39 | #include "src/Geometry/Homogeneous.h" 40 | #include "src/Geometry/RotationBase.h" 41 | #include "src/Geometry/Rotation2D.h" 42 | #include "src/Geometry/Quaternion.h" 43 | #include "src/Geometry/AngleAxis.h" 44 | #include "src/Geometry/Transform.h" 45 | #include "src/Geometry/Translation.h" 46 | #include "src/Geometry/Scaling.h" 47 | #include "src/Geometry/Hyperplane.h" 48 | #include "src/Geometry/ParametrizedLine.h" 49 | #include "src/Geometry/AlignedBox.h" 50 | #include "src/Geometry/Umeyama.h" 51 | 52 | // Use the SSE optimized version whenever possible. 53 | #if (defined EIGEN_VECTORIZE_SSE) || (defined EIGEN_VECTORIZE_NEON) 54 | #include "src/Geometry/arch/Geometry_SIMD.h" 55 | #endif 56 | 57 | #include "src/Core/util/ReenableStupidWarnings.h" 58 | 59 | #endif // EIGEN_GEOMETRY_MODULE_H 60 | -------------------------------------------------------------------------------- /docs/docs/rabitq/rabitq.md: -------------------------------------------------------------------------------- 1 | # RaBitQ 2 | 3 | The RaBitQ algorithm is a drop-in replacement of binary quantization and (uniform) scalar quantization, with its [1-bit version](https://arxiv.org/abs/2405.12497) (released in May 2024) and [multi-bit version](https://arxiv.org/abs/2409.09913) (released in Sep 2024), respectively. 4 | 5 | 6 | 7 | 8 | 9 | The key advantages of RaBitQ include 10 | 11 | - **High Accuracy with Tiny Space** - RaBitQ achieves the state-of-the-art accuracy under diverse space budgets for the estimation of similarity metrics. It produces promising accuracy with even **1-bit per dimension**. 12 | - **Fast Distance Estimation** - RaBitQ supports to estimate the similarity metrics with high efficiency based on bitwise operations or [FastScan](https://arxiv.org/abs/1704.07355). 13 | - **Theoretical Error Bound** - RaBitQ provides an asymptotically optimal error bound for the estimation of distances and inner product. The error bound can be used for reliable ordering and [reranking](reranking.md). 14 | 15 | 16 | ## Workflow 17 | 18 | The RaBitQ algorithm includes two steps: 19 | 20 | 1. **Random Rotation** - Sample a random rotation and apply it to all vectors (including the raw data vectors, the center vector and the raw query vectors). See [Rotator](rotator.md) for more details. 21 | 22 | 2. **Quantization** - After the random rotation, the quantization algorithm quantizes a vector of floating-point numbers into a vector of low-bit unsigned integers. See [Quantizer](quantizer.md) for more details. 23 | 24 | 25 | After the quantization, we can estimate the similarity metrics including Euclidean distance, inner product and cosine similarity based on the code vector $\mathbf{x}_u$ and the rescaling factor $\Delta_x$. See [Estimator](estimator.md) for more details. 26 | 27 | -------------------------------------------------------------------------------- /include/rabitqlib/defines.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include "rabitqlib/third/Eigen/Dense" 6 | 7 | #define BIT_ID(x) (__builtin_popcount((x) - 1)) 8 | #define LOWBIT(x) ((x) & (-(x))) 9 | 10 | namespace rabitqlib { 11 | 12 | using PID = uint32_t; 13 | 14 | constexpr uint32_t kPidMax = 0xFFFFFFFF; 15 | 16 | template 17 | using RowMajorMatrix = Eigen::Matrix; 18 | 19 | template 20 | using RowMajorMatrixMap = Eigen::Map>; 21 | 22 | template 23 | using ConstRowMajorMatrixMap = Eigen::Map>; 24 | 25 | template 26 | using RowMajorArray = Eigen::Array; 27 | 28 | template 29 | using Vector = Eigen::Matrix; 30 | 31 | template 32 | using RowMajorArrayMap = Eigen::Map>; 33 | 34 | template 35 | using ConstRowMajorArrayMap = Eigen::Map>; 36 | 37 | template 38 | using VectorMap = Eigen::Map>; 39 | 40 | template 41 | using ConstVectorMap = Eigen::Map>; 42 | 43 | template 44 | struct AnnCandidate { 45 | TP id = 0; 46 | T distance = std::numeric_limits::max(); 47 | 48 | AnnCandidate() = default; 49 | explicit AnnCandidate(TP vec_id, T dis) : id(vec_id), distance(dis) {} 50 | 51 | friend bool operator<(const AnnCandidate& first, const AnnCandidate& second) { 52 | return first.distance < second.distance; 53 | } 54 | friend bool operator>(const AnnCandidate& first, const AnnCandidate& second) { 55 | return first.distance > second.distance; 56 | } 57 | friend bool operator>=(const AnnCandidate& first, const AnnCandidate& second) { 58 | return first.distance >= second.distance; 59 | } 60 | friend bool operator<=(const AnnCandidate& first, const AnnCandidate& second) { 61 | return first.distance <= second.distance; 62 | } 63 | }; 64 | 65 | enum MetricType : std::uint8_t { METRIC_L2, METRIC_IP }; 66 | enum ScalarQuantizerType : std::uint8_t { RECONSTRUCTION, UNBIASED_ESTIMATION, PLAIN }; 67 | } // namespace rabitqlib -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/IterativeLinearSolvers: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // This Source Code Form is subject to the terms of the Mozilla 5 | // Public License v. 2.0. If a copy of the MPL was not distributed 6 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 7 | 8 | #ifndef EIGEN_ITERATIVELINEARSOLVERS_MODULE_H 9 | #define EIGEN_ITERATIVELINEARSOLVERS_MODULE_H 10 | 11 | #include "SparseCore" 12 | #include "OrderingMethods" 13 | 14 | #include "src/Core/util/DisableStupidWarnings.h" 15 | 16 | /** 17 | * \defgroup IterativeLinearSolvers_Module IterativeLinearSolvers module 18 | * 19 | * This module currently provides iterative methods to solve problems of the form \c A \c x = \c b, where \c A is a squared matrix, usually very large and sparse. 20 | * Those solvers are accessible via the following classes: 21 | * - ConjugateGradient for selfadjoint (hermitian) matrices, 22 | * - LeastSquaresConjugateGradient for rectangular least-square problems, 23 | * - BiCGSTAB for general square matrices. 24 | * 25 | * These iterative solvers are associated with some preconditioners: 26 | * - IdentityPreconditioner - not really useful 27 | * - DiagonalPreconditioner - also called Jacobi preconditioner, work very well on diagonal dominant matrices. 28 | * - IncompleteLUT - incomplete LU factorization with dual thresholding 29 | * 30 | * Such problems can also be solved using the direct sparse decomposition modules: SparseCholesky, CholmodSupport, UmfPackSupport, SuperLUSupport. 31 | * 32 | \code 33 | #include 34 | \endcode 35 | */ 36 | 37 | #include "src/IterativeLinearSolvers/SolveWithGuess.h" 38 | #include "src/IterativeLinearSolvers/IterativeSolverBase.h" 39 | #include "src/IterativeLinearSolvers/BasicPreconditioners.h" 40 | #include "src/IterativeLinearSolvers/ConjugateGradient.h" 41 | #include "src/IterativeLinearSolvers/LeastSquareConjugateGradient.h" 42 | #include "src/IterativeLinearSolvers/BiCGSTAB.h" 43 | #include "src/IterativeLinearSolvers/IncompleteLUT.h" 44 | #include "src/IterativeLinearSolvers/IncompleteCholesky.h" 45 | 46 | #include "src/Core/util/ReenableStupidWarnings.h" 47 | 48 | #endif // EIGEN_ITERATIVELINEARSOLVERS_MODULE_H 49 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/SparseLU/SparseLU_Utils.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2012 Désiré Nuentsa-Wakam 5 | // 6 | // This Source Code Form is subject to the terms of the Mozilla 7 | // Public License v. 2.0. If a copy of the MPL was not distributed 8 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | 10 | 11 | #ifndef EIGEN_SPARSELU_UTILS_H 12 | #define EIGEN_SPARSELU_UTILS_H 13 | 14 | namespace Eigen { 15 | namespace internal { 16 | 17 | /** 18 | * \brief Count Nonzero elements in the factors 19 | */ 20 | template 21 | void SparseLUImpl::countnz(const Index n, Index& nnzL, Index& nnzU, GlobalLU_t& glu) 22 | { 23 | nnzL = 0; 24 | nnzU = (glu.xusub)(n); 25 | Index nsuper = (glu.supno)(n); 26 | Index jlen; 27 | Index i, j, fsupc; 28 | if (n <= 0 ) return; 29 | // For each supernode 30 | for (i = 0; i <= nsuper; i++) 31 | { 32 | fsupc = glu.xsup(i); 33 | jlen = glu.xlsub(fsupc+1) - glu.xlsub(fsupc); 34 | 35 | for (j = fsupc; j < glu.xsup(i+1); j++) 36 | { 37 | nnzL += jlen; 38 | nnzU += j - fsupc + 1; 39 | jlen--; 40 | } 41 | } 42 | } 43 | 44 | /** 45 | * \brief Fix up the data storage lsub for L-subscripts. 46 | * 47 | * It removes the subscripts sets for structural pruning, 48 | * and applies permutation to the remaining subscripts 49 | * 50 | */ 51 | template 52 | void SparseLUImpl::fixupL(const Index n, const IndexVector& perm_r, GlobalLU_t& glu) 53 | { 54 | Index fsupc, i, j, k, jstart; 55 | 56 | StorageIndex nextl = 0; 57 | Index nsuper = (glu.supno)(n); 58 | 59 | // For each supernode 60 | for (i = 0; i <= nsuper; i++) 61 | { 62 | fsupc = glu.xsup(i); 63 | jstart = glu.xlsub(fsupc); 64 | glu.xlsub(fsupc) = nextl; 65 | for (j = jstart; j < glu.xlsub(fsupc + 1); j++) 66 | { 67 | glu.lsub(nextl) = perm_r(glu.lsub(j)); // Now indexed into P*A 68 | nextl++; 69 | } 70 | for (k = fsupc+1; k < glu.xsup(i+1); k++) 71 | glu.xlsub(k) = nextl; // other columns in supernode i 72 | } 73 | 74 | glu.xlsub(n) = nextl; 75 | } 76 | 77 | } // end namespace internal 78 | 79 | } // end namespace Eigen 80 | #endif // EIGEN_SPARSELU_UTILS_H 81 | -------------------------------------------------------------------------------- /include/rabitqlib/index/lut.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #include "rabitqlib/fastscan/fastscan.hpp" 8 | #include "rabitqlib/fastscan/highacc_fastscan.hpp" 9 | #include "rabitqlib/utils/space.hpp" 10 | 11 | namespace rabitqlib { 12 | 13 | template 14 | class Lut { 15 | static constexpr size_t kNumBits = 8; 16 | static constexpr size_t kNumBitsHacc = 16; 17 | static_assert(std::is_floating_point_v, "T must be an floating type in Lut"); 18 | 19 | private: 20 | size_t table_length_ = 0; 21 | std::vector lut_; 22 | T delta_; 23 | T sum_vl_lut_; 24 | 25 | public: 26 | explicit Lut() = default; 27 | explicit Lut(const T* rotated_query, size_t padded_dim, bool use_hacc = false) 28 | : table_length_(padded_dim << 2) 29 | , lut_(table_length_ * (static_cast(use_hacc) + 1)) { 30 | // quantize float lut 31 | std::vector lut_float(table_length_); 32 | fastscan::pack_lut(padded_dim, rotated_query, lut_float.data()); 33 | T vl_lut; 34 | T vr_lut; 35 | data_range(lut_float.data(), table_length_, vl_lut, vr_lut); 36 | 37 | if (use_hacc) { 38 | delta_ = (vr_lut - vl_lut) / ((1 << kNumBitsHacc) - 1); 39 | 40 | // quantize float lut into uint16 then change to split table 41 | std::vector lut_u16(table_length_); 42 | scalar_quantize( 43 | lut_u16.data(), lut_float.data(), table_length_, vl_lut, delta_ 44 | ); 45 | fastscan::transfer_lut_hacc(lut_u16.data(), padded_dim, lut_.data()); 46 | } else { 47 | delta_ = (vr_lut - vl_lut) / ((1 << kNumBits) - 1); 48 | scalar_quantize(lut_.data(), lut_float.data(), table_length_, vl_lut, delta_); 49 | } 50 | 51 | size_t num_table = table_length_ / 16; 52 | sum_vl_lut_ = vl_lut * static_cast(num_table); 53 | } 54 | Lut& operator=(Lut&& other) noexcept { 55 | lut_ = std::move(other.lut_); 56 | delta_ = other.delta_; 57 | sum_vl_lut_ = other.sum_vl_lut_; 58 | return *this; 59 | } 60 | 61 | [[nodiscard]] const uint8_t* lut() const { return lut_.data(); }; 62 | [[nodiscard]] T delta() const { return delta_; }; 63 | [[nodiscard]] T sum_vl() const { return sum_vl_lut_; }; 64 | }; 65 | } // namespace rabitqlib -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/SparseCore/MappedSparseMatrix.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2008-2014 Gael Guennebaud 5 | // 6 | // This Source Code Form is subject to the terms of the Mozilla 7 | // Public License v. 2.0. If a copy of the MPL was not distributed 8 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | 10 | #ifndef EIGEN_MAPPED_SPARSEMATRIX_H 11 | #define EIGEN_MAPPED_SPARSEMATRIX_H 12 | 13 | namespace Eigen { 14 | 15 | /** \deprecated Use Map > 16 | * \class MappedSparseMatrix 17 | * 18 | * \brief Sparse matrix 19 | * 20 | * \param _Scalar the scalar type, i.e. the type of the coefficients 21 | * 22 | * See http://www.netlib.org/linalg/html_templates/node91.html for details on the storage scheme. 23 | * 24 | */ 25 | namespace internal { 26 | template 27 | struct traits > : traits > 28 | {}; 29 | } // end namespace internal 30 | 31 | template 32 | class MappedSparseMatrix 33 | : public Map > 34 | { 35 | typedef Map > Base; 36 | 37 | public: 38 | 39 | typedef typename Base::StorageIndex StorageIndex; 40 | typedef typename Base::Scalar Scalar; 41 | 42 | inline MappedSparseMatrix(Index rows, Index cols, Index nnz, StorageIndex* outerIndexPtr, StorageIndex* innerIndexPtr, Scalar* valuePtr, StorageIndex* innerNonZeroPtr = 0) 43 | : Base(rows, cols, nnz, outerIndexPtr, innerIndexPtr, valuePtr, innerNonZeroPtr) 44 | {} 45 | 46 | /** Empty destructor */ 47 | inline ~MappedSparseMatrix() {} 48 | }; 49 | 50 | namespace internal { 51 | 52 | template 53 | struct evaluator > 54 | : evaluator > > 55 | { 56 | typedef MappedSparseMatrix<_Scalar,_Options,_StorageIndex> XprType; 57 | typedef evaluator > Base; 58 | 59 | evaluator() : Base() {} 60 | explicit evaluator(const XprType &mat) : Base(mat) {} 61 | }; 62 | 63 | } 64 | 65 | } // end namespace Eigen 66 | 67 | #endif // EIGEN_MAPPED_SPARSEMATRIX_H 68 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/SuperLUSupport: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // This Source Code Form is subject to the terms of the Mozilla 5 | // Public License v. 2.0. If a copy of the MPL was not distributed 6 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 7 | 8 | #ifndef EIGEN_SUPERLUSUPPORT_MODULE_H 9 | #define EIGEN_SUPERLUSUPPORT_MODULE_H 10 | 11 | #include "SparseCore" 12 | 13 | #include "src/Core/util/DisableStupidWarnings.h" 14 | 15 | #ifdef EMPTY 16 | #define EIGEN_EMPTY_WAS_ALREADY_DEFINED 17 | #endif 18 | 19 | typedef int int_t; 20 | #include 21 | #include 22 | #include 23 | 24 | // slu_util.h defines a preprocessor token named EMPTY which is really polluting, 25 | // so we remove it in favor of a SUPERLU_EMPTY token. 26 | // If EMPTY was already defined then we don't undef it. 27 | 28 | #if defined(EIGEN_EMPTY_WAS_ALREADY_DEFINED) 29 | # undef EIGEN_EMPTY_WAS_ALREADY_DEFINED 30 | #elif defined(EMPTY) 31 | # undef EMPTY 32 | #endif 33 | 34 | #define SUPERLU_EMPTY (-1) 35 | 36 | namespace Eigen { struct SluMatrix; } 37 | 38 | /** \ingroup Support_modules 39 | * \defgroup SuperLUSupport_Module SuperLUSupport module 40 | * 41 | * This module provides an interface to the SuperLU library. 42 | * It provides the following factorization class: 43 | * - class SuperLU: a supernodal sequential LU factorization. 44 | * - class SuperILU: a supernodal sequential incomplete LU factorization (to be used as a preconditioner for iterative methods). 45 | * 46 | * \warning This wrapper requires at least versions 4.0 of SuperLU. The 3.x versions are not supported. 47 | * 48 | * \warning When including this module, you have to use SUPERLU_EMPTY instead of EMPTY which is no longer defined because it is too polluting. 49 | * 50 | * \code 51 | * #include 52 | * \endcode 53 | * 54 | * In order to use this module, the superlu headers must be accessible from the include paths, and your binary must be linked to the superlu library and its dependencies. 55 | * The dependencies depend on how superlu has been compiled. 56 | * For a cmake based project, you can use our FindSuperLU.cmake module to help you in this task. 57 | * 58 | */ 59 | 60 | #include "src/SuperLUSupport/SuperLUSupport.h" 61 | 62 | #include "src/Core/util/ReenableStupidWarnings.h" 63 | 64 | #endif // EIGEN_SUPERLUSUPPORT_MODULE_H 65 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/SparseCore: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // This Source Code Form is subject to the terms of the Mozilla 5 | // Public License v. 2.0. If a copy of the MPL was not distributed 6 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 7 | 8 | #ifndef EIGEN_SPARSECORE_MODULE_H 9 | #define EIGEN_SPARSECORE_MODULE_H 10 | 11 | #include "Core" 12 | 13 | #include "src/Core/util/DisableStupidWarnings.h" 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | /** 22 | * \defgroup SparseCore_Module SparseCore module 23 | * 24 | * This module provides a sparse matrix representation, and basic associated matrix manipulations 25 | * and operations. 26 | * 27 | * See the \ref TutorialSparse "Sparse tutorial" 28 | * 29 | * \code 30 | * #include 31 | * \endcode 32 | * 33 | * This module depends on: Core. 34 | */ 35 | 36 | #include "src/SparseCore/SparseUtil.h" 37 | #include "src/SparseCore/SparseMatrixBase.h" 38 | #include "src/SparseCore/SparseAssign.h" 39 | #include "src/SparseCore/CompressedStorage.h" 40 | #include "src/SparseCore/AmbiVector.h" 41 | #include "src/SparseCore/SparseCompressedBase.h" 42 | #include "src/SparseCore/SparseMatrix.h" 43 | #include "src/SparseCore/SparseMap.h" 44 | #include "src/SparseCore/MappedSparseMatrix.h" 45 | #include "src/SparseCore/SparseVector.h" 46 | #include "src/SparseCore/SparseRef.h" 47 | #include "src/SparseCore/SparseCwiseUnaryOp.h" 48 | #include "src/SparseCore/SparseCwiseBinaryOp.h" 49 | #include "src/SparseCore/SparseTranspose.h" 50 | #include "src/SparseCore/SparseBlock.h" 51 | #include "src/SparseCore/SparseDot.h" 52 | #include "src/SparseCore/SparseRedux.h" 53 | #include "src/SparseCore/SparseView.h" 54 | #include "src/SparseCore/SparseDiagonalProduct.h" 55 | #include "src/SparseCore/ConservativeSparseSparseProduct.h" 56 | #include "src/SparseCore/SparseSparseProductWithPruning.h" 57 | #include "src/SparseCore/SparseProduct.h" 58 | #include "src/SparseCore/SparseDenseProduct.h" 59 | #include "src/SparseCore/SparseSelfAdjointView.h" 60 | #include "src/SparseCore/SparseTriangularView.h" 61 | #include "src/SparseCore/TriangularSolver.h" 62 | #include "src/SparseCore/SparsePermutation.h" 63 | #include "src/SparseCore/SparseFuzzy.h" 64 | #include "src/SparseCore/SparseSolverBase.h" 65 | 66 | #include "src/Core/util/ReenableStupidWarnings.h" 67 | 68 | #endif // EIGEN_SPARSECORE_MODULE_H 69 | 70 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/Core/arch/AVX512/TypeCasting.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2019 Rasmus Munk Larsen 5 | // 6 | // This Source Code Form is subject to the terms of the Mozilla 7 | // Public License v. 2.0. If a copy of the MPL was not distributed 8 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | 10 | #ifndef EIGEN_TYPE_CASTING_AVX512_H 11 | #define EIGEN_TYPE_CASTING_AVX512_H 12 | 13 | namespace Eigen { 14 | 15 | namespace internal { 16 | 17 | template<> EIGEN_STRONG_INLINE Packet16i pcast(const Packet16f& a) { 18 | return _mm512_cvttps_epi32(a); 19 | } 20 | 21 | template<> EIGEN_STRONG_INLINE Packet16f pcast(const Packet16i& a) { 22 | return _mm512_cvtepi32_ps(a); 23 | } 24 | 25 | template<> EIGEN_STRONG_INLINE Packet16i preinterpret(const Packet16f& a) { 26 | return _mm512_castps_si512(a); 27 | } 28 | 29 | template<> EIGEN_STRONG_INLINE Packet16f preinterpret(const Packet16i& a) { 30 | return _mm512_castsi512_ps(a); 31 | } 32 | 33 | template <> 34 | struct type_casting_traits { 35 | enum { 36 | VectorizedCast = 1, 37 | SrcCoeffRatio = 1, 38 | TgtCoeffRatio = 1 39 | }; 40 | }; 41 | 42 | template<> EIGEN_STRONG_INLINE Packet16f pcast(const Packet16h& a) { 43 | return half2float(a); 44 | } 45 | 46 | template <> 47 | struct type_casting_traits { 48 | enum { 49 | VectorizedCast = 1, 50 | SrcCoeffRatio = 1, 51 | TgtCoeffRatio = 1 52 | }; 53 | }; 54 | 55 | template<> EIGEN_STRONG_INLINE Packet16h pcast(const Packet16f& a) { 56 | return float2half(a); 57 | } 58 | 59 | template <> 60 | struct type_casting_traits { 61 | enum { 62 | VectorizedCast = 1, 63 | SrcCoeffRatio = 1, 64 | TgtCoeffRatio = 1 65 | }; 66 | }; 67 | 68 | template<> EIGEN_STRONG_INLINE Packet16f pcast(const Packet16bf& a) { 69 | return Bf16ToF32(a); 70 | } 71 | 72 | template <> 73 | struct type_casting_traits { 74 | enum { 75 | VectorizedCast = 1, 76 | SrcCoeffRatio = 1, 77 | TgtCoeffRatio = 1 78 | }; 79 | }; 80 | 81 | template<> EIGEN_STRONG_INLINE Packet16bf pcast(const Packet16f& a) { 82 | return F32ToBf16(a); 83 | } 84 | 85 | } // end namespace internal 86 | 87 | } // end namespace Eigen 88 | 89 | #endif // EIGEN_TYPE_CASTING_AVX512_H 90 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/Core/arch/GPU/TypeCasting.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2016 Benoit Steiner 5 | // 6 | // This Source Code Form is subject to the terms of the Mozilla 7 | // Public License v. 2.0. If a copy of the MPL was not distributed 8 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | 10 | #ifndef EIGEN_TYPE_CASTING_GPU_H 11 | #define EIGEN_TYPE_CASTING_GPU_H 12 | 13 | namespace Eigen { 14 | 15 | namespace internal { 16 | 17 | #if (defined(EIGEN_HAS_CUDA_FP16) && defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 300) || \ 18 | (defined(EIGEN_HAS_HIP_FP16) && defined(EIGEN_HIP_DEVICE_COMPILE)) 19 | 20 | 21 | template <> 22 | struct type_casting_traits { 23 | enum { 24 | VectorizedCast = 1, 25 | SrcCoeffRatio = 1, 26 | TgtCoeffRatio = 2 27 | }; 28 | }; 29 | 30 | template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pcast(const half2& a, const half2& b) { 31 | float2 r1 = __half22float2(a); 32 | float2 r2 = __half22float2(b); 33 | return make_float4(r1.x, r1.y, r2.x, r2.y); 34 | } 35 | 36 | 37 | template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 pcast(const float4& a, const float4& b) { 38 | Packet4h2 r; 39 | half2* r_alias=reinterpret_cast(&r); 40 | r_alias[0]=__floats2half2_rn(a.x,a.y); 41 | r_alias[1]=__floats2half2_rn(a.z,a.w); 42 | r_alias[2]=__floats2half2_rn(b.x,b.y); 43 | r_alias[3]=__floats2half2_rn(b.z,b.w); 44 | return r; 45 | } 46 | 47 | template <> 48 | struct type_casting_traits { 49 | enum { 50 | VectorizedCast = 1, 51 | SrcCoeffRatio = 2, 52 | TgtCoeffRatio = 1 53 | }; 54 | }; 55 | 56 | template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pcast(const Packet4h2& a) { 57 | // Simply discard the second half of the input 58 | float4 r; 59 | const half2* a_alias=reinterpret_cast(&a); 60 | float2 r1 = __half22float2(a_alias[0]); 61 | float2 r2 = __half22float2(a_alias[1]); 62 | r.x=static_cast(r1.x); 63 | r.y=static_cast(r1.y); 64 | r.z=static_cast(r2.x); 65 | r.w=static_cast(r2.y); 66 | return r; 67 | } 68 | 69 | template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pcast(const float4& a) { 70 | // Simply discard the second half of the input 71 | return __floats2half2_rn(a.x, a.y); 72 | } 73 | 74 | #endif 75 | 76 | } // end namespace internal 77 | 78 | } // end namespace Eigen 79 | 80 | #endif // EIGEN_TYPE_CASTING_GPU_H 81 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/Core/arch/AltiVec/MathFunctions.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2007 Julien Pommier 5 | // Copyright (C) 2009 Gael Guennebaud 6 | // Copyright (C) 2016 Konstantinos Margaritis 7 | // 8 | // This Source Code Form is subject to the terms of the Mozilla 9 | // Public License v. 2.0. If a copy of the MPL was not distributed 10 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | 12 | #ifndef EIGEN_MATH_FUNCTIONS_ALTIVEC_H 13 | #define EIGEN_MATH_FUNCTIONS_ALTIVEC_H 14 | 15 | namespace Eigen { 16 | 17 | namespace internal { 18 | 19 | template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED 20 | Packet4f plog(const Packet4f& _x) 21 | { 22 | return plog_float(_x); 23 | } 24 | 25 | template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED 26 | Packet4f pexp(const Packet4f& _x) 27 | { 28 | return pexp_float(_x); 29 | } 30 | 31 | template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED 32 | Packet4f psin(const Packet4f& _x) 33 | { 34 | return psin_float(_x); 35 | } 36 | 37 | template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED 38 | Packet4f pcos(const Packet4f& _x) 39 | { 40 | return pcos_float(_x); 41 | } 42 | 43 | #ifndef EIGEN_COMP_CLANG 44 | template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED 45 | Packet4f prsqrt(const Packet4f& x) 46 | { 47 | return vec_rsqrt(x); 48 | } 49 | #endif 50 | 51 | #ifdef __VSX__ 52 | #ifndef EIGEN_COMP_CLANG 53 | template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED 54 | Packet2d prsqrt(const Packet2d& x) 55 | { 56 | return vec_rsqrt(x); 57 | } 58 | #endif 59 | 60 | template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED 61 | Packet4f psqrt(const Packet4f& x) 62 | { 63 | return vec_sqrt(x); 64 | } 65 | 66 | template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED 67 | Packet2d psqrt(const Packet2d& x) 68 | { 69 | return vec_sqrt(x); 70 | } 71 | 72 | template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED 73 | Packet2d pexp(const Packet2d& _x) 74 | { 75 | return pexp_double(_x); 76 | } 77 | #endif 78 | 79 | // Hyperbolic Tangent function. 80 | template <> 81 | EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f 82 | ptanh(const Packet4f& x) { 83 | return internal::generic_fast_tanh_float(x); 84 | } 85 | 86 | } // end namespace internal 87 | 88 | } // end namespace Eigen 89 | 90 | #endif // EIGEN_MATH_FUNCTIONS_ALTIVEC_H 91 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/OrderingMethods: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // This Source Code Form is subject to the terms of the Mozilla 5 | // Public License v. 2.0. If a copy of the MPL was not distributed 6 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 7 | 8 | #ifndef EIGEN_ORDERINGMETHODS_MODULE_H 9 | #define EIGEN_ORDERINGMETHODS_MODULE_H 10 | 11 | #include "SparseCore" 12 | 13 | #include "src/Core/util/DisableStupidWarnings.h" 14 | 15 | /** 16 | * \defgroup OrderingMethods_Module OrderingMethods module 17 | * 18 | * This module is currently for internal use only 19 | * 20 | * It defines various built-in and external ordering methods for sparse matrices. 21 | * They are typically used to reduce the number of elements during 22 | * the sparse matrix decomposition (LLT, LU, QR). 23 | * Precisely, in a preprocessing step, a permutation matrix P is computed using 24 | * those ordering methods and applied to the columns of the matrix. 25 | * Using for instance the sparse Cholesky decomposition, it is expected that 26 | * the nonzeros elements in LLT(A*P) will be much smaller than that in LLT(A). 27 | * 28 | * 29 | * Usage : 30 | * \code 31 | * #include 32 | * \endcode 33 | * 34 | * A simple usage is as a template parameter in the sparse decomposition classes : 35 | * 36 | * \code 37 | * SparseLU > solver; 38 | * \endcode 39 | * 40 | * \code 41 | * SparseQR > solver; 42 | * \endcode 43 | * 44 | * It is possible as well to call directly a particular ordering method for your own purpose, 45 | * \code 46 | * AMDOrdering ordering; 47 | * PermutationMatrix perm; 48 | * SparseMatrix A; 49 | * //Fill the matrix ... 50 | * 51 | * ordering(A, perm); // Call AMD 52 | * \endcode 53 | * 54 | * \note Some of these methods (like AMD or METIS), need the sparsity pattern 55 | * of the input matrix to be symmetric. When the matrix is structurally unsymmetric, 56 | * Eigen computes internally the pattern of \f$A^T*A\f$ before calling the method. 57 | * If your matrix is already symmetric (at leat in structure), you can avoid that 58 | * by calling the method with a SelfAdjointView type. 59 | * 60 | * \code 61 | * // Call the ordering on the pattern of the lower triangular matrix A 62 | * ordering(A.selfadjointView(), perm); 63 | * \endcode 64 | */ 65 | 66 | #include "src/OrderingMethods/Amd.h" 67 | #include "src/OrderingMethods/Ordering.h" 68 | #include "src/Core/util/ReenableStupidWarnings.h" 69 | 70 | #endif // EIGEN_ORDERINGMETHODS_MODULE_H 71 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/Core/NestByValue.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2008 Gael Guennebaud 5 | // Copyright (C) 2006-2008 Benoit Jacob 6 | // 7 | // This Source Code Form is subject to the terms of the Mozilla 8 | // Public License v. 2.0. If a copy of the MPL was not distributed 9 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 10 | 11 | #ifndef EIGEN_NESTBYVALUE_H 12 | #define EIGEN_NESTBYVALUE_H 13 | 14 | namespace Eigen { 15 | 16 | namespace internal { 17 | template 18 | struct traits > : public traits 19 | { 20 | enum { 21 | Flags = traits::Flags & ~NestByRefBit 22 | }; 23 | }; 24 | } 25 | 26 | /** \class NestByValue 27 | * \ingroup Core_Module 28 | * 29 | * \brief Expression which must be nested by value 30 | * 31 | * \tparam ExpressionType the type of the object of which we are requiring nesting-by-value 32 | * 33 | * This class is the return type of MatrixBase::nestByValue() 34 | * and most of the time this is the only way it is used. 35 | * 36 | * \sa MatrixBase::nestByValue() 37 | */ 38 | template class NestByValue 39 | : public internal::dense_xpr_base< NestByValue >::type 40 | { 41 | public: 42 | 43 | typedef typename internal::dense_xpr_base::type Base; 44 | EIGEN_DENSE_PUBLIC_INTERFACE(NestByValue) 45 | 46 | EIGEN_DEVICE_FUNC explicit inline NestByValue(const ExpressionType& matrix) : m_expression(matrix) {} 47 | 48 | EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index rows() const EIGEN_NOEXCEPT { return m_expression.rows(); } 49 | EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR inline Index cols() const EIGEN_NOEXCEPT { return m_expression.cols(); } 50 | 51 | EIGEN_DEVICE_FUNC operator const ExpressionType&() const { return m_expression; } 52 | 53 | EIGEN_DEVICE_FUNC const ExpressionType& nestedExpression() const { return m_expression; } 54 | 55 | protected: 56 | const ExpressionType m_expression; 57 | }; 58 | 59 | /** \returns an expression of the temporary version of *this. 60 | */ 61 | template 62 | EIGEN_DEVICE_FUNC inline const NestByValue 63 | DenseBase::nestByValue() const 64 | { 65 | return NestByValue(derived()); 66 | } 67 | 68 | namespace internal { 69 | 70 | // Evaluator of Solve -> eval into a temporary 71 | template 72 | struct evaluator > 73 | : public evaluator 74 | { 75 | typedef evaluator Base; 76 | 77 | EIGEN_DEVICE_FUNC explicit evaluator(const NestByValue& xpr) 78 | : Base(xpr.nestedExpression()) 79 | {} 80 | }; 81 | } 82 | 83 | } // end namespace Eigen 84 | 85 | #endif // EIGEN_NESTBYVALUE_H 86 | -------------------------------------------------------------------------------- /include/rabitqlib/utils/io.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | namespace rabitqlib { 12 | // get num of bytes 13 | inline size_t get_filesize(const char* filename) noexcept { 14 | try { 15 | return std::filesystem::file_size(filename); 16 | } catch (const std::filesystem::filesystem_error& e) { 17 | // Log the error and return -1 to maintain original behavior on error. 18 | std::cerr << "Error getting file size for '" << filename << "': " << e.what() 19 | << '\n'; 20 | return static_cast(-1); 21 | } 22 | } 23 | 24 | inline bool file_exists(const char* filename) { return std::filesystem::exists(filename); } 25 | 26 | // load .*vecs file to a matrix (e.g., RowMajorFloatMat) 27 | template 28 | void load_vecs(const char* filename, M& row_mat) { 29 | if (!file_exists(filename)) { 30 | std::cerr << "File " << filename << " not exists\n"; 31 | exit(1); 32 | } 33 | 34 | assert((std::is_same_v> == true)); 35 | 36 | uint32_t tmp; 37 | size_t file_size = get_filesize(filename); 38 | std::ifstream input(filename, std::ios::binary); 39 | 40 | input.read(reinterpret_cast(&tmp), sizeof(uint32_t)); 41 | 42 | size_t cols = tmp; 43 | size_t rows = file_size / (cols * sizeof(T) + sizeof(uint32_t)); 44 | row_mat = M(rows, cols); 45 | 46 | input.seekg(0, std::ifstream::beg); 47 | 48 | for (size_t i = 0; i < rows; i++) { 49 | input.read(reinterpret_cast(&tmp), sizeof(uint32_t)); 50 | input.read(reinterpret_cast(&row_mat(i, 0)), sizeof(T) * cols); 51 | } 52 | 53 | std::cout << "File " << filename << " loaded\n"; 54 | std::cout << "Rows " << rows << " Cols " << cols << '\n' << std::flush; 55 | input.close(); 56 | } 57 | 58 | // load .*bin file to a matrix (e.g., RowMajorFloatMat) 59 | template 60 | void load_bin(const char* filename, M& row_mat) { 61 | if (!file_exists(filename)) { 62 | std::cerr << "File " << filename << " not exists\n"; 63 | exit(1); 64 | } 65 | 66 | assert((std::is_same_v> == true)); 67 | 68 | uint32_t rows; 69 | uint32_t cols; 70 | std::ifstream input(filename, std::ios::binary); 71 | 72 | input.read(reinterpret_cast(&rows), sizeof(uint32_t)); 73 | input.read(reinterpret_cast(&cols), sizeof(uint32_t)); 74 | 75 | row_mat = M(rows, cols); 76 | 77 | for (size_t i = 0; i < rows; i++) { 78 | input.read(reinterpret_cast(&row_mat(i, 0)), sizeof(T) * cols); 79 | } 80 | 81 | std::cout << "File " << filename << " loaded\n"; 82 | std::cout << "Rows " << rows << " Cols " << cols << '\n' << std::flush; 83 | input.close(); 84 | } 85 | } // namespace rabitqlib -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/Core/arch/SYCL/TypeCasting.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Mehdi Goli Codeplay Software Ltd. 5 | // Ralph Potter Codeplay Software Ltd. 6 | // Luke Iwanski Codeplay Software Ltd. 7 | // Contact: 8 | // 9 | // This Source Code Form is subject to the terms of the Mozilla 10 | // Public License v. 2.0. If a copy of the MPL was not distributed 11 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 12 | 13 | /***************************************************************** 14 | * TypeCasting.h 15 | * 16 | * \brief: 17 | * TypeCasting 18 | * 19 | *****************************************************************/ 20 | 21 | #ifndef EIGEN_TYPE_CASTING_SYCL_H 22 | #define EIGEN_TYPE_CASTING_SYCL_H 23 | 24 | namespace Eigen { 25 | 26 | namespace internal { 27 | #ifdef SYCL_DEVICE_ONLY 28 | template <> 29 | struct type_casting_traits { 30 | enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; 31 | }; 32 | 33 | template <> 34 | EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE cl::sycl::cl_int4 35 | pcast(const cl::sycl::cl_float4& a) { 36 | return a 37 | .template convert(); 38 | } 39 | 40 | template <> 41 | struct type_casting_traits { 42 | enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; 43 | }; 44 | 45 | template <> 46 | EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE cl::sycl::cl_float4 47 | pcast(const cl::sycl::cl_int4& a) { 48 | return a.template convert(); 50 | } 51 | 52 | template <> 53 | struct type_casting_traits { 54 | enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 }; 55 | }; 56 | 57 | template <> 58 | EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE cl::sycl::cl_float4 59 | pcast( 60 | const cl::sycl::cl_double2& a, const cl::sycl::cl_double2& b) { 61 | auto a1 = a.template convert(); 63 | auto b1 = b.template convert(); 65 | return cl::sycl::float4(a1.x(), a1.y(), b1.x(), b1.y()); 66 | } 67 | 68 | template <> 69 | struct type_casting_traits { 70 | enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 }; 71 | }; 72 | 73 | template <> 74 | EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE cl::sycl::cl_double2 75 | pcast(const cl::sycl::cl_float4& a) { 76 | // Simply discard the second half of the input 77 | return cl::sycl::cl_double2(a.x(), a.y()); 78 | } 79 | 80 | #endif 81 | } // end namespace internal 82 | 83 | } // end namespace Eigen 84 | 85 | #endif // EIGEN_TYPE_CASTING_SYCL_H 86 | -------------------------------------------------------------------------------- /docs/docs/kernel_ip.md: -------------------------------------------------------------------------------- 1 | # Kernel - Inner Product 2 | 3 | 4 | > The implementation of computing inner product between binary codes and query vectors heavily affects the efficiency. The best implementation may vary largely across platforms and the dimensionality of datasets. For now, we only include the implementation with `__builtin_popcountll` and expect that compilers will automatically vectorize it. 5 | 6 | This part introduces how to compute the inner product between quantization codes and rotated query vectors i.e., $\left< \mathbf{x}_0,\mathbf{q}_r'\right>$ and $\left< \mathbf{x}_u,\mathbf{q}_r'\right>$. The implementation includes two types: 7 | 8 | 1. The inner product between binary codes and $\mathbf{q}_r'$. 9 | 2. The inner product between multi-bit codes and $\mathbf{q}_r'$. 10 | 11 | 12 | ## The Kernel for Binary Codes 13 | 14 | ### Single Code 15 | We compute the inner product between a single binary vector $\mathbf{x}_0$ and a floating-point vector $\mathbf{q}_r'$ via bitwise-and `&` and `popcnt`. We first quantize $\mathbf{q}_r'$ into a vector of 4-bit unsigned integers (based on a fast version of RaBitQ). 16 | 17 | $$ 18 | \mathbf{q}_r'\approx \Delta_q \mathbf{q}_u + v_q\cdot \mathbf{1}_D 19 | $$ 20 | 21 | Let $\mathbf{q}_u^{(i)}$ be the $i$-th bit of $\mathbf{q}_u$. Then with the quantized vector $\mathbf{q}_u$, we can compute the inner product based on the following formula: 22 | 23 | $$ 24 | \begin{align} 25 | \left< \mathbf{x}_0,\mathbf{q}_r'\right> 26 | &\approx \left< \mathbf{x}_0,\Delta_q \mathbf{q}_u + v_q\cdot \mathbf{1}_D\right> 27 | \\&=\Delta_q \left< \mathbf{x}_0,\mathbf{q}_u\right> + v_q \cdot \mathrm{popcnt}(\mathbf{x}_0) 28 | \\ &=\Delta_q \sum_{i=0}^{B_q-1} \left( 2^i\left< \mathbf{x}_0,\mathbf{q}_u^{(i)}\right> \right) + v_q \cdot\mathrm{popcnt}(\mathbf{x}_0) 29 | \\ &=\Delta_q \sum_{i=0}^{B_q-1} 2^i\cdot \mathrm{popcnt}(\mathbf{x}_0\ \&\ \mathbf{q}_u^{(i)}) + v_q \cdot \mathrm{popcnt}(\mathbf{x}_0) 30 | \end{align} 31 | $$ 32 | 33 | 34 | ### Batch Code 35 | We compute the inner product between a batch of binary vectors $\mathbf{x}_0$ and a floating-point vector $\mathbf{q}_r'$ via `FastScan`. Here we provide a brief introduction and refer readers to a detailed [tutorial by Faiss](https://github.com/facebookresearch/faiss/wiki/Fast-accumulation-of-PQ-and-AQ-codes-(FastScan)). 36 | 37 | For a $D$-bit binary, we split it into $M=D/4$ segments. We prepare look-up-tables for each segment. 38 | - $\mathrm{LUT}[m][mask]$: the inner product with $\mathbf{q}_r'$ in the $m$-th segment, i.e., the $(4m)$-th to the $(4m+3)$-th dimensions, when the code in the $m$-th segment equals to $mask$. 39 | 40 | Based on the look-up-tables, we can compute the inner product as follows: 41 | 42 | $$ 43 | \begin{align} 44 | \left< \mathbf{x}_0,\mathbf{q}_r'\right>= \sum_{m=0}^{M-1} \mathrm{LUT}[m][\mathbf{x}_{0}[4m:4m+3]] 45 | \end{align} 46 | $$ 47 | 48 | 49 | ## The Kernel for Multi-bit Codes 50 | 51 | For the multi-bit codes, we convert the unsigned integer codes to floating point numbers with native instructions of AVX512. -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/Core/Swap.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2006-2008 Benoit Jacob 5 | // 6 | // This Source Code Form is subject to the terms of the Mozilla 7 | // Public License v. 2.0. If a copy of the MPL was not distributed 8 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | 10 | #ifndef EIGEN_SWAP_H 11 | #define EIGEN_SWAP_H 12 | 13 | namespace Eigen { 14 | 15 | namespace internal { 16 | 17 | // Overload default assignPacket behavior for swapping them 18 | template 19 | class generic_dense_assignment_kernel, Specialized> 20 | : public generic_dense_assignment_kernel, BuiltIn> 21 | { 22 | protected: 23 | typedef generic_dense_assignment_kernel, BuiltIn> Base; 24 | using Base::m_dst; 25 | using Base::m_src; 26 | using Base::m_functor; 27 | 28 | public: 29 | typedef typename Base::Scalar Scalar; 30 | typedef typename Base::DstXprType DstXprType; 31 | typedef swap_assign_op Functor; 32 | 33 | EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE 34 | generic_dense_assignment_kernel(DstEvaluatorTypeT &dst, const SrcEvaluatorTypeT &src, const Functor &func, DstXprType& dstExpr) 35 | : Base(dst, src, func, dstExpr) 36 | {} 37 | 38 | template 39 | EIGEN_STRONG_INLINE void assignPacket(Index row, Index col) 40 | { 41 | PacketType tmp = m_src.template packet(row,col); 42 | const_cast(m_src).template writePacket(row,col, m_dst.template packet(row,col)); 43 | m_dst.template writePacket(row,col,tmp); 44 | } 45 | 46 | template 47 | EIGEN_STRONG_INLINE void assignPacket(Index index) 48 | { 49 | PacketType tmp = m_src.template packet(index); 50 | const_cast(m_src).template writePacket(index, m_dst.template packet(index)); 51 | m_dst.template writePacket(index,tmp); 52 | } 53 | 54 | // TODO find a simple way not to have to copy/paste this function from generic_dense_assignment_kernel, by simple I mean no CRTP (Gael) 55 | template 56 | EIGEN_STRONG_INLINE void assignPacketByOuterInner(Index outer, Index inner) 57 | { 58 | Index row = Base::rowIndexByOuterInner(outer, inner); 59 | Index col = Base::colIndexByOuterInner(outer, inner); 60 | assignPacket(row, col); 61 | } 62 | }; 63 | 64 | } // namespace internal 65 | 66 | } // end namespace Eigen 67 | 68 | #endif // EIGEN_SWAP_H 69 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/misc/Kernel.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2009 Benoit Jacob 5 | // 6 | // This Source Code Form is subject to the terms of the Mozilla 7 | // Public License v. 2.0. If a copy of the MPL was not distributed 8 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | 10 | #ifndef EIGEN_MISC_KERNEL_H 11 | #define EIGEN_MISC_KERNEL_H 12 | 13 | namespace Eigen { 14 | 15 | namespace internal { 16 | 17 | /** \class kernel_retval_base 18 | * 19 | */ 20 | template 21 | struct traits > 22 | { 23 | typedef typename DecompositionType::MatrixType MatrixType; 24 | typedef Matrix< 25 | typename MatrixType::Scalar, 26 | MatrixType::ColsAtCompileTime, // the number of rows in the "kernel matrix" 27 | // is the number of cols of the original matrix 28 | // so that the product "matrix * kernel = zero" makes sense 29 | Dynamic, // we don't know at compile-time the dimension of the kernel 30 | MatrixType::Options, 31 | MatrixType::MaxColsAtCompileTime, // see explanation for 2nd template parameter 32 | MatrixType::MaxColsAtCompileTime // the kernel is a subspace of the domain space, 33 | // whose dimension is the number of columns of the original matrix 34 | > ReturnType; 35 | }; 36 | 37 | template struct kernel_retval_base 38 | : public ReturnByValue > 39 | { 40 | typedef _DecompositionType DecompositionType; 41 | typedef ReturnByValue Base; 42 | 43 | explicit kernel_retval_base(const DecompositionType& dec) 44 | : m_dec(dec), 45 | m_rank(dec.rank()), 46 | m_cols(m_rank==dec.cols() ? 1 : dec.cols() - m_rank) 47 | {} 48 | 49 | inline Index rows() const { return m_dec.cols(); } 50 | inline Index cols() const { return m_cols; } 51 | inline Index rank() const { return m_rank; } 52 | inline const DecompositionType& dec() const { return m_dec; } 53 | 54 | template inline void evalTo(Dest& dst) const 55 | { 56 | static_cast*>(this)->evalTo(dst); 57 | } 58 | 59 | protected: 60 | const DecompositionType& m_dec; 61 | Index m_rank, m_cols; 62 | }; 63 | 64 | } // end namespace internal 65 | 66 | #define EIGEN_MAKE_KERNEL_HELPERS(DecompositionType) \ 67 | typedef typename DecompositionType::MatrixType MatrixType; \ 68 | typedef typename MatrixType::Scalar Scalar; \ 69 | typedef typename MatrixType::RealScalar RealScalar; \ 70 | typedef Eigen::internal::kernel_retval_base Base; \ 71 | using Base::dec; \ 72 | using Base::rank; \ 73 | using Base::rows; \ 74 | using Base::cols; \ 75 | kernel_retval(const DecompositionType& dec) : Base(dec) {} 76 | 77 | } // end namespace Eigen 78 | 79 | #endif // EIGEN_MISC_KERNEL_H 80 | -------------------------------------------------------------------------------- /sample/ivf_rabitq_indexing.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "rabitqlib/defines.hpp" 5 | #include "rabitqlib/index/ivf/ivf.hpp" 6 | #include "rabitqlib/utils/io.hpp" 7 | #include "rabitqlib/utils/stopw.hpp" 8 | 9 | using PID = rabitqlib::PID; 10 | using index_type = rabitqlib::ivf::IVF; 11 | using data_type = rabitqlib::RowMajorArray; 12 | using gt_type = rabitqlib::RowMajorArray; 13 | 14 | int main(int argc, char** argv) { 15 | if (argc < 6) { 16 | std::cerr << "Usage: " << argv[0] << " \n" 17 | << "arg1: path for data file, format .fvecs\n" 18 | << "arg2: path for centroids file generated by ivf.py\n" 19 | << "arg3: path for cluster ids file generated by ivf.py\n" 20 | << "arg4: total number of bits for quantization\n" 21 | << "arg5: path for saving index\n" 22 | << "arg6: metric type (\"l2\" or \"ip\"), l2 by default\n" 23 | << "arg7: if use faster quantization (\"true\" or \"false\"), false by " 24 | "default\n"; 25 | exit(1); 26 | } 27 | 28 | rabitqlib::MetricType metric_type = rabitqlib::METRIC_L2; 29 | if (argc > 6) { 30 | std::string metric_str(argv[6]); 31 | if (metric_str == "ip" || metric_str == "IP") { 32 | metric_type = rabitqlib::METRIC_IP; 33 | } 34 | } 35 | if (metric_type == rabitqlib::METRIC_IP) { 36 | std::cout << "Metric Type: IP\n"; 37 | } else if (metric_type == rabitqlib::METRIC_L2) { 38 | std::cout << "Metric Type: L2\n"; 39 | } 40 | 41 | bool faster_quant = false; 42 | if (argc > 7) { 43 | std::string faster_str(argv[7]); 44 | if (faster_str == "true") { 45 | faster_quant = true; 46 | std::cout << "Using faster quantize for indexing...\n"; 47 | } 48 | } 49 | 50 | char* data_file = argv[1]; 51 | char* centroids_file = argv[2]; 52 | char* cids_file = argv[3]; 53 | size_t total_bits = atoi(argv[4]); 54 | char* index_file = argv[5]; 55 | 56 | data_type data; 57 | data_type centroids; 58 | gt_type cids; 59 | 60 | rabitqlib::load_vecs(data_file, data); 61 | rabitqlib::load_vecs(centroids_file, centroids); 62 | rabitqlib::load_vecs(cids_file, cids); 63 | 64 | size_t num_points = data.rows(); 65 | size_t dim = data.cols(); 66 | size_t k = centroids.rows(); 67 | 68 | std::cout << "data loaded\n"; 69 | std::cout << "\tN: " << num_points << '\n'; 70 | std::cout << "\tDIM: " << dim << '\n'; 71 | 72 | rabitqlib::StopW stopw; 73 | index_type ivf(num_points, dim, k, total_bits, metric_type); 74 | ivf.construct(data.data(), centroids.data(), cids.data(), faster_quant); 75 | float miniutes = stopw.get_elapsed_mili() / 1000 / 60; 76 | std::cout << "ivf constructed \n"; 77 | ivf.save(index_file); 78 | 79 | std::cout << "Indexing time " << miniutes << '\n'; 80 | 81 | return 0; 82 | } -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/Core/Assign.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2007 Michael Olbrich 5 | // Copyright (C) 2006-2010 Benoit Jacob 6 | // Copyright (C) 2008 Gael Guennebaud 7 | // 8 | // This Source Code Form is subject to the terms of the Mozilla 9 | // Public License v. 2.0. If a copy of the MPL was not distributed 10 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 11 | 12 | #ifndef EIGEN_ASSIGN_H 13 | #define EIGEN_ASSIGN_H 14 | 15 | namespace Eigen { 16 | 17 | template 18 | template 19 | EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& DenseBase 20 | ::lazyAssign(const DenseBase& other) 21 | { 22 | enum{ 23 | SameType = internal::is_same::value 24 | }; 25 | 26 | EIGEN_STATIC_ASSERT_LVALUE(Derived) 27 | EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(Derived,OtherDerived) 28 | EIGEN_STATIC_ASSERT(SameType,YOU_MIXED_DIFFERENT_NUMERIC_TYPES__YOU_NEED_TO_USE_THE_CAST_METHOD_OF_MATRIXBASE_TO_CAST_NUMERIC_TYPES_EXPLICITLY) 29 | 30 | eigen_assert(rows() == other.rows() && cols() == other.cols()); 31 | internal::call_assignment_no_alias(derived(),other.derived()); 32 | 33 | return derived(); 34 | } 35 | 36 | template 37 | template 38 | EIGEN_DEVICE_FUNC 39 | EIGEN_STRONG_INLINE Derived& DenseBase::operator=(const DenseBase& other) 40 | { 41 | internal::call_assignment(derived(), other.derived()); 42 | return derived(); 43 | } 44 | 45 | template 46 | EIGEN_DEVICE_FUNC 47 | EIGEN_STRONG_INLINE Derived& DenseBase::operator=(const DenseBase& other) 48 | { 49 | internal::call_assignment(derived(), other.derived()); 50 | return derived(); 51 | } 52 | 53 | template 54 | EIGEN_DEVICE_FUNC 55 | EIGEN_STRONG_INLINE Derived& MatrixBase::operator=(const MatrixBase& other) 56 | { 57 | internal::call_assignment(derived(), other.derived()); 58 | return derived(); 59 | } 60 | 61 | template 62 | template 63 | EIGEN_DEVICE_FUNC 64 | EIGEN_STRONG_INLINE Derived& MatrixBase::operator=(const DenseBase& other) 65 | { 66 | internal::call_assignment(derived(), other.derived()); 67 | return derived(); 68 | } 69 | 70 | template 71 | template 72 | EIGEN_DEVICE_FUNC 73 | EIGEN_STRONG_INLINE Derived& MatrixBase::operator=(const EigenBase& other) 74 | { 75 | internal::call_assignment(derived(), other.derived()); 76 | return derived(); 77 | } 78 | 79 | template 80 | template 81 | EIGEN_DEVICE_FUNC 82 | EIGEN_STRONG_INLINE Derived& MatrixBase::operator=(const ReturnByValue& other) 83 | { 84 | other.derived().evalTo(derived()); 85 | return derived(); 86 | } 87 | 88 | } // end namespace Eigen 89 | 90 | #endif // EIGEN_ASSIGN_H 91 | -------------------------------------------------------------------------------- /sample/symqg_querying.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "rabitqlib/defines.hpp" 4 | #include "rabitqlib/index/symqg/qg.hpp" 5 | #include "rabitqlib/utils/io.hpp" 6 | #include "rabitqlib/utils/stopw.hpp" 7 | 8 | using PID = rabitqlib::PID; 9 | using index_type = rabitqlib::symqg::QuantizedGraph; 10 | using data_type = rabitqlib::RowMajorArray; 11 | using gt_type = rabitqlib::RowMajorArray; 12 | 13 | std::vector efs = { 14 | 10, 20, 40, 50, 60, 80, 100, 150, 170, 190, 200, 250, 300, 400, 500, 600, 700, 800, 1500 15 | }; 16 | size_t test_round = 3; 17 | size_t topk = 10; 18 | 19 | int main(int argc, char** argv) { 20 | if (argc != 4) { 21 | std::cerr << "Usage: " << argv[0] << " \n" 22 | << "arg1: path for index \n" 23 | << "arg2: path for query file, format .fvecs\n" 24 | << "arg3: path for groundtruth file format .ivecs\n"; 25 | exit(1); 26 | } 27 | 28 | char* index_file = argv[1]; 29 | char* query_file = argv[2]; 30 | char* gt_file = argv[3]; 31 | 32 | data_type query; 33 | gt_type gt; 34 | rabitqlib::load_vecs(query_file, query); 35 | rabitqlib::load_vecs(gt_file, gt); 36 | size_t nq = query.rows(); 37 | size_t total_count = nq * topk; 38 | 39 | index_type qg; 40 | qg.load(index_file); 41 | 42 | rabitqlib::StopW stopw; 43 | 44 | std::vector> all_qps(test_round, std::vector(efs.size())); 45 | std::vector> all_recall(test_round, std::vector(efs.size())); 46 | 47 | for (size_t r = 0; r < test_round; r++) { 48 | for (size_t i = 0; i < efs.size(); ++i) { 49 | size_t ef = efs[i]; 50 | size_t total_correct = 0; 51 | float total_time = 0; 52 | qg.set_ef(ef); 53 | std::vector results(topk); 54 | for (size_t z = 0; z < nq; z++) { 55 | stopw.reset(); 56 | qg.search(&query(z, 0), topk, results.data()); 57 | total_time += stopw.get_elapsed_micro(); 58 | for (size_t y = 0; y < topk; y++) { 59 | for (size_t k = 0; k < topk; k++) { 60 | if (gt(z, k) == results[y]) { 61 | total_correct++; 62 | break; 63 | } 64 | } 65 | } 66 | } 67 | float qps = static_cast(nq) / (total_time / 1e6F); 68 | float recall = 69 | static_cast(total_correct) / static_cast(total_count); 70 | 71 | all_qps[r][i] = qps; 72 | all_recall[r][i] = recall; 73 | } 74 | } 75 | 76 | auto avg_qps = rabitqlib::horizontal_avg(all_qps); 77 | auto avg_recall = rabitqlib::horizontal_avg(all_recall); 78 | 79 | std::cout << "EF\tQPS\tRecall\n"; 80 | for (size_t i = 0; i < avg_qps.size(); ++i) { 81 | std::cout << efs[i] << '\t' << avg_qps[i] << '\t' << avg_recall[i] << '\n'; 82 | } 83 | 84 | return 0; 85 | } -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/StlSupport/details.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2009 Gael Guennebaud 5 | // Copyright (C) 2009 Hauke Heibel 6 | // 7 | // This Source Code Form is subject to the terms of the Mozilla 8 | // Public License v. 2.0. If a copy of the MPL was not distributed 9 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 10 | 11 | #ifndef EIGEN_STL_DETAILS_H 12 | #define EIGEN_STL_DETAILS_H 13 | 14 | #ifndef EIGEN_ALIGNED_ALLOCATOR 15 | #define EIGEN_ALIGNED_ALLOCATOR Eigen::aligned_allocator 16 | #endif 17 | 18 | namespace Eigen { 19 | 20 | // This one is needed to prevent reimplementing the whole std::vector. 21 | template 22 | class aligned_allocator_indirection : public EIGEN_ALIGNED_ALLOCATOR 23 | { 24 | public: 25 | typedef std::size_t size_type; 26 | typedef std::ptrdiff_t difference_type; 27 | typedef T* pointer; 28 | typedef const T* const_pointer; 29 | typedef T& reference; 30 | typedef const T& const_reference; 31 | typedef T value_type; 32 | 33 | template 34 | struct rebind 35 | { 36 | typedef aligned_allocator_indirection other; 37 | }; 38 | 39 | aligned_allocator_indirection() {} 40 | aligned_allocator_indirection(const aligned_allocator_indirection& ) : EIGEN_ALIGNED_ALLOCATOR() {} 41 | aligned_allocator_indirection(const EIGEN_ALIGNED_ALLOCATOR& ) {} 42 | template 43 | aligned_allocator_indirection(const aligned_allocator_indirection& ) {} 44 | template 45 | aligned_allocator_indirection(const EIGEN_ALIGNED_ALLOCATOR& ) {} 46 | ~aligned_allocator_indirection() {} 47 | }; 48 | 49 | #if EIGEN_COMP_MSVC 50 | 51 | // sometimes, MSVC detects, at compile time, that the argument x 52 | // in std::vector::resize(size_t s,T x) won't be aligned and generate an error 53 | // even if this function is never called. Whence this little wrapper. 54 | #define EIGEN_WORKAROUND_MSVC_STL_SUPPORT(T) \ 55 | typename Eigen::internal::conditional< \ 56 | Eigen::internal::is_arithmetic::value, \ 57 | T, \ 58 | Eigen::internal::workaround_msvc_stl_support \ 59 | >::type 60 | 61 | namespace internal { 62 | template struct workaround_msvc_stl_support : public T 63 | { 64 | inline workaround_msvc_stl_support() : T() {} 65 | inline workaround_msvc_stl_support(const T& other) : T(other) {} 66 | inline operator T& () { return *static_cast(this); } 67 | inline operator const T& () const { return *static_cast(this); } 68 | template 69 | inline T& operator=(const OtherT& other) 70 | { T::operator=(other); return *this; } 71 | inline workaround_msvc_stl_support& operator=(const workaround_msvc_stl_support& other) 72 | { T::operator=(other); return *this; } 73 | }; 74 | } 75 | 76 | #else 77 | 78 | #define EIGEN_WORKAROUND_MSVC_STL_SUPPORT(T) T 79 | 80 | #endif 81 | 82 | } 83 | 84 | #endif // EIGEN_STL_DETAILS_H 85 | -------------------------------------------------------------------------------- /docs/docs/compact_code.md: -------------------------------------------------------------------------------- 1 | # Compact Storage of Codes 2 | 3 | This section describes the compact storage of codes. RaBitQLib supports to quantize codes with different bit widths, i.e., 1, 2, 3, 4, 5, 6, 7 and 8. These bit widths except 8 are unaligned with byte alignment. Thus, we need to design a specialized compact storage format for the code vector for each bit width. We pad the dimensionality to a multiple of 64 for the ease of alignment. The implementation can be found in `rabitqlib/quantization/pack_excode.hpp`. 4 | 5 | Example 6 | ```cpp 7 | #include 8 | #include 9 | #include 10 | 11 | int main(){ 12 | size_t dim = 768; 13 | size_t bits = 4; 14 | 15 | std::vector code(dim); 16 | // Generate random 4-bit values (0-15) for each dimension 17 | for (size_t i = 0; i < dim; ++i) { 18 | code[i] = rand() % 16; // 4-bit values range from 0 to 15 19 | } 20 | 21 | std::vector compact_code(dim * bits / 8); 22 | 23 | rabitqlib::quant::rabitq_impl::ex_bits::packing_rabitqplus_code( 24 | code.data(), compact_code.data(), dim, bits 25 | ); 26 | } 27 | 28 | ``` 29 | 30 | The following shows the compact storage format for every 64 dimensions. 31 | 32 | 33 | 34 | ## 1-bit 35 | 36 | The code sequentially stores the binary value for each of the 64 dimensions. 37 | 38 | ## 2-bit 39 | 40 | ![2-bit Storage](assets/img/compact/2bit.png){ width=300px align=center } 41 | 42 | The code is stored in a byte array of length 16. Each row in the figure represents a byte. The 0-th byte stores the 2-bit codes of the 0-th, 16-th, 32-th and 48-th dimensions. The 1-th byte stores the 2-bit codes of the 1-th, 17-th, 33-th and 49-th dimensions, so on and so forth. This storage allows efficient unpacking with SIMD, i.e., shifting and masking with `SSE`. 43 | 44 | ## 3-bit = 2-bit + 1-bit 45 | 46 | ## 4-bit 47 | 48 | ![4-bit Storage](assets/img/compact/4bit.png){ width=600px align=center } 49 | 50 | 51 | The code is stored in a byte array of length 32. The 0-th byte stores the 4-bit codes of the 0-th and 16-th dimensions. The 1-th byte stores the 4-bit codes of the 1-th and 17-th dimensions, so on and so forth. This storage allows efficient unpacking with SIMD, i.e., shifting and masking with `SSE`. 52 | 53 | ## 5-bit = 4-bit + 1-bit 54 | 55 | ## 6-bit 56 | 57 | ![6-bit Storage](assets/img/compact/6bit.png){ width=900px align=center } 58 | 59 | The code is stored in a byte array of length 48. 60 | 61 | - The first 16 bytes store the 6-bit codes of the 0-th to the 15-th dimensions and the upper 2-bit codes of the 32-th to 47-th dimensions. 62 | - The second 16 bytes store the 6-bit codes of the 16-th to the 31-th dimensions and the upper 2-bit codes of the 48-th to 63-th dimensions. 63 | - The third 16 bytes store the lower 4-bit codes of the 32-th to 47-th dimensions and the lower 4-bit codes of the 48-th to 63-th dimensions. 64 | 65 | This storage allows efficient unpacking with SIMD, i.e., shifting and masking with `SSE`. 66 | 67 | ## 7-bit = 6-bit + 1-bit 68 | 69 | ## 8-bit 70 | 71 | The code of 8-bit is aligned with byte arrays and needs no specialized design. 72 | 73 | 74 | 75 | 76 | 77 | 78 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/Core/arch/AVX/TypeCasting.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2015 Benoit Steiner 5 | // 6 | // This Source Code Form is subject to the terms of the Mozilla 7 | // Public License v. 2.0. If a copy of the MPL was not distributed 8 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | 10 | #ifndef EIGEN_TYPE_CASTING_AVX_H 11 | #define EIGEN_TYPE_CASTING_AVX_H 12 | 13 | namespace Eigen { 14 | 15 | namespace internal { 16 | 17 | // For now we use SSE to handle integers, so we can't use AVX instructions to cast 18 | // from int to float 19 | template <> 20 | struct type_casting_traits { 21 | enum { 22 | VectorizedCast = 0, 23 | SrcCoeffRatio = 1, 24 | TgtCoeffRatio = 1 25 | }; 26 | }; 27 | 28 | template <> 29 | struct type_casting_traits { 30 | enum { 31 | VectorizedCast = 0, 32 | SrcCoeffRatio = 1, 33 | TgtCoeffRatio = 1 34 | }; 35 | }; 36 | 37 | 38 | #ifndef EIGEN_VECTORIZE_AVX512 39 | 40 | template <> 41 | struct type_casting_traits { 42 | enum { 43 | VectorizedCast = 1, 44 | SrcCoeffRatio = 1, 45 | TgtCoeffRatio = 1 46 | }; 47 | }; 48 | 49 | 50 | template <> 51 | struct type_casting_traits { 52 | enum { 53 | VectorizedCast = 1, 54 | SrcCoeffRatio = 1, 55 | TgtCoeffRatio = 1 56 | }; 57 | }; 58 | 59 | template <> 60 | struct type_casting_traits { 61 | enum { 62 | VectorizedCast = 1, 63 | SrcCoeffRatio = 1, 64 | TgtCoeffRatio = 1 65 | }; 66 | }; 67 | 68 | template <> 69 | struct type_casting_traits { 70 | enum { 71 | VectorizedCast = 1, 72 | SrcCoeffRatio = 1, 73 | TgtCoeffRatio = 1 74 | }; 75 | }; 76 | 77 | #endif // EIGEN_VECTORIZE_AVX512 78 | 79 | template<> EIGEN_STRONG_INLINE Packet8i pcast(const Packet8f& a) { 80 | return _mm256_cvttps_epi32(a); 81 | } 82 | 83 | template<> EIGEN_STRONG_INLINE Packet8f pcast(const Packet8i& a) { 84 | return _mm256_cvtepi32_ps(a); 85 | } 86 | 87 | template<> EIGEN_STRONG_INLINE Packet8i preinterpret(const Packet8f& a) { 88 | return _mm256_castps_si256(a); 89 | } 90 | 91 | template<> EIGEN_STRONG_INLINE Packet8f preinterpret(const Packet8i& a) { 92 | return _mm256_castsi256_ps(a); 93 | } 94 | 95 | template<> EIGEN_STRONG_INLINE Packet8f pcast(const Packet8h& a) { 96 | return half2float(a); 97 | } 98 | 99 | template<> EIGEN_STRONG_INLINE Packet8f pcast(const Packet8bf& a) { 100 | return Bf16ToF32(a); 101 | } 102 | 103 | template<> EIGEN_STRONG_INLINE Packet8h pcast(const Packet8f& a) { 104 | return float2half(a); 105 | } 106 | 107 | template<> EIGEN_STRONG_INLINE Packet8bf pcast(const Packet8f& a) { 108 | return F32ToBf16(a); 109 | } 110 | 111 | } // end namespace internal 112 | 113 | } // end namespace Eigen 114 | 115 | #endif // EIGEN_TYPE_CASTING_AVX_H 116 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/SparseLU/SparseLU_relax_snode.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2012 Désiré Nuentsa-Wakam 5 | // 6 | // This Source Code Form is subject to the terms of the Mozilla 7 | // Public License v. 2.0. If a copy of the MPL was not distributed 8 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | 10 | /* This file is a modified version of heap_relax_snode.c file in SuperLU 11 | * -- SuperLU routine (version 3.0) -- 12 | * Univ. of California Berkeley, Xerox Palo Alto Research Center, 13 | * and Lawrence Berkeley National Lab. 14 | * October 15, 2003 15 | * 16 | * Copyright (c) 1994 by Xerox Corporation. All rights reserved. 17 | * 18 | * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY 19 | * EXPRESSED OR IMPLIED. ANY USE IS AT YOUR OWN RISK. 20 | * 21 | * Permission is hereby granted to use or copy this program for any 22 | * purpose, provided the above notices are retained on all copies. 23 | * Permission to modify the code and to distribute modified code is 24 | * granted, provided the above notices are retained, and a notice that 25 | * the code was modified is included with the above copyright notice. 26 | */ 27 | 28 | #ifndef SPARSELU_RELAX_SNODE_H 29 | #define SPARSELU_RELAX_SNODE_H 30 | 31 | namespace Eigen { 32 | 33 | namespace internal { 34 | 35 | /** 36 | * \brief Identify the initial relaxed supernodes 37 | * 38 | * This routine is applied to a column elimination tree. 39 | * It assumes that the matrix has been reordered according to the postorder of the etree 40 | * \param n the number of columns 41 | * \param et elimination tree 42 | * \param relax_columns Maximum number of columns allowed in a relaxed snode 43 | * \param descendants Number of descendants of each node in the etree 44 | * \param relax_end last column in a supernode 45 | */ 46 | template 47 | void SparseLUImpl::relax_snode (const Index n, IndexVector& et, const Index relax_columns, IndexVector& descendants, IndexVector& relax_end) 48 | { 49 | 50 | // compute the number of descendants of each node in the etree 51 | Index parent; 52 | relax_end.setConstant(emptyIdxLU); 53 | descendants.setZero(); 54 | for (Index j = 0; j < n; j++) 55 | { 56 | parent = et(j); 57 | if (parent != n) // not the dummy root 58 | descendants(parent) += descendants(j) + 1; 59 | } 60 | // Identify the relaxed supernodes by postorder traversal of the etree 61 | Index snode_start; // beginning of a snode 62 | for (Index j = 0; j < n; ) 63 | { 64 | parent = et(j); 65 | snode_start = j; 66 | while ( parent != n && descendants(parent) < relax_columns ) 67 | { 68 | j = parent; 69 | parent = et(j); 70 | } 71 | // Found a supernode in postordered etree, j is the last column 72 | relax_end(snode_start) = StorageIndex(j); // Record last column 73 | j++; 74 | // Search for a new leaf 75 | while (descendants(j) != 0 && j < n) j++; 76 | } // End postorder traversal of the etree 77 | 78 | } 79 | 80 | } // end namespace internal 81 | 82 | } // end namespace Eigen 83 | #endif 84 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/misc/Image.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2009 Benoit Jacob 5 | // 6 | // This Source Code Form is subject to the terms of the Mozilla 7 | // Public License v. 2.0. If a copy of the MPL was not distributed 8 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | 10 | #ifndef EIGEN_MISC_IMAGE_H 11 | #define EIGEN_MISC_IMAGE_H 12 | 13 | namespace Eigen { 14 | 15 | namespace internal { 16 | 17 | /** \class image_retval_base 18 | * 19 | */ 20 | template 21 | struct traits > 22 | { 23 | typedef typename DecompositionType::MatrixType MatrixType; 24 | typedef Matrix< 25 | typename MatrixType::Scalar, 26 | MatrixType::RowsAtCompileTime, // the image is a subspace of the destination space, whose 27 | // dimension is the number of rows of the original matrix 28 | Dynamic, // we don't know at compile time the dimension of the image (the rank) 29 | MatrixType::Options, 30 | MatrixType::MaxRowsAtCompileTime, // the image matrix will consist of columns from the original matrix, 31 | MatrixType::MaxColsAtCompileTime // so it has the same number of rows and at most as many columns. 32 | > ReturnType; 33 | }; 34 | 35 | template struct image_retval_base 36 | : public ReturnByValue > 37 | { 38 | typedef _DecompositionType DecompositionType; 39 | typedef typename DecompositionType::MatrixType MatrixType; 40 | typedef ReturnByValue Base; 41 | 42 | image_retval_base(const DecompositionType& dec, const MatrixType& originalMatrix) 43 | : m_dec(dec), m_rank(dec.rank()), 44 | m_cols(m_rank == 0 ? 1 : m_rank), 45 | m_originalMatrix(originalMatrix) 46 | {} 47 | 48 | inline Index rows() const { return m_dec.rows(); } 49 | inline Index cols() const { return m_cols; } 50 | inline Index rank() const { return m_rank; } 51 | inline const DecompositionType& dec() const { return m_dec; } 52 | inline const MatrixType& originalMatrix() const { return m_originalMatrix; } 53 | 54 | template inline void evalTo(Dest& dst) const 55 | { 56 | static_cast*>(this)->evalTo(dst); 57 | } 58 | 59 | protected: 60 | const DecompositionType& m_dec; 61 | Index m_rank, m_cols; 62 | const MatrixType& m_originalMatrix; 63 | }; 64 | 65 | } // end namespace internal 66 | 67 | #define EIGEN_MAKE_IMAGE_HELPERS(DecompositionType) \ 68 | typedef typename DecompositionType::MatrixType MatrixType; \ 69 | typedef typename MatrixType::Scalar Scalar; \ 70 | typedef typename MatrixType::RealScalar RealScalar; \ 71 | typedef Eigen::internal::image_retval_base Base; \ 72 | using Base::dec; \ 73 | using Base::originalMatrix; \ 74 | using Base::rank; \ 75 | using Base::rows; \ 76 | using Base::cols; \ 77 | image_retval(const DecompositionType& dec, const MatrixType& originalMatrix) \ 78 | : Base(dec, originalMatrix) {} 79 | 80 | } // end namespace Eigen 81 | 82 | #endif // EIGEN_MISC_IMAGE_H 83 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/Core/arch/GPU/MathFunctions.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2014 Benoit Steiner 5 | // 6 | // This Source Code Form is subject to the terms of the Mozilla 7 | // Public License v. 2.0. If a copy of the MPL was not distributed 8 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | 10 | #ifndef EIGEN_MATH_FUNCTIONS_GPU_H 11 | #define EIGEN_MATH_FUNCTIONS_GPU_H 12 | 13 | namespace Eigen { 14 | 15 | namespace internal { 16 | 17 | // Make sure this is only available when targeting a GPU: we don't want to 18 | // introduce conflicts between these packet_traits definitions and the ones 19 | // we'll use on the host side (SSE, AVX, ...) 20 | #if defined(EIGEN_GPUCC) && defined(EIGEN_USE_GPU) 21 | template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE 22 | float4 plog(const float4& a) 23 | { 24 | return make_float4(logf(a.x), logf(a.y), logf(a.z), logf(a.w)); 25 | } 26 | 27 | template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE 28 | double2 plog(const double2& a) 29 | { 30 | using ::log; 31 | return make_double2(log(a.x), log(a.y)); 32 | } 33 | 34 | template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE 35 | float4 plog1p(const float4& a) 36 | { 37 | return make_float4(log1pf(a.x), log1pf(a.y), log1pf(a.z), log1pf(a.w)); 38 | } 39 | 40 | template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE 41 | double2 plog1p(const double2& a) 42 | { 43 | return make_double2(log1p(a.x), log1p(a.y)); 44 | } 45 | 46 | template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE 47 | float4 pexp(const float4& a) 48 | { 49 | return make_float4(expf(a.x), expf(a.y), expf(a.z), expf(a.w)); 50 | } 51 | 52 | template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE 53 | double2 pexp(const double2& a) 54 | { 55 | using ::exp; 56 | return make_double2(exp(a.x), exp(a.y)); 57 | } 58 | 59 | template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE 60 | float4 pexpm1(const float4& a) 61 | { 62 | return make_float4(expm1f(a.x), expm1f(a.y), expm1f(a.z), expm1f(a.w)); 63 | } 64 | 65 | template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE 66 | double2 pexpm1(const double2& a) 67 | { 68 | return make_double2(expm1(a.x), expm1(a.y)); 69 | } 70 | 71 | template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE 72 | float4 psqrt(const float4& a) 73 | { 74 | return make_float4(sqrtf(a.x), sqrtf(a.y), sqrtf(a.z), sqrtf(a.w)); 75 | } 76 | 77 | template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE 78 | double2 psqrt(const double2& a) 79 | { 80 | using ::sqrt; 81 | return make_double2(sqrt(a.x), sqrt(a.y)); 82 | } 83 | 84 | template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE 85 | float4 prsqrt(const float4& a) 86 | { 87 | return make_float4(rsqrtf(a.x), rsqrtf(a.y), rsqrtf(a.z), rsqrtf(a.w)); 88 | } 89 | 90 | template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE 91 | double2 prsqrt(const double2& a) 92 | { 93 | return make_double2(rsqrt(a.x), rsqrt(a.y)); 94 | } 95 | 96 | 97 | #endif 98 | 99 | } // end namespace internal 100 | 101 | } // end namespace Eigen 102 | 103 | #endif // EIGEN_MATH_FUNCTIONS_GPU_H 104 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/QR/HouseholderQR_LAPACKE.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2011, Intel Corporation. All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, 5 | are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright notice, 10 | this list of conditions and the following disclaimer in the documentation 11 | and/or other materials provided with the distribution. 12 | * Neither the name of Intel Corporation nor the names of its contributors may 13 | be used to endorse or promote products derived from this software without 14 | specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 17 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 20 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 23 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | 27 | ******************************************************************************** 28 | * Content : Eigen bindings to LAPACKe 29 | * Householder QR decomposition of a matrix w/o pivoting based on 30 | * LAPACKE_?geqrf function. 31 | ******************************************************************************** 32 | */ 33 | 34 | #ifndef EIGEN_QR_LAPACKE_H 35 | #define EIGEN_QR_LAPACKE_H 36 | 37 | namespace Eigen { 38 | 39 | namespace internal { 40 | 41 | /** \internal Specialization for the data types supported by LAPACKe */ 42 | 43 | #define EIGEN_LAPACKE_QR_NOPIV(EIGTYPE, LAPACKE_TYPE, LAPACKE_PREFIX) \ 44 | template \ 45 | struct householder_qr_inplace_blocked \ 46 | { \ 47 | static void run(MatrixQR& mat, HCoeffs& hCoeffs, Index = 32, \ 48 | typename MatrixQR::Scalar* = 0) \ 49 | { \ 50 | lapack_int m = (lapack_int) mat.rows(); \ 51 | lapack_int n = (lapack_int) mat.cols(); \ 52 | lapack_int lda = (lapack_int) mat.outerStride(); \ 53 | lapack_int matrix_order = (MatrixQR::IsRowMajor) ? LAPACK_ROW_MAJOR : LAPACK_COL_MAJOR; \ 54 | LAPACKE_##LAPACKE_PREFIX##geqrf( matrix_order, m, n, (LAPACKE_TYPE*)mat.data(), lda, (LAPACKE_TYPE*)hCoeffs.data()); \ 55 | hCoeffs.adjointInPlace(); \ 56 | } \ 57 | }; 58 | 59 | EIGEN_LAPACKE_QR_NOPIV(double, double, d) 60 | EIGEN_LAPACKE_QR_NOPIV(float, float, s) 61 | EIGEN_LAPACKE_QR_NOPIV(dcomplex, lapack_complex_double, z) 62 | EIGEN_LAPACKE_QR_NOPIV(scomplex, lapack_complex_float, c) 63 | 64 | } // end namespace internal 65 | 66 | } // end namespace Eigen 67 | 68 | #endif // EIGEN_QR_LAPACKE_H 69 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/Core/arch/NEON/MathFunctions.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // This Source Code Form is subject to the terms of the Mozilla 5 | // Public License v. 2.0. If a copy of the MPL was not distributed 6 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 7 | 8 | #ifndef EIGEN_MATH_FUNCTIONS_NEON_H 9 | #define EIGEN_MATH_FUNCTIONS_NEON_H 10 | 11 | namespace Eigen { 12 | 13 | namespace internal { 14 | 15 | template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2f pexp(const Packet2f& x) 16 | { return pexp_float(x); } 17 | template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f pexp(const Packet4f& x) 18 | { return pexp_float(x); } 19 | 20 | template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2f plog(const Packet2f& x) 21 | { return plog_float(x); } 22 | template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f plog(const Packet4f& x) 23 | { return plog_float(x); } 24 | 25 | template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2f psin(const Packet2f& x) 26 | { return psin_float(x); } 27 | template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f psin(const Packet4f& x) 28 | { return psin_float(x); } 29 | 30 | template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2f pcos(const Packet2f& x) 31 | { return pcos_float(x); } 32 | template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f pcos(const Packet4f& x) 33 | { return pcos_float(x); } 34 | 35 | // Hyperbolic Tangent function. 36 | template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2f ptanh(const Packet2f& x) 37 | { return internal::generic_fast_tanh_float(x); } 38 | template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet4f ptanh(const Packet4f& x) 39 | { return internal::generic_fast_tanh_float(x); } 40 | 41 | BF16_PACKET_FUNCTION(Packet4f, Packet4bf, psin) 42 | BF16_PACKET_FUNCTION(Packet4f, Packet4bf, pcos) 43 | BF16_PACKET_FUNCTION(Packet4f, Packet4bf, plog) 44 | BF16_PACKET_FUNCTION(Packet4f, Packet4bf, pexp) 45 | BF16_PACKET_FUNCTION(Packet4f, Packet4bf, ptanh) 46 | 47 | template <> 48 | EIGEN_STRONG_INLINE Packet4bf pfrexp(const Packet4bf& a, Packet4bf& exponent) { 49 | Packet4f fexponent; 50 | const Packet4bf out = F32ToBf16(pfrexp(Bf16ToF32(a), fexponent)); 51 | exponent = F32ToBf16(fexponent); 52 | return out; 53 | } 54 | 55 | template <> 56 | EIGEN_STRONG_INLINE Packet4bf pldexp(const Packet4bf& a, const Packet4bf& exponent) { 57 | return F32ToBf16(pldexp(Bf16ToF32(a), Bf16ToF32(exponent))); 58 | } 59 | 60 | //---------- double ---------- 61 | 62 | #if EIGEN_ARCH_ARM64 && !EIGEN_APPLE_DOUBLE_NEON_BUG 63 | template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2d pexp(const Packet2d& x) 64 | { return pexp_double(x); } 65 | 66 | template<> EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED Packet2d plog(const Packet2d& x) 67 | { return plog_double(x); } 68 | 69 | #endif 70 | 71 | } // end namespace internal 72 | 73 | } // end namespace Eigen 74 | 75 | #endif // EIGEN_MATH_FUNCTIONS_NEON_H 76 | -------------------------------------------------------------------------------- /sample/hnsw_rabitq_indexing.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "rabitqlib/index/hnsw/hnsw.hpp" 5 | #include "rabitqlib/utils/io.hpp" 6 | #include "rabitqlib/utils/stopw.hpp" 7 | 8 | using PID = rabitqlib::PID; 9 | using index_type = rabitqlib::hnsw::HierarchicalNSW; 10 | using data_type = rabitqlib::RowMajorArray; 11 | using gt_type = rabitqlib::RowMajorArray; 12 | 13 | int main(int argc, char* argv[]) { 14 | if (argc < 8) { 15 | std::cerr << "Usage: " << argv[0] 16 | << " \n" 17 | << "arg1: path for data file, format .fvecs\n" 18 | << "arg2: path for centroids file, format .fvecs\n" 19 | << "arg3: path for cluster_ids file, format .ivecs\n" 20 | << "arg4: m (degree bound) for hnsw\n" 21 | << "arg5: ef for indexing \n" 22 | << "arg6: total number of bits for quantization\n" 23 | << "arg7: path for saving index\n" 24 | << "arg8: metric type (\"l2\" or \"ip\"), l2 by default\n" 25 | << "arg9: if use faster quantization (\"true\" or \"false\"), false by " 26 | "default\n"; 27 | exit(1); 28 | } 29 | 30 | char* data_file = argv[1]; 31 | char* centroid_file = argv[2]; 32 | char* cid_file = argv[3]; 33 | size_t m = atoi(argv[4]); 34 | size_t ef = atoi(argv[5]); 35 | size_t total_bits = atoi(argv[6]); 36 | char* index_file = argv[7]; 37 | 38 | rabitqlib::MetricType metric_type = rabitqlib::METRIC_L2; 39 | if (argc > 8) { 40 | std::string metric_str(argv[8]); 41 | if (metric_str == "ip" || metric_str == "IP") { 42 | metric_type = rabitqlib::METRIC_IP; 43 | } 44 | } 45 | if (metric_type == rabitqlib::METRIC_IP) { 46 | std::cout << "Metric Type: IP\n"; 47 | } else if (metric_type == rabitqlib::METRIC_L2) { 48 | std::cout << "Metric Type: L2\n"; 49 | } 50 | 51 | bool faster_quant = false; 52 | if (argc > 9) { 53 | std::string faster_str(argv[9]); 54 | if (faster_str == "true") { 55 | faster_quant = true; 56 | std::cout << "Using faster quantize for indexing...\n"; 57 | } 58 | } 59 | 60 | data_type data; 61 | data_type centroids; 62 | gt_type cluster_id; 63 | 64 | rabitqlib::load_vecs(data_file, data); 65 | rabitqlib::load_vecs(centroid_file, centroids); 66 | rabitqlib::load_vecs(cid_file, cluster_id); 67 | 68 | size_t num_points = data.rows(); 69 | size_t dim = data.cols(); 70 | 71 | size_t random_seed = 100; // by default 100 72 | auto* hnsw = new rabitqlib::hnsw::HierarchicalNSW( 73 | num_points, dim, total_bits, m, ef, random_seed, metric_type 74 | ); 75 | 76 | rabitqlib::StopW stopw; 77 | stopw.reset(); 78 | 79 | hnsw->construct( 80 | centroids.rows(), 81 | centroids.data(), 82 | num_points, 83 | data.data(), 84 | cluster_id.data(), 85 | 0, 86 | faster_quant 87 | ); 88 | 89 | float total_time = stopw.get_elapsed_micro(); 90 | total_time /= 1e6; 91 | 92 | std::cout << "indexing time = " << total_time << "s" << '\n'; 93 | hnsw->save(index_file); 94 | 95 | std::cout << "index saved..." << '\n'; 96 | 97 | return 0; 98 | } -------------------------------------------------------------------------------- /include/rabitqlib/utils/hashset.hpp: -------------------------------------------------------------------------------- 1 | // This code is modified based on NGT from Yahoo Japan 2 | // https://github.com/yahoojapan/NGT 3 | // 4 | // Copyright (C) 2015 Yahoo Japan Corporation 5 | // 6 | // Licensed under the Apache License, Version 2.0 (the "License"); 7 | // you may not use this file except in compliance with the License. 8 | // You may obtain a copy of the License at 9 | // 10 | // http://www.apache.org/licenses/LICENSE-2.0 11 | // 12 | // Unless required by applicable law or agreed to in writing, software 13 | // distributed under the License is distributed on an "AS IS" BASIS, 14 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // See the License for the specific language governing permissions and 16 | // limitations under the License. 17 | // 18 | 19 | #pragma once 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | #include "rabitqlib/defines.hpp" 27 | #include "rabitqlib/utils/memory.hpp" 28 | 29 | namespace rabitqlib { 30 | /** 31 | * @brief hash set to record visited vertices 32 | * 33 | */ 34 | class HashBasedBooleanSet { 35 | private: 36 | size_t table_size_ = 0; 37 | PID mask_ = 0; 38 | std::vector> table_; 39 | std::unordered_set stl_hash_; 40 | 41 | [[nodiscard]] auto hash1(const PID value) const { return value & mask_; } 42 | 43 | public: 44 | HashBasedBooleanSet() = default; 45 | ~HashBasedBooleanSet() = default; 46 | 47 | HashBasedBooleanSet(const HashBasedBooleanSet&) = default; 48 | HashBasedBooleanSet(HashBasedBooleanSet&&) noexcept = default; 49 | HashBasedBooleanSet& operator=(HashBasedBooleanSet&&) noexcept = default; 50 | 51 | explicit HashBasedBooleanSet(size_t size) { 52 | size_t bit_size = 0; 53 | size_t bit = size; 54 | while (bit != 0) { 55 | bit_size++; 56 | bit >>= 1; 57 | } 58 | size_t bucket_size = 0x1 << ((bit_size + 4) / 2 + 3); 59 | initialize(bucket_size); 60 | } 61 | 62 | void initialize(const size_t table_size) { 63 | table_size_ = table_size; 64 | mask_ = static_cast(table_size_ - 1); 65 | const PID check_val = hash1(static_cast(table_size)); 66 | if (check_val != 0) { 67 | std::cerr << "[WARN] table size is not 2^N : " << table_size << '\n'; 68 | } 69 | 70 | table_ = std::vector>(table_size); 71 | std::fill(table_.begin(), table_.end(), kPidMax); 72 | stl_hash_.clear(); 73 | } 74 | 75 | void clear() { 76 | std::fill(table_.begin(), table_.end(), kPidMax); 77 | stl_hash_.clear(); 78 | } 79 | 80 | // get if data_id is in the hashset 81 | [[nodiscard]] bool get(PID data_id) const { 82 | PID val = this->table_[hash1(data_id)]; 83 | if (val == data_id) { 84 | return true; 85 | } 86 | return (val != kPidMax && stl_hash_.find(data_id) != stl_hash_.end()); 87 | } 88 | 89 | void set(PID data_id) { 90 | PID& val = table_[hash1(data_id)]; 91 | if (val == data_id) { 92 | return; 93 | } 94 | if (val == kPidMax) { 95 | val = data_id; 96 | } else { 97 | stl_hash_.emplace(data_id); 98 | } 99 | } 100 | }; 101 | } // namespace rabitqlib -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/SparseCore/SparseDot.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2008 Gael Guennebaud 5 | // 6 | // This Source Code Form is subject to the terms of the Mozilla 7 | // Public License v. 2.0. If a copy of the MPL was not distributed 8 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | 10 | #ifndef EIGEN_SPARSE_DOT_H 11 | #define EIGEN_SPARSE_DOT_H 12 | 13 | namespace Eigen { 14 | 15 | template 16 | template 17 | typename internal::traits::Scalar 18 | SparseMatrixBase::dot(const MatrixBase& other) const 19 | { 20 | EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) 21 | EIGEN_STATIC_ASSERT_VECTOR_ONLY(OtherDerived) 22 | EIGEN_STATIC_ASSERT_SAME_VECTOR_SIZE(Derived,OtherDerived) 23 | EIGEN_STATIC_ASSERT((internal::is_same::value), 24 | YOU_MIXED_DIFFERENT_NUMERIC_TYPES__YOU_NEED_TO_USE_THE_CAST_METHOD_OF_MATRIXBASE_TO_CAST_NUMERIC_TYPES_EXPLICITLY) 25 | 26 | eigen_assert(size() == other.size()); 27 | eigen_assert(other.size()>0 && "you are using a non initialized vector"); 28 | 29 | internal::evaluator thisEval(derived()); 30 | typename internal::evaluator::InnerIterator i(thisEval, 0); 31 | Scalar res(0); 32 | while (i) 33 | { 34 | res += numext::conj(i.value()) * other.coeff(i.index()); 35 | ++i; 36 | } 37 | return res; 38 | } 39 | 40 | template 41 | template 42 | typename internal::traits::Scalar 43 | SparseMatrixBase::dot(const SparseMatrixBase& other) const 44 | { 45 | EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) 46 | EIGEN_STATIC_ASSERT_VECTOR_ONLY(OtherDerived) 47 | EIGEN_STATIC_ASSERT_SAME_VECTOR_SIZE(Derived,OtherDerived) 48 | EIGEN_STATIC_ASSERT((internal::is_same::value), 49 | YOU_MIXED_DIFFERENT_NUMERIC_TYPES__YOU_NEED_TO_USE_THE_CAST_METHOD_OF_MATRIXBASE_TO_CAST_NUMERIC_TYPES_EXPLICITLY) 50 | 51 | eigen_assert(size() == other.size()); 52 | 53 | internal::evaluator thisEval(derived()); 54 | typename internal::evaluator::InnerIterator i(thisEval, 0); 55 | 56 | internal::evaluator otherEval(other.derived()); 57 | typename internal::evaluator::InnerIterator j(otherEval, 0); 58 | 59 | Scalar res(0); 60 | while (i && j) 61 | { 62 | if (i.index()==j.index()) 63 | { 64 | res += numext::conj(i.value()) * j.value(); 65 | ++i; ++j; 66 | } 67 | else if (i.index() 76 | inline typename NumTraits::Scalar>::Real 77 | SparseMatrixBase::squaredNorm() const 78 | { 79 | return numext::real((*this).cwiseAbs2().sum()); 80 | } 81 | 82 | template 83 | inline typename NumTraits::Scalar>::Real 84 | SparseMatrixBase::norm() const 85 | { 86 | using std::sqrt; 87 | return sqrt(squaredNorm()); 88 | } 89 | 90 | template 91 | inline typename NumTraits::Scalar>::Real 92 | SparseMatrixBase::blueNorm() const 93 | { 94 | return internal::blueNorm_impl(*this); 95 | } 96 | } // end namespace Eigen 97 | 98 | #endif // EIGEN_SPARSE_DOT_H 99 | -------------------------------------------------------------------------------- /include/rabitqlib/utils/buffer.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | #include "rabitqlib/defines.hpp" 7 | #include "rabitqlib/utils/memory.hpp" 8 | 9 | namespace rabitqlib::buffer { 10 | /** 11 | * @brief sorted linear buffer, used as beam set for graph-based ANN search. In symphonyqg, 12 | * the search buffer may contain duplicate id with different distances 13 | * 14 | */ 15 | template 16 | class SearchBuffer { 17 | private: 18 | std::vector, memory::AlignedAllocator>> data_; 19 | size_t size_ = 0, cur_ = 0, capacity_; 20 | 21 | [[nodiscard]] auto binary_search(T dist) const { 22 | size_t lo = 0; 23 | size_t len = size_; 24 | size_t half; 25 | while (len > 1) { 26 | half = len >> 1; 27 | len -= half; 28 | lo += static_cast(data_[lo + half - 1].distance < dist) * half; 29 | } 30 | return (lo < size_ && data_[lo].distance < dist) ? lo + 1 : lo; 31 | } 32 | 33 | // set top bit to 1 as checked 34 | static void set_checked(PID& data_id) { data_id |= (1 << 31); } 35 | 36 | [[nodiscard]] static auto is_checked(PID data_id) -> bool { 37 | return static_cast(data_id >> 31); 38 | } 39 | 40 | public: 41 | SearchBuffer() = default; 42 | 43 | explicit SearchBuffer(size_t capacity) : data_(capacity + 1), capacity_(capacity) {} 44 | 45 | // insert a data point into buffer 46 | void insert(PID data_id, T dist) { 47 | if (is_full(dist)) { 48 | return; 49 | } 50 | 51 | size_t lo = binary_search(dist); 52 | std::memmove(&data_[lo + 1], &data_[lo], (size_ - lo) * sizeof(AnnCandidate)); 53 | data_[lo] = AnnCandidate(data_id, dist); 54 | size_ += static_cast(size_ < capacity_); 55 | cur_ = lo < cur_ ? lo : cur_; 56 | } 57 | 58 | // get unchecked candidate with minimum distance 59 | PID pop() { 60 | PID cur_id = data_[cur_].id; 61 | set_checked(data_[cur_].id); 62 | ++cur_; 63 | while (cur_ < size_ && is_checked(data_[cur_].id)) { 64 | ++cur_; 65 | } 66 | return cur_id; 67 | } 68 | 69 | void clear() { 70 | size_ = 0; 71 | cur_ = 0; 72 | } 73 | 74 | // return candidate id for next pop() 75 | [[nodiscard]] auto next_id() const { return data_[cur_].id; } 76 | 77 | [[nodiscard]] auto has_next() const -> bool { return cur_ < size_; } 78 | 79 | void resize(size_t new_size) { 80 | this->capacity_ = new_size; 81 | data_ = std::vector, memory::AlignedAllocator>>( 82 | capacity_ + 1 83 | ); 84 | } 85 | 86 | void copy_results(PID* knn) const { 87 | for (size_t i = 0; i < size_; ++i) { 88 | knn[i] = data_[i].id; 89 | } 90 | } 91 | 92 | T top_dist() const { 93 | return is_full() ? data_[size_ - 1].distance : std::numeric_limits::max(); 94 | } 95 | 96 | [[nodiscard]] auto is_full() const -> bool { return size_ == capacity_; } 97 | 98 | // judge if dist can be inserted into buffer 99 | [[nodiscard]] auto is_full(T dist) const -> bool { return dist > top_dist(); } 100 | 101 | const std::vector, memory::AlignedAllocator>>& data() { 102 | return data_; 103 | } 104 | }; 105 | } // namespace rabitqlib::buffer -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/SparseCore/SparseTranspose.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2008-2015 Gael Guennebaud 5 | // 6 | // This Source Code Form is subject to the terms of the Mozilla 7 | // Public License v. 2.0. If a copy of the MPL was not distributed 8 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | 10 | #ifndef EIGEN_SPARSETRANSPOSE_H 11 | #define EIGEN_SPARSETRANSPOSE_H 12 | 13 | namespace Eigen { 14 | 15 | namespace internal { 16 | template 17 | class SparseTransposeImpl 18 | : public SparseMatrixBase > 19 | {}; 20 | 21 | template 22 | class SparseTransposeImpl 23 | : public SparseCompressedBase > 24 | { 25 | typedef SparseCompressedBase > Base; 26 | public: 27 | using Base::derived; 28 | typedef typename Base::Scalar Scalar; 29 | typedef typename Base::StorageIndex StorageIndex; 30 | 31 | inline Index nonZeros() const { return derived().nestedExpression().nonZeros(); } 32 | 33 | inline const Scalar* valuePtr() const { return derived().nestedExpression().valuePtr(); } 34 | inline const StorageIndex* innerIndexPtr() const { return derived().nestedExpression().innerIndexPtr(); } 35 | inline const StorageIndex* outerIndexPtr() const { return derived().nestedExpression().outerIndexPtr(); } 36 | inline const StorageIndex* innerNonZeroPtr() const { return derived().nestedExpression().innerNonZeroPtr(); } 37 | 38 | inline Scalar* valuePtr() { return derived().nestedExpression().valuePtr(); } 39 | inline StorageIndex* innerIndexPtr() { return derived().nestedExpression().innerIndexPtr(); } 40 | inline StorageIndex* outerIndexPtr() { return derived().nestedExpression().outerIndexPtr(); } 41 | inline StorageIndex* innerNonZeroPtr() { return derived().nestedExpression().innerNonZeroPtr(); } 42 | }; 43 | } 44 | 45 | template class TransposeImpl 46 | : public internal::SparseTransposeImpl 47 | { 48 | protected: 49 | typedef internal::SparseTransposeImpl Base; 50 | }; 51 | 52 | namespace internal { 53 | 54 | template 55 | struct unary_evaluator, IteratorBased> 56 | : public evaluator_base > 57 | { 58 | typedef typename evaluator::InnerIterator EvalIterator; 59 | public: 60 | typedef Transpose XprType; 61 | 62 | inline Index nonZerosEstimate() const { 63 | return m_argImpl.nonZerosEstimate(); 64 | } 65 | 66 | class InnerIterator : public EvalIterator 67 | { 68 | public: 69 | EIGEN_STRONG_INLINE InnerIterator(const unary_evaluator& unaryOp, Index outer) 70 | : EvalIterator(unaryOp.m_argImpl,outer) 71 | {} 72 | 73 | Index row() const { return EvalIterator::col(); } 74 | Index col() const { return EvalIterator::row(); } 75 | }; 76 | 77 | enum { 78 | CoeffReadCost = evaluator::CoeffReadCost, 79 | Flags = XprType::Flags 80 | }; 81 | 82 | explicit unary_evaluator(const XprType& op) :m_argImpl(op.nestedExpression()) {} 83 | 84 | protected: 85 | evaluator m_argImpl; 86 | }; 87 | 88 | } // end namespace internal 89 | 90 | } // end namespace Eigen 91 | 92 | #endif // EIGEN_SPARSETRANSPOSE_H 93 | -------------------------------------------------------------------------------- /docs/docs/index/qg.md: -------------------------------------------------------------------------------- 1 | # QG + RaBitQ (SymphonyQG) 2 | 3 | [QG](https://medium.com/@masajiro.iwasaki/fusion-of-graph-based-indexing-and-product-quantization-for-ann-search-7d1f0336d0d0) is a graph-based index originated from the [NGT library](https://github.com/yahoojapan/NGT). Different from HNSW, it creates multiple quantization codes for every vector and carefully re-organizes their layout to minimize random memory accesses in querying. RaBitQ + QG in developped from our research project [SymphonyQG](https://dl.acm.org/doi/10.1145/3709730). Unlike IVF + RaBitQ and HNSW + RaBitQ, which consumes less memory than the raw datasets, RaBitQ + QG consumes more memory to pursue the best time-accuracy trade-off. 4 | Here, we offer a toy example for the indexing and querying of QG. 5 | To test QG on real-world datasets, please refer to `sample/symqg_indexing.cpp` and `sample/symqg_querying.cpp` for 6 | detailed information 7 | 8 | ## Index Construction 9 | 10 | We build the QG by iteratively refining the graph structure. 11 | Since the QG is more complicated than other indices, we need a QGBuilder to help us construct the index. 12 | 13 | At the beginning, we need to intialize a QG and a QGBuilder by following construtor. 14 | ```cpp 15 | QuantizedGraph::QuantizedGraph( 16 | size_t num, 17 | size_t dim, 18 | size_t max_deg, 19 | RotatorType type = RotatorType::FhtKacRotator 20 | ); 21 | 22 | QGBuilder::QGBuilder( 23 | QuantizedGraph& index, 24 | uint32_t ef_build, 25 | const float* data, 26 | size_t num_threads = std::numeric_limits::max() 27 | ) 28 | ``` 29 | - **num**: Number of vertices (vectors) in the dataset. 30 | - **dim**: Dimension of the dataset. 31 | - **max_deg**: Degree bound of QG, must be a multiple of 32. 32 | - **index**: Previously initialized QG. 33 | - **ef_build**: Search window size during indexing. 34 | - **data**: Pointer to the dataset, size of num * dim. 35 | - **num_threads**: Number of threads to use (default: std::numeric_limits::max(), which auto-selects). 36 | ```cpp 37 | size_t rows = 1000000; 38 | size_t cols = 128; 39 | size_t degree = 32; 40 | size_t ef = 200 41 | 42 | float* data = new float[rows * cols]; // only for illustration 43 | 44 | QuantizedGraph qg(rows, cols, degree); // init qg 45 | 46 | QGBuilder builder(qg, ef, data.data()); // init builder 47 | ``` 48 | 49 | Then, we can use the builder to construct the index. Then we can save the index. 50 | ```cpp 51 | builder.build(); // build index interatively 52 | 53 | const char* index_file = "./qg_example.index" 54 | qg.save(index_file); // save index 55 | ``` 56 | 57 | ### Data Layout 58 | 59 | Each indexed element is stored in the following layout. 60 | ``` 61 | [Raw data vector] 62 | [Batch data for QG] 63 | [Edges] 64 | ``` 65 | 66 | ## Querying 67 | 68 | For querying, code is pretty simple. 69 | ```cpp 70 | void QuantizedGraph::search( 71 | const T* __restrict__ query, 72 | uint32_t k, 73 | uint32_t* __restrict__ results); 74 | ``` 75 | - **query**: Query vector. 76 | - **k**: Top-k. 77 | - **results**: Result buffer, size of k. 78 | Then we can use a pre-constructed index to search. 79 | ```cpp 80 | QuantizedGraph qg; 81 | qg.load("./qg_example.index"); // load pre-constructed index 82 | 83 | 84 | size_t ef = 100; 85 | size_t topk = 10; 86 | std::vector results(topk); // result buffer 87 | float* query = new float[cols]; // query vector (only for illustration) 88 | 89 | qg.set_ef(ef); // set search window size 90 | qg.search(query, topk, results.data()); // search knn, result will be stored in results 91 | ``` -------------------------------------------------------------------------------- /docs/docs/rabitq/rotator.md: -------------------------------------------------------------------------------- 1 | # Random Rotation 2 | Random rotation (i.e., Johnson Lindenstrauss Transformation) is a crucial step to ensure robust performance and theoretical error bounds of RaBitQ. It is applied to all vectors (including raw data vectors, center vectors and raw query vectors) as a preprocessing step. This section describes the usage of the random rotation. 3 | 4 | RaBitQLib provides two types of random rotation. All implementations sample and store a random rotation at first. Then they apply the sampled random rotation to every input vector and return the rotated vector. 5 | 6 | By default, the library uses the `FFHT + Kac’s Walk` method. 7 | 8 | The implementation can be found in `rotator.hpp`. 9 | 10 | ```css 11 | . 12 | ├── rabitqlib 13 | │ ├── ... 14 | │ └── utils 15 | │ ├── ... 16 | │ └── rotator.hpp 17 | └── ... 18 | ``` 19 | 20 | ### Example 21 | 22 | ```cpp 23 | // Initialize a rotator 24 | // Version 1 - the default rotator 25 | // vectors are padded to the smallest multiple of 64 26 | // storage - 4D bits, time - O(D * log D) 27 | rabitqlib::Rotator* rotator = rabitqlib::choose_rotator( 28 | dim = dim, 29 | RotatorType type = RotatorType::FhtKacRotator); 30 | 31 | // Initialize a rotator 32 | // Version 2 - the random orthogonal transformation 33 | // vectors are padded to the smallest multiple of 64 34 | // storage - D * D floats, time - O(D * D) 35 | rabitqlib::Rotator* rotator = rabitqlib::choose_rotator( 36 | dim = dim, 37 | RotatorType type = RotatorType::MatrixRotator); 38 | 39 | // Apply a rotator to a vector 40 | size_t dim = 768; 41 | std::vector x(dim); 42 | std::vector x_prime(dim); 43 | ... 44 | rotator -> rotate(x.data(), x_prime.data()) 45 | 46 | 47 | 48 | ``` 49 | 50 | ## FFHT + Kac’s Walk 51 | ### Description 52 | This method is a combination of the well-known Fast Johnson-Lindenstrauss Transformation algorithms based on [Fast Hadamard Transform](https://www.cs.princeton.edu/~chazelle/pubs/FJLT-sicomp09.pdf) and ideas in [Kac’s Walk](https://projecteuclid.org/journals/annals-of-applied-probability/volume-27/issue-1/Kacs-walk-on-n-sphere-mixes-in-nlog-n-steps/10.1214/16-AAP1214.full). 53 | It first samples 4 sequences of random signs (i.e., Rademacher random variables). Then for each vector, it repeats the following procedures 4 times. 54 | 55 | 1. Flip its coordinates with the $i$-th sequence of sampled random signs. 56 | 2. Apply FFHT on the first/last $2^k$ coordinates (alternately), where $2^k$ is the maximum power of 2 that is less than or equal to the dimensionality of the vector. 57 | 3. Apply Givens rotation with a fixed angle $\theta = \frac{\pi}{4}$ to the 1st and the 2nd halves of coordinates. 58 | 59 | The following table summarizes the space and time complexity of this method. 60 | 61 | | Space Consumption | Time Complexity | 62 | | ----------------- | --------------- | 63 | | $4D$ binary values ($4D$ bits) | $O(D\log D)$ | 64 | 65 | This implementation is based on the [FFHT library](https://github.com/FALCONN-LIB/FFHT) developed by Alexandr Andoni, Piotr Indyk, Thijs Laarhoven, Ilya Razenshteyn and Ludwig Schmidt. 66 | 67 | 68 | ## Random Orthogonal Transformation 69 | ### Description 70 | This method is the classical Johnson-Lindenstrauss Transformation. It first samples a random gaussian matrix and orthogonalizes it with QR decomposition. Then it multiplies the matrix to every vector. 71 | 72 | The following table summarizes the space and time complexity of this method. 73 | 74 | | Space Consumption | Time Complexity | 75 | | ----------------- | --------------- | 76 | | $D^2$ floating-point numbers | $O(D^2)$ | 77 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/plugins/MatrixCwiseUnaryOps.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2008-2009 Gael Guennebaud 5 | // Copyright (C) 2006-2008 Benoit Jacob 6 | // 7 | // This Source Code Form is subject to the terms of the Mozilla 8 | // Public License v. 2.0. If a copy of the MPL was not distributed 9 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 10 | 11 | // This file is included into the body of the base classes supporting matrix specific coefficient-wise functions. 12 | // This include MatrixBase and SparseMatrixBase. 13 | 14 | 15 | typedef CwiseUnaryOp, const Derived> CwiseAbsReturnType; 16 | typedef CwiseUnaryOp, const Derived> CwiseAbs2ReturnType; 17 | typedef CwiseUnaryOp, const Derived> CwiseArgReturnType; 18 | typedef CwiseUnaryOp, const Derived> CwiseSqrtReturnType; 19 | typedef CwiseUnaryOp, const Derived> CwiseSignReturnType; 20 | typedef CwiseUnaryOp, const Derived> CwiseInverseReturnType; 21 | 22 | /// \returns an expression of the coefficient-wise absolute value of \c *this 23 | /// 24 | /// Example: \include MatrixBase_cwiseAbs.cpp 25 | /// Output: \verbinclude MatrixBase_cwiseAbs.out 26 | /// 27 | EIGEN_DOC_UNARY_ADDONS(cwiseAbs,absolute value) 28 | /// 29 | /// \sa cwiseAbs2() 30 | /// 31 | EIGEN_DEVICE_FUNC 32 | EIGEN_STRONG_INLINE const CwiseAbsReturnType 33 | cwiseAbs() const { return CwiseAbsReturnType(derived()); } 34 | 35 | /// \returns an expression of the coefficient-wise squared absolute value of \c *this 36 | /// 37 | /// Example: \include MatrixBase_cwiseAbs2.cpp 38 | /// Output: \verbinclude MatrixBase_cwiseAbs2.out 39 | /// 40 | EIGEN_DOC_UNARY_ADDONS(cwiseAbs2,squared absolute value) 41 | /// 42 | /// \sa cwiseAbs() 43 | /// 44 | EIGEN_DEVICE_FUNC 45 | EIGEN_STRONG_INLINE const CwiseAbs2ReturnType 46 | cwiseAbs2() const { return CwiseAbs2ReturnType(derived()); } 47 | 48 | /// \returns an expression of the coefficient-wise square root of *this. 49 | /// 50 | /// Example: \include MatrixBase_cwiseSqrt.cpp 51 | /// Output: \verbinclude MatrixBase_cwiseSqrt.out 52 | /// 53 | EIGEN_DOC_UNARY_ADDONS(cwiseSqrt,square-root) 54 | /// 55 | /// \sa cwisePow(), cwiseSquare() 56 | /// 57 | EIGEN_DEVICE_FUNC 58 | inline const CwiseSqrtReturnType 59 | cwiseSqrt() const { return CwiseSqrtReturnType(derived()); } 60 | 61 | /// \returns an expression of the coefficient-wise signum of *this. 62 | /// 63 | /// Example: \include MatrixBase_cwiseSign.cpp 64 | /// Output: \verbinclude MatrixBase_cwiseSign.out 65 | /// 66 | EIGEN_DOC_UNARY_ADDONS(cwiseSign,sign function) 67 | /// 68 | EIGEN_DEVICE_FUNC 69 | inline const CwiseSignReturnType 70 | cwiseSign() const { return CwiseSignReturnType(derived()); } 71 | 72 | 73 | /// \returns an expression of the coefficient-wise inverse of *this. 74 | /// 75 | /// Example: \include MatrixBase_cwiseInverse.cpp 76 | /// Output: \verbinclude MatrixBase_cwiseInverse.out 77 | /// 78 | EIGEN_DOC_UNARY_ADDONS(cwiseInverse,inverse) 79 | /// 80 | /// \sa cwiseProduct() 81 | /// 82 | EIGEN_DEVICE_FUNC 83 | inline const CwiseInverseReturnType 84 | cwiseInverse() const { return CwiseInverseReturnType(derived()); } 85 | 86 | /// \returns an expression of the coefficient-wise phase angle of \c *this 87 | /// 88 | /// Example: \include MatrixBase_cwiseArg.cpp 89 | /// Output: \verbinclude MatrixBase_cwiseArg.out 90 | /// 91 | EIGEN_DOC_UNARY_ADDONS(cwiseArg,arg) 92 | 93 | EIGEN_DEVICE_FUNC 94 | inline const CwiseArgReturnType 95 | cwiseArg() const { return CwiseArgReturnType(derived()); } 96 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/Core/VectorBlock.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2008-2010 Gael Guennebaud 5 | // Copyright (C) 2006-2008 Benoit Jacob 6 | // 7 | // This Source Code Form is subject to the terms of the Mozilla 8 | // Public License v. 2.0. If a copy of the MPL was not distributed 9 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 10 | 11 | #ifndef EIGEN_VECTORBLOCK_H 12 | #define EIGEN_VECTORBLOCK_H 13 | 14 | namespace Eigen { 15 | 16 | namespace internal { 17 | template 18 | struct traits > 19 | : public traits::Flags & RowMajorBit ? 1 : Size, 21 | traits::Flags & RowMajorBit ? Size : 1> > 22 | { 23 | }; 24 | } 25 | 26 | /** \class VectorBlock 27 | * \ingroup Core_Module 28 | * 29 | * \brief Expression of a fixed-size or dynamic-size sub-vector 30 | * 31 | * \tparam VectorType the type of the object in which we are taking a sub-vector 32 | * \tparam Size size of the sub-vector we are taking at compile time (optional) 33 | * 34 | * This class represents an expression of either a fixed-size or dynamic-size sub-vector. 35 | * It is the return type of DenseBase::segment(Index,Index) and DenseBase::segment(Index) and 36 | * most of the time this is the only way it is used. 37 | * 38 | * However, if you want to directly manipulate sub-vector expressions, 39 | * for instance if you want to write a function returning such an expression, you 40 | * will need to use this class. 41 | * 42 | * Here is an example illustrating the dynamic case: 43 | * \include class_VectorBlock.cpp 44 | * Output: \verbinclude class_VectorBlock.out 45 | * 46 | * \note Even though this expression has dynamic size, in the case where \a VectorType 47 | * has fixed size, this expression inherits a fixed maximal size which means that evaluating 48 | * it does not cause a dynamic memory allocation. 49 | * 50 | * Here is an example illustrating the fixed-size case: 51 | * \include class_FixedVectorBlock.cpp 52 | * Output: \verbinclude class_FixedVectorBlock.out 53 | * 54 | * \sa class Block, DenseBase::segment(Index,Index,Index,Index), DenseBase::segment(Index,Index) 55 | */ 56 | template class VectorBlock 57 | : public Block::Flags & RowMajorBit ? 1 : Size, 59 | internal::traits::Flags & RowMajorBit ? Size : 1> 60 | { 61 | typedef Block::Flags & RowMajorBit ? 1 : Size, 63 | internal::traits::Flags & RowMajorBit ? Size : 1> Base; 64 | enum { 65 | IsColVector = !(internal::traits::Flags & RowMajorBit) 66 | }; 67 | public: 68 | EIGEN_DENSE_PUBLIC_INTERFACE(VectorBlock) 69 | 70 | using Base::operator=; 71 | 72 | /** Dynamic-size constructor 73 | */ 74 | EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE 75 | VectorBlock(VectorType& vector, Index start, Index size) 76 | : Base(vector, 77 | IsColVector ? start : 0, IsColVector ? 0 : start, 78 | IsColVector ? size : 1, IsColVector ? 1 : size) 79 | { 80 | EIGEN_STATIC_ASSERT_VECTOR_ONLY(VectorBlock); 81 | } 82 | 83 | /** Fixed-size constructor 84 | */ 85 | EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE 86 | VectorBlock(VectorType& vector, Index start) 87 | : Base(vector, IsColVector ? start : 0, IsColVector ? 0 : start) 88 | { 89 | EIGEN_STATIC_ASSERT_VECTOR_ONLY(VectorBlock); 90 | } 91 | }; 92 | 93 | 94 | } // end namespace Eigen 95 | 96 | #endif // EIGEN_VECTORBLOCK_H 97 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/LU/PartialPivLU_LAPACKE.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2011, Intel Corporation. All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, 5 | are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright notice, 10 | this list of conditions and the following disclaimer in the documentation 11 | and/or other materials provided with the distribution. 12 | * Neither the name of Intel Corporation nor the names of its contributors may 13 | be used to endorse or promote products derived from this software without 14 | specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 17 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 20 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 23 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | 27 | ******************************************************************************** 28 | * Content : Eigen bindings to LAPACKe 29 | * LU decomposition with partial pivoting based on LAPACKE_?getrf function. 30 | ******************************************************************************** 31 | */ 32 | 33 | #ifndef EIGEN_PARTIALLU_LAPACK_H 34 | #define EIGEN_PARTIALLU_LAPACK_H 35 | 36 | namespace Eigen { 37 | 38 | namespace internal { 39 | 40 | /** \internal Specialization for the data types supported by LAPACKe */ 41 | 42 | #define EIGEN_LAPACKE_LU_PARTPIV(EIGTYPE, LAPACKE_TYPE, LAPACKE_PREFIX) \ 43 | template \ 44 | struct partial_lu_impl \ 45 | { \ 46 | /* \internal performs the LU decomposition in-place of the matrix represented */ \ 47 | static lapack_int blocked_lu(Index rows, Index cols, EIGTYPE* lu_data, Index luStride, lapack_int* row_transpositions, lapack_int& nb_transpositions, lapack_int maxBlockSize=256) \ 48 | { \ 49 | EIGEN_UNUSED_VARIABLE(maxBlockSize);\ 50 | lapack_int matrix_order, first_zero_pivot; \ 51 | lapack_int m, n, lda, *ipiv, info; \ 52 | EIGTYPE* a; \ 53 | /* Set up parameters for ?getrf */ \ 54 | matrix_order = StorageOrder==RowMajor ? LAPACK_ROW_MAJOR : LAPACK_COL_MAJOR; \ 55 | lda = convert_index(luStride); \ 56 | a = lu_data; \ 57 | ipiv = row_transpositions; \ 58 | m = convert_index(rows); \ 59 | n = convert_index(cols); \ 60 | nb_transpositions = 0; \ 61 | \ 62 | info = LAPACKE_##LAPACKE_PREFIX##getrf( matrix_order, m, n, (LAPACKE_TYPE*)a, lda, ipiv ); \ 63 | \ 64 | for(int i=0;i= 0); \ 67 | /* something should be done with nb_transpositions */ \ 68 | \ 69 | first_zero_pivot = info; \ 70 | return first_zero_pivot; \ 71 | } \ 72 | }; 73 | 74 | EIGEN_LAPACKE_LU_PARTPIV(double, double, d) 75 | EIGEN_LAPACKE_LU_PARTPIV(float, float, s) 76 | EIGEN_LAPACKE_LU_PARTPIV(dcomplex, lapack_complex_double, z) 77 | EIGEN_LAPACKE_LU_PARTPIV(scomplex, lapack_complex_float, c) 78 | 79 | } // end namespace internal 80 | 81 | } // end namespace Eigen 82 | 83 | #endif // EIGEN_PARTIALLU_LAPACK_H 84 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/Eigenvalues/RealSchur_LAPACKE.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2011, Intel Corporation. All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, 5 | are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright notice, 10 | this list of conditions and the following disclaimer in the documentation 11 | and/or other materials provided with the distribution. 12 | * Neither the name of Intel Corporation nor the names of its contributors may 13 | be used to endorse or promote products derived from this software without 14 | specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 17 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 20 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 23 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | 27 | ******************************************************************************** 28 | * Content : Eigen bindings to LAPACKe 29 | * Real Schur needed to real unsymmetrical eigenvalues/eigenvectors. 30 | ******************************************************************************** 31 | */ 32 | 33 | #ifndef EIGEN_REAL_SCHUR_LAPACKE_H 34 | #define EIGEN_REAL_SCHUR_LAPACKE_H 35 | 36 | namespace Eigen { 37 | 38 | /** \internal Specialization for the data types supported by LAPACKe */ 39 | 40 | #define EIGEN_LAPACKE_SCHUR_REAL(EIGTYPE, LAPACKE_TYPE, LAPACKE_PREFIX, LAPACKE_PREFIX_U, EIGCOLROW, LAPACKE_COLROW) \ 41 | template<> template inline \ 42 | RealSchur >& \ 43 | RealSchur >::compute(const EigenBase& matrix, bool computeU) \ 44 | { \ 45 | eigen_assert(matrix.cols() == matrix.rows()); \ 46 | \ 47 | lapack_int n = internal::convert_index(matrix.cols()), sdim, info; \ 48 | lapack_int matrix_order = LAPACKE_COLROW; \ 49 | char jobvs, sort='N'; \ 50 | LAPACK_##LAPACKE_PREFIX_U##_SELECT2 select = 0; \ 51 | jobvs = (computeU) ? 'V' : 'N'; \ 52 | m_matU.resize(n, n); \ 53 | lapack_int ldvs = internal::convert_index(m_matU.outerStride()); \ 54 | m_matT = matrix; \ 55 | lapack_int lda = internal::convert_index(m_matT.outerStride()); \ 56 | Matrix wr, wi; \ 57 | wr.resize(n, 1); wi.resize(n, 1); \ 58 | info = LAPACKE_##LAPACKE_PREFIX##gees( matrix_order, jobvs, sort, select, n, (LAPACKE_TYPE*)m_matT.data(), lda, &sdim, (LAPACKE_TYPE*)wr.data(), (LAPACKE_TYPE*)wi.data(), (LAPACKE_TYPE*)m_matU.data(), ldvs ); \ 59 | if(info == 0) \ 60 | m_info = Success; \ 61 | else \ 62 | m_info = NoConvergence; \ 63 | \ 64 | m_isInitialized = true; \ 65 | m_matUisUptodate = computeU; \ 66 | return *this; \ 67 | \ 68 | } 69 | 70 | EIGEN_LAPACKE_SCHUR_REAL(double, double, d, D, ColMajor, LAPACK_COL_MAJOR) 71 | EIGEN_LAPACKE_SCHUR_REAL(float, float, s, S, ColMajor, LAPACK_COL_MAJOR) 72 | EIGEN_LAPACKE_SCHUR_REAL(double, double, d, D, RowMajor, LAPACK_ROW_MAJOR) 73 | EIGEN_LAPACKE_SCHUR_REAL(float, float, s, S, RowMajor, LAPACK_ROW_MAJOR) 74 | 75 | } // end namespace Eigen 76 | 77 | #endif // EIGEN_REAL_SCHUR_LAPACKE_H 78 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/LU/Determinant.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2008 Benoit Jacob 5 | // 6 | // This Source Code Form is subject to the terms of the Mozilla 7 | // Public License v. 2.0. If a copy of the MPL was not distributed 8 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | 10 | #ifndef EIGEN_DETERMINANT_H 11 | #define EIGEN_DETERMINANT_H 12 | 13 | namespace Eigen { 14 | 15 | namespace internal { 16 | 17 | template 18 | EIGEN_DEVICE_FUNC 19 | inline const typename Derived::Scalar bruteforce_det3_helper 20 | (const MatrixBase& matrix, int a, int b, int c) 21 | { 22 | return matrix.coeff(0,a) 23 | * (matrix.coeff(1,b) * matrix.coeff(2,c) - matrix.coeff(1,c) * matrix.coeff(2,b)); 24 | } 25 | 26 | template struct determinant_impl 29 | { 30 | static inline typename traits::Scalar run(const Derived& m) 31 | { 32 | if(Derived::ColsAtCompileTime==Dynamic && m.rows()==0) 33 | return typename traits::Scalar(1); 34 | return m.partialPivLu().determinant(); 35 | } 36 | }; 37 | 38 | template struct determinant_impl 39 | { 40 | static inline EIGEN_DEVICE_FUNC 41 | typename traits::Scalar run(const Derived& m) 42 | { 43 | return m.coeff(0,0); 44 | } 45 | }; 46 | 47 | template struct determinant_impl 48 | { 49 | static inline EIGEN_DEVICE_FUNC 50 | typename traits::Scalar run(const Derived& m) 51 | { 52 | return m.coeff(0,0) * m.coeff(1,1) - m.coeff(1,0) * m.coeff(0,1); 53 | } 54 | }; 55 | 56 | template struct determinant_impl 57 | { 58 | static inline EIGEN_DEVICE_FUNC 59 | typename traits::Scalar run(const Derived& m) 60 | { 61 | return bruteforce_det3_helper(m,0,1,2) 62 | - bruteforce_det3_helper(m,1,0,2) 63 | + bruteforce_det3_helper(m,2,0,1); 64 | } 65 | }; 66 | 67 | template struct determinant_impl 68 | { 69 | typedef typename traits::Scalar Scalar; 70 | static EIGEN_DEVICE_FUNC 71 | Scalar run(const Derived& m) 72 | { 73 | Scalar d2_01 = det2(m, 0, 1); 74 | Scalar d2_02 = det2(m, 0, 2); 75 | Scalar d2_03 = det2(m, 0, 3); 76 | Scalar d2_12 = det2(m, 1, 2); 77 | Scalar d2_13 = det2(m, 1, 3); 78 | Scalar d2_23 = det2(m, 2, 3); 79 | Scalar d3_0 = det3(m, 1,d2_23, 2,d2_13, 3,d2_12); 80 | Scalar d3_1 = det3(m, 0,d2_23, 2,d2_03, 3,d2_02); 81 | Scalar d3_2 = det3(m, 0,d2_13, 1,d2_03, 3,d2_01); 82 | Scalar d3_3 = det3(m, 0,d2_12, 1,d2_02, 2,d2_01); 83 | return internal::pmadd(-m(0,3),d3_0, m(1,3)*d3_1) + 84 | internal::pmadd(-m(2,3),d3_2, m(3,3)*d3_3); 85 | } 86 | protected: 87 | static EIGEN_DEVICE_FUNC 88 | Scalar det2(const Derived& m, Index i0, Index i1) 89 | { 90 | return m(i0,0) * m(i1,1) - m(i1,0) * m(i0,1); 91 | } 92 | 93 | static EIGEN_DEVICE_FUNC 94 | Scalar det3(const Derived& m, Index i0, const Scalar& d0, Index i1, const Scalar& d1, Index i2, const Scalar& d2) 95 | { 96 | return internal::pmadd(m(i0,2), d0, internal::pmadd(-m(i1,2), d1, m(i2,2)*d2)); 97 | } 98 | }; 99 | 100 | } // end namespace internal 101 | 102 | /** \lu_module 103 | * 104 | * \returns the determinant of this matrix 105 | */ 106 | template 107 | EIGEN_DEVICE_FUNC 108 | inline typename internal::traits::Scalar MatrixBase::determinant() const 109 | { 110 | eigen_assert(rows() == cols()); 111 | typedef typename internal::nested_eval::type Nested; 112 | return internal::determinant_impl::type>::run(derived()); 113 | } 114 | 115 | } // end namespace Eigen 116 | 117 | #endif // EIGEN_DETERMINANT_H 118 | -------------------------------------------------------------------------------- /include/rabitqlib/utils/tools.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | namespace rabitqlib { 10 | template 11 | inline void assert_integral() { 12 | static_assert( 13 | std::is_integral_v, 14 | "Template type T must be an integral type (int, long, unsigned, etc.)." 15 | ); 16 | } 17 | 18 | template 19 | inline void assert_floating() { 20 | static_assert( 21 | std::is_floating_point_v, 22 | "Template type T must be a floating-point type (float, double, long double)." 23 | ); 24 | } 25 | 26 | // thread save rand int 27 | template 28 | inline T rand_integer(T min, T max) { 29 | static thread_local std::mt19937 generator( 30 | std::random_device{}() + std::hash()(std::this_thread::get_id()) 31 | ); 32 | std::uniform_int_distribution distribution(min, max); 33 | return distribution(generator); 34 | } 35 | 36 | constexpr size_t div_round_up(size_t val, size_t div) { 37 | return (val / div) + static_cast((val % div) != 0); 38 | } 39 | 40 | constexpr size_t round_up_to_multiple(size_t val, size_t multiple_of) { 41 | return multiple_of * (div_round_up(val, multiple_of)); 42 | } 43 | 44 | inline size_t floor_log2(size_t x) { 45 | size_t ret = 0; 46 | while (x > 1) { 47 | ret++; 48 | x >>= 1; 49 | } 50 | return ret; 51 | } 52 | 53 | inline size_t ceil_log2(size_t x) { 54 | size_t ret = floor_log2(x); 55 | return (1UL << ret) < x ? ret + 1 : ret; 56 | } 57 | 58 | inline bool is_powerof2(size_t n) { return n > 0 && (n & (n - 1)) == 0; } 59 | 60 | template 61 | constexpr T div_round_up(T x, T divisor) { 62 | static_assert(std::is_integral_v, "T must be an integral type"); 63 | return (x / divisor) + static_cast((x % divisor) != 0); 64 | } 65 | 66 | template 67 | constexpr T round_up_to_multiple_of(size_t x, size_t multiple_of) { 68 | return multiple_of * (div_round_up(x, multiple_of)); 69 | } 70 | 71 | // get number of threads of current sys 72 | inline size_t total_threads() { 73 | const auto threads = std::thread::hardware_concurrency(); 74 | return threads == 0 ? 1 : threads; 75 | } 76 | 77 | template 78 | float distance_ratio( 79 | const T* data, 80 | const T* query, 81 | const TP* gt, 82 | const TP* ann_results, 83 | size_t k, 84 | size_t dim, 85 | T (*dist_func)(const T*, const T*, size_t) 86 | ) { 87 | std::priority_queue gt_distances; 88 | std::priority_queue ann_distances; 89 | 90 | for (size_t i = 0; i < k; ++i) { 91 | TP gt_id = gt[i]; 92 | TP ann_id = ann_results[i]; 93 | gt_distances.emplace(dist_func(query, data + (gt_id * dim), dim)); 94 | ann_distances.emplace(dist_func(query, data + (ann_id * dim), dim)); 95 | } 96 | 97 | float ret = 0; 98 | size_t valid_k = 0; 99 | 100 | while (!gt_distances.empty()) { 101 | if (gt_distances.top() > 1e-5) { 102 | ret += std::sqrt(ann_distances.top() / gt_distances.top()); 103 | ++valid_k; 104 | } 105 | gt_distances.pop(); 106 | ann_distances.pop(); 107 | } 108 | 109 | if (valid_k == 0) { 110 | return static_cast(k); 111 | } 112 | return ret * static_cast(k) / static_cast(valid_k); 113 | } 114 | 115 | template 116 | std::vector horizontal_avg(const std::vector>& data) { 117 | size_t rows = data.size(); 118 | size_t cols = data[0].size(); 119 | 120 | std::vector avg(cols, 0); 121 | for (auto& row : data) { 122 | for (size_t j = 0; j < cols; ++j) { 123 | avg[j] += row[j]; 124 | } 125 | } 126 | 127 | for (size_t j = 0; j < cols; ++j) { 128 | avg[j] /= rows; 129 | } 130 | 131 | return avg; 132 | } 133 | } // namespace rabitqlib -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/Core/Inverse.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2014-2019 Gael Guennebaud 5 | // 6 | // This Source Code Form is subject to the terms of the Mozilla 7 | // Public License v. 2.0. If a copy of the MPL was not distributed 8 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | 10 | #ifndef EIGEN_INVERSE_H 11 | #define EIGEN_INVERSE_H 12 | 13 | namespace Eigen { 14 | 15 | template class InverseImpl; 16 | 17 | namespace internal { 18 | 19 | template 20 | struct traits > 21 | : traits 22 | { 23 | typedef typename XprType::PlainObject PlainObject; 24 | typedef traits BaseTraits; 25 | enum { 26 | Flags = BaseTraits::Flags & RowMajorBit 27 | }; 28 | }; 29 | 30 | } // end namespace internal 31 | 32 | /** \class Inverse 33 | * 34 | * \brief Expression of the inverse of another expression 35 | * 36 | * \tparam XprType the type of the expression we are taking the inverse 37 | * 38 | * This class represents an abstract expression of A.inverse() 39 | * and most of the time this is the only way it is used. 40 | * 41 | */ 42 | template 43 | class Inverse : public InverseImpl::StorageKind> 44 | { 45 | public: 46 | typedef typename XprType::StorageIndex StorageIndex; 47 | typedef typename XprType::Scalar Scalar; 48 | typedef typename internal::ref_selector::type XprTypeNested; 49 | typedef typename internal::remove_all::type XprTypeNestedCleaned; 50 | typedef typename internal::ref_selector::type Nested; 51 | typedef typename internal::remove_all::type NestedExpression; 52 | 53 | explicit EIGEN_DEVICE_FUNC Inverse(const XprType &xpr) 54 | : m_xpr(xpr) 55 | {} 56 | 57 | EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index rows() const EIGEN_NOEXCEPT { return m_xpr.cols(); } 58 | EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR Index cols() const EIGEN_NOEXCEPT { return m_xpr.rows(); } 59 | 60 | EIGEN_DEVICE_FUNC const XprTypeNestedCleaned& nestedExpression() const { return m_xpr; } 61 | 62 | protected: 63 | XprTypeNested m_xpr; 64 | }; 65 | 66 | // Generic API dispatcher 67 | template 68 | class InverseImpl 69 | : public internal::generic_xpr_base >::type 70 | { 71 | public: 72 | typedef typename internal::generic_xpr_base >::type Base; 73 | typedef typename XprType::Scalar Scalar; 74 | private: 75 | 76 | Scalar coeff(Index row, Index col) const; 77 | Scalar coeff(Index i) const; 78 | }; 79 | 80 | namespace internal { 81 | 82 | /** \internal 83 | * \brief Default evaluator for Inverse expression. 84 | * 85 | * This default evaluator for Inverse expression simply evaluate the inverse into a temporary 86 | * by a call to internal::call_assignment_no_alias. 87 | * Therefore, inverse implementers only have to specialize Assignment, ...> for 88 | * there own nested expression. 89 | * 90 | * \sa class Inverse 91 | */ 92 | template 93 | struct unary_evaluator > 94 | : public evaluator::PlainObject> 95 | { 96 | typedef Inverse InverseType; 97 | typedef typename InverseType::PlainObject PlainObject; 98 | typedef evaluator Base; 99 | 100 | enum { Flags = Base::Flags | EvalBeforeNestingBit }; 101 | 102 | unary_evaluator(const InverseType& inv_xpr) 103 | : m_result(inv_xpr.rows(), inv_xpr.cols()) 104 | { 105 | ::new (static_cast(this)) Base(m_result); 106 | internal::call_assignment_no_alias(m_result, inv_xpr); 107 | } 108 | 109 | protected: 110 | PlainObject m_result; 111 | }; 112 | 113 | } // end namespace internal 114 | 115 | } // end namespace Eigen 116 | 117 | #endif // EIGEN_INVERSE_H 118 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/Core/NoAlias.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2009 Gael Guennebaud 5 | // 6 | // This Source Code Form is subject to the terms of the Mozilla 7 | // Public License v. 2.0. If a copy of the MPL was not distributed 8 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | 10 | #ifndef EIGEN_NOALIAS_H 11 | #define EIGEN_NOALIAS_H 12 | 13 | namespace Eigen { 14 | 15 | /** \class NoAlias 16 | * \ingroup Core_Module 17 | * 18 | * \brief Pseudo expression providing an operator = assuming no aliasing 19 | * 20 | * \tparam ExpressionType the type of the object on which to do the lazy assignment 21 | * 22 | * This class represents an expression with special assignment operators 23 | * assuming no aliasing between the target expression and the source expression. 24 | * More precisely it alloas to bypass the EvalBeforeAssignBit flag of the source expression. 25 | * It is the return type of MatrixBase::noalias() 26 | * and most of the time this is the only way it is used. 27 | * 28 | * \sa MatrixBase::noalias() 29 | */ 30 | template class StorageBase> 31 | class NoAlias 32 | { 33 | public: 34 | typedef typename ExpressionType::Scalar Scalar; 35 | 36 | EIGEN_DEVICE_FUNC 37 | explicit NoAlias(ExpressionType& expression) : m_expression(expression) {} 38 | 39 | template 40 | EIGEN_DEVICE_FUNC 41 | EIGEN_STRONG_INLINE ExpressionType& operator=(const StorageBase& other) 42 | { 43 | call_assignment_no_alias(m_expression, other.derived(), internal::assign_op()); 44 | return m_expression; 45 | } 46 | 47 | template 48 | EIGEN_DEVICE_FUNC 49 | EIGEN_STRONG_INLINE ExpressionType& operator+=(const StorageBase& other) 50 | { 51 | call_assignment_no_alias(m_expression, other.derived(), internal::add_assign_op()); 52 | return m_expression; 53 | } 54 | 55 | template 56 | EIGEN_DEVICE_FUNC 57 | EIGEN_STRONG_INLINE ExpressionType& operator-=(const StorageBase& other) 58 | { 59 | call_assignment_no_alias(m_expression, other.derived(), internal::sub_assign_op()); 60 | return m_expression; 61 | } 62 | 63 | EIGEN_DEVICE_FUNC 64 | ExpressionType& expression() const 65 | { 66 | return m_expression; 67 | } 68 | 69 | protected: 70 | ExpressionType& m_expression; 71 | }; 72 | 73 | /** \returns a pseudo expression of \c *this with an operator= assuming 74 | * no aliasing between \c *this and the source expression. 75 | * 76 | * More precisely, noalias() allows to bypass the EvalBeforeAssignBit flag. 77 | * Currently, even though several expressions may alias, only product 78 | * expressions have this flag. Therefore, noalias() is only useful when 79 | * the source expression contains a matrix product. 80 | * 81 | * Here are some examples where noalias is useful: 82 | * \code 83 | * D.noalias() = A * B; 84 | * D.noalias() += A.transpose() * B; 85 | * D.noalias() -= 2 * A * B.adjoint(); 86 | * \endcode 87 | * 88 | * On the other hand the following example will lead to a \b wrong result: 89 | * \code 90 | * A.noalias() = A * B; 91 | * \endcode 92 | * because the result matrix A is also an operand of the matrix product. Therefore, 93 | * there is no alternative than evaluating A * B in a temporary, that is the default 94 | * behavior when you write: 95 | * \code 96 | * A = A * B; 97 | * \endcode 98 | * 99 | * \sa class NoAlias 100 | */ 101 | template 102 | NoAlias EIGEN_DEVICE_FUNC MatrixBase::noalias() 103 | { 104 | return NoAlias(derived()); 105 | } 106 | 107 | } // end namespace Eigen 108 | 109 | #endif // EIGEN_NOALIAS_H 110 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/Geometry/EulerAngles.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2008 Gael Guennebaud 5 | // 6 | // This Source Code Form is subject to the terms of the Mozilla 7 | // Public License v. 2.0. If a copy of the MPL was not distributed 8 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | 10 | #ifndef EIGEN_EULERANGLES_H 11 | #define EIGEN_EULERANGLES_H 12 | 13 | namespace Eigen { 14 | 15 | /** \geometry_module \ingroup Geometry_Module 16 | * 17 | * 18 | * \returns the Euler-angles of the rotation matrix \c *this using the convention defined by the triplet (\a a0,\a a1,\a a2) 19 | * 20 | * Each of the three parameters \a a0,\a a1,\a a2 represents the respective rotation axis as an integer in {0,1,2}. 21 | * For instance, in: 22 | * \code Vector3f ea = mat.eulerAngles(2, 0, 2); \endcode 23 | * "2" represents the z axis and "0" the x axis, etc. The returned angles are such that 24 | * we have the following equality: 25 | * \code 26 | * mat == AngleAxisf(ea[0], Vector3f::UnitZ()) 27 | * * AngleAxisf(ea[1], Vector3f::UnitX()) 28 | * * AngleAxisf(ea[2], Vector3f::UnitZ()); \endcode 29 | * This corresponds to the right-multiply conventions (with right hand side frames). 30 | * 31 | * The returned angles are in the ranges [0:pi]x[-pi:pi]x[-pi:pi]. 32 | * 33 | * \sa class AngleAxis 34 | */ 35 | template 36 | EIGEN_DEVICE_FUNC inline Matrix::Scalar,3,1> 37 | MatrixBase::eulerAngles(Index a0, Index a1, Index a2) const 38 | { 39 | EIGEN_USING_STD(atan2) 40 | EIGEN_USING_STD(sin) 41 | EIGEN_USING_STD(cos) 42 | /* Implemented from Graphics Gems IV */ 43 | EIGEN_STATIC_ASSERT_MATRIX_SPECIFIC_SIZE(Derived,3,3) 44 | 45 | Matrix res; 46 | typedef Matrix Vector2; 47 | 48 | const Index odd = ((a0+1)%3 == a1) ? 0 : 1; 49 | const Index i = a0; 50 | const Index j = (a0 + 1 + odd)%3; 51 | const Index k = (a0 + 2 - odd)%3; 52 | 53 | if (a0==a2) 54 | { 55 | res[0] = atan2(coeff(j,i), coeff(k,i)); 56 | if((odd && res[0]Scalar(0))) 57 | { 58 | if(res[0] > Scalar(0)) { 59 | res[0] -= Scalar(EIGEN_PI); 60 | } 61 | else { 62 | res[0] += Scalar(EIGEN_PI); 63 | } 64 | Scalar s2 = Vector2(coeff(j,i), coeff(k,i)).norm(); 65 | res[1] = -atan2(s2, coeff(i,i)); 66 | } 67 | else 68 | { 69 | Scalar s2 = Vector2(coeff(j,i), coeff(k,i)).norm(); 70 | res[1] = atan2(s2, coeff(i,i)); 71 | } 72 | 73 | // With a=(0,1,0), we have i=0; j=1; k=2, and after computing the first two angles, 74 | // we can compute their respective rotation, and apply its inverse to M. Since the result must 75 | // be a rotation around x, we have: 76 | // 77 | // c2 s1.s2 c1.s2 1 0 0 78 | // 0 c1 -s1 * M = 0 c3 s3 79 | // -s2 s1.c2 c1.c2 0 -s3 c3 80 | // 81 | // Thus: m11.c1 - m21.s1 = c3 & m12.c1 - m22.s1 = s3 82 | 83 | Scalar s1 = sin(res[0]); 84 | Scalar c1 = cos(res[0]); 85 | res[2] = atan2(c1*coeff(j,k)-s1*coeff(k,k), c1*coeff(j,j) - s1 * coeff(k,j)); 86 | } 87 | else 88 | { 89 | res[0] = atan2(coeff(j,k), coeff(k,k)); 90 | Scalar c2 = Vector2(coeff(i,i), coeff(i,j)).norm(); 91 | if((odd && res[0]Scalar(0))) { 92 | if(res[0] > Scalar(0)) { 93 | res[0] -= Scalar(EIGEN_PI); 94 | } 95 | else { 96 | res[0] += Scalar(EIGEN_PI); 97 | } 98 | res[1] = atan2(-coeff(i,k), -c2); 99 | } 100 | else 101 | res[1] = atan2(-coeff(i,k), c2); 102 | Scalar s1 = sin(res[0]); 103 | Scalar c1 = cos(res[0]); 104 | res[2] = atan2(s1*coeff(k,i)-c1*coeff(j,i), c1*coeff(j,j) - s1 * coeff(k,j)); 105 | } 106 | if (!odd) 107 | res = -res; 108 | 109 | return res; 110 | } 111 | 112 | } // end namespace Eigen 113 | 114 | #endif // EIGEN_EULERANGLES_H 115 | -------------------------------------------------------------------------------- /include/rabitqlib/third/Eigen/src/SparseLU/SparseLU_copy_to_ucol.h: -------------------------------------------------------------------------------- 1 | // This file is part of Eigen, a lightweight C++ template library 2 | // for linear algebra. 3 | // 4 | // Copyright (C) 2012 Désiré Nuentsa-Wakam 5 | // 6 | // This Source Code Form is subject to the terms of the Mozilla 7 | // Public License v. 2.0. If a copy of the MPL was not distributed 8 | // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 | /* 10 | 11 | * NOTE: This file is the modified version of [s,d,c,z]copy_to_ucol.c file in SuperLU 12 | 13 | * -- SuperLU routine (version 2.0) -- 14 | * Univ. of California Berkeley, Xerox Palo Alto Research Center, 15 | * and Lawrence Berkeley National Lab. 16 | * November 15, 1997 17 | * 18 | * Copyright (c) 1994 by Xerox Corporation. All rights reserved. 19 | * 20 | * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY 21 | * EXPRESSED OR IMPLIED. ANY USE IS AT YOUR OWN RISK. 22 | * 23 | * Permission is hereby granted to use or copy this program for any 24 | * purpose, provided the above notices are retained on all copies. 25 | * Permission to modify the code and to distribute modified code is 26 | * granted, provided the above notices are retained, and a notice that 27 | * the code was modified is included with the above copyright notice. 28 | */ 29 | #ifndef SPARSELU_COPY_TO_UCOL_H 30 | #define SPARSELU_COPY_TO_UCOL_H 31 | 32 | namespace Eigen { 33 | namespace internal { 34 | 35 | /** 36 | * \brief Performs numeric block updates (sup-col) in topological order 37 | * 38 | * \param jcol current column to update 39 | * \param nseg Number of segments in the U part 40 | * \param segrep segment representative ... 41 | * \param repfnz First nonzero column in each row ... 42 | * \param perm_r Row permutation 43 | * \param dense Store the full representation of the column 44 | * \param glu Global LU data. 45 | * \return 0 - successful return 46 | * > 0 - number of bytes allocated when run out of space 47 | * 48 | */ 49 | template 50 | Index SparseLUImpl::copy_to_ucol(const Index jcol, const Index nseg, IndexVector& segrep, 51 | BlockIndexVector repfnz ,IndexVector& perm_r, BlockScalarVector dense, GlobalLU_t& glu) 52 | { 53 | Index ksub, krep, ksupno; 54 | 55 | Index jsupno = glu.supno(jcol); 56 | 57 | // For each nonzero supernode segment of U[*,j] in topological order 58 | Index k = nseg - 1, i; 59 | StorageIndex nextu = glu.xusub(jcol); 60 | Index kfnz, isub, segsize; 61 | Index new_next,irow; 62 | Index fsupc, mem; 63 | for (ksub = 0; ksub < nseg; ksub++) 64 | { 65 | krep = segrep(k); k--; 66 | ksupno = glu.supno(krep); 67 | if (jsupno != ksupno ) // should go into ucol(); 68 | { 69 | kfnz = repfnz(krep); 70 | if (kfnz != emptyIdxLU) 71 | { // Nonzero U-segment 72 | fsupc = glu.xsup(ksupno); 73 | isub = glu.xlsub(fsupc) + kfnz - fsupc; 74 | segsize = krep - kfnz + 1; 75 | new_next = nextu + segsize; 76 | while (new_next > glu.nzumax) 77 | { 78 | mem = memXpand(glu.ucol, glu.nzumax, nextu, UCOL, glu.num_expansions); 79 | if (mem) return mem; 80 | mem = memXpand(glu.usub, glu.nzumax, nextu, USUB, glu.num_expansions); 81 | if (mem) return mem; 82 | 83 | } 84 | 85 | for (i = 0; i < segsize; i++) 86 | { 87 | irow = glu.lsub(isub); 88 | glu.usub(nextu) = perm_r(irow); // Unlike the L part, the U part is stored in its final order 89 | glu.ucol(nextu) = dense(irow); 90 | dense(irow) = Scalar(0.0); 91 | nextu++; 92 | isub++; 93 | } 94 | 95 | } // end nonzero U-segment 96 | 97 | } // end if jsupno 98 | 99 | } // end for each segment 100 | glu.xusub(jcol + 1) = nextu; // close U(*,jcol) 101 | return 0; 102 | } 103 | 104 | } // namespace internal 105 | } // end namespace Eigen 106 | 107 | #endif // SPARSELU_COPY_TO_UCOL_H 108 | --------------------------------------------------------------------------------