├── test
    ├── unittest_main.cpp
    ├── bigtest.h
    ├── unittest_alex_multimap.h
    ├── unittest_alex_map.h
    ├── unittest_glin_piecewise.h
    ├── unittest_geos.h
    ├── unittest_glin_line_projct.h
    ├── unittest_alex.h
    └── unittest_nodes.h
├── glin
    ├── libmorton
    │   ├── morton_common.h
    │   ├── morton_BMI.h
    │   ├── morton.h
    │   ├── morton2D_LUTs.h
    │   ├── morton_AVX512BITALG.h
    │   ├── morton2D.h
    │   ├── morton3D.h
    │   └── morton3D_LUTs.h
    ├── index_num_nodes.h
    ├── index_size.h
    ├── Encoder.h
    ├── hilbert
    │   └── hilbert.h
    └── piecewise.h
├── src
    ├── benchmark
    │   ├── utils.h
    │   ├── flags.h
    │   ├── zipf.h
    │   └── main.cpp
    ├── examples
    │   └── main.cpp
    └── core
    │   ├── projection.h
    │   ├── alex_multimap.h
    │   ├── alex_map.h
    │   └── alex_base.h
├── CMakeLists.txt
├── cmake
    └── FindGEOS.cmake
└── README.md


/test/unittest_main.cpp:
--------------------------------------------------------------------------------
 1 | // Copyright (c) Microsoft Corporation.
 2 | // Licensed under the MIT license.
 3 | 
 4 | #define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN
 5 | #include "doctest.h"
 6 | 
 7 | #define private public
 8 | #include "unittest_alex.h"
 9 | #include "unittest_alex_map.h"
10 | #include "unittest_alex_multimap.h"
11 | #include "unittest_geos.h"
12 | #include "unittest_nodes.h"
13 | #include "unittest_glin.h"
14 | #include "unittest_glin_insert.h"
15 | #include "unittest_glin_piecewise.h"
16 | #include "unittest_glin_line_projct.h"


--------------------------------------------------------------------------------
/glin/libmorton/morton_common.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | // Libmorton - Common helper methods needed in Morton encoding/decoding
 4 | 
 5 | #include <stdint.h>
 6 | #if defined(_MSC_VER)
 7 | #include <intrin.h>
 8 | #endif
 9 | 
10 | namespace libmorton {
11 | 	template<typename morton>
12 | 	inline bool findFirstSetBitZeroIdx(const morton x, unsigned long* firstbit_location) {
13 | #if defined(_MSC_VER) && !defined(_WIN64)
14 | 		// 32 BIT on 32 BIT
15 | 		if (sizeof(morton) <= 4) {
16 | 			return _BitScanReverse(firstbit_location, x) != 0;
17 | 		}
18 | 		// 64 BIT on 32 BIT
19 | 		else {
20 | 			*firstbit_location = 0;
21 | 			if (_BitScanReverse(firstbit_location, (x >> 32))) { // check first part
22 | 				*firstbit_location += 32;
23 | 				return true;
24 | 			}
25 | 			return _BitScanReverse(firstbit_location, (x & 0xFFFFFFFF)) != 0;
26 | 		}
27 | #elif defined(_MSC_VER) && defined(_WIN64)
28 | 		// 32 or 64 BIT on 64 BIT
29 | 		return _BitScanReverse64(firstbit_location, x) != 0;
30 | #elif defined(__GNUC__)
31 | 		if (x == 0) {
32 | 			return false;
33 | 		}
34 | 		else {
35 | 			*firstbit_location = static_cast<unsigned long>((sizeof(morton) * 8) - __builtin_clzll(x) - 1);
36 | 			return true;
37 | 		}
38 | #endif
39 | 	}
40 | 
41 | 	template<typename morton>
42 | 	inline bool findFirstSetBit(const morton x, unsigned long* firstbit_location) {
43 | 		if (findFirstSetBitZeroIdx(x, firstbit_location)) {
44 | 			*firstbit_location += 1;
45 | 			return true;
46 | 		}
47 | 		return false;
48 | 	}
49 | }


--------------------------------------------------------------------------------
/test/bigtest.h:
--------------------------------------------------------------------------------
 1 | //
 2 | // Created by Juno Wang on 6/30/21.
 3 | //
 4 | 
 5 | #ifndef UNITTEST_ALEX_MULTIMAP_H_BIGTEST_H
 6 | #define UNITTEST_ALEX_MULTIMAP_H_BIGTEST_H
 7 | #include <memory>
 8 | #include <geos/geom/Envelope.h>
 9 | 
10 | using geos::geom::Coordinate;
11 | using geos::geom::Envelope;
12 | using namespace std;
13 | using namespace geos;
14 | 
15 | // Forward declaration
16 | namespace geos {
17 |     namespace geom {
18 |         class Polygon;
19 |         class CoordinateSequence;
20 |         class GeometryFactory;
21 |     }
22 | }
23 | 
24 | class GeometryTestFactory {
25 | public:
26 |     static geom::Polygon* createBox(geom::GeometryFactory* fact, double minx, double miny, int nSide, double segLen);
27 |     static geom::CoordinateSequence* createBox(double minx, double miny, int nSide, double segLen);
28 |     static geom::CoordinateSequence* createCircle(double basex, double basey, double size, uint32_t nPts);
29 |     static geom::Polygon* createCircle(geom::GeometryFactory* fact, double basex, double basey, double size, uint32_t nPts);
30 |     static geom::CoordinateSequence* createSineStar(double basex, double basey, double size, double armLen, int nArms,
31 |                                                     int nPts);
32 |     static geom::Polygon* createSineStar(geom::GeometryFactory* fact, double basex, double basey, double size,
33 |                                          double armLen, int nArms, int nPts);
34 | 
35 | };
36 | 
37 | #endif //UNITTEST_ALEX_MULTIMAP_H_BIGTEST_H
38 | 


--------------------------------------------------------------------------------
/src/benchmark/utils.h:
--------------------------------------------------------------------------------
 1 | // Copyright (c) Microsoft Corporation.
 2 | // Licensed under the MIT license.
 3 | 
 4 | #include "zipf.h"
 5 | 
 6 | template <class T>
 7 | bool load_binary_data(T data[], int length, const std::string& file_path) {
 8 |   std::ifstream is(file_path.c_str(), std::ios::binary | std::ios::in);
 9 |   if (!is.is_open()) {
10 |     return false;
11 |   }
12 |   is.read(reinterpret_cast<char*>(data), std::streamsize(length * sizeof(T)));
13 |   is.close();
14 |   return true;
15 | }
16 | 
17 | template <class T>
18 | bool load_text_data(T array[], int length, const std::string& file_path) {
19 |   std::ifstream is(file_path.c_str());
20 |   if (!is.is_open()) {
21 |     return false;
22 |   }
23 |   int i = 0;
24 |   std::string str;
25 |   while (std::getline(is, str) && i < length) {
26 |     std::istringstream ss(str);
27 |     ss >> array[i];
28 |     i++;
29 |   }
30 |   is.close();
31 |   return true;
32 | }
33 | 
34 | template <class T>
35 | T* get_search_keys(T array[], int num_keys, int num_searches) {
36 |   std::mt19937_64 gen(std::random_device{}());
37 |   std::uniform_int_distribution<int> dis(0, num_keys - 1);
38 |   auto* keys = new T[num_searches];
39 |   for (int i = 0; i < num_searches; i++) {
40 |     int pos = dis(gen);
41 |     keys[i] = array[pos];
42 |   }
43 |   return keys;
44 | }
45 | 
46 | template <class T>
47 | T* get_search_keys_zipf(T array[], int num_keys, int num_searches) {
48 |   auto* keys = new T[num_searches];
49 |   ScrambledZipfianGenerator zipf_gen(num_keys);
50 |   for (int i = 0; i < num_searches; i++) {
51 |     int pos = zipf_gen.nextValue();
52 |     keys[i] = array[pos];
53 |   }
54 |   return keys;
55 | }
56 | 


--------------------------------------------------------------------------------
/CMakeLists.txt:
--------------------------------------------------------------------------------
 1 | cmake_minimum_required(VERSION 3.12)
 2 | project(glin)
 3 | set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake) # find geos path
 4 | if(APPLE) # This has to be manually setup
 5 | #    set(Boost_INCLUDE_DIR /usr/local/Cellar/boost/1.75.0/include)
 6 |     set(CMAKE_FIND_FRAMEWORK LAST)
 7 |     set(CMAKE_FIND_APPBUNDLE LAST)
 8 | endif()
 9 | set(Boost_USE_MULTITHREADED ON)
10 | # FindGEOS.cmake is copied from https://github.com/valhalla/valhalla/blob/master/cmake/FindGEOS.cmake
11 | find_package(GEOS REQUIRED)# if not find geo, cannot compile find geos.cmake
12 | find_package(Boost REQUIRED)
13 | set(CMAKE_CXX_STANDARD 14)
14 | 
15 | 
16 | # SIMD
17 | if(MSVC)
18 |     set(CMAKE_CXX_FLAGS "/O2 /arch:AVX2 /W1 /EHsc")
19 | elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
20 |     set(CMAKE_CXX_FLAGS "-O3 -xHost")
21 | else()
22 |     # clang and gcc
23 |     set(CMAKE_CXX_FLAGS "-O3 -march=native -Wall -Wextra")
24 | endif()
25 | 
26 | 
27 | add_executable(test_glin test/unittest_main.cpp glin/hilbert/hilbert.h glin/hilbert/hilbert.cpp)
28 | target_link_libraries(test_glin PRIVATE ${GEOS_LIBRARY} ${Boost_SYSTEM_LIBRARY})
29 | target_include_directories(test_glin PUBLIC "$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src/core;${GEOS_INCLUDE_DIR};${Boost_INCLUDE_DIRS}>")
30 | 
31 | add_executable(test_glin_piece test/unittest_main.cpp glin/hilbert/hilbert.h glin/hilbert/hilbert.cpp)
32 | target_compile_definitions(test_glin_piece PRIVATE PIECE)
33 | target_link_libraries(test_glin_piece PRIVATE ${GEOS_LIBRARY} ${Boost_SYSTEM_LIBRARY})
34 | target_include_directories(test_glin_piece PUBLIC "$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src/core;${GEOS_INCLUDE_DIR};${Boost_INCLUDE_DIRS}>")
35 | 
36 | enable_testing()
37 | add_test(test_glin test_glin_piece) #running test while compile


--------------------------------------------------------------------------------
/src/benchmark/flags.h:
--------------------------------------------------------------------------------
 1 | // Copyright (c) Microsoft Corporation.
 2 | // Licensed under the MIT license.
 3 | 
 4 | #include <iostream>
 5 | #include <map>
 6 | #include <sstream>
 7 | #include <string>
 8 | 
 9 | std::map<std::string, std::string> parse_flags(int argc, char** argv) {
10 |   std::map<std::string, std::string> flags;
11 |   for (int i = 1; i < argc; i++) {
12 |     std::string arg(argv[i]);
13 |     size_t equals = arg.find("=");
14 |     size_t dash = arg.find("--");
15 |     if (dash != 0) {
16 |       std::cout << "Bad flag '" << argv[i] << "'. Expected --key=value"
17 |                 << std::endl;
18 |       continue;
19 |     }
20 |     std::string key = arg.substr(2, equals - 2);
21 |     std::string val;
22 |     if (equals == std::string::npos) {
23 |       val = "";
24 |       std::cout << "found flag " << key << std::endl;
25 |     } else {
26 |       val = arg.substr(equals + 1);
27 |       std::cout << "found flag " << key << " = " << val << std::endl;
28 |     }
29 |     flags[key] = val;
30 |   }
31 |   return flags;
32 | }
33 | 
34 | std::string get_with_default(const std::map<std::string, std::string>& m,
35 |                              const std::string& key,
36 |                              const std::string& defval) {
37 |   auto it = m.find(key);
38 |   if (it == m.end()) {
39 |     return defval;
40 |   }
41 |   return it->second;
42 | }
43 | 
44 | std::string get_required(const std::map<std::string, std::string>& m,
45 |                          const std::string& key) {
46 |   auto it = m.find(key);
47 |   if (it == m.end()) {
48 |     std::cout << "Required flag --" << key << " was not found" << std::endl;
49 |   }
50 |   return it->second;
51 | }
52 | 
53 | bool get_boolean_flag(const std::map<std::string, std::string>& m,
54 |                       const std::string& key) {
55 |   return m.find(key) != m.end();
56 | }
57 | 
58 | std::vector<std::string> get_comma_separated(
59 |     std::map<std::string, std::string>& m, const std::string& key) {
60 |   std::vector<std::string> vals;
61 |   auto it = m.find(key);
62 |   if (it == m.end()) {
63 |     return vals;
64 |   }
65 |   std::istringstream s(m[key]);
66 |   std::string val;
67 |   while (std::getline(s, val, ',')) {
68 |     vals.push_back(val);
69 |     std::cout << "parsed csv val " << val << std::endl;
70 |   }
71 |   return vals;
72 | }
73 | 


--------------------------------------------------------------------------------
/src/benchmark/zipf.h:
--------------------------------------------------------------------------------
 1 | // Copyright (c) Microsoft Corporation.
 2 | // Licensed under the MIT license.
 3 | 
 4 | // Zipf generator, inspired by
 5 | // https://github.com/brianfrankcooper/YCSB/blob/master/core/src/main/java/site/ycsb/generator/ScrambledZipfianGenerator.java
 6 | // https://github.com/brianfrankcooper/YCSB/blob/master/core/src/main/java/site/ycsb/generator/ZipfianGenerator.java
 7 | 
 8 | class ScrambledZipfianGenerator {
 9 |  public:
10 |   static constexpr double ZETAN = 26.46902820178302;
11 |   static constexpr double ZIPFIAN_CONSTANT = 0.99;
12 | 
13 |   int num_keys_;
14 |   double alpha_;
15 |   double eta_;
16 |   std::mt19937_64 gen_;
17 |   std::uniform_real_distribution<double> dis_;
18 | 
19 |   explicit ScrambledZipfianGenerator(int num_keys)
20 |       : num_keys_(num_keys), gen_(std::random_device{}()), dis_(0, 1) {
21 |     double zeta2theta = zeta(2);
22 |     alpha_ = 1. / (1. - ZIPFIAN_CONSTANT);
23 |     eta_ = (1 - std::pow(2. / num_keys_, 1 - ZIPFIAN_CONSTANT)) /
24 |            (1 - zeta2theta / ZETAN);
25 |   }
26 | 
27 |   int nextValue() {
28 |     double u = dis_(gen_);
29 |     double uz = u * ZETAN;
30 | 
31 |     int ret;
32 |     if (uz < 1.0) {
33 |       ret = 0;
34 |     } else if (uz < 1.0 + std::pow(0.5, ZIPFIAN_CONSTANT)) {
35 |       ret = 1;
36 |     } else {
37 |       ret = (int)(num_keys_ * std::pow(eta_ * u - eta_ + 1, alpha_));
38 |     }
39 | 
40 |     ret = fnv1a(ret) % num_keys_;
41 |     return ret;
42 |   }
43 | 
44 |   double zeta(long n) {
45 |     double sum = 0.0;
46 |     for (long i = 0; i < n; i++) {
47 |       sum += 1 / std::pow(i + 1, ZIPFIAN_CONSTANT);
48 |     }
49 |     return sum;
50 |   }
51 | 
52 |   // FNV hash from https://create.stephan-brumme.com/fnv-hash/
53 |   static const uint32_t PRIME = 0x01000193;  //   16777619
54 |   static const uint32_t SEED = 0x811C9DC5;   // 2166136261
55 |   /// hash a single byte
56 |   inline uint32_t fnv1a(unsigned char oneByte, uint32_t hash = SEED) {
57 |     return (oneByte ^ hash) * PRIME;
58 |   }
59 |   /// hash a 32 bit integer (four bytes)
60 |   inline uint32_t fnv1a(int fourBytes, uint32_t hash = SEED) {
61 |     const unsigned char* ptr = (const unsigned char*)&fourBytes;
62 |     hash = fnv1a(*ptr++, hash);
63 |     hash = fnv1a(*ptr++, hash);
64 |     hash = fnv1a(*ptr++, hash);
65 |     return fnv1a(*ptr, hash);
66 |   }
67 | };


--------------------------------------------------------------------------------
/glin/libmorton/morton_BMI.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | #if defined(__BMI2__) || (defined(__AVX2__) && defined(_MSC_VER))
 3 | #include <immintrin.h>
 4 | #include <stdint.h>
 5 | 
 6 | namespace libmorton {
 7 | 
 8 | 	namespace bmi2_detail {
 9 | 		inline uint32_t pdep(uint32_t source, uint32_t mask) noexcept {
10 | 			return _pdep_u32(source, mask);
11 | 		}
12 | 		inline uint64_t pdep(uint64_t source, uint64_t mask) noexcept {
13 | 			return _pdep_u64(source, mask);
14 | 		}
15 | 		inline uint32_t pext(uint32_t source, uint32_t mask) noexcept {
16 | 			return _pext_u32(source, mask);
17 | 		}
18 | 		inline uint64_t pext(uint64_t source, uint64_t mask) noexcept {
19 | 			return _pext_u64(source, mask);
20 | 		}
21 | 	}  // namespace bmi2_detail
22 | 
23 | #define BMI_2D_X_MASK 0x5555555555555555
24 | #define BMI_2D_Y_MASK 0xAAAAAAAAAAAAAAAA
25 | 
26 | 	template<typename morton, typename coord>
27 | 	inline morton m2D_e_BMI(const coord x, const coord y) {
28 | 		morton m = 0;
29 | 		m |= bmi2_detail::pdep(static_cast<morton>(x), static_cast<morton>(BMI_2D_X_MASK))
30 | 			| bmi2_detail::pdep(static_cast<morton>(y), static_cast<morton>(BMI_2D_Y_MASK));
31 | 		return m;
32 | 	}
33 | 
34 | 	template<typename morton, typename coord>
35 | 	inline void m2D_d_BMI(const morton m, coord& x, coord& y) {
36 | 		x = static_cast<coord>(bmi2_detail::pext(m, static_cast<morton>(BMI_2D_X_MASK)));
37 | 		y = static_cast<coord>(bmi2_detail::pext(m, static_cast<morton>(BMI_2D_Y_MASK)));
38 | 	}
39 | 
40 | #define BMI_3D_X_MASK 0x9249249249249249
41 | #define BMI_3D_Y_MASK 0x2492492492492492
42 | #define BMI_3D_Z_MASK 0x4924924924924924
43 | 
44 | 	template<typename morton, typename coord>
45 | 	inline morton m3D_e_BMI(const coord x, const coord y, const coord z) {
46 | 		morton m = 0;
47 | 		m |= bmi2_detail::pdep(static_cast<morton>(x), static_cast<morton>(BMI_3D_X_MASK))
48 | 			| bmi2_detail::pdep(static_cast<morton>(y), static_cast<morton>(BMI_3D_Y_MASK))
49 | 			| bmi2_detail::pdep(static_cast<morton>(z), static_cast<morton>(BMI_3D_Z_MASK));
50 | 		return m;
51 | 	}
52 | 
53 | 	template<typename morton, typename coord>
54 | 	inline void m3D_d_BMI(const morton m, coord& x, coord& y, coord& z) {
55 | 		x = static_cast<coord>(bmi2_detail::pext(m, static_cast<morton>(BMI_3D_X_MASK)));
56 | 		y = static_cast<coord>(bmi2_detail::pext(m, static_cast<morton>(BMI_3D_Y_MASK)));
57 | 		z = static_cast<coord>(bmi2_detail::pext(m, static_cast<morton>(BMI_3D_Z_MASK)));
58 | 	}
59 | }
60 | #endif
61 | 


--------------------------------------------------------------------------------
/glin/index_num_nodes.h:
--------------------------------------------------------------------------------
 1 | // Boost.Geometry Index
 2 | //
 3 | // R-tree visitor calculating index size
 4 | //
 5 | // Copyright (c) 2022 Jia Yu
 6 | //
 7 | // This file was modified by Oracle on 2019-2020.
 8 | // Modifications copyright (c) 2019-2020 Oracle and/or its affiliates.
 9 | // Contributed and/or modified by Adam Wulkiewicz, on behalf of Oracle
10 | //
11 | // Use, modification and distribution is subject to the Boost Software License,
12 | // Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
13 | // http://www.boost.org/LICENSE_1_0.txt)
14 | 
15 | #include <algorithm>
16 | #include <tuple>
17 | #include <boost/geometry.hpp>
18 | #include <boost/geometry/geometries/point.hpp>
19 | #include <boost/geometry/geometries/box.hpp>
20 | #include <boost/geometry/index/rtree.hpp>
21 | #include <boost/geometry/index/detail/rtree/utilities/view.hpp>
22 | 
23 | 
24 | namespace boost { namespace geometry { namespace index { namespace detail { namespace rtree { namespace utilities {
25 | 
26 | namespace visitors {
27 | 
28 | template <typename MembersHolder>
29 | struct index_num_nodes
30 |     : public MembersHolder::visitor_const
31 | {
32 |     typedef typename MembersHolder::internal_node internal_node;
33 |     typedef typename MembersHolder::leaf leaf;
34 | 
35 |     inline index_num_nodes()
36 |         : internal_size(0)
37 |         , leaf_size(0)
38 |     {}
39 | 
40 |     inline void operator()(internal_node const& n)
41 |     {
42 |         typedef typename rtree::elements_type<internal_node>::type elements_type;
43 |         typedef typename boost::geometry::model::box<boost::geometry::model::point
44 |                 <double, 2, boost::geometry::cs::cartesian>> box_type;
45 |         elements_type const& elements = rtree::elements(n);
46 | 
47 |         // Count this internal node
48 |         internal_size++;
49 |         for (typename elements_type::const_iterator it = elements.begin();
50 |             it != elements.end(); ++it)
51 |         {
52 |             rtree::apply_visitor(*this, *it->second);
53 |         }
54 |     }
55 | 
56 |     inline void operator()(leaf const& n)
57 |     {   
58 |         leaf_size++;
59 |     }
60 |     
61 |     std::size_t internal_size;
62 |     std::size_t leaf_size;
63 | };
64 | 
65 | } // namespace visitors
66 | 
67 | template <typename Rtree> inline
68 | std::tuple<std::size_t, std::size_t>
69 | index_num_nodes(Rtree const& tree)
70 | {
71 |     typedef utilities::view<Rtree> RTV;
72 |     RTV rtv(tree);
73 | 
74 |     visitors::index_num_nodes< typename RTV::members_holder> index_num_nodes_v;
75 | 
76 |     rtv.apply_visitor(index_num_nodes_v);
77 |     
78 |     return std::make_tuple(index_num_nodes_v.internal_size, index_num_nodes_v.leaf_size);
79 | }
80 | 
81 | }}}}}} // namespace boost::geometry::index::detail::rtree::utilities


--------------------------------------------------------------------------------
/cmake/FindGEOS.cmake:
--------------------------------------------------------------------------------
 1 | #---
 2 | # File: FindGEOS.cmake
 3 | #
 4 | # Find the native GEOS(Geometry Engine - Open Source) includes and libraries.
 5 | #
 6 | # This module defines:
 7 | #
 8 | # GEOS_INCLUDE_DIR, where to find geos.h, etc.
 9 | # GEOS_LIBRARY, libraries to link against to use GEOS.  Currently there are
10 | # two looked for, geos and geos_c libraries.
11 | # GEOS_FOUND, True if found, false if one of the above are not found.
12 | #
13 | # For ossim, typically geos will be system installed which should be found;
14 | # or found in the ossim 3rd party dependencies directory from a geos build
15 | # and install.  If the latter it will rely on CMAKE_INCLUDE_PATH and
16 | # CMAKE_LIBRARY_PATH having the path to the party dependencies directory.
17 | #
18 | # NOTE:
19 | # This script is specialized for ossim, e.g. looking in /usr/local/ossim.
20 | #
21 | # $Id$
22 | #---
23 | 
24 | #---
25 | # Find include path:
26 | # Note: Ubuntu 14.04+ did not have geos.h (not included in any ossim src).
27 | # Instead looking for Geometry.h
28 | #---
29 | 
30 | find_path( GEOS_INCLUDE_DIR geos/geom/Geometry.h
31 |            PATHS
32 |            ${CMAKE_INSTALL_PREFIX}/include
33 |            ${GEOS_DIR}/include
34 |            /usr/include
35 |            /usr/local/include
36 |            /usr/local/ossim/include )
37 | 
38 | # Find GEOS library:
39 | find_library( GEOS_LIB NAMES geos
40 |               PATHS
41 |               ${CMAKE_INSTALL_PREFIX}/lib
42 |               ${GEOS_DIR}/lib
43 |               /usr/lib64
44 |               /usr/lib
45 |               /usr/local/lib
46 |               /usr/local/ossim/lib )
47 | 
48 | # Find GEOS C library:
49 | find_library( GEOS_C_LIB NAMES geos_c
50 |               PATHS
51 |               ${CMAKE_INSTALL_PREFIX}/lib
52 |               ${GEOS_DIR}/lib
53 |               /usr/lib64
54 |               /usr/lib
55 |               /usr/local/lib
56 |               /usr/local/ossim/lib )
57 | 
58 | # Set the GEOS_LIBRARY:
59 | if( GEOS_LIB AND GEOS_C_LIB )
60 |    set( GEOS_LIBRARY ${GEOS_LIB} ${GEOS_C_LIB} CACHE STRING INTERNAL )
61 | endif(GEOS_LIB AND GEOS_C_LIB )
62 | 
63 | #---
64 | # This function sets GEOS_FOUND if variables are valid.
65 | #---
66 | include(FindPackageHandleStandardArgs)
67 | find_package_handle_standard_args( GEOS DEFAULT_MSG
68 |                                    GEOS_LIBRARY
69 |                                    GEOS_INCLUDE_DIR )
70 | 
71 | add_library(GEOS::GEOS INTERFACE IMPORTED)
72 | if( GEOS_FOUND )
73 |   if( NOT GEOS_FIND_QUIETLY )
74 |     message( STATUS "Found GEOS..." )
75 |   endif( NOT GEOS_FIND_QUIETLY )
76 | 
77 |   set_target_properties(GEOS::GEOS PROPERTIES
78 |     INTERFACE_LINK_LIBRARIES "${GEOS_LIBRARY}"
79 |     INTERFACE_INCLUDE_DIRECTORIES "${GEOS_INCLUDE_DIR}")
80 | else( GEOS_FOUND )
81 |    if( NOT GEOS_FIND_QUIETLY )
82 |       message( WARNING "Could not find GEOS" )
83 |    endif( NOT GEOS_FIND_QUIETLY )
84 | endif( GEOS_FOUND )
85 | 
86 | if( NOT GEOS_FIND_QUIETLY )
87 |    message( STATUS "GEOS_INCLUDE_DIR=${GEOS_INCLUDE_DIR}" )
88 |    message( STATUS "GEOS_LIBRARY=${GEOS_LIBRARY}" )
89 | endif( NOT GEOS_FIND_QUIETLY )
90 | 


--------------------------------------------------------------------------------
/glin/index_size.h:
--------------------------------------------------------------------------------
 1 | // Boost.Geometry Index
 2 | //
 3 | // R-tree visitor calculating index size
 4 | //
 5 | // Copyright (c) 2022 Jia Yu
 6 | //
 7 | // This file was modified by Oracle on 2019-2020.
 8 | // Modifications copyright (c) 2019-2020 Oracle and/or its affiliates.
 9 | // Contributed and/or modified by Adam Wulkiewicz, on behalf of Oracle
10 | //
11 | // Use, modification and distribution is subject to the Boost Software License,
12 | // Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
13 | // http://www.boost.org/LICENSE_1_0.txt)
14 | 
15 | #include <algorithm>
16 | #include <tuple>
17 | #include <boost/geometry.hpp>
18 | #include <boost/geometry/geometries/point.hpp>
19 | #include <boost/geometry/geometries/box.hpp>
20 | #include <boost/geometry/index/rtree.hpp>
21 | #include <boost/geometry/index/detail/rtree/utilities/view.hpp>
22 | 
23 | namespace boost { namespace geometry { namespace index { namespace detail { namespace rtree { namespace utilities {
24 | 
25 | namespace visitors {
26 | 
27 | template <typename MembersHolder>
28 | struct index_size
29 |     : public MembersHolder::visitor_const
30 | {
31 |     typedef typename MembersHolder::internal_node internal_node;
32 |     typedef typename MembersHolder::leaf leaf;
33 | 
34 |     inline index_size()
35 |         : internal_size(0)
36 |         , leaf_size(0)
37 |     {}
38 | 
39 |     inline void operator()(internal_node const& n)
40 |     {
41 |         typedef typename rtree::elements_type<internal_node>::type elements_type;
42 |         typedef typename boost::geometry::model::box<boost::geometry::model::point
43 |                 <double, 2, boost::geometry::cs::cartesian>> box_type;
44 |         elements_type const& elements = rtree::elements(n);
45 | 
46 |         // Calculate this internal node size
47 |         internal_size += sizeof(internal_node);
48 |         for (typename elements_type::const_iterator it = elements.begin();
49 |             it != elements.end(); ++it)
50 |         {
51 |             // Count the size (in bytes) of child node box pointer and child node pointer
52 |             internal_size += sizeof(box_type *);
53 |             internal_size += sizeof(elements_type *);
54 |             // Count the size of box. Each box has 2 corners, each of which has 2 double-type members
55 |             internal_size += sizeof(box_type);
56 |             rtree::apply_visitor(*this, *it->second);
57 |         }
58 |     }
59 | 
60 |     inline void operator()(leaf const& n)
61 |     {   
62 |         leaf_size += sizeof(leaf);
63 |     }
64 |     
65 |     std::size_t internal_size;
66 |     std::size_t leaf_size;
67 | };
68 | 
69 | } // namespace visitors
70 | 
71 | template <typename Rtree> inline
72 | std::tuple<std::size_t, std::size_t>
73 | index_size(Rtree const& tree)
74 | {
75 |     typedef utilities::view<Rtree> RTV;
76 |     RTV rtv(tree);
77 | 
78 |     visitors::index_size<
79 |         typename RTV::members_holder
80 |     > index_size_v;
81 | 
82 |     rtv.apply_visitor(index_size_v);
83 |     
84 |     return std::make_tuple(index_size_v.internal_size, index_size_v.leaf_size);
85 | }
86 | 
87 | }}}}}} // namespace boost::geometry::index::detail::rtree::utilities


--------------------------------------------------------------------------------
/test/unittest_alex_multimap.h:
--------------------------------------------------------------------------------
  1 | // Copyright (c) Microsoft Corporation.
  2 | // Licensed under the MIT license.
  3 | 
  4 | #include "doctest.h"
  5 | 
  6 | #include "alex_multimap.h"
  7 | 
  8 | using namespace alex;
  9 | 
 10 | TEST_SUITE("AlexMultimap") {
 11 | 
 12 | TEST_CASE("TestFind") {
 13 |   AlexMultimap<int, int> index;
 14 | 
 15 |   AlexMultimap<int, int>::V values[500];
 16 |   // even numbers from 0 to 998 inclusive
 17 |   for (int i = 0; i < 500; i++) {
 18 |     values[i].first = i * 2;
 19 |     values[i].second = i;
 20 |   }
 21 | 
 22 |   std::sort(values, values + 500);
 23 |   index.bulk_load(values, 500);
 24 | 
 25 |   // Find existent keys
 26 |   for (int i = 0; i < 500; i++) {
 27 |     auto it = index.find(values[i].first);
 28 |     CHECK(!it.is_end());
 29 |     CHECK_EQ(values[i].first, it.key());
 30 |   }
 31 | 
 32 |   // Find non-existent keys
 33 |   for (int i = 1; i < 100; i += 2) {
 34 |     auto it = index.find(i);
 35 |     CHECK(it.is_end());
 36 |   }
 37 | }
 38 | 
 39 | TEST_CASE("TestRandomInserts") {
 40 |   AlexMultimap<int, int> index;
 41 | 
 42 |   AlexMultimap<int, int>::V values[200];
 43 |   for (int i = 0; i < 200; i++) {
 44 |     values[i].first = rand() % 500;
 45 |     values[i].second = i;
 46 |   }
 47 | 
 48 |   std::sort(values, values + 25);
 49 |   index.bulk_load(values, 25);
 50 | 
 51 |   for (int i = 25; i < 200; i++) {
 52 |     auto it = index.insert(values[i].first, values[i].second);
 53 |     CHECK_EQ(it.key(), values[i].first);
 54 |   }
 55 | 
 56 |   // Check that getting the key is correct.
 57 |   for (int i = 0; i < 200; i++) {
 58 |     auto it = index.find(values[i].first);
 59 |     CHECK(!it.is_end());
 60 |     CHECK_EQ(values[i].first, it.key());
 61 |   }
 62 | }
 63 | 
 64 | TEST_CASE("TestRandomErases") {
 65 |   AlexMultimap<int, int> index;
 66 | 
 67 |   AlexMultimap<int, int>::V values[200];
 68 |   for (int i = 0; i < 200; i++) {
 69 |     values[i].first = i;
 70 |     values[i].second = i;
 71 |   }
 72 | 
 73 |   std::sort(values, values + 200);
 74 |   index.bulk_load(values, 200);
 75 | 
 76 |   // Try to erase a nonexistent key
 77 |   CHECK_EQ(index.erase(1000), 0);
 78 | 
 79 |   // Erase with key
 80 |   for (int i = 0; i < 100; i++) {
 81 |     int num_erased = index.erase(values[i].first);
 82 |     CHECK_EQ(num_erased, 1);
 83 |   }
 84 | 
 85 |   // Erase with iterator
 86 |   for (int i = 100; i < 200; i++) {
 87 |     auto it = index.lower_bound(values[i].first);
 88 |     CHECK(!it.is_end());
 89 |     index.erase(it);
 90 |   }
 91 | 
 92 |   CHECK_EQ(index.get_stats().num_keys, 0);
 93 | }
 94 | 
 95 | TEST_CASE("TestRangeScan") {
 96 |   AlexMultimap<int, int> index;
 97 | 
 98 |   AlexMultimap<int, int>::V values[200];
 99 |   for (int i = 0; i < 200; i++) {
100 |     values[i].first = i;
101 |     values[i].second = i;
102 |   }
103 | 
104 |   std::sort(values, values + 200);
105 |   index.bulk_load(values, 200);
106 | 
107 |   std::vector<int> results;
108 |   int sum = 0;
109 |   for (auto it = index.begin(); it != index.end(); it++) {
110 |     results.push_back((*it).second);
111 |     sum += (*it).second;
112 |   }
113 |   CHECK_EQ(results.size(), 200);
114 |   CHECK_EQ(sum, 19900);
115 | 
116 |   std::vector<int> results2;
117 |   int sum2 = 0;
118 |   for (auto it = index.find(10), it_end = index.find(100); it != it_end; it++) {
119 |     results2.push_back((*it).second);
120 |     sum2 += (*it).second;
121 |   }
122 |   CHECK_EQ(results2.size(), 90);
123 |   CHECK_EQ(sum2, 4905);
124 | }
125 | }


--------------------------------------------------------------------------------
/src/examples/main.cpp:
--------------------------------------------------------------------------------
  1 | // Copyright (c) Microsoft Corporation.
  2 | // Licensed under the MIT license.
  3 | 
  4 | /*
  5 |  * This short sample program demonstrates ALEX's API.
  6 |  */
  7 | 
  8 | #include "../core/alex.h"
  9 | 
 10 | #define KEY_TYPE int
 11 | #define PAYLOAD_TYPE int
 12 | 
 13 | int main(int, char**) {
 14 |   // Create some synthetic data: keys are dense integers between 0 and 99, and
 15 |   // payloads are random values
 16 |   const int num_keys = 100;
 17 |   std::pair<KEY_TYPE, PAYLOAD_TYPE> values[num_keys];
 18 |   std::mt19937_64 gen(std::random_device{}());
 19 |   std::uniform_int_distribution<PAYLOAD_TYPE> dis;
 20 |   for (int i = 0; i < num_keys; i++) {
 21 |     values[i].first = i;
 22 |     values[i].second = dis(gen);
 23 |   }
 24 | 
 25 |   alex::Alex<KEY_TYPE, PAYLOAD_TYPE> index;
 26 | 
 27 |   // Bulk load the keys [0, 100)
 28 |   index.bulk_load(values, num_keys);
 29 | 
 30 |   // Insert the keys [100, 200). Now there are 200 keys.
 31 |   for (int i = num_keys; i < 2 * num_keys; i++) {
 32 |     KEY_TYPE new_key = i;
 33 |     PAYLOAD_TYPE new_payload = dis(gen);
 34 |     index.insert(new_key, new_payload);
 35 |   }
 36 | 
 37 |   // Erase the keys [0, 10). Now there are 190 keys.
 38 |   for (int i = 0; i < 10; i++) {
 39 |     index.erase(i);
 40 |   }
 41 | 
 42 |   // Iterate through all entries in the index and update their payload if the
 43 |   // key is even
 44 |   int num_entries = 0;
 45 |   for (auto it = index.begin(); it != index.end(); it++) {
 46 |     if (it.key() % 2 == 0) {  // it.key() is equivalent to (*it).first
 47 |       it.payload() = dis(gen);
 48 |     }
 49 |     num_entries++;
 50 |   }
 51 |   if (num_entries != 190) {
 52 |     std::cout << "Error! There should be 190 entries in the index."
 53 |               << std::endl;
 54 |   }
 55 | 
 56 |   // Iterate through all entries with keys between 50 (inclusive) and 100
 57 |   // (exclusive)
 58 |   num_entries = 0;
 59 |   for (auto it = index.lower_bound(50); it != index.lower_bound(100); it++) {
 60 |     num_entries++;
 61 |   }
 62 |   if (num_entries != 50) {
 63 |     std::cout
 64 |         << "Error! There should be 50 entries with keys in the range [50, 100)."
 65 |         << std::endl;
 66 |   }
 67 | 
 68 |   // Equivalent way of iterating through all entries with keys between 50
 69 |   // (inclusive) and 100 (exclusive)
 70 |   num_entries = 0;
 71 |   auto it = index.lower_bound(50);
 72 |   while (it.key() < 100 && it != index.end()) {
 73 |     num_entries++;
 74 |     it++;
 75 |   }
 76 |   if (num_entries != 50) {
 77 |     std::cout
 78 |         << "Error! There should be 50 entries with keys in the range [50, 100)."
 79 |         << std::endl;
 80 |   }
 81 | 
 82 |   // Insert 9 more keys with value 42. Now there are 199 keys.
 83 |   for (int i = 0; i < 9; i++) {
 84 |     KEY_TYPE new_key = 42;
 85 |     PAYLOAD_TYPE new_payload = dis(gen);
 86 |     index.insert(new_key, new_payload);
 87 |   }
 88 | 
 89 |   // Iterate through all 10 entries with keys of value 42
 90 |   int num_duplicates = 0;
 91 |   for (auto it = index.lower_bound(42); it != index.upper_bound(42); it++) {
 92 |     num_duplicates++;
 93 |   }
 94 |   if (num_duplicates != 10) {
 95 |     std::cout << "Error! There should be 10 entries with key of value 42."
 96 |               << std::endl;
 97 |   }
 98 | 
 99 |   // Check if a non-existent key exists
100 |   it = index.find(1337);
101 |   if (it != index.end()) {
102 |     std::cout << "Error! Key with value 1337 should not exist." << std::endl;
103 |   }
104 | 
105 |   // Look at some stats
106 |   auto stats = index.get_stats();
107 |   std::cout << "Final num keys: " << stats.num_keys
108 |             << std::endl;  // expected: 199
109 |   std::cout << "Num inserts: " << stats.num_inserts
110 |             << std::endl;  // expected: 109
111 | }


--------------------------------------------------------------------------------
/src/core/projection.h:
--------------------------------------------------------------------------------
 1 | //
 2 | // Created by Juno Wang on 7/2/21.
 3 | //
 4 | #include <geos/geom/GeometryFactory.h>
 5 | #include <geos/geom/Geometry.h>
 6 | #include <geos/geom/Coordinate.h>
 7 | #include <geos/geom/CoordinateSequence.h>
 8 | #include <geos/geom/CoordinateArraySequence.h>
 9 | #include <geos/geom/Dimension.h>
10 | #include <geos/geom/PrecisionModel.h>
11 | #include <geos/util/IllegalArgumentException.h>
12 | #include <geos/geom/LineSegment.h>
13 | #include "../../glin/Encoder.h"
14 | /*
15 |  * project geometry on certain line(y = kx+b)
16 |  */
17 | void shape_projection(geos::geom::Geometry *geom, geos::geom::LineSegment segment, long double &dist_min, long double &dist_max){
18 |     dist_min = std::numeric_limits<long double>::max();
19 |     dist_max = std::numeric_limits<long double>::min();
20 |     geos::geom::Coordinate original(0,0);
21 |     std::unique_ptr<geos::geom::CoordinateSequence> coordSeq;
22 |     geos::geom::Coordinate proj_result(0,0);
23 |     coordSeq = geom->getCoordinates();
24 | #ifdef DEBUG
25 |     cout <<"coord Sequence" << coordSeq->toString() << endl;
26 | #endif
27 |     //convert the geometry to a range
28 |     for(size_t i = 0; i < coordSeq->size();i++){
29 |         segment.project(coordSeq->getAt(i),proj_result);
30 |         long double dist = proj_result.distance(original);
31 |         // if the results of projection have
32 |             if (dist < dist_min){
33 |                 dist_min = dist;
34 | 
35 |             }
36 |             if(dist> dist_max){
37 |                 dist_max = dist;
38 |             }
39 |     }
40 | #ifdef DEBUG
41 |     cout<<"max dist result" << dist_max << endl;
42 |     cout<<"min dist result" << dist_min << endl;
43 | #endif
44 | }
45 | /*
46 |  * project geometry on moore hilbert curve or z-order curve
47 |  */
48 | void curve_shape_projection(geos::geom::Geometry *geometry, std::string curve_type,
49 |                             double cell_xmin, double cell_ymin, double cell_x_intvl,double cell_y_intvl,double &dist_start, double &dist_end){
50 |     dist_start = std::numeric_limits<double>::max();
51 |     dist_end = std::numeric_limits<double>::min();
52 |     const geos::geom::Envelope *envelope = geometry->getEnvelopeInternal();
53 |     double minX = envelope->getMinX();
54 |     double minY = envelope->getMinY();
55 |     double maxX = envelope->getMaxX();
56 |     double maxY = envelope->getMaxY();
57 | 
58 | 
59 |     auto encoder = new Encoder<double>(cell_xmin, cell_x_intvl, cell_ymin, cell_y_intvl );
60 | 
61 |     if(curve_type == "h"){
62 |             auto index_double_h = encoder->encode_h(minX,minY,maxX,maxY);
63 |             dist_start = index_double_h.first;
64 |             dist_end = index_double_h.second;
65 |     }
66 |     if(curve_type == "z"){
67 |         auto index_z = encoder->encode_z(minX,minY,maxX,maxY);
68 |         dist_start = index_z.first;
69 |         dist_end = index_z.second;
70 |     }
71 | 
72 | }
73 | void curve_shape_projection(geos::geom::Envelope *envelope, std::string curve_type, double cell_xmin, double cell_ymin, double cell_x_intvl,double cell_y_intvl,double &dist_start, double &dist_end){
74 |     dist_start = std::numeric_limits<double>::max();
75 |     dist_end = std::numeric_limits<double>::min();
76 |     double minX = envelope->getMinX();
77 |     double minY = envelope->getMinY();
78 |     double maxX = envelope->getMaxX();
79 |     double maxY = envelope->getMaxY();
80 | 
81 |     auto encoder = new Encoder<double>(cell_xmin, cell_x_intvl, cell_ymin, cell_y_intvl );
82 | 
83 |     if(curve_type == "h"){
84 |         auto index_double_h = encoder->encode_h(minX,minY,maxX,maxY);
85 |         dist_start = index_double_h.first;
86 |         dist_end = index_double_h.second;
87 |     }
88 |     if(curve_type == "z"){
89 |         auto index_z = encoder->encode_z(minX,minY,maxX,maxY);
90 |         dist_start = index_z.first;
91 |         dist_end = index_z.second;
92 |     }
93 | 
94 | }
95 | 


--------------------------------------------------------------------------------
/test/unittest_alex_map.h:
--------------------------------------------------------------------------------
  1 | // Copyright (c) Microsoft Corporation.
  2 | // Licensed under the MIT license.
  3 | 
  4 | #include "doctest.h"
  5 | 
  6 | #include "alex_map.h"
  7 | 
  8 | using namespace alex;
  9 | 
 10 | TEST_SUITE("AlexMap") {
 11 | 
 12 | TEST_CASE("TestFind") {
 13 |   AlexMap<int, int> index;
 14 | 
 15 |   AlexMap<int, int>::V values[500];
 16 |   // even numbers from 0 to 998 inclusive
 17 |   for (int i = 0; i < 500; i++) {
 18 |     values[i].first = i * 2;
 19 |     values[i].second = i;
 20 |   }
 21 | 
 22 |   std::sort(values, values + 500);
 23 |   index.bulk_load(values, 500);
 24 | 
 25 |   // Find existent keys
 26 |   for (int i = 0; i < 500; i++) {
 27 |     // Three different ways of finding
 28 |     if (i % 3 == 0) {
 29 |       auto it = index.find(values[i].first);
 30 |       CHECK(!it.is_end());
 31 |       CHECK_EQ(values[i].first, it.key());
 32 |     } else if (i % 3 == 1) {
 33 |       int &payload = index.at(values[i].first);
 34 |       CHECK_EQ(values[i].second, payload);
 35 |     } else {
 36 |       int &payload = index[values[i].first];
 37 |       CHECK_EQ(values[i].second, payload);
 38 |     }
 39 |   }
 40 | 
 41 |   // Find non-existent keys
 42 |   for (int i = 1; i < 100; i += 2) {
 43 |     auto it = index.find(i);
 44 |     CHECK(it.is_end());
 45 |   }
 46 | }
 47 | 
 48 | TEST_CASE("TestRandomInserts") {
 49 |   AlexMap<int, int> index;
 50 | 
 51 |   AlexMap<int, int>::V values[200];
 52 |   for (int i = 0; i < 200; i++) {
 53 |     values[i].first = rand() % 500;
 54 |     values[i].second = i;
 55 |   }
 56 | 
 57 |   std::sort(values, values + 25);
 58 |   index.bulk_load(values, 25);
 59 | 
 60 |   for (int i = 25; i < 200; i++) {
 61 |     // Two different ways of inserting
 62 |     if (i % 2 == 0) {
 63 |       auto ret = index.insert(values[i].first, values[i].second);
 64 |       CHECK_EQ(ret.first.key(), values[i].first);
 65 |     } else {
 66 |       index[values[i].first] = values[i].second;
 67 |       CHECK_EQ(index[values[i].first], values[i].second);
 68 |     }
 69 |   }
 70 | 
 71 |   // Check that getting the key is correct.
 72 |   for (int i = 0; i < 200; i++) {
 73 |     auto it = index.find(values[i].first);
 74 |     CHECK(!it.is_end());
 75 |     CHECK_EQ(values[i].first, it.key());
 76 |   }
 77 | }
 78 | 
 79 | TEST_CASE("TestRandomErases") {
 80 |   AlexMap<int, int> index;
 81 | 
 82 |   AlexMap<int, int>::V values[200];
 83 |   for (int i = 0; i < 200; i++) {
 84 |     values[i].first = i;
 85 |     values[i].second = i;
 86 |   }
 87 | 
 88 |   std::sort(values, values + 200);
 89 |   index.bulk_load(values, 200);
 90 | 
 91 |   // Try to erase a nonexistent key
 92 |   CHECK_EQ(index.erase(1000), 0);
 93 | 
 94 |   // Erase with key
 95 |   for (int i = 0; i < 100; i++) {
 96 |     int num_erased = index.erase(values[i].first);
 97 |     CHECK_EQ(num_erased, 1);
 98 |   }
 99 | 
100 |   // Erase with iterator
101 |   for (int i = 100; i < 200; i++) {
102 |     auto it = index.lower_bound(values[i].first);
103 |     CHECK(!it.is_end());
104 |     index.erase(it);
105 |   }
106 | 
107 |   CHECK_EQ(index.get_stats().num_keys, 0);
108 | }
109 | 
110 | TEST_CASE("TestRangeScan") {
111 |   AlexMap<int, int> index;
112 | 
113 |   AlexMap<int, int>::V values[200];
114 |   for (int i = 0; i < 200; i++) {
115 |     values[i].first = i;
116 |     values[i].second = i;
117 |   }
118 | 
119 |   std::sort(values, values + 200);
120 |   index.bulk_load(values, 200);
121 | 
122 |   std::vector<int> results;
123 |   int sum = 0;
124 |   for (auto it = index.begin(); it != index.end(); it++) {
125 |     results.push_back((*it).second);
126 |     sum += (*it).second;
127 |   }
128 |   CHECK_EQ(results.size(), 200);
129 |   CHECK_EQ(sum, 19900);
130 | 
131 |   std::vector<int> results2;
132 |   int sum2 = 0;
133 |   for (auto it = index.find(10), it_end = index.find(100); it != it_end; it++) {
134 |     results2.push_back((*it).second);
135 |     sum2 += (*it).second;
136 |   }
137 |   CHECK_EQ(results2.size(), 90);
138 |   CHECK_EQ(sum2, 4905);
139 | }
140 | }


--------------------------------------------------------------------------------
/test/unittest_glin_piecewise.h:
--------------------------------------------------------------------------------
  1 | //
  2 | // Created by Congying Wang on 10/25/21.
  3 | //
  4 | #include "doctest.h"
  5 | #include <map>
  6 | 
  7 | TEST_SUITE("piecewise") {
  8 |     TEST_CASE("piecewise with zero range") {
  9 |         const int size = 7;
 10 |         std::pair<double, double> records[size];
 11 |         records[0].first = 2;
 12 |         records[0].second = 2;
 13 |         records[1].first = 2;
 14 |         records[1].second = 2;
 15 |         records[2].first = 2;
 16 |         records[2].second = 2;
 17 |         records[3].first = 5;
 18 |         records[3].second = 5;
 19 |         records[4].first = 3;
 20 |         records[4].second = 3;
 21 |         records[5].first = 4;
 22 |         records[5].second = 4;
 23 |         records[6].first = 5;
 24 |         records[6].second = 5;
 25 |         std::vector<std::tuple<double, double, double, double>> pieces;
 26 |         piecewise(records, size, 3, pieces);
 27 |         CHECK(!pieces.empty());
 28 |         CHECK(pieces.size() == 3);
 29 |         CHECK(std::get<0>(pieces[1]) == 2.0);
 30 |         CHECK(std::get<1>(pieces[1]) == 2.0);
 31 |         CHECK(std::get<2>(pieces[1]) == 3.0);
 32 |         CHECK(std::get<3>(pieces[1]) == 6.0);
 33 | 
 34 | #ifdef DEBUG
 35 |         for (int i = 0; i < pieces.size(); i++) {
 36 |             std::cout << get<0>(pieces[i]) << "  " << get<1>(pieces[i]) << "  " << get<2>(pieces[i]) << "  "
 37 |                       << get<3>(pieces[i]) << endl;
 38 |         }
 39 | #endif
 40 | 
 41 |     }
 42 | 
 43 | 
 44 | TEST_CASE(" piecewise") {
 45 |     const int size = 6;
 46 |     std::pair<double, double> records[size];
 47 |     records[0].first = 2;
 48 |     records[0].second = 5;
 49 |     records[1].first = 2;
 50 |     records[1].second = 4;
 51 |     records[2].first = 2;
 52 |     records[2].second = 8;
 53 |     records[3].first = 3;
 54 |     records[3].second = 9;
 55 |     records[4].first = 4;
 56 |     records[4].second = 9;
 57 |     records[5].first = 5;
 58 |     records[5].second = 7;
 59 |     std::vector<std::tuple<double, double, double, double>> pieces;
 60 |     piecewise(records, size, 3, pieces);
 61 |     CHECK(!pieces.empty());
 62 |     CHECK(pieces.size() == 3);
 63 |     CHECK(std::get<0>(pieces[1]) == 7);
 64 |     CHECK(std::get<1>(pieces[1]) == 2);
 65 |     CHECK(std::get<2>(pieces[1]) == 3);
 66 |     CHECK(std::get<0>(pieces[2]) == 9);
 67 |     CHECK(std::get<1>(pieces[2]) == 2);
 68 |     CHECK(std::get<2>(pieces[2]) == 3);
 69 | 
 70 | #ifdef  DEBUG
 71 |     for (int i = 0; i < pieces.size(); i++) {
 72 |         std::cout << get<0>(pieces[i]) << "  " << get<1>(pieces[i]) << "  " << get<2>(pieces[i]) << "  "
 73 |                   << get<3>(pieces[i]) << endl;
 74 |         }
 75 | #endif
 76 | }
 77 | 
 78 | TEST_CASE("Using loop form data and perform piecewise") {
 79 |     pair<double, double> records[200];
 80 |     std::vector<std::tuple<double, double, double, double>> orig_pieces;
 81 |     double start_number = 100.50;
 82 |     for (int i = 0; i < 200; i++) {
 83 |         records[i].first = start_number - i;
 84 |         records[i].second = start_number;
 85 |         start_number--;
 86 | //        std::cout << records[i].first << " " << records[i].second << endl;
 87 |     }
 88 | 
 89 |     piecewise(records, 200, 10, orig_pieces);
 90 |     double count = 0;
 91 |     for (int i = 0; i < orig_pieces.size(); i++) {
 92 |         if (i != 0) {
 93 |             count += std::get<2>(orig_pieces[i]);
 94 |             CHECK(std::get<2>(orig_pieces[i]) <= 10);
 95 |         }
 96 | 
 97 |     }
 98 |     CHECK(count == 200);
 99 | }
100 | 
101 | TEST_CASE("piecewise with repeating data") {
102 |         /*
103 |          * this records contains large amount of data with repeating end.
104 |          * For repeating data, we ignore the error bound and put all data with same end together until repeating ends
105 |          */
106 |         // test piecewise ffunction by push reeesult back to
107 |         // build up the record
108 |         const int size = 100;
109 |         std::pair<double, double> records1[size];
110 |         for (int i = 0; i < size; i++) {
111 |             records1[i].first = i;
112 |             records1[i].second = i + 15;
113 |             if (i % 2 == 0) {
114 |                 records1[i].second = i + 10;
115 |             }
116 |             if (i % 5 == 0) {
117 |                 records1[i].second = 100;
118 |             }
119 |         }
120 |         std::vector<std::tuple<double, double, double, double>> pieces;
121 |         double piece_limit = 50;
122 |         piecewise(records1, 100, piece_limit, pieces);
123 |         CHECK(!pieces.empty());
124 |         CHECK(pieces.size() == 3);
125 |         for (int i = 0; i < pieces.size(); i++) {
126 | #ifdef DEBUG
127 | 
128 |             std::cout << get<0>(pieces[i]) << "  " << get<1>(pieces[i]) << "  " << get<2>(pieces[i]) << "  "
129 |                       << get<3>(pieces[i]) << endl;
130 | #endif
131 |             if (i != 0) {
132 |                 CHECK(get<2>(pieces[i]) >= piece_limit);
133 |             }
134 |         }
135 |     }
136 | };


--------------------------------------------------------------------------------
/glin/Encoder.h:
--------------------------------------------------------------------------------
  1 | #include "hilbert/hilbert.h"
  2 | #include "libmorton/morton.h"
  3 | 
  4 | template <class INPUT>
  5 | class Encoder {
  6 | private:
  7 |     INPUT xmin; // The minimum X value in the entire space
  8 |     INPUT xintvl; // X interval which is the cell size on X
  9 |     INPUT ymin; // The minimum X value in the entire space
 10 |     INPUT yintvl; // Y interval which is the cell size on Y
 11 | public:
 12 |     // Build an encoder using min, cell interval (aka cell size)
 13 |     Encoder(INPUT xmin, INPUT xintvl, INPUT ymin, INPUT yintvl): xmin(xmin), xintvl(xintvl), ymin(ymin), yintvl(yintvl){
 14 |     }
 15 | 
 16 |     // Build an encoder using min, max and number of cells
 17 |     Encoder(INPUT xmin, INPUT xmax, int xnum, INPUT ymin, INPUT ymax, int ynum):
 18 |     xmin(xmin), xintvl((xmax - xmin) * 1.0 / xnum), ymin(ymin), yintvl((ymax - ymin) * 1.0 / ynum){
 19 |     }
 20 | 
 21 |     template<class OUTPUT>
 22 |     inline OUTPUT d2i (INPUT d, INPUT min, INPUT interval) {
 23 |         auto i = (INPUT)((d - min) / interval);
 24 |         return i;
 25 |     }
 26 |     
 27 |     inline uint_fast64_t encode_z(INPUT x, INPUT y) {
 28 |         auto x_int = d2i<uint_fast32_t>(x, xmin, xintvl);
 29 |         auto y_int = d2i<uint_fast32_t>(y, ymin, yintvl);
 30 |         return libmorton::morton2D_64_encode(x_int, y_int);
 31 |     }
 32 | 
 33 |     inline bitmask_t encode_h(INPUT x, INPUT y) {
 34 |         auto x_int = d2i<bitmask_t>(x, xmin, xintvl);
 35 |         auto y_int = d2i<bitmask_t>(y, ymin, yintvl);
 36 |         bitmask_t coord[2] = {x_int, y_int};
 37 |         return hilbert_c2i(2, 32, coord);
 38 |     }
 39 | 
 40 |     /**
 41 |      * The min max Z order curve range of a rectangle are the IDs of minXY and maxXY
 42 |      * See https://stackoverflow.com/questions/30170783/how-to-use-morton-orderz-order-curve-in-range-search
 43 |      * See https://aws.amazon.com/blogs/database/z-order-indexing-for-multifaceted-queries-in-amazon-dynamodb-part-1/?sc_channel=sm&sc_campaign=zackblog&sc_country=global&sc_geo=global&sc_category=rds&sc_outcome=aware&adbsc=awsdbblog_social_20170517_72417147&adbid=864895517733470208&adbpl=tw&adbpr=66780587
 44 |      * @param x1_dbl minX
 45 |      * @param y1_dbl minY
 46 |      * @param x2_dbl maxX
 47 |      * @param y2_dbl maxY
 48 |      * @return
 49 |      */
 50 |     inline std::pair<uint_fast64_t, uint_fast64_t> encode_z(INPUT x1_dbl, INPUT y1_dbl, INPUT x2_dbl, INPUT y2_dbl){
 51 |         auto x1 = d2i<uint_fast32_t>(x1_dbl, xmin, xintvl);
 52 |         auto y1 = d2i<uint_fast32_t>(y1_dbl, ymin, yintvl);
 53 |         auto x2 = d2i<uint_fast32_t>(x2_dbl, xmin, xintvl);
 54 |         auto y2 = d2i<uint_fast32_t>(y2_dbl, ymin, yintvl);
 55 |         auto min = libmorton::morton2D_64_encode(x1, y1);
 56 |         auto max = libmorton::morton2D_64_encode(x2, y2);
 57 |         return std::make_pair(min, max);
 58 |     }
 59 | 
 60 |     /**
 61 |      * The min and max Hilbert curve ID ranges of a rectangle lie on the boundary of a rectangle
 62 |      * See https://stackoverflow.com/questions/12772893/how-to-use-morton-order-in-range-search
 63 |      * See https://github.com/davidmoten/hilbert-curve
 64 |      * @param x1_dbl minX
 65 |      * @param y1_dbl minY
 66 |      * @param x2_dbl maxX
 67 |      * @param y2_dbl maxY
 68 |      * @return
 69 |      */
 70 |     inline std::pair<bitmask_t, bitmask_t> encode_h(INPUT x1_dbl, INPUT y1_dbl, INPUT x2_dbl, INPUT y2_dbl){
 71 |         auto x1 = d2i<bitmask_t>(x1_dbl, xmin, xintvl);
 72 |         auto y1 = d2i<bitmask_t>(y1_dbl, ymin, yintvl);
 73 |         auto x2 = d2i<bitmask_t>(x2_dbl, xmin, xintvl);
 74 |         auto y2 = d2i<bitmask_t>(y2_dbl, ymin, yintvl);
 75 |         /**
 76 |          * ********
 77 |          * *      *
 78 |          * *      *
 79 |          * ********
 80 |          */
 81 |         bitmask_t working_co[2] = {0, 0};
 82 |         auto min = std::numeric_limits<bitmask_t>::max();
 83 |         auto max = std::numeric_limits<bitmask_t>::min();
 84 |         for (bitmask_t i = x1; i <= x2; ++i) {
 85 |             working_co[0] = i;
 86 |             working_co[1] = y1;
 87 |             auto working_id = hilbert_c2i(2, 32, working_co);
 88 |             min = std::min(min, working_id);
 89 |             max = std::max(max, working_id);
 90 |         }
 91 |         for (bitmask_t i = x1; i <= x2; ++i) {
 92 |             working_co[0] = i;
 93 |             working_co[1] = y2;
 94 |             auto working_id = hilbert_c2i(2, 32, working_co);
 95 |             min = std::min(min, working_id);
 96 |             max = std::max(max, working_id);
 97 |         }
 98 |         for (bitmask_t i = y1 + 1; i <= y2 -1 ; ++i) {
 99 |             working_co[0] = x1;
100 |             working_co[1] = i;
101 |             auto working_id = hilbert_c2i(2, 32, working_co);
102 |             min = std::min(min, working_id);
103 |             max = std::max(max, working_id);
104 |         }
105 |         for (bitmask_t i = y1 + 1; i <= y2 -1 ; ++i) {
106 |             working_co[0] = x2;
107 |             working_co[1] = i;
108 |             auto working_id = hilbert_c2i(2, 32, working_co);
109 |             min = std::min(min, working_id);
110 |             max = std::max(max, working_id);
111 |         }
112 |         return std::make_pair(min, max);
113 |     }
114 | };


--------------------------------------------------------------------------------
/glin/libmorton/morton.h:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | // This file will always contain inline functions which point to the fastest Morton encoding/decoding implementation
  4 | // IF you just want to use the fastest method to encode/decode morton codes, include this header.
  5 | 
  6 | // If you want to experiment with alternative methods (which might be slower, all depending on hardware / your data set)
  7 | // check the individual headers below.
  8 | 
  9 | #include "morton2D.h"
 10 | #include "morton3D.h"
 11 | 
 12 | #if defined(__BMI2__) || (defined(__AVX2__) && defined(_MSC_VER))
 13 | #include "morton_BMI.h"
 14 | #elif defined(__AVX512BITALG__)
 15 | #include "morton_AVX512BITALG.h"
 16 | #endif
 17 | 
 18 | namespace libmorton {
 19 | 	// Functions under this are stubs which will always point to fastest implementation at the moment
 20 | 	//-----------------------------------------------------------------------------------------------
 21 | 
 22 | 	// ENCODING
 23 | #if defined(__BMI2__) || (defined(__AVX2__) && defined(_MSC_VER))
 24 | 	inline uint_fast32_t morton2D_32_encode(const uint_fast16_t x, const uint_fast16_t y) {
 25 | 		return m2D_e_BMI<uint_fast32_t, uint_fast16_t>(x, y);
 26 | 	}
 27 | 	inline uint_fast64_t morton2D_64_encode(const uint_fast32_t x, const uint_fast32_t y) {
 28 | 		return m2D_e_BMI<uint_fast64_t, uint_fast32_t>(x, y);
 29 | 	}
 30 | 	inline uint_fast32_t morton3D_32_encode(const uint_fast16_t x, const uint_fast16_t y, const uint_fast16_t z) {
 31 | 		return m3D_e_BMI<uint_fast32_t, uint_fast16_t>(x, y, z);
 32 | 	}
 33 | 	inline uint_fast64_t morton3D_64_encode(const uint_fast32_t x, const uint_fast32_t y, const uint_fast32_t z) {
 34 | 		return m3D_e_BMI<uint_fast64_t, uint_fast32_t>(x, y, z);
 35 | 	}
 36 | #elif defined(__AVX512BITALG__)
 37 | 	inline uint_fast32_t morton2D_32_encode(const uint_fast16_t x, const uint_fast16_t y) {
 38 | 		return m2D_e_BITALG<uint_fast32_t, uint_fast16_t>(x, y);
 39 | 	}
 40 | 	inline uint_fast64_t morton2D_64_encode(const uint_fast32_t x, const uint_fast32_t y) {
 41 | 		return m2D_e_BITALG<uint_fast64_t, uint_fast32_t>(x, y);
 42 | 	}
 43 | 	inline uint_fast32_t morton3D_32_encode(const uint_fast16_t x, const uint_fast16_t y, const uint_fast16_t z) {
 44 | 		return m3D_e_BITALG<uint_fast32_t, uint_fast16_t>(x, y, z);
 45 | 	}
 46 | 	inline uint_fast64_t morton3D_64_encode(const uint_fast32_t x, const uint_fast32_t y, const uint_fast32_t z) {
 47 | 		return m3D_e_BITALG<uint_fast64_t, uint_fast32_t>(x, y, z);
 48 | 	}
 49 | #else
 50 | 	inline uint_fast32_t morton2D_32_encode(const uint_fast16_t x, const uint_fast16_t y) {
 51 | 		return m2D_e_sLUT<uint_fast32_t, uint_fast16_t>(x, y);
 52 | 	}
 53 | 	inline uint_fast64_t morton2D_64_encode(const uint_fast32_t x, const uint_fast32_t y) {
 54 | 		return m2D_e_sLUT<uint_fast64_t, uint_fast32_t>(x, y);
 55 | 	}
 56 | 	inline uint_fast32_t morton3D_32_encode(const uint_fast16_t x, const uint_fast16_t y, const uint_fast16_t z) {
 57 | 		return m3D_e_sLUT<uint_fast32_t, uint_fast16_t>(x, y, z);
 58 | 	}
 59 | 	inline uint_fast64_t morton3D_64_encode(const uint_fast32_t x, const uint_fast32_t y, const uint_fast32_t z) {
 60 | 		return m3D_e_sLUT<uint_fast64_t, uint_fast32_t>(x, y, z);
 61 | 	}
 62 | #endif
 63 | 
 64 | 	// DECODING
 65 | 
 66 | #if defined(__BMI2__) || (defined(__AVX2__) && defined(_MSC_VER))
 67 | 	inline void morton2D_32_decode(const uint_fast32_t morton, uint_fast16_t& x, uint_fast16_t& y) {
 68 | 		m2D_d_BMI<uint_fast32_t, uint_fast16_t>(morton, x, y);
 69 | 	}
 70 | 	inline void morton2D_64_decode(const uint_fast64_t morton, uint_fast32_t& x, uint_fast32_t& y) {
 71 | 		m2D_d_BMI<uint_fast64_t, uint_fast32_t>(morton, x, y);
 72 | 	}
 73 | 	inline void morton3D_32_decode(const uint_fast32_t morton, uint_fast16_t& x, uint_fast16_t& y, uint_fast16_t& z) {
 74 | 		m3D_d_BMI<uint_fast32_t, uint_fast16_t>(morton, x, y, z);
 75 | 	}
 76 | 	inline void morton3D_64_decode(const uint_fast64_t morton, uint_fast32_t& x, uint_fast32_t& y, uint_fast32_t& z) {
 77 | 		m3D_d_BMI<uint_fast64_t, uint_fast32_t>(morton, x, y, z);
 78 | 	}
 79 | #elif defined(__AVX512BITALG__)
 80 | 	inline void morton2D_32_decode(const uint_fast32_t morton, uint_fast16_t& x, uint_fast16_t& y) {
 81 | 		m2D_d_BITALG<uint_fast32_t, uint_fast16_t>(morton, x, y);
 82 | 	}
 83 | 	inline void morton2D_64_decode(const uint_fast64_t morton, uint_fast32_t& x, uint_fast32_t& y) {
 84 | 		m2D_d_BITALG<uint_fast64_t, uint_fast32_t>(morton, x, y);
 85 | 	}
 86 | 	inline void morton3D_32_decode(const uint_fast32_t morton, uint_fast16_t& x, uint_fast16_t& y, uint_fast16_t& z) {
 87 | 		m3D_d_BITALG<uint_fast32_t, uint_fast16_t>(morton, x, y, z);
 88 | 	}
 89 | 	inline void morton3D_64_decode(const uint_fast64_t morton, uint_fast32_t& x, uint_fast32_t& y, uint_fast32_t& z) {
 90 | 		m3D_d_BITALG<uint_fast64_t, uint_fast32_t>(morton, x, y, z);
 91 | }
 92 | #else
 93 | 	inline void morton2D_32_decode(const uint_fast32_t morton, uint_fast16_t& x, uint_fast16_t& y) {
 94 | 		m2D_d_sLUT<uint_fast32_t, uint_fast16_t>(morton, x, y);
 95 | 	}
 96 | 	inline void morton2D_64_decode(const uint_fast64_t morton, uint_fast32_t& x, uint_fast32_t& y) {
 97 | 		m2D_d_sLUT<uint_fast64_t, uint_fast32_t>(morton, x, y);
 98 | 	}
 99 | 	inline void morton3D_32_decode(const uint_fast32_t morton, uint_fast16_t& x, uint_fast16_t& y, uint_fast16_t& z) {
100 | 		m3D_d_sLUT<uint_fast32_t, uint_fast16_t>(morton, x, y, z);
101 | 	}
102 | 	inline void morton3D_64_decode(const uint_fast64_t morton, uint_fast32_t& x, uint_fast32_t& y, uint_fast32_t& z) {
103 | 		m3D_d_sLUT<uint_fast64_t, uint_fast32_t>(morton, x, y, z);
104 | 	}
105 | #endif
106 | }


--------------------------------------------------------------------------------
/glin/libmorton/morton2D_LUTs.h:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <stdint.h>
  4 | 
  5 | namespace libmorton {
  6 | 
  7 | 	// LUT for Morton2D encode X
  8 | 	static const uint_fast16_t Morton2D_encode_x_256[256] =
  9 | 	{
 10 | 	0, 1, 4, 5, 16, 17, 20, 21,
 11 | 	64, 65, 68, 69, 80, 81, 84, 85,
 12 | 	256, 257, 260, 261, 272, 273, 276, 277,
 13 | 	320, 321, 324, 325, 336, 337, 340, 341,
 14 | 	1024, 1025, 1028, 1029, 1040, 1041, 1044, 1045,
 15 | 	1088, 1089, 1092, 1093, 1104, 1105, 1108, 1109,
 16 | 	1280, 1281, 1284, 1285, 1296, 1297, 1300, 1301,
 17 | 	1344, 1345, 1348, 1349, 1360, 1361, 1364, 1365,
 18 | 	4096, 4097, 4100, 4101, 4112, 4113, 4116, 4117,
 19 | 	4160, 4161, 4164, 4165, 4176, 4177, 4180, 4181,
 20 | 	4352, 4353, 4356, 4357, 4368, 4369, 4372, 4373,
 21 | 	4416, 4417, 4420, 4421, 4432, 4433, 4436, 4437,
 22 | 	5120, 5121, 5124, 5125, 5136, 5137, 5140, 5141,
 23 | 	5184, 5185, 5188, 5189, 5200, 5201, 5204, 5205,
 24 | 	5376, 5377, 5380, 5381, 5392, 5393, 5396, 5397,
 25 | 	5440, 5441, 5444, 5445, 5456, 5457, 5460, 5461,
 26 | 	16384, 16385, 16388, 16389, 16400, 16401, 16404, 16405,
 27 | 	16448, 16449, 16452, 16453, 16464, 16465, 16468, 16469,
 28 | 	16640, 16641, 16644, 16645, 16656, 16657, 16660, 16661,
 29 | 	16704, 16705, 16708, 16709, 16720, 16721, 16724, 16725,
 30 | 	17408, 17409, 17412, 17413, 17424, 17425, 17428, 17429,
 31 | 	17472, 17473, 17476, 17477, 17488, 17489, 17492, 17493,
 32 | 	17664, 17665, 17668, 17669, 17680, 17681, 17684, 17685,
 33 | 	17728, 17729, 17732, 17733, 17744, 17745, 17748, 17749,
 34 | 	20480, 20481, 20484, 20485, 20496, 20497, 20500, 20501,
 35 | 	20544, 20545, 20548, 20549, 20560, 20561, 20564, 20565,
 36 | 	20736, 20737, 20740, 20741, 20752, 20753, 20756, 20757,
 37 | 	20800, 20801, 20804, 20805, 20816, 20817, 20820, 20821,
 38 | 	21504, 21505, 21508, 21509, 21520, 21521, 21524, 21525,
 39 | 	21568, 21569, 21572, 21573, 21584, 21585, 21588, 21589,
 40 | 	21760, 21761, 21764, 21765, 21776, 21777, 21780, 21781,
 41 | 	21824, 21825, 21828, 21829, 21840, 21841, 21844, 21845
 42 | 	};
 43 | 
 44 | 	// LUT for Morton2D encode Y
 45 | 	static const uint_fast16_t Morton2D_encode_y_256[256] =
 46 | 	{
 47 | 	0, 2, 8, 10, 32, 34, 40, 42,
 48 | 	128, 130, 136, 138, 160, 162, 168, 170,
 49 | 	512, 514, 520, 522, 544, 546, 552, 554,
 50 | 	640, 642, 648, 650, 672, 674, 680, 682,
 51 | 	2048, 2050, 2056, 2058, 2080, 2082, 2088, 2090,
 52 | 	2176, 2178, 2184, 2186, 2208, 2210, 2216, 2218,
 53 | 	2560, 2562, 2568, 2570, 2592, 2594, 2600, 2602,
 54 | 	2688, 2690, 2696, 2698, 2720, 2722, 2728, 2730,
 55 | 	8192, 8194, 8200, 8202, 8224, 8226, 8232, 8234,
 56 | 	8320, 8322, 8328, 8330, 8352, 8354, 8360, 8362,
 57 | 	8704, 8706, 8712, 8714, 8736, 8738, 8744, 8746,
 58 | 	8832, 8834, 8840, 8842, 8864, 8866, 8872, 8874,
 59 | 	10240, 10242, 10248, 10250, 10272, 10274, 10280, 10282,
 60 | 	10368, 10370, 10376, 10378, 10400, 10402, 10408, 10410,
 61 | 	10752, 10754, 10760, 10762, 10784, 10786, 10792, 10794,
 62 | 	10880, 10882, 10888, 10890, 10912, 10914, 10920, 10922,
 63 | 	32768, 32770, 32776, 32778, 32800, 32802, 32808, 32810,
 64 | 	32896, 32898, 32904, 32906, 32928, 32930, 32936, 32938,
 65 | 	33280, 33282, 33288, 33290, 33312, 33314, 33320, 33322,
 66 | 	33408, 33410, 33416, 33418, 33440, 33442, 33448, 33450,
 67 | 	34816, 34818, 34824, 34826, 34848, 34850, 34856, 34858,
 68 | 	34944, 34946, 34952, 34954, 34976, 34978, 34984, 34986,
 69 | 	35328, 35330, 35336, 35338, 35360, 35362, 35368, 35370,
 70 | 	35456, 35458, 35464, 35466, 35488, 35490, 35496, 35498,
 71 | 	40960, 40962, 40968, 40970, 40992, 40994, 41000, 41002,
 72 | 	41088, 41090, 41096, 41098, 41120, 41122, 41128, 41130,
 73 | 	41472, 41474, 41480, 41482, 41504, 41506, 41512, 41514,
 74 | 	41600, 41602, 41608, 41610, 41632, 41634, 41640, 41642,
 75 | 	43008, 43010, 43016, 43018, 43040, 43042, 43048, 43050,
 76 | 	43136, 43138, 43144, 43146, 43168, 43170, 43176, 43178,
 77 | 	43520, 43522, 43528, 43530, 43552, 43554, 43560, 43562,
 78 | 	43648, 43650, 43656, 43658, 43680, 43682, 43688, 43690
 79 | 	};
 80 | 
 81 | 	// LUT for Morton2D decode X
 82 | 	static const uint_fast8_t Morton2D_decode_x_256[256] = {
 83 | 	0,1,0,1,2,3,2,3,0,1,0,1,2,3,2,3,
 84 | 	4,5,4,5,6,7,6,7,4,5,4,5,6,7,6,7,
 85 | 	0,1,0,1,2,3,2,3,0,1,0,1,2,3,2,3,
 86 | 	4,5,4,5,6,7,6,7,4,5,4,5,6,7,6,7,
 87 | 	8,9,8,9,10,11,10,11,8,9,8,9,10,11,10,11,
 88 | 	12,13,12,13,14,15,14,15,12,13,12,13,14,15,14,15,
 89 | 	8,9,8,9,10,11,10,11,8,9,8,9,10,11,10,11,
 90 | 	12,13,12,13,14,15,14,15,12,13,12,13,14,15,14,15,
 91 | 	0,1,0,1,2,3,2,3,0,1,0,1,2,3,2,3,
 92 | 	4,5,4,5,6,7,6,7,4,5,4,5,6,7,6,7,
 93 | 	0,1,0,1,2,3,2,3,0,1,0,1,2,3,2,3,
 94 | 	4,5,4,5,6,7,6,7,4,5,4,5,6,7,6,7,
 95 | 	8,9,8,9,10,11,10,11,8,9,8,9,10,11,10,11,
 96 | 	12,13,12,13,14,15,14,15,12,13,12,13,14,15,14,15,
 97 | 	8,9,8,9,10,11,10,11,8,9,8,9,10,11,10,11,
 98 | 	12,13,12,13,14,15,14,15,12,13,12,13,14,15,14,15
 99 | 	};
100 | 
101 | 	// LUT for Morton2D decode Y
102 | 	static const uint_fast8_t Morton2D_decode_y_256[256] = {
103 | 	0,0,1,1,0,0,1,1,2,2,3,3,2,2,3,3,
104 | 	0,0,1,1,0,0,1,1,2,2,3,3,2,2,3,3,
105 | 	4,4,5,5,4,4,5,5,6,6,7,7,6,6,7,7,
106 | 	4,4,5,5,4,4,5,5,6,6,7,7,6,6,7,7,
107 | 	0,0,1,1,0,0,1,1,2,2,3,3,2,2,3,3,
108 | 	0,0,1,1,0,0,1,1,2,2,3,3,2,2,3,3,
109 | 	4,4,5,5,4,4,5,5,6,6,7,7,6,6,7,7,
110 | 	4,4,5,5,4,4,5,5,6,6,7,7,6,6,7,7,
111 | 	8,8,9,9,8,8,9,9,10,10,11,11,10,10,11,11,
112 | 	8,8,9,9,8,8,9,9,10,10,11,11,10,10,11,11,
113 | 	12,12,13,13,12,12,13,13,14,14,15,15,14,14,15,15,
114 | 	12,12,13,13,12,12,13,13,14,14,15,15,14,14,15,15,
115 | 	8,8,9,9,8,8,9,9,10,10,11,11,10,10,11,11,
116 | 	8,8,9,9,8,8,9,9,10,10,11,11,10,10,11,11,
117 | 	12,12,13,13,12,12,13,13,14,14,15,15,14,14,15,15,
118 | 	12,12,13,13,12,12,13,13,14,14,15,15,14,14,15,15
119 | 	};
120 | }
121 | 


--------------------------------------------------------------------------------
/test/unittest_geos.h:
--------------------------------------------------------------------------------
  1 | //
  2 | // Created by Juno Wang on 6/29/21.
  3 | //
  4 | 
  5 | #include <geos/geom/GeometryFactory.h>
  6 | #include <geos/geom/Geometry.h>
  7 | #include <geos/geom/Polygon.h>
  8 | #include <geos/io/WKTReader.h>
  9 | #include "bigtest.h"
 10 | #include <geos/geom/Point.h>
 11 | #include <geos/index/strtree/SimpleSTRtree.h>
 12 | #include <geos/index/strtree/GeometryItemDistance.h>
 13 | #include <geos/index/ItemVisitor.h>
 14 | #include <geos/geom/Envelope.h>
 15 | #include <geos/geom/Coordinate.h>
 16 | #include <geos/geom/CoordinateSequence.h>
 17 | #include <geos/geom/CoordinateArraySequence.h>
 18 | #include <geos/geom/Dimension.h>
 19 | #include <geos/geom/PrecisionModel.h>
 20 | #include <geos/util/IllegalArgumentException.h>
 21 | #include <geos/geom/LineSegment.h>
 22 | #include <geos/index/quadtree/Quadtree.h>
 23 | 
 24 | 
 25 | 
 26 | 
 27 | using namespace geos::geom;
 28 | using geos::geom::Coordinate;
 29 | using geos::geom::Envelope;
 30 | using Polygon = geos::geom::Polygon;
 31 | 
 32 | // Nickname for classes
 33 | typedef std::unique_ptr<geos::geom::Geometry> GeomPtr;
 34 | typedef geos::geom::GeometryFactory GeometryFactory;
 35 | /*
 36 | * Generate a random point in the range of
 37 | * POINT(0..range, 0..range). Caller must
 38 | * free.
 39 | */
 40 | 
 41 | TEST_SUITE("Basic GEOS ops") {
 42 |     TEST_CASE("Test covers") {
 43 |         GeometryFactory::Ptr factory = GeometryFactory::create();
 44 |         geos::io::WKTReader wkt_reader(*factory);
 45 |         std::string wkt_a("POLYGON ((0 0, 0 100, 100 100, 100 0, 0 0))");
 46 |         std::string wkt_b("POLYGON ((0 0, 0 100, 90 90, 90 0, 0 0))");
 47 |         GeomPtr g1(wkt_reader.read(wkt_a));
 48 |         GeomPtr g2(wkt_reader.read(wkt_b));
 49 |         CHECK(g1->covers(g2.get()));
 50 |     }
 51 | 
 52 |     TEST_CASE("Tree"){
 53 |         index::strtree::SimpleSTRtree t(10);
 54 |         std::vector<std::unique_ptr<geom::Geometry>> geoms;
 55 |         const int gridSize = 10;
 56 | 
 57 |         auto gf = geom::GeometryFactory::create();
 58 |         for (int i = 0; i < gridSize; ++i) {
 59 |             for (int j = 0; j < gridSize; ++j) {
 60 |                 geom::Coordinate c((double)i, (double)j);
 61 |                 geom::Point* pt = gf->createPoint(c);
 62 |                 geoms.emplace_back(pt);
 63 |                 t.insert(pt);
 64 |             }
 65 |         }
 66 | 
 67 |         geom::Envelope qe(-0.5, 1.5, -0.5, 1.5);
 68 |         std::vector<void*> matches;
 69 |         t.query(&qe, matches);
 70 |         // std::cout << matches.size() << std::endl;
 71 |         CHECK(matches.size() == 4);
 72 |         t.query(&qe, matches);
 73 |         // std::cout << matches.size() << std::endl;
 74 |         CHECK(matches.size() == 8);
 75 | 
 76 |         // std::cout << t << std::endl;
 77 | 
 78 |     }
 79 |     TEST_CASE("Polygan covers point WKT"){
 80 |         GeometryFactory::Ptr factory = GeometryFactory::create();
 81 |         geos::io::WKTReader reader(*factory);
 82 |         GeomPtr g1(reader.read(
 83 |                 "POLYGON ((-1.183864 52.951915, -1.183862 52.951903, -1.183890 52.951900, -1.183924 52.951897, -1.183958 52.951894, -1.183954 52.951880, -1.183954 52.951878, -1.183932 52.951841, -1.183904 52.951844, -1.183870 52.951847, -1.183832 52.951852, -1.183824 52.951838, -1.183820 52.951830, -1.183870 52.951819, -1.183886 52.951815, -1.183890 52.951819, -1.183929 52.951810, -1.183909 52.951776, -1.183861 52.951787, -1.183853 52.951788, -1.183842 52.951770, -1.183970 52.951742, -1.183983 52.951763, -1.183963 52.951768, -1.183975 52.951788, -1.183994 52.951785, -1.184009 52.951807, -1.184002 52.951808, -1.184009 52.951835, -1.183990 52.951836, -1.183990 52.951836, -1.183990 52.951838, -1.184001 52.951880, -1.184018 52.951954, -1.184020 52.951956, -1.183998 52.951957, -1.183998 52.951956, -1.183996 52.951948, -1.183970 52.951906, -1.183936 52.951909, -1.183864 52.951915))"
 84 |         ));
 85 |         GeomPtr g2(reader.read(
 86 |                 "POINT (-1.183972 52.951871)"
 87 |         ));
 88 |         CHECK(g1->covers(g2.get()));
 89 |         CHECK(g2->coveredBy(g1.get()));
 90 |         CHECK(g1->contains(g2.get()));
 91 |     }
 92 |     TEST_CASE("LineSegment Class with line reverse"){
 93 |         auto gf = geos::geom::GeometryFactory::create();
 94 |         geos::geom::Coordinate proj_point(5, 2.0);
 95 |         geos::geom::Coordinate proj_result(0,0);
 96 |         geos::geom::Coordinate pv1(0,0);
 97 |         geos::geom::Coordinate pv2(5,5);
 98 |         geos::geom::LineSegment seg1(pv1,pv2);
 99 |         CHECK(seg1[0] == pv1);
100 |         seg1.reverse();
101 |         CHECK(seg1[1] == pv1);
102 | 
103 | 
104 |         // can call reverse but dont know why cannot call project
105 |     }
106 |     TEST_CASE("LineSegment projection y=x"){
107 |         geos::geom::Coordinate proj_point(5, 2.0);
108 |         geos::geom::Coordinate proj_result(0,0);
109 |         geos::geom::Coordinate pv1(0,0);
110 |         geos::geom::Coordinate pv2(5,5);
111 |         geos::geom::LineSegment seg1(pv1,pv2);
112 |         seg1.project(proj_point,proj_result);
113 |         CHECK(proj_result.x == 3.5);
114 |         CHECK((proj_result.y) == 3.5);
115 |     }
116 |     TEST_CASE("LineSegment porject y = 3x+2"){
117 |         geos::geom::Coordinate proj_point(5, 2.0);
118 |         geos::geom::Coordinate proj_result(0,0);
119 |         geos::geom::Coordinate pv1(0,2);
120 |         geos::geom::Coordinate pv2(5,17);
121 |         geos::geom::LineSegment seg2(pv1,pv2);
122 |         seg2.project(proj_point,proj_result);
123 |         CHECK(proj_result.x == 0.5);
124 |         CHECK(proj_result.y == 3.5);
125 |     }
126 | }
127 | 
128 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # GLIN
 2 | 
 3 | Check out our preprint: [GLIN: A Lightweight Learned Indexing Mechanism for Complex Geometries](https://arxiv.org/abs/2207.07745)
 4 | 
 5 | ## Introduction
 6 | This paper introduces GLIN, a generic learned index for spatial range queries on complex geometries. To achieve that, GLIN transforms geometries to Z-address intervals, and builds a hierarchical model to learn the cumulative distribution function between these intervals and the record positions. The lightweight hierarchical model greatly shortens the index probing time. Furthermore, GLIN augments spatial range query using an add-on piecewise function to guarantee the query accuracy for both 𝐶𝑜𝑛𝑡𝑎𝑖𝑛𝑠 and 𝐼𝑛𝑡𝑒𝑟𝑠𝑒𝑐𝑡𝑠 spatial relationships.
 7 | 
 8 | ## Highlights
 9 | - GLIN transforms geometries to 1-dimensional sortable values,namely Z-address intervals, using Z-order curve.
10 | - GLIN is the first learned index that handles non-point geospatial values.
11 | - GLIN updates its index structure efficiently with query accuracy guaranteed.
12 | 
13 | ## Code Structure
14 | - GLIN: contains core glin fucntion other than Recursive model index(RMI) as following
15 |   - Z-Address Interval transformation function
16 |   - Index search function and update function (insertion and deletion)
17 |   - Piecewise function initialization, query window augmentation, piecewise function updates
18 | - src: GLIN-customized ALEX recursive model, including nodes, node updates function, and model training.
19 | - test: tests related to GLIN-customized ALEX recursive model and GLIN's function
20 | 
21 | ## Dependencies 
22 | - GEOS: brew install geos
23 | - BOOST C++ library : brew install boost
24 | 
25 | ## Build and Run test
26 | - Compile the project using CMake. It will build a new direcotory named as ``` build```
27 |   ```./build.sh ```
28 | - Run unit tests without query augmentation, for Contains relationship only
29 |  ``` ./build/test_glin ```
30 |   - run unit tests with query augmentation. It supports both Contains and Intersects relationship
31 |   
32 |   ``` ./build/test_glin_piece```
33 | - GitHub Action automatically tests every commit using all unit tests.
34 | 
35 | ## API Doc
36 | - To construct a index for geometry colomn 
37 | 
38 |  ```alex::Glin<double, geos::geom::Geometry *> index```
39 |  - Bulkload to Index
40 | 
41 | ```index.glin_bulk_load(poly_vec, piecelimitation, "z", cell_xmin,cell_ymin, cell_x_intvl, cell_y_intvl, pieces)```
42 | 
43 | - Parameters:
44 |   - poly_vec : a vector of Geometries in std::vector<geos::geom::Geometry *>
45 |   - piecelimitation: a double, limits how many records is summarized in a single piece
46 |   - space-filling-curve : z-order-filling-curve
47 |   - cell_xmin: the minimal value of x in a geometry, for real-world data, set cell_xmin = -180
48 |   - cell_ymin: the minimal value of y in a geometry, for real-world data, set cell_ymin = -180
49 |   - cell_x_intvl, cell_y_intvl:: a small enough value is valid for all data, by default we set to 0.0000005, but it is tunable parameter
50 |   - pieces: vector of piecewise function. If the user does not choose PIECEWISE setting, this value is empty.
51 | 
52 | * Search in index
53 | 
54 | `   index.glin_find(query, "z",  cell_xmin,cell_ymin, cell_x_intvl, cell_y_intvl, pieces, find_result, count_filter)`
55 | * Parameters:
56 |     * querywindow: the query window to perform search with
57 |     * space-filling-curve :  z-order-filling-curve
58 |     * cell_xmin:  the minimal value of x in a geometry, for real-world data, set cell_xmin = -180
59 |     * cell_ymin:  the minimal value of y in a geometry, for real-world data, set cell_ymin = -180
60 |     * cell_x_intvl, cell_y_intvl:: a small enough value is valid for all data, by default we set to 0.0000005, but it is tunable parameter
61 |     * pieces: vector of piecewise function. If the user does not choose PIECEWISE setting, this value is empty.
62 |     * find result:  The result after refine
63 |     * count_filter :  the number of result before refine
64 | 
65 | 
66 | - Insert geometry into index
67 | 
68 | ```index.glin_insert(insert_tuple, "z", cell_xmin,cell_ymin, cell_x_intvl, cell_y_intvl, piece_limit, pieces);```
69 | 
70 |  - Parameters
71 |    - insert_tuple:  A tuple of to-be-inserted-Geometry and its MBR  std::tuple<geos::geom::Geometry*, geos::geom::Envelope*>
72 |    - space-filling-curve :  z-order-filling-curve
73 |    - cell_xmin:  the minimal value of x in a geometry, for real-world data, set cell_xmin = -180
74 |    - cell_ymin:  the minimal value of y in a geometry, for real-world data, set cell_ymin = -180
75 |    - cell_x_intvl, cell_y_intvl:: a small enough value is valid for all data, by default we set to 0.0000005, but it is tunable parameter
76 |    - piece_limit: piece_limitation
77 |    - pieces: vector of piecewise function. If the user does not choose PIECEWISE setting, this value is empty.
78 | 
79 | 
80 | - Delete geometry from index
81 | 
82 |   ` int num_erase = index.erase(poly, "z", cell_xmin,cell_ymin, cell_x_intvl, cell_y_intvl, piecelimitation, pieces);`
83 |     
84 | * Parameters:
85 |   * poly:  A to-be-remove geometry
86 |   * space-filling-curve :  z-order-filling-curve
87 |   * cell_xmin:  the minimal value of x in a geometry, for real-world data, set cell_xmin = -180
88 |   * cell_ymin:  the minimal value of y in a geometry, for real-world data, set cell_ymin = -180
89 |   * cell_x_intvl, cell_y_intvl:: a small enough value is valid for all data, by default we set to 0.0000005, but it is tunable parameter
90 |   * piece_limit: piece_limitation
91 |   * pieces: vector of piecewise function. If the user does not choose PIECEWISE setting, this value is empty.
92 |   * This function return a number of record are erased
93 | 
94 | Example can be found at unittest_glin.h and unittest_glin_maintenance.h
95 | 


--------------------------------------------------------------------------------
/glin/hilbert/hilbert.h:
--------------------------------------------------------------------------------
  1 | /* C header file for Hilbert curve functions */
  2 | #if !defined(_hilbert_h_)
  3 | #define _hilbert_h_
  4 | 
  5 | #ifdef __cplusplus
  6 | extern "C" {
  7 | #endif
  8 | 
  9 | /* define the bitmask_t type as an integer of sufficient size */
 10 | typedef unsigned long long bitmask_t;
 11 | /* define the halfmask_t type as an integer of 1/2 the size of bitmask_t */
 12 | typedef unsigned long halfmask_t;
 13 | 
 14 | /*****************************************************************
 15 |  * hilbert_i2c
 16 |  * 
 17 |  * Convert an index into a Hilbert curve to a set of coordinates.
 18 |  * Inputs:
 19 |  *  nDims:      Number of coordinate axes.
 20 |  *  nBits:      Number of bits per axis.
 21 |  *  index:      The index, contains nDims*nBits bits (so nDims*nBits must be <= 8*sizeof(bitmask_t)).
 22 |  * Outputs:
 23 |  *  coord:      The list of nDims coordinates, each with nBits bits.
 24 |  * Assumptions:
 25 |  *      nDims*nBits <= (sizeof index) * (bits_per_byte)
 26 |  */
 27 | 
 28 | void hilbert_i2c(unsigned nDims, unsigned nBits, bitmask_t index, bitmask_t coord[]);
 29 | 
 30 | /*****************************************************************
 31 |  * hilbert_c2i
 32 |  * 
 33 |  * Convert coordinates of a point on a Hilbert curve to its index.
 34 |  * Inputs:
 35 |  *  nDims:      Number of coordinates.
 36 |  *  nBits:      Number of bits/coordinate.
 37 |  *  coord:      Array of n nBits-bit coordinates.
 38 |  * Outputs:
 39 |  *  index:      Output index value.  nDims*nBits bits.
 40 |  * Assumptions:
 41 |  *      nDims*nBits <= (sizeof bitmask_t) * (bits_per_byte)
 42 |  */
 43 | 
 44 | bitmask_t hilbert_c2i(unsigned nDims, unsigned nBits, bitmask_t const coord[]);
 45 | 
 46 | /*****************************************************************
 47 |  * hilbert_cmp, hilbert_ieee_cmp
 48 |  * 
 49 |  * Determine which of two points lies further along the Hilbert curve
 50 |  * Inputs:
 51 |  *  nDims:      Number of coordinates.
 52 |  *  nBytes:     Number of bytes of storage/coordinate (hilbert_cmp only)
 53 |  *  nBits:      Number of bits/coordinate. (hilbert_cmp only)
 54 |  *  coord1:     Array of nDims nBytes-byte coordinates (or doubles for ieee_cmp).
 55 |  *  coord2:     Array of nDims nBytes-byte coordinates (or doubles for ieee_cmp).
 56 |  * Return value:
 57 |  *      -1, 0, or 1 according to whether
 58 |            coord1<coord2, coord1==coord2, coord1>coord2
 59 |  * Assumptions:
 60 |  *      nBits <= (sizeof bitmask_t) * (bits_per_byte)
 61 |  */
 62 | 
 63 | int hilbert_cmp(unsigned nDims, unsigned nBytes, unsigned nBits, void const* coord1, void const* coord2);
 64 | int hilbert_ieee_cmp(unsigned nDims, double const* coord1, double const* coord2);
 65 | 
 66 | /*****************************************************************
 67 |  * hilbert_box_vtx
 68 |  * 
 69 |  * Determine the first or last vertex of a box to lie on a Hilbert curve
 70 |  * Inputs:
 71 |  *  nDims:      Number of coordinates.
 72 |  *  nBytes:     Number of bytes/coordinate.
 73 |  *  nBits:      Number of bits/coordinate. (hilbert_cmp only)
 74 |  *  findMin:    Is it the least vertex sought?
 75 |  *  coord1:     Array of nDims nBytes-byte coordinates - one corner of box
 76 |  *  coord2:     Array of nDims nBytes-byte coordinates - opposite corner
 77 |  * Output:
 78 |  *      c1 and c2 modified to refer to selected corner
 79 |  *      value returned is log2 of size of largest power-of-two-aligned box that
 80 |  *      contains the selected corner and no other corners
 81 |  * Assumptions:
 82 |  *      nBits <= (sizeof bitmask_t) * (bits_per_byte)
 83 |  */
 84 | unsigned
 85 | hilbert_box_vtx(unsigned nDims, unsigned nBytes, unsigned nBits,
 86 |                 int findMin, void* c1, void* c2);
 87 | unsigned
 88 | hilbert_ieee_box_vtx(unsigned nDims,
 89 |                      int findMin, double* c1, double* c2);
 90 | 
 91 | /*****************************************************************
 92 |  * hilbert_box_pt
 93 |  * 
 94 |  * Determine the first or last point of a box to lie on a Hilbert curve
 95 |  * Inputs:
 96 |  *  nDims:      Number of coordinates.
 97 |  *  nBytes:     Number of bytes/coordinate.
 98 |  *  nBits:      Number of bits/coordinate.
 99 |  *  findMin:    Is it the least vertex sought?
100 |  *  coord1:     Array of nDims nBytes-byte coordinates - one corner of box
101 |  *  coord2:     Array of nDims nBytes-byte coordinates - opposite corner
102 |  * Output:
103 |  *      c1 and c2 modified to refer to least point
104 |  * Assumptions:
105 |  *      nBits <= (sizeof bitmask_t) * (bits_per_byte)
106 |  */
107 | unsigned
108 | hilbert_box_pt(unsigned nDims, unsigned nBytes, unsigned nBits,
109 |                int findMin, void* coord1, void* coord2);
110 | unsigned
111 | hilbert_ieee_box_pt(unsigned nDims,
112 |                     int findMin, double* c1, double* c2);
113 | 
114 | /*****************************************************************
115 |  * hilbert_nextinbox
116 |  * 
117 |  * Determine the first point of a box after a given point to lie on a Hilbert curve
118 |  * Inputs:
119 |  *  nDims:      Number of coordinates.
120 |  *  nBytes:     Number of bytes/coordinate.
121 |  *  nBits:      Number of bits/coordinate.
122 |  *  findPrev:   Is the previous point sought?
123 |  *  coord1:     Array of nDims nBytes-byte coordinates - one corner of box
124 |  *  coord2:     Array of nDims nBytes-byte coordinates - opposite corner
125 |  *  point:      Array of nDims nBytes-byte coordinates - lower bound on point returned
126 |  *  
127 |  * Output:
128 |       if returns 1:
129 |  *      c1 and c2 modified to refer to least point after "point" in box
130 |       else returns 0:
131 |         arguments unchanged; "point" is beyond the last point of the box
132 |  * Assumptions:
133 |  *      nBits <= (sizeof bitmask_t) * (bits_per_byte)
134 |  */
135 | int
136 | hilbert_nextinbox(unsigned nDims, unsigned nBytes, unsigned nBits,
137 |                   int findPrev, void* coord1, void* coord2,
138 |                   void const* point);
139 | 
140 | /*****************************************************************
141 |  * hilbert_incr
142 |  * 
143 |  * Advance from one point to its successor on a Hilbert curve
144 |  * Inputs:
145 |  *  nDims:      Number of coordinates.
146 |  *  nBits:      Number of bits/coordinate.
147 |  *  coord:      Array of nDims nBits-bit coordinates.
148 |  * Output:
149 |  *  coord:      Next point on Hilbert curve
150 |  * Assumptions:
151 |  *      nBits <= (sizeof bitmask_t) * (bits_per_byte)
152 |  */
153 | 
154 | void
155 | hilbert_incr(unsigned nDims, unsigned nBits, bitmask_t coord[]);
156 | 
157 | #ifdef __cplusplus
158 | }
159 | #endif
160 | 
161 | #endif /* _hilbert_h_ */


--------------------------------------------------------------------------------
/test/unittest_glin_line_projct.h:
--------------------------------------------------------------------------------
  1 | //
  2 | // Created by Congying Wang on 10/25/21.
  3 | //
  4 | #include "doctest.h"
  5 | #include <geos/geom/GeometryFactory.h>
  6 | #include <geos/geom/Geometry.h>
  7 | #include <geos/geom/Polygon.h>
  8 | #include <geos/geom/Coordinate.h>
  9 | #include <geos/geom/CoordinateSequence.h>
 10 | #include <geos/geom/CoordinateArraySequence.h>
 11 | #include <geos/geom/LinearRing.h>
 12 | 
 13 | TEST_SUITE("Glin projection") {
 14 |     TEST_CASE("projection") {
 15 |         // We will use a coordinate list to build the linearring
 16 |         std::unique_ptr<geos::geom::PrecisionModel> pm(new geos::geom::PrecisionModel());
 17 |         geos::geom::GeometryFactory::Ptr global_factory = geos::geom::GeometryFactory::create(pm.get(), -1);;
 18 |         geos::geom::CoordinateArraySequence *cl = new CoordinateArraySequence();
 19 |         cl->add(geos::geom::Coordinate(0, 0));
 20 |         cl->add(geos::geom::Coordinate(0, 6));
 21 |         cl->add(geos::geom::Coordinate(6, 6));
 22 |         cl->add(geos::geom::Coordinate(6, 0));
 23 |         cl->add(geos::geom::Coordinate(0, 0));
 24 | 
 25 |         /*
 26 |          * Now that we have a CoordinateSequence we can create the linearring.
 27 |          * The newly created LinearRing will take ownershipof the CoordinateSequence.
 28 |          * linearring for outer shell
 29 |          */
 30 |         geos::geom::LinearRing *lr;
 31 |         lr = global_factory->createLinearRing(cl);
 32 |         // We need a LinearRing for the polygon shell
 33 |         geos::geom::LinearRing *outer = lr;
 34 |         //linearring for inner hole
 35 |         geos::geom::CoordinateArraySequence *cl1 = new CoordinateArraySequence();
 36 |         cl1->add(geos::geom::Coordinate(2, 2));
 37 |         cl1->add(geos::geom::Coordinate(2, 4));
 38 |         cl1->add(geos::geom::Coordinate(4, 4));
 39 |         cl1->add(geos::geom::Coordinate(4, 2));
 40 |         cl1->add(geos::geom::Coordinate(2, 2));
 41 |         geos::geom::LinearRing *lr1 = global_factory->createLinearRing(cl1);
 42 |         geos::geom::LinearRing *inner = lr1;
 43 |         /*
 44 |          * If we need to specify any hole, we do it using
 45 |          a vector of Geometry pointers
 46 |          */
 47 |         vector<geos::geom::LinearRing *> *holes = new vector<geos::geom::LinearRing *>;
 48 | 
 49 |         // We add the newly created geometry to the vector
 50 |         // of holes.
 51 |         holes->push_back(inner);
 52 |         geos::geom::Polygon *poly = global_factory->createPolygon(outer, holes);
 53 |         geos::geom::Coordinate pv1(0, 0);
 54 |         geos::geom::Coordinate pv2(0, 6);
 55 |         geos::geom::LineSegment seg1(pv1, pv2);
 56 |         long double min = 0;
 57 |         long double max = 0;
 58 |         shape_projection(poly, seg1, min, max);
 59 |         max = std::ceil(max * 100.0) / 100.0;
 60 |         min = std::ceil(min * 100.0) / 100.0;
 61 | //        std::cout<< min << std::endl;
 62 | //        std::cout << max << std::endl;
 63 | 
 64 |         // CHECK( max == 8.49);
 65 |         CHECK(min == 0);
 66 |     }
 67 | 
 68 |     TEST_CASE("project shape on y = 3x+2") {
 69 |         // create the polygon
 70 |         std::unique_ptr<geos::geom::PrecisionModel> pm(new geos::geom::PrecisionModel());
 71 |         geos::geom::GeometryFactory::Ptr global_factory = geos::geom::GeometryFactory::create(pm.get(), -1);
 72 |         geos::geom::CoordinateArraySequence *cl = new CoordinateArraySequence();
 73 |         cl->add(geos::geom::Coordinate(0, 0));
 74 |         cl->add(geos::geom::Coordinate(0, 6));
 75 |         cl->add(geos::geom::Coordinate(6, 6));
 76 |         cl->add(geos::geom::Coordinate(6, 0));
 77 |         cl->add(geos::geom::Coordinate(0, 0));
 78 |         geos::geom::LinearRing *lr1 = global_factory->createLinearRing(cl);
 79 |         geos::geom::LinearRing *outer = lr1;
 80 | 
 81 |         vector<geos::geom::LinearRing *> *holes = new vector<geos::geom::LinearRing *>;
 82 |         geos::geom::Polygon *poly = global_factory->createPolygon(outer, holes);
 83 | 
 84 |         geos::geom::Coordinate pv1(0, 0);
 85 |         geos::geom::Coordinate pv2(5, 17);
 86 |         geos::geom::LineSegment seg2(pv1, pv2);
 87 |         long double min;
 88 |         long double max;
 89 |         shape_projection(poly, seg2, min, max);
 90 | 
 91 |         max = std::ceil(max * 100.0) / 100.0;
 92 |         min = std::ceil(min * 100.0) / 100.0;
 93 | //        CHECK(max == 7.44999999999999999982);
 94 |         CHECK(min == 0);
 95 |     }
 96 | 
 97 |     TEST_CASE("Projection result to alex") {
 98 |         // insert project result to alex index
 99 |         std::unique_ptr<geos::geom::PrecisionModel> pm(new geos::geom::PrecisionModel());
100 |         geos::geom::GeometryFactory::Ptr global_factory = geos::geom::GeometryFactory::create(pm.get(), -1);
101 |         geos::geom::CoordinateArraySequence *cl = new CoordinateArraySequence();
102 |         cl->add(geos::geom::Coordinate(0, 0));
103 |         cl->add(geos::geom::Coordinate(0, 6));
104 |         cl->add(geos::geom::Coordinate(6, 6));
105 |         cl->add(geos::geom::Coordinate(6, 0));
106 |         cl->add(geos::geom::Coordinate(0, 0));
107 |         geos::geom::LinearRing *lr1 = global_factory->createLinearRing(cl);
108 |         geos::geom::LinearRing *outer = lr1;
109 | 
110 |         vector<geos::geom::LinearRing *> *holes = new vector<geos::geom::LinearRing *>;
111 |         geos::geom::Polygon *poly = global_factory->createPolygon(outer, holes);
112 |         geos::geom::Coordinate pv1(0, 0);
113 |         geos::geom::Coordinate pv2(5, 17);
114 |         geos::geom::LineSegment seg2(pv1, pv2);
115 |         long double dist_min;
116 |         long double dist_max;
117 |         shape_projection(poly, seg2, dist_min, dist_max);
118 |         dist_max = std::ceil(dist_max * 100.0) / 100.0;
119 |         dist_min = std::ceil(dist_min * 100.0) / 100.0;
120 | 
121 |         geos::geom::Coordinate pv3(0, 0);
122 |         geos::geom::Coordinate pv4(6, 6);
123 |         geos::geom::LineSegment seg1(pv3, pv4);
124 |         long double min;
125 |         long double max;
126 |         shape_projection(poly, seg1, min, max);
127 | 
128 |         Alex<double, double> index;
129 |         Alex<double, double>::V values[2];
130 | 
131 |         values[0].first = min;
132 |         values[0].second = max;
133 |         values[1].first = dist_min;
134 |         values[1].second = dist_max;
135 |         std::sort(values, values + 2);
136 |         index.bulk_load(values, 2);
137 |         for (int i = 0; i < 2; i++) {
138 |             auto it = index.find(values[i].first);
139 |             CHECK_EQ(values[i].first, it.key());
140 |         }
141 |     }
142 |     TEST_CASE("find perpendicular") {
143 |         alex::Glin<double, geos::geom::Geometry *> index1;
144 |         geos::geom::LineSegment seg_45 = index1.create_line_seg(0, 5, 2);
145 |         geos::geom::LineSegment seg_perpen = index1.get_perpendicular_line(seg_45);
146 | //        std::cout<< "original "<< geos::algorithm::Angle::toDegrees(seg_45.angle()) << " perpendicular " <<geos::algorithm::Angle::toDegrees(seg_perpen.angle()) << endl;
147 | //        std::cout << "angle" <<geos::algorithm::Angle::toDegrees(geos::algorithm::Angle::diff(seg_45.angle(),seg_perpen.angle()));
148 |         CHECK(geos::algorithm::Angle::toDegrees(geos::algorithm::Angle::diff(seg_45.angle(), seg_perpen.angle())) == 90);
149 |         double k1 = (seg_45.p1.y - seg_45.p0.y) / (seg_45.p1.x - seg_45.p0.x);
150 | //        std::cout << k1<< endl;
151 |         double k2 = (seg_perpen.p1.y - seg_perpen.p0.y) / (seg_perpen.p1.x - seg_perpen.p0.x);
152 | //        std::cout << k2 << endl;
153 |         CHECK(k1 * k2 == -1);
154 |     }
155 | };
156 | 


--------------------------------------------------------------------------------
/src/core/alex_multimap.h:
--------------------------------------------------------------------------------
  1 | // Copyright (c) Microsoft Corporation.
  2 | // Licensed under the MIT license.
  3 | 
  4 | /*
  5 |  * Implements the STL multimap. It can be used as a near drop-in
  6 |  * replacement for std::multimap, with a few important differences:
  7 |  * 1) The iterators are ForwardIterators instead of BidirectionalIterators.
  8 |  * 2) Keys and payloads are stored separately, so dereferencing the iterator
  9 |  *  does not return a reference.
 10 |  */
 11 | 
 12 | #pragma once
 13 | 
 14 | #include "alex.h"
 15 | 
 16 | namespace alex {
 17 | 
 18 | template <class T, class P, class Compare = AlexCompare,
 19 |           class Alloc = std::allocator<std::pair<T, P>>>
 20 | class AlexMultimap {
 21 |   static_assert(std::is_arithmetic<T>::value, "ALEX key type must be numeric.");
 22 |   static_assert(std::is_same<Compare,AlexCompare>::value, "Must use AlexCompare.");
 23 | 
 24 |  public:
 25 |   // Value type, returned by dereferencing an iterator
 26 |   typedef std::pair<T, P> V;
 27 | 
 28 |   // ALEX class aliases
 29 |   typedef AlexMultimap<T, P, Compare, Alloc> self_type;
 30 |   typedef Alex<T, P, Compare, Alloc, true> alex_impl;
 31 |   typedef typename alex_impl::Iterator iterator;
 32 |   typedef typename alex_impl::ConstIterator const_iterator;
 33 |   typedef typename alex_impl::ReverseIterator reverse_iterator;
 34 |   typedef typename alex_impl::ConstReverseIterator const_reverse_iterator;
 35 | 
 36 |  private:
 37 |   alex_impl alex_;
 38 | 
 39 |   /*** Constructors and setters ***/
 40 | 
 41 |  public:
 42 |   AlexMultimap() : alex_() {}
 43 | 
 44 |   AlexMultimap(const Compare& comp, const Alloc& alloc = Alloc())
 45 |       : alex_(comp, alloc) {}
 46 | 
 47 |   AlexMultimap(const Alloc& alloc) : alex_(alloc) {}
 48 | 
 49 |   ~AlexMultimap() {}
 50 | 
 51 |   // Initializes with range [first, last). The range does not need to be
 52 |   // sorted. This creates a temporary copy of the data. If possible, we
 53 |   // recommend directly using load() instead.
 54 |   template <class InputIterator>
 55 |   explicit AlexMultimap(InputIterator first, InputIterator last,
 56 |                         const Compare& comp, const Alloc& alloc = Alloc())
 57 |       : alex_(first, last, comp, alloc) {}
 58 | 
 59 |   // Initializes with range [first, last). The range does not need to be
 60 |   // sorted. This creates a temporary copy of the data. If possible, we
 61 |   // recommend directly using load() instead.
 62 |   template <class InputIterator>
 63 |   explicit AlexMultimap(InputIterator first, InputIterator last,
 64 |                         const Alloc& alloc = Alloc())
 65 |       : alex_(first, last, alloc) {}
 66 | 
 67 |   explicit AlexMultimap(const self_type& other) : alex_(other.alex_) {}
 68 | 
 69 |   AlexMultimap& operator=(const self_type& other) {
 70 |     if (this != &other) {
 71 |       alex_ = other.alex_;
 72 |     }
 73 |     return *this;
 74 |   }
 75 | 
 76 |   void swap(const self_type& other) { alex_.swap(other.alex_); }
 77 | 
 78 |  public:
 79 |   // When bulk loading, Alex can use provided knowledge of the expected fraction
 80 |   // of operations that will be inserts
 81 |   // For simplicity, operations are either point lookups ("reads") or inserts
 82 |   // ("writes)
 83 |   // i.e., 0 means we expect a read-only workload, 1 means write-only
 84 |   // This is only useful if you set it before bulk loading
 85 |   void set_expected_insert_frac(double expected_insert_frac) {
 86 |     alex_.set_expected_insert_frac(expected_insert_frac);
 87 |   }
 88 | 
 89 |   // Maximum node size, in bytes.
 90 |   // Higher values result in better average throughput, but worse tail/max
 91 |   // insert latency.
 92 |   void set_max_node_size(int max_node_size) {
 93 |     alex_.set_max_node_size(max_node_size);
 94 |   }
 95 | 
 96 |   // Bulk load faster by using sampling to train models.
 97 |   // This is only useful if you set it before bulk loading.
 98 |   void set_approximate_model_computation(bool approximate_model_computation) {
 99 |     alex_.set_approximate_model_computation(approximate_model_computation);
100 |   }
101 | 
102 |   // Bulk load faster by using sampling to compute cost.
103 |   // This is only useful if you set it before bulk loading.
104 |   void set_approximate_cost_computation(bool approximate_cost_computation) {
105 |     alex_.set_approximate_cost_computation(approximate_cost_computation);
106 |   }
107 | 
108 |   /*** Allocators and comparators ***/
109 | 
110 |  public:
111 |   Alloc get_allocator() const { return alex_.get_allocator(); }
112 | 
113 |   Compare key_comp() const { return alex_.key_comp(); }
114 | 
115 |   /*** Bulk loading ***/
116 | 
117 |  public:
118 |   // values should be the sorted array of key-payload pairs.
119 |   // The number of elements should be num_keys.
120 |   // The index must be empty when calling this method.
121 |   void bulk_load(const V values[], int num_keys) {
122 |     alex_.bulk_load(values, num_keys);
123 |   }
124 | 
125 |   /*** Lookup ***/
126 | 
127 |  public:
128 |   // Looks for an exact match of the key
129 |   // If the key does not exist, returns an end iterator
130 |   // If there are multiple keys with the same value, returns an iterator to the
131 |   // right-most key
132 |   // If you instead want an iterator to the left-most key with the input value,
133 |   // use lower_bound()
134 |   iterator find(const T& key) { return alex_.find(key); }
135 | 
136 |   const_iterator find(const T& key) const { return alex_.find(key); }
137 | 
138 |   size_t count(const T& key) { return alex_.size(key); }
139 | 
140 |   // Returns an iterator to the first key no less than the input value
141 |   iterator lower_bound(const T& key) { return alex_.lower_bound(key); }
142 | 
143 |   const_iterator lower_bound(const T& key) const {
144 |     return alex_.lower_bound(key);
145 |   }
146 | 
147 |   // Returns an iterator to the first key greater than the input value
148 |   iterator upper_bound(const T& key) { return alex_.upper_bound(key); }
149 | 
150 |   const_iterator upper_bound(const T& key) const {
151 |     return alex_.upper_bound(key);
152 |   }
153 | 
154 |   std::pair<iterator, iterator> equal_range(const T& key) {
155 |     return alex_.equal_range(key);
156 |   }
157 | 
158 |   std::pair<const_iterator, const_iterator> equal_range(const T& key) const {
159 |     return alex_.equal_range(key);
160 |   }
161 | 
162 |   iterator begin() { return alex_.begin(); }
163 | 
164 |   iterator end() { return alex_.end(); }
165 | 
166 |   const_iterator cbegin() const { return alex_.cbegin(); }
167 | 
168 |   const_iterator cend() const { return alex_.cend(); }
169 | 
170 |   reverse_iterator rbegin() { return alex_.rbegin(); }
171 | 
172 |   reverse_iterator rend() { return alex_.rend(); }
173 | 
174 |   const_reverse_iterator crbegin() const { return alex_.crbegin(); }
175 | 
176 |   const_reverse_iterator crend() const { return alex_.crend(); }
177 | 
178 |   /*** Insert ***/
179 | 
180 |  public:
181 |   iterator insert(const V& value) { return alex_.insert(value).first; }
182 | 
183 |   template <class InputIterator>
184 |   void insert(InputIterator first, InputIterator last) {
185 |     alex_.insert(first, last);
186 |   }
187 | 
188 |   // This will NOT do an update of an existing key.
189 |   // To perform an update or read-modify-write, do a lookup and modify the
190 |   // payload's value.
191 |   iterator insert(const T& key, const P& payload) {
192 |     return alex_.insert(key, payload).first;
193 |   }
194 | 
195 |   /*** Delete ***/
196 | 
197 |  public:
198 |   // Erases all keys with a certain key value
199 |   int erase(const T& key) { return alex_.erase(key); }
200 | 
201 |   // Erases element pointed to by iterator
202 |   void erase(iterator it) { alex_.erase(it); }
203 | 
204 |   // Removes all elements
205 |   void clear() { alex_.clear(); }
206 | 
207 |   /*** Stats ***/
208 | 
209 |  public:
210 |   // Number of elements
211 |   size_t size() const { return alex_.size(); }
212 | 
213 |   // True if there are no elements
214 |   bool empty() const { return alex_.empty(); }
215 | 
216 |   // This is just a function required by the STL standard. ALEX can hold more
217 |   // items.
218 |   size_t max_size() const { return alex_.max_size(); }
219 | 
220 |   // Return a const reference to the current statistics
221 |   const struct alex_impl::Stats& get_stats() const { return alex_.stats_; }
222 | };
223 | }
224 | 


--------------------------------------------------------------------------------
/src/benchmark/main.cpp:
--------------------------------------------------------------------------------
  1 | // Copyright (c) Microsoft Corporation.
  2 | // Licensed under the MIT license.
  3 | 
  4 | /*
  5 |  * Simple benchmark that runs a mixture of point lookups and inserts on ALEX.
  6 |  */
  7 | 
  8 | #include "../core/alex.h"
  9 | 
 10 | #include <iomanip>
 11 | 
 12 | #include "flags.h"
 13 | #include "utils.h"
 14 | 
 15 | // Modify these if running your own workload
 16 | #define KEY_TYPE double
 17 | #define PAYLOAD_TYPE double
 18 | 
 19 | /*
 20 |  * Required flags:
 21 |  * --keys_file              path to the file that contains keys
 22 |  * --keys_file_type         file type of keys_file (options: binary or text)
 23 |  * --init_num_keys          number of keys to bulk load with
 24 |  * --total_num_keys         total number of keys in the keys file
 25 |  * --batch_size             number of operations (lookup or insert) per batch
 26 |  *
 27 |  * Optional flags:
 28 |  * --insert_frac            fraction of operations that are inserts (instead of
 29 |  * lookups)
 30 |  * --lookup_distribution    lookup keys distribution (options: uniform or zipf)
 31 |  * --time_limit             time limit, in minutes
 32 |  * --print_batch_stats      whether to output stats for each batch
 33 |  */
 34 | int main(int argc, char* argv[]) {
 35 |   auto flags = parse_flags(argc, argv);
 36 |   std::string keys_file_path = get_required(flags, "keys_file");
 37 |   std::string keys_file_type = get_required(flags, "keys_file_type");
 38 |   auto init_num_keys = stoi(get_required(flags, "init_num_keys"));
 39 |   auto total_num_keys = stoi(get_required(flags, "total_num_keys"));
 40 |   auto batch_size = stoi(get_required(flags, "batch_size"));
 41 |   auto insert_frac = stod(get_with_default(flags, "insert_frac", "0.5"));
 42 |   std::string lookup_distribution =
 43 |       get_with_default(flags, "lookup_distribution", "zipf");
 44 |   auto time_limit = stod(get_with_default(flags, "time_limit", "0.5"));
 45 |   bool print_batch_stats = get_boolean_flag(flags, "print_batch_stats");
 46 | 
 47 |   // Read keys from file
 48 |   auto keys = new KEY_TYPE[total_num_keys];
 49 |   if (keys_file_type == "binary") {
 50 |     load_binary_data(keys, total_num_keys, keys_file_path);
 51 |   } else if (keys_file_type == "text") {
 52 |     load_text_data(keys, total_num_keys, keys_file_path);
 53 |   } else {
 54 |     std::cerr << "--keys_file_type must be either 'binary' or 'text'"
 55 |               << std::endl;
 56 |     return 1;
 57 |   }
 58 | 
 59 |   // Combine bulk loaded keys with randomly generated payloads
 60 |   auto values = new std::pair<KEY_TYPE, PAYLOAD_TYPE>[init_num_keys];
 61 |   std::mt19937_64 gen_payload(std::random_device{}());
 62 |   for (int i = 0; i < init_num_keys; i++) {
 63 |     values[i].first = keys[i];
 64 |     values[i].second = static_cast<PAYLOAD_TYPE>(gen_payload());
 65 |   }
 66 | 
 67 |   // Create ALEX and bulk load
 68 |   alex::Alex<KEY_TYPE, PAYLOAD_TYPE> index;
 69 |   std::sort(values, values + init_num_keys,
 70 |             [](auto const& a, auto const& b) { return a.first < b.first; });
 71 |   index.bulk_load(values, init_num_keys);
 72 | 
 73 |   // Run workload
 74 |   int i = init_num_keys;
 75 |   long long cumulative_inserts = 0;
 76 |   long long cumulative_lookups = 0;
 77 |   int num_inserts_per_batch = static_cast<int>(batch_size * insert_frac);
 78 |   int num_lookups_per_batch = batch_size - num_inserts_per_batch;
 79 |   double cumulative_insert_time = 0;
 80 |   double cumulative_lookup_time = 0;
 81 | 
 82 |   auto workload_start_time = std::chrono::high_resolution_clock::now();
 83 |   int batch_no = 0;
 84 |   PAYLOAD_TYPE sum = 0;
 85 |   std::cout << std::scientific;
 86 |   std::cout << std::setprecision(3);
 87 |   while (true) {
 88 |     batch_no++;
 89 | 
 90 |     // Do lookups
 91 |     double batch_lookup_time = 0.0;
 92 |     if (i > 0) {
 93 |       KEY_TYPE* lookup_keys = nullptr;
 94 |       if (lookup_distribution == "uniform") {
 95 |         lookup_keys = get_search_keys(keys, i, num_lookups_per_batch);
 96 |       } else if (lookup_distribution == "zipf") {
 97 |         lookup_keys = get_search_keys_zipf(keys, i, num_lookups_per_batch);
 98 |       } else {
 99 |         std::cerr << "--lookup_distribution must be either 'uniform' or 'zipf'"
100 |                   << std::endl;
101 |         return 1;
102 |       }
103 |       auto lookups_start_time = std::chrono::high_resolution_clock::now();
104 |       for (int j = 0; j < num_lookups_per_batch; j++) {
105 |         KEY_TYPE key = lookup_keys[j];
106 |         PAYLOAD_TYPE* payload = index.get_payload(key);
107 |         if (payload) {
108 |           sum += *payload;
109 |         }
110 |       }
111 |       auto lookups_end_time = std::chrono::high_resolution_clock::now();
112 |       batch_lookup_time = std::chrono::duration_cast<std::chrono::nanoseconds>(
113 |                               lookups_end_time - lookups_start_time)
114 |                               .count();
115 |       cumulative_lookup_time += batch_lookup_time;
116 |       cumulative_lookups += num_lookups_per_batch;
117 |       delete[] lookup_keys;
118 |     }
119 | 
120 |     // Do inserts
121 |     int num_actual_inserts =
122 |         std::min(num_inserts_per_batch, total_num_keys - i);
123 |     int num_keys_after_batch = i + num_actual_inserts;
124 |     auto inserts_start_time = std::chrono::high_resolution_clock::now();
125 |     for (; i < num_keys_after_batch; i++) {
126 |       index.insert(keys[i], static_cast<PAYLOAD_TYPE>(gen_payload()));
127 |     }
128 |     auto inserts_end_time = std::chrono::high_resolution_clock::now();
129 |     double batch_insert_time =
130 |         std::chrono::duration_cast<std::chrono::nanoseconds>(inserts_end_time -
131 |                                                              inserts_start_time)
132 |             .count();
133 |     cumulative_insert_time += batch_insert_time;
134 |     cumulative_inserts += num_actual_inserts;
135 | 
136 |     if (print_batch_stats) {
137 |       int num_batch_operations = num_lookups_per_batch + num_actual_inserts;
138 |       double batch_time = batch_lookup_time + batch_insert_time;
139 |       long long cumulative_operations = cumulative_lookups + cumulative_inserts;
140 |       double cumulative_time = cumulative_lookup_time + cumulative_insert_time;
141 |       std::cout << "Batch " << batch_no
142 |                 << ", cumulative ops: " << cumulative_operations
143 |                 << "\n\tbatch throughput:\t"
144 |                 << num_lookups_per_batch / batch_lookup_time * 1e9
145 |                 << " lookups/sec,\t"
146 |                 << num_actual_inserts / batch_insert_time * 1e9
147 |                 << " inserts/sec,\t" << num_batch_operations / batch_time * 1e9
148 |                 << " ops/sec"
149 |                 << "\n\tcumulative throughput:\t"
150 |                 << cumulative_lookups / cumulative_lookup_time * 1e9
151 |                 << " lookups/sec,\t"
152 |                 << cumulative_inserts / cumulative_insert_time * 1e9
153 |                 << " inserts/sec,\t"
154 |                 << cumulative_operations / cumulative_time * 1e9 << " ops/sec"
155 |                 << std::endl;
156 |     }
157 | 
158 |     // Check for workload end conditions
159 |     if (num_actual_inserts < num_inserts_per_batch) {
160 |       // End if we have inserted all keys in a workload with inserts
161 |       break;
162 |     }
163 |     double workload_elapsed_time =
164 |         std::chrono::duration_cast<std::chrono::nanoseconds>(
165 |             std::chrono::high_resolution_clock::now() - workload_start_time)
166 |             .count();
167 |     if (workload_elapsed_time > time_limit * 1e9 * 60) {
168 |       break;
169 |     }
170 |   }
171 | 
172 |   long long cumulative_operations = cumulative_lookups + cumulative_inserts;
173 |   double cumulative_time = cumulative_lookup_time + cumulative_insert_time;
174 |   std::cout << "Cumulative stats: " << batch_no << " batches, "
175 |             << cumulative_operations << " ops (" << cumulative_lookups
176 |             << " lookups, " << cumulative_inserts << " inserts)"
177 |             << "\n\tcumulative throughput:\t"
178 |             << cumulative_lookups / cumulative_lookup_time * 1e9
179 |             << " lookups/sec,\t"
180 |             << cumulative_inserts / cumulative_insert_time * 1e9
181 |             << " inserts/sec,\t"
182 |             << cumulative_operations / cumulative_time * 1e9 << " ops/sec"
183 |             << std::endl;
184 | 
185 |   delete[] keys;
186 |   delete[] values;
187 | }
188 | 


--------------------------------------------------------------------------------
/src/core/alex_map.h:
--------------------------------------------------------------------------------
  1 | // Copyright (c) Microsoft Corporation.
  2 | // Licensed under the MIT license.
  3 | 
  4 | /*
  5 |  * Implements the STL map. It can be used as a near drop-in
  6 |  * replacement for std::map, with a few important differences:
  7 |  * 1) The iterators are ForwardIterators instead of BidirectionalIterators.
  8 |  * 2) Keys and payloads are stored separately, so dereferencing the iterator
  9 |  *  does not return a reference.
 10 |  */
 11 | 
 12 | #pragma once
 13 | 
 14 | #include "alex.h"
 15 | 
 16 | namespace alex {
 17 | 
 18 | template <class T, class P, class Compare = AlexCompare,
 19 |           class Alloc = std::allocator<std::pair<T, P>>>
 20 | class AlexMap {
 21 |   static_assert(std::is_arithmetic<T>::value, "ALEX key type must be numeric.");
 22 |   static_assert(std::is_same<Compare,AlexCompare>::value, "Must use AlexCompare.");
 23 | 
 24 |  public:
 25 |   // Value type, returned by dereferencing an iterator
 26 |   typedef std::pair<T, P> V;
 27 | 
 28 |   // ALEX class aliases
 29 |   typedef AlexMap<T, P, Compare, Alloc> self_type;
 30 |   typedef Alex<T, P, Compare, Alloc, false> alex_impl;
 31 |   typedef typename alex_impl::Iterator iterator;
 32 |   typedef typename alex_impl::ConstIterator const_iterator;
 33 |   typedef typename alex_impl::ReverseIterator reverse_iterator;
 34 |   typedef typename alex_impl::ConstReverseIterator const_reverse_iterator;
 35 | 
 36 |  private:
 37 |   alex_impl alex_;
 38 | 
 39 |   /*** Constructors and setters ***/
 40 | 
 41 |  public:
 42 |   AlexMap() : alex_() {}
 43 | 
 44 |   AlexMap(const Compare& comp, const Alloc& alloc = Alloc())
 45 |       : alex_(comp, alloc) {}
 46 | 
 47 |   AlexMap(const Alloc& alloc) : alex_(alloc) {}
 48 | 
 49 |   ~AlexMap() {}
 50 | 
 51 |   // Initializes with range [first, last). The range does not need to be
 52 |   // sorted. This creates a temporary copy of the data. If possible, we
 53 |   // recommend directly using load() instead.
 54 |   template <class InputIterator>
 55 |   explicit AlexMap(InputIterator first, InputIterator last, const Compare& comp,
 56 |                    const Alloc& alloc = Alloc())
 57 |       : alex_(first, last, comp, alloc) {}
 58 | 
 59 |   // Initializes with range [first, last). The range does not need to be
 60 |   // sorted. This creates a temporary copy of the data. If possible, we
 61 |   // recommend directly using load() instead.
 62 |   template <class InputIterator>
 63 |   explicit AlexMap(InputIterator first, InputIterator last,
 64 |                    const Alloc& alloc = Alloc())
 65 |       : alex_(first, last, alloc) {}
 66 | 
 67 |   explicit AlexMap(const self_type& other) : alex_(other.alex_) {}
 68 | 
 69 |   AlexMap& operator=(const self_type& other) {
 70 |     if (this != &other) {
 71 |       alex_ = other.alex_;
 72 |     }
 73 |     return *this;
 74 |   }
 75 | 
 76 |   void swap(const self_type& other) { alex_.swap(other.alex_); }
 77 | 
 78 |  public:
 79 |   // When bulk loading, Alex can use provided knowledge of the expected fraction
 80 |   // of operations that will be inserts
 81 |   // For simplicity, operations are either point lookups ("reads") or inserts
 82 |   // ("writes)
 83 |   // i.e., 0 means we expect a read-only workload, 1 means write-only
 84 |   // This is only useful if you set it before bulk loading
 85 |   void set_expected_insert_frac(double expected_insert_frac) {
 86 |     alex_.set_expected_insert_frac(expected_insert_frac);
 87 |   }
 88 | 
 89 |   // Maximum node size, in bytes.
 90 |   // Higher values result in better average throughput, but worse tail/max
 91 |   // insert latency.
 92 |   void set_max_node_size(int max_node_size) {
 93 |     alex_.set_max_node_size(max_node_size);
 94 |   }
 95 | 
 96 |   // Bulk load faster by using sampling to train models.
 97 |   // This is only useful if you set it before bulk loading.
 98 |   void set_approximate_model_computation(bool approximate_model_computation) {
 99 |     alex_.set_approximate_model_computation(approximate_model_computation);
100 |   }
101 | 
102 |   // Bulk load faster by using sampling to compute cost.
103 |   // This is only useful if you set it before bulk loading.
104 |   void set_approximate_cost_computation(bool approximate_cost_computation) {
105 |     alex_.set_approximate_cost_computation(approximate_cost_computation);
106 |   }
107 | 
108 |   /*** Allocators and comparators ***/
109 | 
110 |  public:
111 |   Alloc get_allocator() const { return alex_.get_allocator(); }
112 | 
113 |   Compare key_comp() const { return alex_.key_comp(); }
114 | 
115 |   /*** Bulk loading ***/
116 | 
117 |  public:
118 |   // values should be the sorted array of key-payload pairs.
119 |   // The number of elements should be num_keys.
120 |   // The index must be empty when calling this method.
121 |   void bulk_load(const V values[], int num_keys) {
122 |     alex_.bulk_load(values, num_keys);
123 |   }
124 | 
125 |   /*** Element access ***/
126 | 
127 |  public:
128 |   P& operator[](const T& key) { return alex_.insert(key, P()).first.payload(); }
129 | 
130 |   P& at(const T& key) {
131 |     P* payload = alex_.get_payload(key);
132 |     if (payload == nullptr) {
133 |       throw std::out_of_range("AlexMap::at: input does not match any key.");
134 |     } else {
135 |       return *payload;
136 |     }
137 |   }
138 | 
139 |   const P& at(const T& key) const {
140 |     P* payload = alex_.get_payload(key);
141 |     if (payload == nullptr) {
142 |       throw std::out_of_range("AlexMap::at: input does not match any key.");
143 |     } else {
144 |       return *payload;
145 |     }
146 |   }
147 | 
148 |   /*** Lookup ***/
149 | 
150 |  public:
151 |   // Looks for an exact match of the key
152 |   // If the key does not exist, returns an end iterator
153 |   // If there are multiple keys with the same value, returns an iterator to the
154 |   // right-most key
155 |   // If you instead want an iterator to the left-most key with the input value,
156 |   // use lower_bound()
157 |   iterator find(const T& key) { return alex_.find(key); }
158 | 
159 |   const_iterator find(const T& key) const { return alex_.find(key); }
160 | 
161 |   size_t count(const T& key) { return alex_.size(key); }
162 | 
163 |   // Returns an iterator to the first key no less than the input value
164 |   iterator lower_bound(const T& key) { return alex_.lower_bound(key); }
165 | 
166 |   const_iterator lower_bound(const T& key) const {
167 |     return alex_.lower_bound(key);
168 |   }
169 | 
170 |   // Returns an iterator to the first key greater than the input value
171 |   iterator upper_bound(const T& key) { return alex_.upper_bound(key); }
172 | 
173 |   const_iterator upper_bound(const T& key) const {
174 |     return alex_.upper_bound(key);
175 |   }
176 | 
177 |   std::pair<iterator, iterator> equal_range(const T& key) {
178 |     return alex_.equal_range(key);
179 |   }
180 | 
181 |   std::pair<const_iterator, const_iterator> equal_range(const T& key) const {
182 |     return alex_.equal_range(key);
183 |   }
184 | 
185 |   iterator begin() { return alex_.begin(); }
186 | 
187 |   iterator end() { return alex_.end(); }
188 | 
189 |   const_iterator cbegin() const { return alex_.cbegin(); }
190 | 
191 |   const_iterator cend() const { return alex_.cend(); }
192 | 
193 |   reverse_iterator rbegin() { return alex_.rbegin(); }
194 | 
195 |   reverse_iterator rend() { return alex_.rend(); }
196 | 
197 |   const_reverse_iterator crbegin() const { return alex_.crbegin(); }
198 | 
199 |   const_reverse_iterator crend() const { return alex_.crend(); }
200 | 
201 |   /*** Insert ***/
202 | 
203 |  public:
204 |   std::pair<iterator, bool> insert(const V& value) {
205 |     return alex_.insert(value);
206 |   }
207 | 
208 |   template <class InputIterator>
209 |   void insert(InputIterator first, InputIterator last) {
210 |     alex_.insert(first, last);
211 |   }
212 | 
213 |   // This will NOT do an update of an existing key.
214 |   // To perform an update or read-modify-write, do a lookup and modify the
215 |   // payload's value.
216 |   std::pair<iterator, bool> insert(const T& key, const P& payload) {
217 |     return alex_.insert(key, payload);
218 |   }
219 | 
220 |   /*** Delete ***/
221 | 
222 |  public:
223 |   // Erases all keys with a certain key value
224 |   int erase(const T& key) { return alex_.erase(key); }
225 | 
226 |   // Erases element pointed to by iterator
227 |   void erase(iterator it) { alex_.erase(it); }
228 | 
229 |   // Removes all elements
230 |   void clear() { alex_.clear(); }
231 | 
232 |   /*** Stats ***/
233 | 
234 |  public:
235 |   // Number of elements
236 |   size_t size() const { return alex_.size(); }
237 | 
238 |   // True if there are no elements
239 |   bool empty() const { return alex_.empty(); }
240 | 
241 |   // This is just a function required by the STL standard. ALEX can hold more
242 |   // items.
243 |   size_t max_size() const { return alex_.max_size(); }
244 | 
245 |   // Return a const reference to the current statistics
246 |   const struct alex_impl::Stats& get_stats() const { return alex_.stats_; }
247 | };
248 | }
249 | 


--------------------------------------------------------------------------------
/glin/libmorton/morton_AVX512BITALG.h:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | #if defined(__AVX512BITALG__)
  3 | #include <immintrin.h>
  4 | #include <stdint.h>
  5 | 
  6 | namespace libmorton {
  7 | 
  8 | 	namespace bitalg_detail {
  9 | 		// "Zip" and interleave an m-vector of n-bit integers into a
 10 | 		// new n*m-bit integer
 11 | 		// 2D MORTONS
 12 | 		inline void bitunzip2D(const uint32_t morton, uint32_t& x, uint32_t& y) noexcept {
 13 | 			// Unpack bits into upper and lower half of 32-bit integer in parallel
 14 | 			// into 16-bit components
 15 | 			const uint32_t Unzipped = _cvtmask32_u32(
 16 | 				_mm256_bitshuffle_epi64_mask(
 17 | 					_mm256_set1_epi32(morton),
 18 | 					_mm256_set_epi8(
 19 | 						// Every odd bit
 20 | 						31, 29, 27, 25, 23, 21, 19, 17, 
 21 | 						15, 13, 11,  9,  7,  5,  3,  1, 
 22 | 						// Every even bit
 23 | 						30, 28, 26, 24, 22, 20, 18, 16, 
 24 | 						14, 12, 10,  8,  6,  4,  2,  0
 25 | 					)
 26 | 				)
 27 | 			);
 28 | 			x = static_cast<uint16_t>(Unzipped >>  0);
 29 | 			y = static_cast<uint16_t>(Unzipped >> 16);
 30 | 		}
 31 | 		inline void bitunzip2D(const uint64_t morton, uint64_t& x, uint64_t& y) noexcept {
 32 | 			// Unpack bits into upper and lower half of 64-bit integer in parallel
 33 | 			// into 32-bit components
 34 | 			const uint64_t Unzipped = _cvtmask64_u64(
 35 | 				_mm512_bitshuffle_epi64_mask(
 36 | 					_mm512_set1_epi64(morton),
 37 | 					_mm512_set_epi8(
 38 | 						// Every odd bit
 39 | 						63, 61, 59, 57, 55, 53, 51, 49, 
 40 | 						47, 45, 43, 41, 39, 37, 35, 33, 
 41 | 						31, 29, 27, 25, 23, 21, 19, 17, 
 42 | 						15, 13, 11,  9,  7,  5,  3,  1, 
 43 | 						// Every even bit
 44 | 						62, 60, 58, 56, 54, 52, 50, 48, 
 45 | 						46, 44, 42, 40, 38, 36, 34, 32, 
 46 | 						30, 28, 26, 24, 22, 20, 18, 16, 
 47 | 						14, 12, 10,  8,  6,  4,  2,  0
 48 | 					)
 49 | 				)
 50 | 			);
 51 | 			x = static_cast<uint32_t>(Unzipped >>  0);
 52 | 			y = static_cast<uint32_t>(Unzipped >> 32);
 53 | 		}
 54 | 		inline uint32_t bitzip2D(uint32_t x, uint32_t y) noexcept {
 55 | 			// Put both 32-bit integer into each 64-bit lane
 56 | 			const __m256i CoordVec = _mm256_set1_epi64x(
 57 | 				(static_cast<uint64_t>(y) << 32u) | x
 58 | 			);
 59 | 			// Interleave bits from 32-bit X and Y coordinate
 60 | 			const __mmask32 Interleave = _mm256_bitshuffle_epi64_mask(
 61 | 				CoordVec,
 62 | 				_mm256_set_epi16(
 63 | 					0x1000 + 0x0101 * 15, 0x1000 + 0x0101 * 14,
 64 | 					0x1000 + 0x0101 * 13, 0x1000 + 0x0101 * 12,
 65 | 					0x1000 + 0x0101 * 11, 0x1000 + 0x0101 * 10,
 66 | 					0x1000 + 0x0101 *  9, 0x1000 + 0x0101 *  8,
 67 | 					0x1000 + 0x0101 *  7, 0x1000 + 0x0101 *  6,
 68 | 					0x1000 + 0x0101 *  5, 0x1000 + 0x0101 *  4,
 69 | 					0x1000 + 0x0101 *  3, 0x1000 + 0x0101 *  2,
 70 | 					0x1000 + 0x0101 *  1, 0x1000 + 0x0101 *  0
 71 | 				)
 72 | 			);
 73 | 			return _cvtmask32_u32(Interleave);
 74 | 		}
 75 | 
 76 | 		inline uint64_t bitzip2D(uint64_t x, uint64_t y) noexcept {
 77 | 			const __m512i CoordVec = _mm512_set1_epi64(
 78 | 				(static_cast<uint64_t>(y) << 32u) | x
 79 | 			);
 80 | 			// Interleave bits from 32-bit X and Y coordinate
 81 | 			const __mmask64 Interleave = _mm512_bitshuffle_epi64_mask(
 82 | 				CoordVec,
 83 | 				_mm512_set_epi16(
 84 | 					0x2000 + 0x0101 * 31, 0x2000 + 0x0101 * 30,
 85 | 					0x2000 + 0x0101 * 29, 0x2000 + 0x0101 * 28,
 86 | 					0x2000 + 0x0101 * 27, 0x2000 + 0x0101 * 26,
 87 | 					0x2000 + 0x0101 * 25, 0x2000 + 0x0101 * 24,
 88 | 					0x2000 + 0x0101 * 23, 0x2000 + 0x0101 * 22,
 89 | 					0x2000 + 0x0101 * 21, 0x2000 + 0x0101 * 20,
 90 | 					0x2000 + 0x0101 * 19, 0x2000 + 0x0101 * 18,
 91 | 					0x2000 + 0x0101 * 17, 0x2000 + 0x0101 * 16,
 92 | 					0x2000 + 0x0101 * 15, 0x2000 + 0x0101 * 14,
 93 | 					0x2000 + 0x0101 * 13, 0x2000 + 0x0101 * 12,
 94 | 					0x2000 + 0x0101 * 11, 0x2000 + 0x0101 * 10,
 95 | 					0x2000 + 0x0101 *  9, 0x2000 + 0x0101 *  8,
 96 | 					0x2000 + 0x0101 *  7, 0x2000 + 0x0101 *  6,
 97 | 					0x2000 + 0x0101 *  5, 0x2000 + 0x0101 *  4,
 98 | 					0x2000 + 0x0101 *  3, 0x2000 + 0x0101 *  2,
 99 | 					0x2000 + 0x0101 *  1, 0x2000 + 0x0101 *  0
100 | 				)
101 | 			);
102 | 			return _cvtmask64_u64(Interleave);
103 | 		}
104 | 		// 3D MORTONS
105 | 		inline void bitunzip3D(const uint32_t morton, uint32_t& x, uint32_t& y, uint32_t& z) noexcept {
106 | 			// Unpack 32-bit integer in parallel into 10-bit components, within 16-bit lanes
107 | 			const uint64_t Unzipped = _cvtmask64_u64(
108 | 				_mm512_bitshuffle_epi64_mask(
109 | 					_mm512_set1_epi64(morton),
110 | 					_mm512_set_epi8(
111 | 						~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, 
112 | 						~0, ~0, ~0, ~0, ~0, ~0, 29, 26, 23, 20, 17, 14, 11,  8,  5,  2,
113 | 						~0, ~0, ~0, ~0, ~0, 31, 28, 25, 22, 19, 16, 13, 10,  7,  4,  1,
114 | 						~0, ~0, ~0, ~0, ~0, 30, 27, 24, 21, 18, 15, 12,  9,  6,  3,  0
115 | 					)
116 | 				)
117 | 			);
118 | 			x = static_cast<uint16_t>(Unzipped >>  0);
119 | 			y = static_cast<uint16_t>(Unzipped >> 16);
120 | 			z = static_cast<uint16_t>(Unzipped >> 32);
121 | 		}
122 | 		inline void bitunzip3D(const uint64_t morton, uint64_t& x, uint64_t& y, uint64_t& z) noexcept {
123 | 			// Unpack 64-bit integer in parallel into 21-bit components
124 | 			const uint64_t Unzipped = _cvtmask64_u64(
125 | 				_mm512_bitshuffle_epi64_mask(
126 | 					_mm512_set1_epi64(morton),
127 | 					_mm512_set_epi8(
128 | 						~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, 61, 58, 55, 52, 49, 46, 43, 40, 37, 34, 31, 28, 25, 22, 19, 16, 13, 10, 7, 4, 1,
129 | 						~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, 63, 60, 57, 54, 51, 48, 45, 42, 39, 36, 33, 30, 27, 24, 21, 18, 15, 12, 9, 6, 3, 0
130 | 					)
131 | 				)
132 | 			);
133 | 			x = static_cast<uint32_t>(Unzipped >>  0);
134 | 			y = static_cast<uint32_t>(Unzipped >> 32);
135 | 			z = bmi2_detail::pext(morton, 0x4924924924924924);
136 | 			//z = static_cast<uint32_t>(Unzipped >> 64);
137 | 		}
138 | 		inline uint32_t bitzip3D(uint32_t x, uint32_t y, uint32_t z) noexcept {
139 | 			const __m256i CoordVec = _mm256_broadcastsi128_si256(
140 | 				_mm_set_epi32(0, z, y, x)
141 | 			);
142 | 			const __m256i ShuffleVec = _mm256_permutexvar_epi8(
143 | 				_mm256_set_epi64x(
144 | 					0xFFFFFFFFFF100800ul + 0x010101 * 1, // Lane 3 | ...000 | z[1] | y[1] | x[1]
145 | 					0xFFFFFFFFFF100800ul + 0x010101 * 0, // Lane 2 | ...000 | z[0] | y[0] | x[0]
146 | 					0xFFFFFFFFFF100800ul + 0x010101 * 0, // Lane 1 | ...000 | z[0] | y[0] | x[0]
147 | 					0xFFFFFFFFFF100800ul + 0x010101 * 0  // Lane 0 | ...000 | z[0] | y[0] | x[0]
148 | 				),
149 | 				CoordVec
150 | 			);
151 | 			const __mmask32 Interleave = _mm256_bitshuffle_epi64_mask(
152 | 				ShuffleVec,
153 | 				_mm256_set_epi64x(
154 | 					0x0202010101000000 + 0x0100020100020100 * 8,
155 | 					0x0707070606060505 + 0x0201000201000201 * 8,
156 | 					0x0504040403030302 + 0x0002010002010002 * 8,
157 | 					0x0202010101000000 + 0x0100020100020100 * 8 
158 | 				)
159 | 			);
160 | 			return _cvtmask32_u32(Interleave);
161 | 		}
162 | 		inline uint64_t bitzip3D(uint64_t x, uint64_t y, uint64_t z) noexcept {
163 | 			// Put both 32-bit integers into each 64-bit lane
164 | 			// Todo: _mm512_shuffle_epi8 version, 128-bit lane woes
165 | 			const __m512i CoordVec = _mm512_set_epi64(
166 | 				0, 0, 0, 0, 0, z, y, x
167 | 			);
168 | 			const __m512i ShuffleVec = _mm512_permutexvar_epi8(
169 | 				_mm512_set_epi64(
170 | 					0xFFFFFFFFFF100800ul + 0x010101 * 2, // Lane 7 | ...000 | z[2] | y[2] | x[2]
171 | 					0xFFFFFFFFFF100800ul + 0x010101 * 2, // Lane 6 | ...000 | z[2] | y[2] | x[2]
172 | 					0xFFFFFFFFFF100800ul + 0x010101 * 1, // Lane 5 | ...000 | z[1] | y[1] | x[1]
173 | 					0xFFFFFFFFFF100800ul + 0x010101 * 1, // Lane 4 | ...000 | z[1] | y[1] | x[1]
174 | 					0xFFFFFFFFFF100800ul + 0x010101 * 1, // Lane 3 | ...000 | z[1] | y[1] | x[1]
175 | 					0xFFFFFFFFFF100800ul + 0x010101 * 0, // Lane 2 | ...000 | z[0] | y[0] | x[0]
176 | 					0xFFFFFFFFFF100800ul + 0x010101 * 0, // Lane 1 | ...000 | z[0] | y[0] | x[0]
177 | 					0xFFFFFFFFFF100800ul + 0x010101 * 0  // Lane 0 | ...000 | z[0] | y[0] | x[0]
178 | 				),
179 | 				CoordVec
180 | 			);
181 | 			// Interleave bits from 32-bit X and Y and Z coordinate
182 | 			const __mmask64 Interleave = _mm512_bitshuffle_epi64_mask(
183 | 				ShuffleVec,
184 | 				_mm512_set_epi64(
185 | 					0x0504040403030302 + 0x0002010002010002 * 8,
186 | 					0x0202010101000000 + 0x0100020100020100 * 8,
187 | 					0x0707070606060505 + 0x0201000201000201 * 8,
188 | 					0x0504040403030302 + 0x0002010002010002 * 8,
189 | 					0x0202010101000000 + 0x0100020100020100 * 8,
190 | 					0x0707070606060505 + 0x0201000201000201 * 8,
191 | 					0x0504040403030302 + 0x0002010002010002 * 8,
192 | 					0x0202010101000000 + 0x0100020100020100 * 8 
193 | 				)
194 | 			);
195 | 			return _cvtmask64_u64(Interleave);
196 | 		}
197 | 	}  // namespace bitalg_detail
198 | 
199 | 	template<typename morton, typename coord>
200 | 	inline morton m2D_e_BITALG(const coord x, const coord y) {
201 | 		return bitalg_detail::bitzip2D(
202 | 			static_cast<coord>(x), static_cast<coord>(y)
203 | 		);
204 | 	}
205 | 
206 | 	template<typename morton, typename coord>
207 | 	inline void m2D_d_BITALG(const morton m, coord& x, coord& y) {
208 | 		bitalg_detail::bitunzip2D(m, x, y);
209 | 	}
210 | 
211 | 	template<typename morton, typename coord>
212 | 	inline morton m3D_e_BITALG(const coord x, const coord y, const coord z) {
213 | 		return bitalg_detail::bitzip3D(
214 | 			static_cast<coord>(x), static_cast<coord>(y),  static_cast<coord>(z)
215 | 		);
216 | 	}
217 | 
218 | 	template<typename morton, typename coord>
219 | 	inline void m3D_d_BITALG(const morton m, coord& x, coord& y, coord& z) {
220 | 		bitalg_detail::bitunzip3D(m, x, y, z);
221 | 	}
222 | }
223 | #endif


--------------------------------------------------------------------------------
/glin/libmorton/morton2D.h:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | // Libmorton - Methods to encode/decode 64-bit morton codes from/to 32-bit (x,y) coordinates
  4 | // Warning: morton.h will always point to the functions that use the fastest available method.
  5 | 
  6 | #include <algorithm>
  7 | #include <stdint.h>
  8 | #include <math.h>
  9 | #include "morton2D_LUTs.h"
 10 | #include "morton_common.h"
 11 | 
 12 | #define EIGHTBITMASK (morton) 0x000000FF
 13 | 
 14 | namespace libmorton {
 15 | 
 16 | 	// Encode methods
 17 | 	template<typename morton, typename coord> inline morton m2D_e_sLUT(const coord x, const coord y);
 18 | 	template<typename morton, typename coord> inline morton m2D_e_sLUT_ET(const coord x, const coord y);
 19 | 	template<typename morton, typename coord> inline morton m2D_e_LUT(const coord x, const coord y);
 20 | 	template<typename morton, typename coord> inline morton m2D_e_LUT_ET(const coord x, const coord y);
 21 | 	template<typename morton, typename coord> inline morton m2D_e_magicbits(const coord x, const coord y);
 22 | 	template<typename morton, typename coord> inline morton m2D_e_for(const coord x, const coord y);
 23 | 	template<typename morton, typename coord> inline morton m2D_e_for_ET(const coord x, const coord y);
 24 | 
 25 | 	// Decode methods
 26 | 	template<typename morton, typename coord> inline void m2D_d_sLUT(const morton m, coord& x, coord& y);
 27 | 	template<typename morton, typename coord> inline void m2D_d_sLUT_ET(const morton m, coord& x, coord& y);
 28 | 	template<typename morton, typename coord> inline void m2D_d_LUT(const morton m, coord& x, coord& y);
 29 | 	template<typename morton, typename coord> inline void m2D_d_LUT_ET(const morton m, coord& x, coord& y);
 30 | 	template<typename morton, typename coord> inline void m2D_d_magicbits(const morton m, coord& x, coord& y);
 31 | 	template<typename morton, typename coord> inline void m2D_d_for(const morton m, coord& x, coord& y);
 32 | 
 33 | 	// ENCODE 2D Morton code : Pre-shifted LookUpTable (sLUT)
 34 | 	template<typename morton, typename coord>
 35 | 	inline morton m2D_e_sLUT(const coord x, const coord y) {
 36 | 		morton answer = 0;
 37 | 		for (unsigned int i = sizeof(coord); i > 0; --i) {
 38 | 			unsigned int shift = (i - 1) * 8;
 39 | 			answer =
 40 | 				answer << 16 |
 41 | 				Morton2D_encode_y_256[(y >> shift) & EIGHTBITMASK] |
 42 | 				Morton2D_encode_x_256[(x >> shift) & EIGHTBITMASK];
 43 | 		}
 44 | 		return answer;
 45 | 	}
 46 | 
 47 | 	// ENCODE 2D Morton code : LookUpTable (LUT)
 48 | 	template<typename morton, typename coord>
 49 | 	inline morton m2D_e_LUT(const coord x, const coord y) {
 50 | 		morton answer = 0;
 51 | 		for (unsigned int i = sizeof(coord); i > 0; --i) {
 52 | 			unsigned int shift = (i - 1) * 8;
 53 | 			answer =
 54 | 				answer << 16 |
 55 | 				(Morton2D_encode_x_256[(y >> shift) & EIGHTBITMASK] << morton(1)) |
 56 | 				(Morton2D_encode_x_256[(x >> shift) & EIGHTBITMASK]);
 57 | 		}
 58 | 		return answer;
 59 | 	}
 60 | 
 61 | 	// HELPER METHOD for Early Termination LUT Encode
 62 | 	template<typename morton, typename coord>
 63 | 	inline morton compute2D_ET_LUT_encode(const coord c, const coord *LUT) {
 64 | 		unsigned long maxbit = 0;
 65 | 		if (findFirstSetBit<coord>(c, &maxbit) == 0) { return 0; }
 66 | 		morton answer = 0;
 67 | 		unsigned int i = 0;
 68 | 		while (maxbit >= i) {
 69 | 			answer |= ((morton)LUT[(c >> i) & EIGHTBITMASK]) << i * 2;
 70 | 			i += 8;
 71 | 		}
 72 | 		return answer;
 73 | 	}
 74 | 
 75 | 	// ENCODE 2D Morton code : Pre-shifted LUT (Early termination version)
 76 | 	// This version tries to terminate early when there are no more bits to process
 77 | 	// Figuring this out is probably too costly in most cases.
 78 | 	template<typename morton, typename coord>
 79 | 	inline morton m2D_e_sLUT_ET(const coord x, const coord y) {
 80 | 		morton answer_x = compute2D_ET_LUT_encode<morton, coord>(x, Morton2D_encode_x_256);
 81 | 		morton answer_y = compute2D_ET_LUT_encode<morton, coord>(y, Morton2D_encode_y_256);
 82 | 		return answer_y | answer_x;
 83 | 	}
 84 | 
 85 | 	// ENCODE 2D Morton code : LUT (Early termination version)
 86 | 	template<typename morton, typename coord>
 87 | 	inline morton m2D_e_LUT_ET(const coord x, const coord y) {
 88 | 		morton answer_x = compute2D_ET_LUT_encode<morton, coord>(x, Morton2D_encode_x_256);
 89 | 		morton answer_y = compute2D_ET_LUT_encode<morton, coord>(y, Morton2D_encode_x_256);
 90 | 		return (answer_y << 1) | answer_x;
 91 | 	}
 92 | 
 93 | 	// Magicbits masks (2D encode)
 94 | 	static uint_fast32_t magicbit2D_masks32[6] = { 0xFFFFFFFF, 0x0000FFFF, 0x00FF00FF, 0x0F0F0F0F, 0x33333333, 0x55555555 };
 95 | 	static uint_fast64_t magicbit2D_masks64[6] = { 0x00000000FFFFFFFF, 0x0000FFFF0000FFFF, 0x00FF00FF00FF00FF, 0x0F0F0F0F0F0F0F0F, 0x3333333333333333, 0x5555555555555555 };
 96 | 
 97 | 	// HELPER METHOD for Magic bits encoding - split by 2
 98 | 	template<typename morton, typename coord>
 99 | 	inline morton morton2D_SplitBy2Bits(const coord a) {
100 | 		const morton* masks = (sizeof(morton) <= 4) ? reinterpret_cast<const morton*>(magicbit2D_masks32) : reinterpret_cast<const morton*>(magicbit2D_masks64);
101 | 		morton x = a;
102 | 		if (sizeof(morton) > 4) { x = (x | (uint_fast64_t)x << 32) & masks[0]; }
103 | 		x = (x | x << 16) & masks[1];
104 | 		x = (x | x << 8) & masks[2];
105 | 		x = (x | x << 4) & masks[3];
106 | 		x = (x | x << 2) & masks[4];
107 | 		x = (x | x << 1) & masks[5];
108 | 		return x;
109 | 	}
110 | 
111 | 	// ENCODE 2D Morton code : Magic bits
112 | 	template<typename morton, typename coord>
113 | 	inline morton m2D_e_magicbits(const coord x, const coord y) {
114 | 		return morton2D_SplitBy2Bits<morton, coord>(x) | (morton2D_SplitBy2Bits<morton, coord>(y) << 1);
115 | 	}
116 | 
117 | 	// ENCODE 2D Morton code : For Loop
118 | 	template<typename morton, typename coord>
119 | 	inline morton m2D_e_for(const coord x, const coord y) {
120 | 		morton answer = 0;
121 | 		unsigned int checkbits = (unsigned int)floor(sizeof(morton) * 4.0f);
122 | 		for (unsigned int i = 0; i < checkbits; ++i) {
123 | 			morton mshifted = static_cast<morton>(0x1) << i; // Here we need to cast 0x1 to 64bits, otherwise there is a bug when morton code is larger than 32 bits
124 | 			unsigned int shift = i; // because you have to shift back i and forth 2*i
125 | 			answer |=
126 | 				((x & mshifted) << shift)
127 | 				| ((y & mshifted) << (shift + 1));
128 | 		}
129 | 		return answer;
130 | 	}
131 | 
132 | 	// ENCODE 2D Morton code : For Loop (Early termination version)
133 | 	template<typename morton, typename coord>
134 | 	inline morton m2D_e_for_ET(const coord x, const coord y) {
135 | 		morton answer = 0;
136 | 		unsigned long x_max = 0, y_max = 0;
137 | 		unsigned int checkbits = sizeof(morton) * 4;
138 | 		findFirstSetBit<morton>(x, &x_max);
139 | 		findFirstSetBit<morton>(y, &y_max);
140 | 		checkbits = std::min(static_cast<unsigned long>(checkbits), std::max(x_max, y_max) + 1ul);
141 | 		for (unsigned int i = 0; i < checkbits; ++i) {
142 | 			morton m_shifted = static_cast<morton>(0x1) << i; // Here we need to cast 0x1 to 64bits, otherwise there is a bug when morton code is larger than 32 bits
143 | 			unsigned int shift = i;
144 | 			answer |= ((x & m_shifted) << shift)
145 | 				| ((y & m_shifted) << (shift + 1));
146 | 		}
147 | 		return answer;
148 | 	}
149 | 
150 | 	// HELPER METHODE for LUT decoding
151 | 	template<typename morton, typename coord>
152 | 	inline coord morton2D_DecodeCoord_LUT256(const morton m, const uint_fast8_t *LUT, const unsigned int startshift) {
153 | 		morton a = 0;
154 | 		unsigned int loops = sizeof(morton);
155 | 		for (unsigned int i = 0; i < loops; ++i) {
156 | 			a |= ((morton)LUT[(m >> ((i * 8) + startshift)) & EIGHTBITMASK] << (4 * i));
157 | 		}
158 | 		return static_cast<coord>(a);
159 | 	}
160 | 
161 | 	// DECODE 2D Morton code : Shifted LUT
162 | 	template<typename morton, typename coord>
163 | 	inline void m2D_d_sLUT(const morton m, coord& x, coord& y) {
164 | 		x = morton2D_DecodeCoord_LUT256<morton, coord>(m, Morton2D_decode_x_256, 0);
165 | 		y = morton2D_DecodeCoord_LUT256<morton, coord>(m, Morton2D_decode_y_256, 0);
166 | 	}
167 | 
168 | 	// DECODE 2D 64-bit morton code : LUT
169 | 	template<typename morton, typename coord>
170 | 	inline void m2D_d_LUT(const morton m, coord& x, coord& y) {
171 | 		x = morton2D_DecodeCoord_LUT256<morton, coord>(m, Morton2D_decode_x_256, 0);
172 | 		y = morton2D_DecodeCoord_LUT256<morton, coord>(m, Morton2D_decode_x_256, 1);
173 | 	}
174 | 
175 | 	// DECODE 2D Morton code : Shifted LUT (early termination)
176 | 	template<typename morton, typename coord>
177 | 	inline void m2D_d_sLUT_ET(const morton m, coord& x, coord& y) {
178 | 		x = 0; y = 0;
179 | 		unsigned long firstbit_location = 0;
180 | 		if (!findFirstSetBit<morton>(m, &firstbit_location)) { return; }
181 | 		unsigned int i = 0;
182 | 		unsigned int shiftback = 0;
183 | 		while (firstbit_location > i) {
184 | 			morton m_shifted = (m >> i) & EIGHTBITMASK;
185 | 			x |= (coord)Morton2D_decode_x_256[m_shifted] << shiftback;
186 | 			y |= (coord)Morton2D_decode_y_256[m_shifted] << shiftback;
187 | 			shiftback += 4;
188 | 			i += 8;
189 | 		}
190 | 	}
191 | 
192 | 	// DECODE 2D Morton code : LUT (early termination)
193 | 	template<typename morton, typename coord>
194 | 	inline void m2D_d_LUT_ET(const morton m, coord& x, coord& y) {
195 | 		x = 0; y = 0;
196 | 		unsigned long firstbit_location = 0;
197 | 		if (!findFirstSetBit<morton>(m, &firstbit_location)) { return; }
198 | 		unsigned int i = 0;
199 | 		unsigned int shiftback = 0;
200 | 		while (firstbit_location > i) {
201 | 			x |= (coord)Morton2D_decode_x_256[(m >> i) & EIGHTBITMASK] << shiftback;
202 | 			y |= (coord)Morton2D_decode_x_256[(m >> (i + 1)) & EIGHTBITMASK] << shiftback;
203 | 			shiftback += 4;
204 | 			i += 8;
205 | 		}
206 | 	}
207 | 
208 | 	// HELPER method for Magicbits decoding
209 | 	template<typename morton, typename coord>
210 | 	static inline coord morton2D_GetSecondBits(const morton m) {
211 | 		morton* masks = (sizeof(morton) <= 4) ? reinterpret_cast<morton*>(magicbit2D_masks32) : reinterpret_cast<morton*>(magicbit2D_masks64);
212 | 		morton x = m & masks[5];
213 | 		x = (x ^ (x >> 1)) & masks[4];
214 | 		x = (x ^ (x >> 2)) & masks[3];
215 | 		x = (x ^ (x >> 4)) & masks[2];
216 | 		x = (x ^ (x >> 8)) & masks[1];
217 | 		if (sizeof(morton) > 4) x = (x ^ (x >> 16)) & masks[0];
218 | 		return static_cast<coord>(x);
219 | 	}
220 | 
221 | 	// DECODE 2D Morton code : Magic bits
222 | 	// This method splits the morton codes bits by using certain patterns (magic bits)
223 | 	template<typename morton, typename coord>
224 | 	inline void m2D_d_magicbits(const morton m, coord& x, coord& y) {
225 | 		x = morton2D_GetSecondBits<morton, coord>(m);
226 | 		y = morton2D_GetSecondBits<morton, coord>(m >> 1);
227 | 	}
228 | 
229 | 
230 | 	// DECODE 2D morton code : For loop
231 | 	template<typename morton, typename coord>
232 | 	inline void m2D_d_for(const morton m, coord& x, coord& y) {
233 | 		x = 0; y = 0;
234 | 		unsigned int checkbits = sizeof(morton) * 4;
235 | 		for (unsigned int i = 0; i <= checkbits; ++i) {
236 | 			morton selector = 1;
237 | 			unsigned int shift_selector = 2 * i;
238 | 			x |= (m & (selector << shift_selector)) >> i;
239 | 			y |= (m & (selector << (shift_selector + 1))) >> (i + 1);
240 | 		}
241 | 	}
242 | 
243 | 	// DECODE 3D Morton code : For loop (Early termination version)
244 | 	template<typename morton, typename coord>
245 | 	inline void m2D_d_for_ET(const morton m, coord& x, coord& y) {
246 | 		x = 0; y = 0;
247 | 		unsigned long firstbit_location = 0;
248 | 		if (!findFirstSetBit<morton>(m, &firstbit_location)) return;
249 | 		float defaultbits = sizeof(morton) * 4;
250 | 		unsigned int checkbits = static_cast<unsigned int>(std::min(defaultbits, firstbit_location / 2.0f));
251 | 		for (unsigned int i = 0; i <= checkbits; ++i) {
252 | 			morton selector = 1;
253 | 			unsigned int shift_selector = 2 * i;
254 | 			x |= (m & (selector << shift_selector)) >> i;
255 | 			y |= (m & (selector << (shift_selector + 1))) >> (i + 1);
256 | 		}
257 | 	}
258 | }
259 | 


--------------------------------------------------------------------------------
/src/core/alex_base.h:
--------------------------------------------------------------------------------
  1 | // Copyright (c) Microsoft Corporation.
  2 | // Licensed under the MIT license.
  3 | 
  4 | /* This file contains the classes for linear models and model builders, helpers
  5 |  * for the bitmap,
  6 |  * cost model weights, statistic accumulators for collecting cost model
  7 |  * statistics,
  8 |  * and other miscellaneous functions
  9 |  */
 10 | 
 11 | #pragma once
 12 | 
 13 | #include <geos/geom/Envelope.h>
 14 | #include <geos/geom/Geometry.h>
 15 | #include <geos/geom/GeometryFactory.h>
 16 | #include <geos/geom/CoordinateArraySequence.h>
 17 | #include <geos/geom/LinearRing.h>
 18 | #include <geos/geom/LineSegment.h>
 19 | #include "projection.h"
 20 | #include <algorithm>
 21 | #include <array>
 22 | #include <chrono>
 23 | #include <cstring>
 24 | #include <iostream>
 25 | #include <limits>
 26 | #include <memory>
 27 | #include <random>
 28 | #include <set>
 29 | #include <string>
 30 | #include <utility>
 31 | #include <vector>
 32 | #ifdef _MSC_VER
 33 | #include <intrin.h>
 34 | #else
 35 | #include <x86intrin.h>
 36 | #endif
 37 | #include <bitset>
 38 | #include <cassert>
 39 | #ifdef _WIN32
 40 | #include <intrin.h>
 41 | #include <limits.h>
 42 | typedef unsigned __int32 uint32_t;
 43 | #else
 44 | #include <stdint.h>
 45 | #endif
 46 | 
 47 | #ifdef _MSC_VER
 48 | #define forceinline __forceinline
 49 | #elif defined(__GNUC__)
 50 | #define forceinline inline __attribute__((__always_inline__))
 51 | #elif defined(__CLANG__)
 52 | #if __has_attribute(__always_inline__)
 53 | #define forceinline inline __attribute__((__always_inline__))
 54 | #else
 55 | #define forceinline inline
 56 | #endif
 57 | #else
 58 | #define forceinline inline
 59 | #endif
 60 | 
 61 | namespace alex {
 62 | 
 63 | /*** Linear model and model builder ***/
 64 | 
 65 | // Forward declaration
 66 | template <class T>
 67 | class LinearModelBuilder;
 68 | 
 69 | // Linear regression model
 70 | template <class T>
 71 | class LinearModel {
 72 |  public:
 73 |   double a_ = 0;  // slope
 74 |   double b_ = 0;  // intercept
 75 | 
 76 |   LinearModel() = default;
 77 |   LinearModel(double a, double b) : a_(a), b_(b) {}
 78 |   explicit LinearModel(const LinearModel& other) : a_(other.a_), b_(other.b_) {}
 79 | 
 80 |   void expand(double expansion_factor) {
 81 |     a_ *= expansion_factor;
 82 |     b_ *= expansion_factor;
 83 |   }
 84 | 
 85 |   inline int predict(T key) const {
 86 |     return static_cast<int>(a_ * static_cast<double>(key) + b_);
 87 |   }
 88 | 
 89 |   inline double predict_double(T key) const {
 90 |     return a_ * static_cast<double>(key) + b_;
 91 |   }
 92 | };
 93 | 
 94 | template <class T>
 95 | class LinearModelBuilder {
 96 |  public:
 97 |   LinearModel<T>* model_;
 98 | 
 99 |   explicit LinearModelBuilder<T>(LinearModel<T>* model) : model_(model) {}
100 | 
101 |   inline void add(T x, int y) {
102 |     count_++;
103 |     x_sum_ += static_cast<long double>(x);
104 |     y_sum_ += static_cast<long double>(y);
105 |     xx_sum_ += static_cast<long double>(x) * x;
106 |     xy_sum_ += static_cast<long double>(x) * y;
107 |     x_min_ = std::min<T>(x, x_min_);
108 |     x_max_ = std::max<T>(x, x_max_);
109 |     y_min_ = std::min<double>(y, y_min_);
110 |     y_max_ = std::max<double>(y, y_max_);
111 |   }
112 | 
113 |   void build() {
114 |     if (count_ <= 1) {
115 |       model_->a_ = 0;
116 |       model_->b_ = static_cast<double>(y_sum_);
117 |       return;
118 |     }
119 | 
120 |     if (static_cast<long double>(count_) * xx_sum_ - x_sum_ * x_sum_ == 0) {
121 |       // all values in a bucket have the same key.
122 |       model_->a_ = 0;
123 |       model_->b_ = static_cast<double>(y_sum_) / count_;
124 |       return;
125 |     }
126 | 
127 |     auto slope = static_cast<double>(
128 |         (static_cast<long double>(count_) * xy_sum_ - x_sum_ * y_sum_) /
129 |         (static_cast<long double>(count_) * xx_sum_ - x_sum_ * x_sum_));
130 |     auto intercept = static_cast<double>(
131 |         (y_sum_ - static_cast<long double>(slope) * x_sum_) / count_);
132 |     model_->a_ = slope;
133 |     model_->b_ = intercept;
134 | 
135 |     // If floating point precision errors, fit spline
136 |     if (model_->a_ <= 0) {
137 |       model_->a_ = (y_max_ - y_min_) / (x_max_ - x_min_);
138 |       model_->b_ = -static_cast<double>(x_min_) * model_->a_;
139 |     }
140 |   }
141 | 
142 |  private:
143 |   int count_ = 0;
144 |   long double x_sum_ = 0;
145 |   long double y_sum_ = 0;
146 |   long double xx_sum_ = 0;
147 |   long double xy_sum_ = 0;
148 |   T x_min_ = std::numeric_limits<T>::max();
149 |   T x_max_ = std::numeric_limits<T>::lowest();
150 |   double y_min_ = std::numeric_limits<double>::max();
151 |   double y_max_ = std::numeric_limits<double>::lowest();
152 | };
153 | 
154 | /*** Comparison ***/
155 | 
156 | struct AlexCompare {
157 |   template <class T1, class T2>
158 |   bool operator()(const T1& x, const T2& y) const {
159 |     static_assert(
160 |         std::is_arithmetic<T1>::value && std::is_arithmetic<T2>::value,
161 |         "Comparison types must be numeric.");
162 |     return x < y;
163 |   }
164 | };
165 | 
166 | /*** Helper methods for bitmap ***/
167 | 
168 | // Extract the rightmost 1 in the binary representation.
169 | // e.g. extract_rightmost_one(010100100) = 000000100
170 | inline uint64_t extract_rightmost_one(uint64_t value) {
171 |   return value & -static_cast<int64_t>(value);
172 | }
173 | 
174 | // Remove the rightmost 1 in the binary representation.
175 | // e.g. remove_rightmost_one(010100100) = 010100000
176 | inline uint64_t remove_rightmost_one(uint64_t value) {
177 |   return value & (value - 1);
178 | }
179 | 
180 | // Count the number of 1s in the binary representation.
181 | // e.g. count_ones(010100100) = 3
182 | inline int count_ones(uint64_t value) {
183 |   return static_cast<int>(_mm_popcnt_u64(value));
184 | }
185 | 
186 | // Get the offset of a bit in a bitmap.
187 | // word_id is the word id of the bit in a bitmap
188 | // bit is the word that contains the bit
189 | inline int get_offset(int word_id, uint64_t bit) {
190 |   return (word_id << 6) + count_ones(bit - 1);
191 | }
192 | 
193 | /*** Cost model weights ***/
194 | 
195 | // Intra-node cost weights
196 | double kExpSearchIterationsWeight = 20;
197 | double kShiftsWeight = 0.5;
198 | 
199 | // TraverseToLeaf cost weights
200 | double kNodeLookupsWeight = 20;
201 | double kModelSizeWeight = 5e-7;
202 | 
203 | /*** Stat Accumulators ***/
204 | 
205 | struct DataNodeStats {
206 |   double num_search_iterations = 0;
207 |   double num_shifts = 0;
208 | };
209 | 
210 | // Used when stats are computed using a sample
211 | struct SampleDataNodeStats {
212 |   double log2_sample_size = 0;
213 |   double num_search_iterations = 0;
214 |   double log2_num_shifts = 0;
215 | };
216 | 
217 | // Accumulates stats that are used in the cost model, based on the actual vs
218 | // predicted position of a key
219 | class StatAccumulator {
220 |  public:
221 |   virtual ~StatAccumulator() = default;
222 |   virtual void accumulate(int actual_position, int predicted_position) = 0;
223 |   virtual double get_stat() = 0;
224 |   virtual void reset() = 0;
225 | };
226 | 
227 | // Mean log error represents the expected number of exponential search
228 | // iterations when doing a lookup
229 | class ExpectedSearchIterationsAccumulator : public StatAccumulator {
230 |  public:
231 |   void accumulate(int actual_position, int predicted_position) override {
232 |     cumulative_log_error_ +=
233 |         std::log2(std::abs(predicted_position - actual_position) + 1);
234 |     count_++;
235 |   }
236 | 
237 |   double get_stat() override {
238 |     if (count_ == 0) return 0;
239 |     return cumulative_log_error_ / count_;
240 |   }
241 | 
242 |   void reset() override {
243 |     cumulative_log_error_ = 0;
244 |     count_ = 0;
245 |   }
246 | 
247 |  public:
248 |   double cumulative_log_error_ = 0;
249 |   int count_ = 0;
250 | };
251 | 
252 | // Mean shifts represents the expected number of shifts when doing an insert
253 | class ExpectedShiftsAccumulator : public StatAccumulator {
254 |  public:
255 |   explicit ExpectedShiftsAccumulator(int data_capacity)
256 |       : data_capacity_(data_capacity) {}
257 | 
258 |   // A dense region of n keys will contribute a total number of expected shifts
259 |   // of approximately
260 |   // ((n-1)/2)((n-1)/2 + 1) = n^2/4 - 1/4
261 |   // This is exact for odd n and off by 0.25 for even n.
262 |   // Therefore, we track n^2/4.
263 |   void accumulate(int actual_position, int) override {
264 |     if (actual_position > last_position_ + 1) {
265 |       long long dense_region_length = last_position_ - dense_region_start_idx_ + 1;
266 |       num_expected_shifts_ += (dense_region_length * dense_region_length) / 4;
267 |       dense_region_start_idx_ = actual_position;
268 |     }
269 |     last_position_ = actual_position;
270 |     count_++;
271 |   }
272 | 
273 |   double get_stat() override {
274 |     if (count_ == 0) return 0;
275 |     // first need to accumulate statistics for current packed region
276 |     long long dense_region_length = last_position_ - dense_region_start_idx_ + 1;
277 |     long long cur_num_expected_shifts =
278 |         num_expected_shifts_ + (dense_region_length * dense_region_length) / 4;
279 |     return cur_num_expected_shifts / static_cast<double>(count_);
280 |   }
281 | 
282 |   void reset() override {
283 |     last_position_ = -1;
284 |     dense_region_start_idx_ = 0;
285 |     num_expected_shifts_ = 0;
286 |     count_ = 0;
287 |   }
288 | 
289 |  public:
290 |   int last_position_ = -1;
291 |   int dense_region_start_idx_ = 0;
292 |   long long num_expected_shifts_ = 0;
293 |   int count_ = 0;
294 |   int data_capacity_ = -1;  // capacity of node
295 | };
296 | 
297 | // Combines ExpectedSearchIterationsAccumulator and ExpectedShiftsAccumulator
298 | class ExpectedIterationsAndShiftsAccumulator : public StatAccumulator {
299 |  public:
300 |   ExpectedIterationsAndShiftsAccumulator() = default;
301 |   explicit ExpectedIterationsAndShiftsAccumulator(int data_capacity)
302 |       : data_capacity_(data_capacity) {}
303 | 
304 |   void accumulate(int actual_position, int predicted_position) override {
305 |     cumulative_log_error_ +=
306 |         std::log2(std::abs(predicted_position - actual_position) + 1);
307 | 
308 |     if (actual_position > last_position_ + 1) {
309 |       long long dense_region_length = last_position_ - dense_region_start_idx_ + 1;
310 |       num_expected_shifts_ += (dense_region_length * dense_region_length) / 4;
311 |       dense_region_start_idx_ = actual_position;
312 |     }
313 |     last_position_ = actual_position;
314 | 
315 |     count_++;
316 |   }
317 | 
318 |   double get_stat() override {
319 |     assert(false);  // this should not be used
320 |     return 0;
321 |   }
322 | 
323 |   double get_expected_num_search_iterations() {
324 |     if (count_ == 0) return 0;
325 |     return cumulative_log_error_ / count_;
326 |   }
327 | 
328 |   double get_expected_num_shifts() {
329 |     if (count_ == 0) return 0;
330 |     long long dense_region_length = last_position_ - dense_region_start_idx_ + 1;
331 |     long long cur_num_expected_shifts =
332 |         num_expected_shifts_ + (dense_region_length * dense_region_length) / 4;
333 |     return cur_num_expected_shifts / static_cast<double>(count_);
334 |   }
335 | 
336 |   void reset() override {
337 |     cumulative_log_error_ = 0;
338 |     last_position_ = -1;
339 |     dense_region_start_idx_ = 0;
340 |     num_expected_shifts_ = 0;
341 |     count_ = 0;
342 |   }
343 | 
344 |  public:
345 |   double cumulative_log_error_ = 0;
346 |   int last_position_ = -1;
347 |   int dense_region_start_idx_ = 0;
348 |   long long num_expected_shifts_ = 0;
349 |   int count_ = 0;
350 |   int data_capacity_ = -1;  // capacity of node
351 | };
352 | 
353 | /*** Miscellaneous helpers ***/
354 | 
355 | // https://stackoverflow.com/questions/364985/algorithm-for-finding-the-smallest-power-of-two-thats-greater-or-equal-to-a-giv
356 | inline int pow_2_round_up(int x) {
357 |   --x;
358 |   x |= x >> 1;
359 |   x |= x >> 2;
360 |   x |= x >> 4;
361 |   x |= x >> 8;
362 |   x |= x >> 16;
363 |   return x + 1;
364 | }
365 | 
366 | // https://stackoverflow.com/questions/994593/how-to-do-an-integer-log2-in-c
367 | inline int log_2_round_down(int x) {
368 |   int res = 0;
369 |   while (x >>= 1) ++res;
370 |   return res;
371 | }
372 | 
373 | // https://stackoverflow.com/questions/1666093/cpuid-implementations-in-c
374 | class CPUID {
375 |   uint32_t regs[4];
376 | 
377 |  public:
378 |   explicit CPUID(unsigned i, unsigned j) {
379 | #ifdef _WIN32
380 |     __cpuidex((int*)regs, (int)i, (int)j);
381 | #else
382 |     asm volatile("cpuid"
383 |                  : "=a"(regs[0]), "=b"(regs[1]), "=c"(regs[2]), "=d"(regs[3])
384 |                  : "a"(i), "c"(j));
385 | #endif
386 |   }
387 | 
388 |   const uint32_t& EAX() const { return regs[0]; }
389 |   const uint32_t& EBX() const { return regs[1]; }
390 |   const uint32_t& ECX() const { return regs[2]; }
391 |   const uint32_t& EDX() const { return regs[3]; }
392 | };
393 | 
394 | // https://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features
395 | bool cpu_supports_bmi() {
396 |   return static_cast<bool>(CPUID(7, 0).EBX() & (1 << 3));
397 | }
398 | }


--------------------------------------------------------------------------------
/glin/libmorton/morton3D.h:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | // Libmorton - Methods to encode/decode 64-bit morton codes from/to 32-bit (x,y,z) coordinates
  4 | // Warning: morton.h will always point to the functions that use the fastest available method.
  5 | 
  6 | #include <algorithm>
  7 | #include <stdint.h>
  8 | #include <math.h>
  9 | #include "morton3D_LUTs.h"
 10 | #include "morton_common.h"
 11 | 
 12 | #define EIGHTBITMASK (morton) 0x000000FF
 13 | #define NINEBITMASK (morton) 0x000001FF
 14 | 
 15 | 
 16 | namespace libmorton {
 17 | 	// AVAILABLE METHODS FOR ENCODING
 18 | 	template<typename morton, typename coord> inline morton m3D_e_sLUT(const coord x, const coord y, const coord z);
 19 | 	template<typename morton, typename coord> inline morton m3D_e_sLUT_ET(const coord x, const coord y, const coord z);
 20 | 	template<typename morton, typename coord> inline morton m3D_e_LUT(const coord x, const coord y, const coord z);
 21 | 	template<typename morton, typename coord> inline morton m3D_e_LUT_ET(const coord x, const coord y, const coord z);
 22 | 	template<typename morton, typename coord> inline morton m3D_e_magicbits(const coord x, const coord y, const coord z);
 23 | 	template<typename morton, typename coord> inline morton m3D_e_for(const coord x, const coord y, const coord z);
 24 | 	template<typename morton, typename coord> inline morton m3D_e_for_ET(const coord x, const coord y, const coord z);
 25 | 
 26 | 	// AVAILABLE METHODS FOR DECODING
 27 | 	template<typename morton, typename coord> inline void m3D_d_sLUT(const morton m, coord& x, coord& y, coord& z);
 28 | 	template<typename morton, typename coord> inline void m3D_d_sLUT_ET(const morton m, coord& x, coord& y, coord& z);
 29 | 	template<typename morton, typename coord> inline void m3D_d_LUT(const morton m, coord& x, coord& y, coord& z);
 30 | 	template<typename morton, typename coord> inline void m3D_d_LUT_ET(const morton m, coord& x, coord& y, coord& z);
 31 | 	template<typename morton, typename coord> inline void m3D_d_magicbits(const morton m, coord& x, coord& y, coord& z);
 32 | 	template<typename morton, typename coord> inline void m3D_d_for(const morton m, coord& x, coord& y, coord& z);
 33 | 	template<typename morton, typename coord> inline void m3D_d_for_ET(const morton m, coord& x, coord& y, coord& z);
 34 | 
 35 | 	// ENCODE 3D Morton code : Pre-Shifted LookUpTable (sLUT)
 36 | 	template<typename morton, typename coord>
 37 | 	inline morton m3D_e_sLUT(const coord x, const coord y, const coord z) {
 38 | 		morton answer = 0;
 39 | 		for (unsigned int i = sizeof(coord); i > 0; --i) {
 40 | 			unsigned int shift = (i - 1) * 8;
 41 | 			answer =
 42 | 				answer << 24 |
 43 | 				(Morton3D_encode_z_256[(z >> shift) & EIGHTBITMASK] |
 44 | 					Morton3D_encode_y_256[(y >> shift) & EIGHTBITMASK] |
 45 | 					Morton3D_encode_x_256[(x >> shift) & EIGHTBITMASK]);
 46 | 		}
 47 | 		return answer;
 48 | 	}
 49 | 
 50 | 	// ENCODE 3D Morton code : LookUpTable (LUT)
 51 | 	template<typename morton, typename coord>
 52 | 	inline morton m3D_e_LUT(const coord x, const coord y, const coord z) {
 53 | 		morton answer = 0;
 54 | 		for (unsigned int i = sizeof(coord); i > 0; --i) {
 55 | 			unsigned int shift = (i - 1) * 8;
 56 | 			answer =
 57 | 				answer << 24 |
 58 | 				(Morton3D_encode_x_256[(z >> shift) & EIGHTBITMASK] << morton(2)) |
 59 | 				(Morton3D_encode_x_256[(y >> shift) & EIGHTBITMASK] << morton(1)) |
 60 | 				Morton3D_encode_x_256[(x >> shift) & EIGHTBITMASK];
 61 | 		}
 62 | 		return answer;
 63 | 	}
 64 | 
 65 | 	// HELPER METHOD for ET LUT encode
 66 | 	template<typename morton, typename coord>
 67 | 	inline morton compute3D_ET_LUT_encode(const coord c, const coord *LUT) {
 68 | 		unsigned long maxbit = 0;
 69 | 		if (findFirstSetBit<coord>(c, &maxbit) == 0) { return 0; }
 70 | 		morton answer = 0;
 71 | 		for (int i = (int)ceil((maxbit + 1) / 8.0f); i >= 0; --i) {
 72 | 			unsigned int shift = i * 8;
 73 | 			answer = answer << 24 | (LUT[(c >> shift) & EIGHTBITMASK]);
 74 | 		}
 75 | 		return answer;
 76 | 	}
 77 | 
 78 | 	// ENCODE 3D Morton code : Pre-shifted LookUpTable (LUT) (Early Termination version)
 79 | 	// This version tries to terminate early when there are no more bits to process
 80 | 	// Figuring this out is probably too costly in most cases.
 81 | 	template<typename morton, typename coord>
 82 | 	inline morton m3D_e_sLUT_ET(const coord x, const coord y, const coord z) {
 83 | 		morton answer_x = compute3D_ET_LUT_encode<morton, coord>(x, Morton3D_encode_x_256);
 84 | 		morton answer_y = compute3D_ET_LUT_encode<morton, coord>(y, Morton3D_encode_y_256);
 85 | 		morton answer_z = compute3D_ET_LUT_encode<morton, coord>(z, Morton3D_encode_z_256);
 86 | 		return answer_z | answer_y | answer_x;
 87 | 	}
 88 | 
 89 | 	// ENCODE 3D Morton code : LookUpTable (LUT) (Early termination version)
 90 | 	// This version tries to terminate early when there are no more bits to process
 91 | 	// Figuring this out is probably too costly in most cases.
 92 | 	template<typename morton, typename coord>
 93 | 	inline morton m3D_e_LUT_ET(const coord x, const coord y, const coord z) {
 94 | 		morton answer_x = compute3D_ET_LUT_encode<morton, coord>(x, Morton3D_encode_x_256);
 95 | 		morton answer_y = compute3D_ET_LUT_encode<morton, coord>(y, Morton3D_encode_x_256);
 96 | 		morton answer_z = compute3D_ET_LUT_encode<morton, coord>(z, Morton3D_encode_x_256);
 97 | 		return (answer_z << 2) | (answer_y << 1) | answer_x;
 98 | 	}
 99 | 
100 | 	// Magicbits masks (3D encode)
101 | 	static uint_fast32_t magicbit3D_masks32_encode[6] = { 0x000003ff, 0, 0x30000ff, 0x0300f00f, 0x30c30c3, 0x9249249 }; // we add a 0 on position 1 in this array to use same code for 32-bit and 64-bit cases
102 | 	static uint_fast64_t magicbit3D_masks64_encode[6] = { 0x1fffff, 0x1f00000000ffff, 0x1f0000ff0000ff, 0x100f00f00f00f00f, 0x10c30c30c30c30c3, 0x1249249249249249 };
103 | 
104 | 	// HELPER METHOD: Magic bits encoding (helper method)
105 | 	template<typename morton, typename coord>
106 | 	static inline morton morton3D_SplitBy3bits(const coord a) {
107 | 		const morton* masks = (sizeof(morton) <= 4) ? reinterpret_cast<const morton*>(magicbit3D_masks32_encode) : reinterpret_cast<const morton*>(magicbit3D_masks64_encode);
108 | 		morton x = ((morton)a) & masks[0];
109 | 		if (sizeof(morton) == 8) { x = (x | (uint_fast64_t)x << 32) & masks[1]; } // for 64-bit case
110 | 		x = (x | x << 16) & masks[2];
111 | 		x = (x | x << 8)  & masks[3];
112 | 		x = (x | x << 4)  & masks[4];
113 | 		x = (x | x << 2)  & masks[5];
114 | 		return x;
115 | 	}
116 | 
117 | 	// ENCODE 3D Morton code : Magic bits method
118 | 	// This method uses certain bit patterns (magic bits) to split bits in the coordinates
119 | 	template<typename morton, typename coord>
120 | 	inline morton m3D_e_magicbits(const coord x, const coord y, const coord z) {
121 | 		return morton3D_SplitBy3bits<morton, coord>(x) | (morton3D_SplitBy3bits<morton, coord>(y) << 1) | (morton3D_SplitBy3bits<morton, coord>(z) << 2);
122 | 	}
123 | 
124 | 	// ENCODE 3D Morton code : For loop
125 | 	// This is the most naive way of encoding coordinates into a morton code
126 | 	template<typename morton, typename coord>
127 | 	inline morton m3D_e_for(const coord x, const coord y, const coord z) {
128 | 		morton answer = 0;
129 | 		unsigned int checkbits = static_cast<unsigned int>(floor((sizeof(morton) * 8.0f / 3.0f)));
130 | 		for (unsigned int i = 0; i < checkbits; ++i) {
131 | 			morton mshifted = static_cast<morton>(1) << i; // Here we need to cast 0x1 to 64bits, otherwise there is a bug when morton code is larger than 32 bits
132 | 			unsigned int shift = 2 * i; // because you have to shift back i and forth 3*i
133 | 			answer |= ((x & mshifted) << shift)
134 | 				| ((y & mshifted) << (shift + 1))
135 | 				| ((z & mshifted) << (shift + 2));
136 | 		}
137 | 		return answer;
138 | 	}
139 | 
140 | 	// ENCODE 3D Morton code : For loop (Early termination version)
141 | 	// In case of the for loop, figuring out when to stop early has huge benefits.
142 | 	template<typename morton, typename coord>
143 | 	inline morton m3D_e_for_ET(const coord x, const coord y, const coord z) {
144 | 		morton answer = 0;
145 | 		unsigned long x_max = 0, y_max = 0, z_max = 0;
146 | 		unsigned int checkbits = static_cast<unsigned int>(floor((sizeof(morton) * 8.0f / 3.0f)));
147 | 		findFirstSetBit<morton>(x, &x_max);
148 | 		findFirstSetBit<morton>(y, &y_max);
149 | 		findFirstSetBit<morton>(z, &z_max);
150 | 		checkbits = std::min((unsigned long)checkbits, std::max(z_max, std::max(x_max, y_max)) + (unsigned long)1);
151 | 		for (unsigned int i = 0; i < checkbits; ++i) {
152 | 			morton m_shifted = static_cast<morton>(1) << i; // Here we need to cast 0x1 to 64bits, otherwise there is a bug when morton code is larger than 32 bits
153 | 			unsigned int shift = 2 * i;
154 | 			answer |= ((x & m_shifted) << shift)
155 | 				| ((y & m_shifted) << (shift + 1))
156 | 				| ((z & m_shifted) << (shift + 2));
157 | 		}
158 | 		return answer;
159 | 	}
160 | 
161 | 
162 | 	// HELPER METHOD for LUT decoding
163 | 	// todo: wouldn't this be better with 8-bit aligned decode LUT?
164 | 	template<typename morton, typename coord>
165 | 	inline coord morton3D_DecodeCoord_LUT256(const morton m, const uint_fast8_t *LUT, const unsigned int startshift) {
166 | 		morton a = 0;
167 | 		unsigned int loops = (sizeof(morton) <= 4) ? 4 : 7; // ceil for 32bit, floor for 64bit
168 | 		for (unsigned int i = 0; i < loops; ++i) {
169 | 			a |= (morton)(LUT[(m >> ((i * 9) + startshift)) & NINEBITMASK] << morton(3 * i));
170 | 		}
171 | 		return static_cast<coord>(a);
172 | 	}
173 | 
174 | 	// DECODE 3D Morton code : Shifted LUT
175 | 	template<typename morton, typename coord>
176 | 	inline void m3D_d_sLUT(const morton m, coord& x, coord& y, coord& z) {
177 | 		x = morton3D_DecodeCoord_LUT256<morton, coord>(m, Morton3D_decode_x_512, 0);
178 | 		y = morton3D_DecodeCoord_LUT256<morton, coord>(m, Morton3D_decode_y_512, 0);
179 | 		z = morton3D_DecodeCoord_LUT256<morton, coord>(m, Morton3D_decode_z_512, 0);
180 | 	}
181 | 
182 | 	// DECODE 3D Morton code : LUT
183 | 	template<typename morton, typename coord>
184 | 	inline void m3D_d_LUT(const morton m, coord& x, coord& y, coord& z) {
185 | 		x = morton3D_DecodeCoord_LUT256<morton, coord>(m, Morton3D_decode_x_512, 0);
186 | 		y = morton3D_DecodeCoord_LUT256<morton, coord>(m, Morton3D_decode_x_512, 1);
187 | 		z = morton3D_DecodeCoord_LUT256<morton, coord>(m, Morton3D_decode_x_512, 2);
188 | 	}
189 | 
190 | 	// DECODE 3D Morton code : Shifted LUT (Early termination version)
191 | 	template<typename morton, typename coord>
192 | 	inline void m3D_d_sLUT_ET(const morton m, coord& x, coord& y, coord& z) {
193 | 		x = 0; y = 0; z = 0;
194 | 		unsigned long firstbit_location = 0;
195 | 		if (!findFirstSetBit<morton>(m, &firstbit_location)) { return; }
196 | 		unsigned int i = 0;
197 | 		unsigned int shiftback = 0;
198 | 		while (firstbit_location > i) {
199 | 			morton m_shifted = (m >> i) & NINEBITMASK;
200 | 			x |= (coord)Morton3D_decode_x_512[m_shifted] << shiftback;
201 | 			y |= (coord)Morton3D_decode_y_512[m_shifted] << shiftback;
202 | 			z |= (coord)Morton3D_decode_z_512[m_shifted] << shiftback;
203 | 			shiftback += 3;
204 | 			i += 9;
205 | 		}
206 | 		return;
207 | 	}
208 | 
209 | 	// DECODE 3D Morton code : LUT (Early termination version)
210 | 	template<typename morton, typename coord>
211 | 	inline void m3D_d_LUT_ET(const morton m, coord& x, coord& y, coord& z) {
212 | 		x = 0; y = 0; z = 0;
213 | 		unsigned long firstbit_location = 0;
214 | 		if (!findFirstSetBit<morton>(m, &firstbit_location)) { return; }
215 | 		unsigned int i = 0;
216 | 		unsigned int shiftback = 0;
217 | 		while (i < firstbit_location) {
218 | 			x = x | (coord)Morton3D_decode_x_512[(m >> i) & NINEBITMASK] << shiftback;
219 | 			y = y | (coord)Morton3D_decode_x_512[(m >> (i + 1)) & NINEBITMASK] << shiftback;
220 | 			z = z | (coord)Morton3D_decode_x_512[(m >> (i + 2)) & NINEBITMASK] << shiftback;
221 | 			i += 9;
222 | 			shiftback += 3;
223 | 		}
224 | 		return;
225 | 	}
226 | 
227 | 	// Magicbits masks (3D decode)
228 | 	static uint_fast32_t magicbit3D_masks32_decode[6] = { 0, 0x000003ff, 0x30000ff, 0x0300f00f, 0x30c30c3, 0x9249249 }; // we add a 0 on position 0 in this array to use same code for 32-bit and 64-bit cases
229 | 	static uint_fast64_t magicbit3D_masks64_decode[6] = { 0x1fffff, 0x1f00000000ffff, 0x1f0000ff0000ff, 0x100f00f00f00f00f, 0x10c30c30c30c30c3, 0x1249249249249249 };
230 | 
231 | 	// HELPER METHOD for Magic bits decoding
232 | 	template<typename morton, typename coord>
233 | 	static inline coord morton3D_GetThirdBits(const morton m) {
234 | 		morton* masks = (sizeof(morton) <= 4) ? reinterpret_cast<morton*>(magicbit3D_masks32_decode) : reinterpret_cast<morton*>(magicbit3D_masks64_decode);
235 | 		morton x = m & masks[5];
236 | 		x = (x ^ (x >> 2)) & masks[4];
237 | 		x = (x ^ (x >> 4)) & masks[3];
238 | 		x = (x ^ (x >> 8)) & masks[2];
239 | 		x = (x ^ (x >> 16)) & masks[1];
240 | 		if (sizeof(morton) > 4) { x = (x ^ ((uint_fast64_t)x >> 32)) & masks[0]; }
241 | 		return static_cast<coord>(x);
242 | 	}
243 | 
244 | 	// DECODE 3D Morton code : Magic bits
245 | 	// This method splits the morton codes bits by using certain patterns (magic bits)
246 | 	template<typename morton, typename coord>
247 | 	inline void m3D_d_magicbits(const morton m, coord& x, coord& y, coord& z) {
248 | 		x = morton3D_GetThirdBits<morton, coord>(m);
249 | 		y = morton3D_GetThirdBits<morton, coord>(m >> 1);
250 | 		z = morton3D_GetThirdBits<morton, coord>(m >> 2);
251 | 	}
252 | 
253 | 	// DECODE 3D Morton code : For loop
254 | 	template<typename morton, typename coord>
255 | 	inline void m3D_d_for(const morton m, coord& x, coord& y, coord& z) {
256 | 		x = 0; y = 0; z = 0;
257 | 		unsigned int checkbits = static_cast<unsigned int>(floor((sizeof(morton) * 8.0f / 3.0f)));
258 | 		for (unsigned int i = 0; i <= checkbits; ++i) {
259 | 			morton selector = 1;
260 | 			unsigned int shift_selector = 3 * i;
261 | 			unsigned int shiftback = 2 * i;
262 | 			x |= (m & (selector << shift_selector)) >> (shiftback);
263 | 			y |= (m & (selector << (shift_selector + 1))) >> (shiftback + 1);
264 | 			z |= (m & (selector << (shift_selector + 2))) >> (shiftback + 2);
265 | 		}
266 | 	}
267 | 
268 | 	// DECODE 3D Morton code : For loop (Early termination version)
269 | 	template<typename morton, typename coord>
270 | 	inline void m3D_d_for_ET(const morton m, coord& x, coord& y, coord& z) {
271 | 		x = 0; y = 0; z = 0;
272 | 		unsigned long firstbit_location = 0;
273 | 		if (!findFirstSetBit<morton>(m, &firstbit_location)) return;
274 | 		float defaultbits = floor((sizeof(morton) * 8.0f / 3.0f));
275 | 		unsigned int checkbits = static_cast<unsigned int>(std::min(defaultbits, firstbit_location / 3.0f));
276 | 		for (unsigned int i = 0; i <= checkbits; ++i) {
277 | 			morton selector = 1;
278 | 			unsigned int shift_selector = 3 * i;
279 | 			unsigned int shiftback = 2 * i;
280 | 			x |= (m & (selector << shift_selector)) >> (shiftback);
281 | 			y |= (m & (selector << (shift_selector + 1))) >> (shiftback + 1);
282 | 			z |= (m & (selector << (shift_selector + 2))) >> (shiftback + 2);
283 | 		}
284 | 	}
285 | }
286 | 


--------------------------------------------------------------------------------
/test/unittest_alex.h:
--------------------------------------------------------------------------------
  1 | // Copyright (c) Microsoft Corporation.
  2 | // Licensed under the MIT license.
  3 | 
  4 | #include "doctest.h"
  5 | 
  6 | #include "alex.h"
  7 | 
  8 | using namespace alex;
  9 | 
 10 | TEST_SUITE("Alex") {
 11 | 
 12 | TEST_CASE("TestBulkLoad") {
 13 |   Alex<int, int> index;
 14 | 
 15 |   Alex<int, int>::V values[500];
 16 |   for (int i = 0; i < 500; i++) {
 17 |     values[i].first = rand() % 5000;
 18 |     values[i].second = i;
 19 |   }
 20 | 
 21 |   std::sort(values, values + 500);
 22 |   index.bulk_load(values, 500);
 23 | 
 24 |   for (int i = 0; i < 500; i++) {
 25 |     auto it = index.find(values[i].first);
 26 |     CHECK(!it.is_end());
 27 |     CHECK_EQ(values[i].first, it.key());
 28 |   }
 29 | 
 30 |   CHECK_EQ(index.get_stats().num_keys, 500);
 31 |   index.clear();
 32 |   CHECK_EQ(index.get_stats().num_keys, 0);
 33 | }
 34 | 
 35 | TEST_CASE("TestConstructors") {
 36 |   Alex<int, int> index;
 37 | 
 38 |   Alex<int, int>::V values[500];
 39 |   for (int i = 0; i < 500; i++) {
 40 |     values[i].first = rand() % 5000;
 41 |     values[i].second = i;
 42 |   }
 43 | 
 44 |   std::sort(values, values + 500);
 45 |   index.bulk_load(values, 500);
 46 | 
 47 |   Alex<int, int> index2(index);  // Copy constructor
 48 |   Alex<int, int> index3;
 49 |   index3 = index;  // Assignment
 50 |   Alex<int, int> index4(std::begin(values), std::end(values));
 51 | 
 52 |   CHECK_NE(index.root_node_, index2.root_node_);
 53 |   CHECK_NE(index.root_node_, index3.root_node_);
 54 | 
 55 |   for (int i = 0; i < 500; i++) {
 56 |     auto it2 = index2.find(values[i].first);
 57 |     CHECK(!it2.is_end());
 58 |     CHECK_EQ(values[i].first, it2.key());
 59 | 
 60 |     auto it3 = index3.find(values[i].first);
 61 |     CHECK(!it3.is_end());
 62 |     CHECK_EQ(values[i].first, it3.key());
 63 | 
 64 |     auto it4 = index4.find(values[i].first);
 65 |     CHECK(!it4.is_end());
 66 |     CHECK_EQ(values[i].first, it4.key());
 67 |   }
 68 | }
 69 | 
 70 | TEST_CASE("TestIterators") {
 71 |   Alex<int, int> index;
 72 | 
 73 |   Alex<int, int>::V values[500];
 74 |   for (int i = 0; i < 250; i++) {
 75 |     values[i].first = i;
 76 |     values[i].second = i;
 77 |   }
 78 |   for (int i = 250; i < 500; i++) {
 79 |     values[i].first = 5 * i;
 80 |     values[i].second = i;
 81 |   }
 82 | 
 83 |   std::sort(values, values + 500);
 84 |   index.bulk_load(values, 500);
 85 | 
 86 |   // Iterator from beginning to end
 87 |   int num_keys = 0;
 88 |   for (auto it = index.begin(); it != index.end(); ++it) {
 89 |     num_keys++;
 90 |   }
 91 |   CHECK_EQ(500, num_keys);
 92 | 
 93 |   // Const iterator from beginning to end
 94 |   num_keys = 0;
 95 |   for (auto it = index.cbegin(); it != index.cend(); ++it) {
 96 |     num_keys++;
 97 |   }
 98 |   CHECK_EQ(500, num_keys);
 99 | 
100 |   // Reverse iterator from beginning to end
101 |   num_keys = 0;
102 |   for (auto it = index.rbegin(); it != index.rend(); ++it) {
103 |     num_keys++;
104 |   }
105 |   CHECK_EQ(500, num_keys);
106 | 
107 |   // Const reverse iterator from beginning to end
108 |   num_keys = 0;
109 |   for (auto it = index.crbegin(); it != index.crend(); ++it) {
110 |     num_keys++;
111 |   }
112 |   CHECK_EQ(500, num_keys);
113 | 
114 |   // Convert iterator to reverse iterator
115 |   auto it = index.find(values[250].first);
116 |   auto rit = Alex<int, int>::ReverseIterator(it);
117 |   num_keys = 0;
118 |   for (; it != index.end(); ++it) {
119 |     num_keys++;
120 |   }
121 |   for (; rit != index.rend(); ++rit) {
122 |     num_keys++;
123 |   }
124 |   CHECK_EQ(501, num_keys);
125 | 
126 |   // Convert const iterator to const reverse iterator
127 |   typename Alex<int, int>::ConstIterator cit = index.find(values[250].first);
128 |   auto crit = Alex<int, int>::ConstReverseIterator(cit);
129 |   num_keys = 0;
130 |   for (; cit != index.cend(); ++cit) {
131 |     num_keys++;
132 |   }
133 |   for (; crit != index.crend(); ++crit) {
134 |     num_keys++;
135 |   }
136 |   CHECK_EQ(501, num_keys);
137 | }
138 | 
139 | TEST_CASE("TestConst") {
140 |   Alex<int, int>::V values[500];
141 |   // even numbers from 0 to 998 inclusive
142 |   for (int i = 0; i < 500; i++) {
143 |     values[i].first = i * 2;
144 |     values[i].second = i;
145 |   }
146 | 
147 |   const Alex<int, int> index(std::begin(values), std::end(values));
148 | 
149 |   // Find existent keys
150 |   for (int i = 0; i < 500; i++) {
151 |     auto it = index.find(values[i].first);
152 |     CHECK(!it.is_end());
153 |     CHECK_EQ(values[i].first, it.key());
154 |   }
155 | }
156 | 
157 | TEST_CASE("TestFind") {
158 |   Alex<int, int> index;
159 | 
160 |   Alex<int, int>::V values[500];
161 |   // even numbers from 0 to 998 inclusive
162 |   for (int i = 0; i < 500; i++) {
163 |     values[i].first = i * 2;
164 |     values[i].second = i;
165 |   }
166 | 
167 |   std::sort(values, values + 500);
168 |   index.bulk_load(values, 500);
169 | 
170 |   // Find existent keys
171 |   for (int i = 0; i < 500; i++) {
172 |     auto it = index.find(values[i].first);
173 |     CHECK(!it.is_end());
174 |     CHECK_EQ(values[i].first, it.key());
175 | 
176 |     int *p = index.get_payload(values[i].first);
177 |     CHECK(p);
178 |     CHECK_EQ(values[i].second, *p);
179 |   }
180 | 
181 |   // Find non-existent keys
182 |   for (int i = 1; i < 100; i += 2) {
183 |     auto it = index.find(i);
184 |     CHECK(it.is_end());
185 | 
186 |     int *p = index.get_payload(i);
187 |     CHECK(!p);
188 |   }
189 | }
190 | 
191 | // Also tests count and equal_range
192 | TEST_CASE("TestLowerUpperBound") {
193 |   Alex<int, int> index;
194 | 
195 |   Alex<int, int>::V values[100];
196 |   // 10 each of 0, 10, ..., 90
197 |   for (int i = 0; i < 100; i += 10) {
198 |     for (int j = 0; j < 10; j++) {
199 |       values[i + j].first = i;
200 |       values[i + j].second = 0;
201 |     }
202 |   }
203 | 
204 |   std::sort(values, values + 100);
205 |   index.bulk_load(values, 100);
206 | 
207 |   // Search for existent keys
208 |   for (int i = 0; i < 100; i += 10) {
209 |     auto it_lb = index.lower_bound(i);
210 |     auto it_ub = index.upper_bound(i);
211 |     CHECK(!it_lb.is_end());
212 |     CHECK_EQ(i, it_lb.key());
213 |     if (i == 90) {
214 |       CHECK(it_ub.is_end());
215 |     } else {
216 |       CHECK(!it_ub.is_end());
217 |       CHECK_EQ(i + 10, it_ub.key());
218 |     }
219 | 
220 |     // Count
221 |     size_t count = index.count(i);
222 |     CHECK_EQ(count, 10);
223 | 
224 |     // Equal range
225 |     auto it_pair = index.equal_range(i);
226 |     CHECK_EQ(it_pair.first, it_lb);
227 |     CHECK_EQ(it_pair.second, it_ub);
228 |   }
229 | 
230 |   // Search for non-existent keys
231 |   for (int i = -5; i <= 95; i += 10) {
232 |     auto it_lb = index.lower_bound(i);
233 |     auto it_ub = index.upper_bound(i);
234 |     if (i > 90) {
235 |       CHECK(it_lb.is_end());
236 |       CHECK(it_ub.is_end());
237 |     } else {
238 |       CHECK(!it_lb.is_end());
239 |       CHECK(!it_ub.is_end());
240 |       CHECK_EQ(i + 5, it_lb.key());
241 |       CHECK_EQ(i + 5, it_ub.key());
242 |     }
243 | 
244 |     // Count
245 |     CHECK_EQ(index.count(i), 0);
246 | 
247 |     // Equal range
248 |     auto it_pair = index.equal_range(i);
249 |     CHECK_EQ(it_pair.first, it_lb);
250 |     CHECK_EQ(it_lb, it_ub);
251 |   }
252 | }
253 | 
254 | TEST_CASE("TestFindLastNoGreaterThan") {
255 |   Alex<int, int> index;
256 | 
257 |   Alex<int, int>::V values[500];
258 |   // even numbers from 0 to 998 inclusive
259 |   for (int i = 0; i < 500; i++) {
260 |     values[i].first = i * 2;
261 |     values[i].second = i;
262 |   }
263 | 
264 |   std::sort(values, values + 500);
265 |   index.bulk_load(values, 500);
266 | 
267 |   // Existent keys
268 |   for (int i = 0; i < 500; i++) {
269 |     auto it = index.find_last_no_greater_than(values[i].first);
270 |     CHECK(!it.is_end());
271 |     CHECK_EQ(values[i].first, it.key());
272 | 
273 |     int *p = index.get_payload_last_no_greater_than(values[i].first);
274 |     CHECK(p);
275 |     CHECK_EQ(values[i].second, *p);
276 |   }
277 | 
278 |   // Non-existent keys
279 |   for (int i = 0; i < 500; i++) {
280 |     int key = values[i].first + 1;
281 |     auto it = index.find_last_no_greater_than(key);
282 |     CHECK(!it.is_end());
283 |     CHECK_LE(it.key(), key);
284 |     it++;
285 |     if (!it.is_end()) {
286 |       CHECK_GT(it.key(), key);
287 |     }
288 | 
289 |     int *p = index.get_payload_last_no_greater_than(key);
290 |     CHECK(p);
291 |     CHECK_EQ(values[i].second, *p);
292 |   }
293 | 
294 |   // Non-existent key smaller than min
295 |   auto it = index.find_last_no_greater_than(-1);
296 |   CHECK(!it.is_end());
297 |   CHECK_EQ(values[0].first, it.key());
298 | }
299 | 
300 | TEST_CASE("TestLargeFindLastNoGreaterThan") {
301 |   Alex<uint64_t, uint64_t> index;
302 |   index.insert(std::make_pair(0ULL, 0ULL));
303 | 
304 |   const uint64_t keys_per_segment = 80523;
305 |   const uint64_t step_size = 43206176;
306 |   const uint64_t num_segments = 16;
307 |   const uint64_t start_keys[] = {698631712, 658125922, 660826308, 663526694,
308 |                                  666227080, 668927466, 671627852, 674328238,
309 |                                  677028624, 679729010, 682429396, 685129782,
310 |                                  687830168, 690530554, 693230940, 695931326};
311 | 
312 |   uint64_t max_key = 0;
313 |   uint64_t max_key_value = 0;
314 |   for (uint64_t segment = 0; segment < num_segments; ++segment) {
315 |     uint64_t curr_key = start_keys[segment];
316 |     for (uint64_t i = 0; i < keys_per_segment; ++i) {
317 |       if (curr_key > max_key) {
318 |         max_key = curr_key;
319 |         max_key_value = i + 1;
320 |       }
321 | 
322 |       index.insert(curr_key, i + 1);
323 |       curr_key += step_size;
324 |     }
325 |   }
326 | 
327 |   // This key is larger than all keys in the index.
328 |   const uint64_t test_key = 3650322694401;
329 |   CHECK_GT(test_key, max_key);
330 | 
331 |   auto it = index.find_last_no_greater_than(test_key);
332 |   CHECK(!it.is_end());
333 |   CHECK_EQ(max_key, it.key());
334 | 
335 |   const uint64_t *p = index.get_payload_last_no_greater_than(test_key);
336 |   CHECK(p);
337 |   CHECK_EQ(max_key_value, *p);
338 | }
339 | 
340 | TEST_CASE("TestReadModifyWrite") {
341 |   Alex<int, int> index;
342 | 
343 |   Alex<int, int>::V values[100];
344 |   for (int i = 0; i < 100; i++) {
345 |     values[i].first = i;
346 |     values[i].second = 0;
347 |   }
348 | 
349 |   std::sort(values, values + 100);
350 |   index.bulk_load(values, 100);
351 | 
352 |   auto it = index.find(50);
353 |   CHECK(!it.is_end());
354 |   CHECK_EQ(50, it.key());
355 |   CHECK_EQ(0, it.payload());
356 | 
357 |   it.payload() = 50;
358 | 
359 |   it = index.find(50);
360 |   CHECK(!it.is_end());
361 |   CHECK_EQ(50, it.key());
362 |   CHECK_EQ(50, it.payload());
363 | }
364 | 
365 | TEST_CASE("TestSequentialInserts") {
366 |   Alex<int, int> index;
367 | 
368 |   Alex<int, int>::V values[50];
369 |   for (int i = 0; i < 50; i++) {
370 |     values[i].first = i;
371 |     values[i].second = i;
372 |   }
373 | 
374 |   std::sort(values, values + 50);
375 |   index.bulk_load(values, 50);
376 | 
377 |   for (int i = 50; i < 200; i++) {
378 |     auto ret = index.insert(i, i);
379 |     CHECK_EQ(ret.first.key(), i);
380 |   }
381 | 
382 |   for (int i = 0; i < 200; i++) {
383 |     auto it = index.find(i);
384 |     CHECK(!it.is_end());
385 |     CHECK_EQ(i, it.key());
386 |   }
387 | }
388 | 
389 | TEST_CASE("TestOrderedInserts") {
390 |   Alex<int, int> index;
391 | 
392 |   Alex<int, int>::V values[100];
393 |   for (int i = 0; i < 100; i++) {
394 |     values[i].first = 2 * i;
395 |     values[i].second = i;
396 |   }
397 | 
398 |   std::sort(values, values + 100);
399 |   index.bulk_load(values, 100);
400 | 
401 |   for (int i = 0; i < 100; i++) {
402 |     auto ret = index.insert(2 * i + 1, i);
403 |     CHECK_EQ(ret.first.key(), 2 * i + 1);
404 |   }
405 | 
406 |   // Check that getting the key is correct.
407 |   for (int i = 0; i < 200; i++) {
408 |     auto it = index.find(i);
409 |     CHECK(!it.is_end());
410 |     CHECK_EQ(i, it.key());
411 |   }
412 | }
413 | 
414 | TEST_CASE("TestRandomInserts") {
415 |   Alex<int, int> index;
416 | 
417 |   Alex<int, int>::V values[200];
418 |   for (int i = 0; i < 200; i++) {
419 |     values[i].first = rand() % 500;
420 |     values[i].second = i;
421 |   }
422 | 
423 |   std::sort(values, values + 25);
424 |   index.bulk_load(values, 25);
425 | 
426 |   for (int i = 25; i < 200; i++) {
427 |     auto ret = index.insert(values[i].first, values[i].second);
428 |     CHECK_EQ(ret.first.key(), values[i].first);
429 |   }
430 | 
431 |   // Check that getting the key is correct.
432 |   for (int i = 0; i < 200; i++) {
433 |     auto it = index.find(values[i].first);
434 |     CHECK(!it.is_end());
435 |     CHECK_EQ(values[i].first, it.key());
436 |   }
437 | }
438 | 
439 | TEST_CASE("TestInsertFromEmpty") {
440 |   Alex<int, int> index;
441 | 
442 |   Alex<int, int>::V values[200];
443 |   for (int i = 0; i < 200; i++) {
444 |     values[i].first = rand() % 500;
445 |     values[i].second = i;
446 |   }
447 | 
448 |   for (int i = 0; i < 200; i++) {
449 |     auto ret = index.insert(values[i].first, values[i].second);
450 |     CHECK_EQ(ret.first.key(), values[i].first);
451 |   }
452 | 
453 |   // Check that getting the key is correct.
454 |   for (int i = 0; i < 200; i++) {
455 |     auto it = index.find(values[i].first);
456 |     CHECK(!it.is_end());
457 |     CHECK_EQ(values[i].first, it.key());
458 |   }
459 | }
460 | 
461 | TEST_CASE("TestRandomErases") {
462 |   Alex<int, int> index;
463 | 
464 |   Alex<int, int>::V values[200];
465 |   for (int i = 0; i < 200; i++) {
466 |     values[i].first = rand() % 500;
467 |     values[i].second = i;
468 |   }
469 | 
470 |   std::sort(values, values + 200);
471 |   index.bulk_load(values, 200);
472 | 
473 |   // Try to erase a nonexistent key
474 |   CHECK_EQ(index.erase_one(1000), 0);
475 | 
476 |   // Erase with key
477 |   for (int i = 0; i < 100; i++) {
478 |     int num_erased = index.erase_one(values[i].first);
479 |     CHECK_EQ(num_erased, 1);
480 |   }
481 | 
482 |   // Erase with iterator
483 |   for (int i = 100; i < 200; i++) {
484 |     auto it = index.lower_bound(values[i].first);
485 |     CHECK(!it.is_end());
486 |     index.erase(it);
487 |   }
488 | 
489 |   CHECK_EQ(index.stats_.num_keys, 0);
490 | }
491 | 
492 | TEST_CASE("TestRangeScan") {
493 |   Alex<int, int> index;
494 | 
495 |   Alex<int, int>::V values[200];
496 |   for (int i = 0; i < 200; i++) {
497 |     values[i].first = i;
498 |     values[i].second = i;
499 |   }
500 | 
501 |   std::sort(values, values + 200);
502 |   index.bulk_load(values, 200);
503 | 
504 |   std::vector<int> results;
505 |   int sum = 0;
506 |   auto it = index.begin();
507 |   for (; it != index.end(); it++) {
508 |     results.push_back((*it).second);
509 |     sum += (*it).second;
510 |   }
511 |   CHECK_EQ(results.size(), 200);
512 |   CHECK_EQ(sum, 19900);
513 | 
514 |   std::vector<int> results2;
515 |   int sum2 = 0;
516 |   auto it2 = index.find(10);
517 |   auto it_end = index.find(100);
518 |   for (; it2 != it_end; it2++) {
519 |     results2.push_back((*it2).second);
520 |     sum2 += (*it2).second;
521 |   }
522 |   CHECK_EQ(results2.size(), 90);
523 |   CHECK_EQ(sum2, 4905);
524 | }
525 | };
526 | 


--------------------------------------------------------------------------------
/glin/piecewise.h:
--------------------------------------------------------------------------------
  1 | //
  2 | // Created by juno on 7/10/21.
  3 | //
  4 | #include "alex.h"
  5 | #include "array"
  6 | 
  7 | #ifndef ALEX_PIECEWISE_H
  8 | #define ALEX_PIECEWISE_H
  9 | 
 10 | //const int size = 0;
 11 | double cal_error(int current_count, double current_max, double current_sum) {
 12 |     double current_average = current_sum / current_count;
 13 |     double error = (current_max - current_average) / current_max;
 14 |     return error;
 15 | }
 16 | 
 17 | bool endpointLess(std::pair<double, double> a, std::pair<double, double> b) { return a.second < b.second; }
 18 | 
 19 | static bool sortbysec_insert(const std::tuple<double, double, double, double> &a,
 20 |                              const std::tuple<double, double, double, double> &b) {
 21 |     return (std::get<0>(a) < std::get<0>(b));
 22 | }
 23 | 
 24 | // the returned pieces aka piecewise function vectors with max range
 25 | //void piecewise(std::pair<double, double> records[], int size, double piece_limitation,
 26 | //               std::vector<std::tuple<double, double, double, double>> &pieces) {
 27 | //    //sort by range end point
 28 | //    std::sort(records, (records + size), endpointLess);
 29 | //    for(int i =0; i < size; i++){
 30 | //        std::cout<<"endpoint,"<< i << ","<< std::to_string( records[i].second )<< std::endl;
 31 | //    }
 32 | //    // maintain 3 parameter for bucket
 33 | //    int current_count = 1;
 34 | //    double current_max = records[0].second - records[0].first;
 35 | //    //push the very first start point as piecewise function's first piece
 36 | //    pieces.emplace_back(records[0].first, std::numeric_limits<double>::min(), std::numeric_limits<double>::min(),
 37 | //                       std::numeric_limits<double>::min());
 38 | //
 39 | //    double current_sum = current_max;
 40 | //
 41 | //    for (int i = 1; i < size; i++) {
 42 | //#ifdef DEBUG
 43 | //        std::cout<<"sorted record is " << records[i].first <<"  " << records[i].second << endl;
 44 | //        assert(records[i].second >= records[i-1].second);
 45 | //#endif
 46 | //
 47 | //        double current_range = records[i].second - records[i].first;
 48 | //
 49 | //        if(current_range != 0 ) {
 50 | //            assert(current_range != 0);
 51 | //            current_count += 1;
 52 | //            if(current_count > piece_limitation) {
 53 | //                double old_max = current_max;
 54 | //                double old_sum = current_sum;
 55 | //                current_max = std::max(current_max, current_range);
 56 | //                current_sum = current_sum + current_range;
 57 | //#ifdef DEBUG
 58 | //                cout<<"the current error to check if start a new pieace "<< current_error << endl;
 59 | //#endif
 60 | //#ifdef DEBUG
 61 | //                    std::cout<< "endpoint is " << records[i-1].second <<" max range " << old_max<<endl;
 62 | //#endif
 63 | //                    int target_offset = i - 1;
 64 | //                    //push where this pieces end to form the piece
 65 | //                    // check if the records share the same end point, if the next record is same, keep reading until next different
 66 | //                    // endpoint record.
 67 | ////                    check if last record has 0 range. if has 0 range, then continue to next record
 68 | //                    if (records[target_offset].second - records[target_offset].first == 0) {
 69 | //                        current_count += 1;
 70 | //                        continue;
 71 | //                    }
 72 | //                    while (records[target_offset].second == records[target_offset + 1].second && target_offset < size) {
 73 | //#ifdef DEBUG
 74 | //                        std::cout << target_offset << "  offset " << endl;
 75 | //#endif
 76 | //                        old_max = std::max(old_max, current_range);
 77 | //                        current_count += 1;
 78 | //                        old_sum = current_sum;
 79 | //                        current_sum += records[target_offset + 1].second - records[target_offset + 1].first;
 80 | //                        target_offset += 1;
 81 | //                    }
 82 | //                    pieces.emplace_back(
 83 | //                            (std::make_tuple(records[target_offset].second, old_max, current_count - 1, old_sum)));
 84 | //#ifdef DEBUG
 85 | //                    cout<<"now having "<<pieces.size()<<" buckets"<<endl;
 86 | //#endif
 87 | //                    current_count = 1;
 88 | //                    current_max = current_range;
 89 | //                    current_sum = current_range;
 90 | //                    i = target_offset + 1;
 91 | //
 92 | //            }else{
 93 | //                current_max = std::max(current_max, current_range);
 94 | //                current_sum = current_sum + current_range;
 95 | //                if(i == size -1){
 96 | //                    pieces.emplace_back(records[size - 1].second, current_max, current_count, current_sum);
 97 | //                }
 98 | //                continue;
 99 | //            }
100 | //        }else{
101 | //            current_count+=1;
102 | //            continue;
103 | //        }
104 | //
105 | //    }
106 | //    if(records[size -1].second == std::get<0>(pieces.back())){
107 | //        return;
108 | //    }else {
109 | //        pieces.emplace_back(records[size - 1].second, current_max, current_count, current_sum);
110 | //    }
111 | //}
112 | 
113 | /*
114 |  * pieceiwise function with Zmin
115 |  */
116 | void piecewise(std::pair<double, double> records[], int size, double piece_limitation,
117 |                std::vector<std::tuple<double, double, double, double>> &pieces) {
118 |     //sort by range end point
119 |     std::sort(records, (records + size), endpointLess);
120 |     //print out for cdf
121 | //    for(int i =0; i < size; i++){
122 | //        std::cout<<"endpoint,"<< i << ","<< std::to_string( records[i].second )<< std::endl;
123 | //    }
124 |     // maintain 3 parameter for bucket
125 |     int current_count = 1;
126 |     double current_min_zmin = records[0].first;
127 |     //push the very first start point as piecewise function's first piece
128 |     pieces.emplace_back(records[0].first, std::numeric_limits<double>::min(), std::numeric_limits<double>::min(),
129 |                         std::numeric_limits<double>::min());
130 | 
131 |     double current_sum = current_min_zmin;
132 | 
133 |     for (int i = 1; i < size; i++) {
134 | #ifdef DEBUG
135 |         std::cout<<"sorted record is " << records[i].first <<"  " << records[i].second << endl;
136 |         assert(records[i].second >= records[i-1].second);
137 | #endif
138 | 
139 |         double current_zmin =  records[i].first;
140 |             current_count += 1;
141 |             if(current_count > piece_limitation) {
142 |                 double old_min_zmin = current_min_zmin;
143 |                 double old_sum = current_sum;
144 |                 current_min_zmin = std::min(current_min_zmin, current_zmin);
145 |                 current_sum = current_sum + current_zmin;
146 | #ifdef DEBUG
147 | 
148 |                 cout<<"the current error to check if start a new pieace "<< current_error << endl;
149 | #endif
150 | #ifdef DEBUG
151 |                 std::cout<< "endpoint is " << records[i-1].second <<" max range " << old_max<<endl;
152 | #endif
153 |                 int target_offset = i - 1;
154 |                 //push where this pieces end to form the piece
155 |                 // check if the records share the same end point, if the next record is same, keep reading until next different
156 |                 // endpoint record.
157 | //                    check if last record has 0 range. if has 0 range, then continue to next record
158 | 
159 |                 while (records[target_offset].second == records[target_offset + 1].second && target_offset < size) {
160 | #ifdef DEBUG
161 |                     std::cout << target_offset << "  offset " << endl;
162 | #endif
163 |                     old_min_zmin = std::min(old_min_zmin, current_zmin);
164 |                     current_count += 1;
165 |                     old_sum = current_sum;
166 |                     current_sum += records[target_offset + 1].first;
167 |                     target_offset += 1;
168 |                 }
169 |                 pieces.emplace_back(
170 |                         (std::make_tuple(records[target_offset].second, old_min_zmin, current_count - 1, old_sum)));
171 | #ifdef DEBUG
172 |                 cout<<"now having "<<pieces.size()<<" buckets"<<endl;
173 | #endif
174 |                 current_count = 1;
175 |                 current_min_zmin = current_zmin;
176 |                 current_sum = current_zmin;
177 |                 i = target_offset + 1;
178 | 
179 |             }else{
180 |                 current_min_zmin = std::min(current_min_zmin, current_zmin);
181 |                 current_sum = current_sum + current_zmin;
182 |                 if(i == size -1){
183 |                     pieces.emplace_back(records[size - 1].second, current_min_zmin, current_count, current_sum);
184 |                 }
185 |                 continue;
186 |             }
187 |     }
188 |     if(records[size -1].second == std::get<0>(pieces.back())){
189 |         return;
190 |     }else {
191 |         pieces.emplace_back(records[size - 1].second, current_min_zmin, current_count, current_sum);
192 |     }
193 | }
194 | 
195 | 
196 | 
197 | // update pieces
198 | void insert_pieces(double range_start, double range_end, double piece_limit,
199 |                    std::vector<std::tuple<double, double, double, double>> &pieces) {
200 | 
201 |     std::vector<std::tuple<double, double, double, double>>::iterator up;
202 |     //need to consider the index is empty
203 |     //when the index is empty
204 |     // check if empty first
205 |     if (pieces.empty()) {
206 |         pieces.emplace_back(range_start, std::numeric_limits<double>::min(), std::numeric_limits<double>::min(),
207 |                                     std::numeric_limits<double>::min()); // numeric limit
208 |         pieces.emplace_back(range_end,  range_start, 1.0, range_start);
209 |         return;
210 |     }
211 |     // lowerbound position that this data should be insert
212 |     // update found piece's count, max, sum
213 |     // lowerbound:  found the most left value that not less than the look up value
214 |     up = std::lower_bound(pieces.begin(), pieces.end(), std::make_tuple(range_end, -1.0, -1.0, -1.0), sortbysec_insert);
215 |     auto piece_position = up - pieces.begin();
216 |     //current distance to the pieces iterator end, end - currentposition should > 0
217 | //    auto dis_to_end = pieces.end() - up;
218 |     // the start is large than the piecewise's start, and end < piecewise's end
219 |     // count/max/sum in the piece, will be update by inserted data
220 |     if (piece_position == pieces.size()) { // the inserted data start is larger than the very last piece
221 |         double piece_min_zmin = std::get<1>(pieces[piece_position - 1]);
222 |         double piece_count = std::get<2>(pieces[piece_position -1]);
223 |         double piece_sum = std::get<3>(pieces[piece_position -1]);
224 |         double current_min_zmin = std::min(piece_min_zmin, range_start);
225 |         double current_count = piece_count + 1;
226 |         double current_sum = piece_sum + (range_start);
227 | //        double current_error = cal_error(current_count, current_min_zmin, current_sum);
228 | 
229 | #ifdef DEBUG
230 |         std::cout << "cal error in insert" << current_error << endl;
231 | #endif
232 |         // add new record will make very last piece's count > piece limit
233 |         if (current_count > piece_limit) {
234 |             // directly push new record as new bucket
235 |             pieces.emplace_back((std::make_tuple(range_end, range_start, 1,
236 |                                         range_start)));
237 |         } else { //new record make last piece count < piece limit
238 |             //update all parameters accordingly
239 |             std::get<0>(pieces[piece_position -1]) = range_end;
240 |             std::get<1>(pieces[piece_position-1]) = current_min_zmin;
241 |             std::get<2>(pieces[piece_position -1]) = current_count;
242 |             //  std::cout << " updated count2 " << get<2>(pieces[piece_position]) << endl;
243 |             std::get<3>(pieces[piece_position-1]) = current_sum;
244 |         }
245 |         return;
246 |     }
247 |         //or insert into the begnning
248 |     else if (piece_position == 0) { // head insert
249 |         //get the piece info in pieces[1]
250 |         if (range_end != std::get<0>(pieces[0])) {
251 |             double piece_min_zmin = std::get<1>(pieces[piece_position + 1]);
252 |             double piece_count = std::get<2>(pieces[piece_position + 1]);
253 |             double piece_sum = std::get<3>(pieces[piece_position + 1]);
254 |             double current_min_zmin = std::min(piece_min_zmin, range_start);
255 |             double current_count = piece_count + 1;
256 |             double current_sum = piece_sum +  range_start;
257 | //            double current_error = cal_error(current_count, current_min_zmin, current_sum);
258 | 
259 |             if (current_count > piece_limit) {
260 |                 std::get<0>(pieces[0]) = range_end;
261 |                 std::get<1>(pieces[0]) = range_start;
262 |                 std::get<2>(pieces[0]) = 1;
263 |                 std::get<3>(pieces[0]) = range_start;
264 |                 pieces.insert(pieces.begin(),std:: make_tuple(range_start, std::numeric_limits<double>::min(),
265 |                                                          std::numeric_limits<double>::min(),
266 |                                                          std::numeric_limits<double>::min()));
267 |             } else {
268 |                 //piece 0 is dummy piece
269 |                 std::get<0>(pieces[0]) = range_start;
270 |                 std::get<1>(pieces[0]) = current_min_zmin;
271 |                 std::get<2>(pieces[0]) = current_count;
272 |                 std::get<3>(pieces[0]) = current_sum;
273 |                 std::get<1>(pieces[piece_position]) = current_min_zmin;
274 |                 std::get<2>(pieces[piece_position]) = current_count;
275 |                 std::get<3>(pieces[piece_position]) = current_sum;
276 | 
277 |             }
278 |             return;
279 |         } else {
280 |             piece_position++;
281 |         }
282 | 
283 |     }
284 | 
285 |     double piece_min_zmin = std::get<1>(pieces[piece_position]);
286 |     double piece_count = std::get<2>(pieces[piece_position]);
287 |     double piece_sum = std::get<3>(pieces[piece_position]);
288 |     double current_min_zmin = std::min(piece_min_zmin,  range_start);
289 |     double current_count = piece_count + 1;
290 |     //    std::cout<< "current count "<< piece_count << endl;
291 |     double current_sum = piece_sum +  range_start;
292 |     // double current_error = cal_error(current_count,current_min_zmin,current_sum);
293 |     // finish the update of piece that tobe inserted
294 |     std::get<1>(pieces[piece_position]) = current_min_zmin;
295 |     std::get<2>(pieces[piece_position]) = current_count;
296 | #ifdef DEBUG
297 |     assert(current_count = piece_count+1);
298 | #endif
299 | //     std::cout << " updated count " << get<2>(pieces[piece_position]) << endl;
300 |    std::get<3>(pieces[piece_position]) = current_sum;
301 | 
302 | 
303 | }
304 | 
305 | #endif //ALEX_PIECEWISE_H
306 | 


--------------------------------------------------------------------------------
/glin/libmorton/morton3D_LUTs.h:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <stdint.h>
  4 | 
  5 | namespace libmorton {
  6 | 	// LUT for Morton3D encode X
  7 | 	static const uint_fast32_t Morton3D_encode_x_256[256] =
  8 | 	{
  9 | 		0x00000000,
 10 | 		0x00000001, 0x00000008, 0x00000009, 0x00000040, 0x00000041, 0x00000048, 0x00000049, 0x00000200,
 11 | 		0x00000201, 0x00000208, 0x00000209, 0x00000240, 0x00000241, 0x00000248, 0x00000249, 0x00001000,
 12 | 		0x00001001, 0x00001008, 0x00001009, 0x00001040, 0x00001041, 0x00001048, 0x00001049, 0x00001200,
 13 | 		0x00001201, 0x00001208, 0x00001209, 0x00001240, 0x00001241, 0x00001248, 0x00001249, 0x00008000,
 14 | 		0x00008001, 0x00008008, 0x00008009, 0x00008040, 0x00008041, 0x00008048, 0x00008049, 0x00008200,
 15 | 		0x00008201, 0x00008208, 0x00008209, 0x00008240, 0x00008241, 0x00008248, 0x00008249, 0x00009000,
 16 | 		0x00009001, 0x00009008, 0x00009009, 0x00009040, 0x00009041, 0x00009048, 0x00009049, 0x00009200,
 17 | 		0x00009201, 0x00009208, 0x00009209, 0x00009240, 0x00009241, 0x00009248, 0x00009249, 0x00040000,
 18 | 		0x00040001, 0x00040008, 0x00040009, 0x00040040, 0x00040041, 0x00040048, 0x00040049, 0x00040200,
 19 | 		0x00040201, 0x00040208, 0x00040209, 0x00040240, 0x00040241, 0x00040248, 0x00040249, 0x00041000,
 20 | 		0x00041001, 0x00041008, 0x00041009, 0x00041040, 0x00041041, 0x00041048, 0x00041049, 0x00041200,
 21 | 		0x00041201, 0x00041208, 0x00041209, 0x00041240, 0x00041241, 0x00041248, 0x00041249, 0x00048000,
 22 | 		0x00048001, 0x00048008, 0x00048009, 0x00048040, 0x00048041, 0x00048048, 0x00048049, 0x00048200,
 23 | 		0x00048201, 0x00048208, 0x00048209, 0x00048240, 0x00048241, 0x00048248, 0x00048249, 0x00049000,
 24 | 		0x00049001, 0x00049008, 0x00049009, 0x00049040, 0x00049041, 0x00049048, 0x00049049, 0x00049200,
 25 | 		0x00049201, 0x00049208, 0x00049209, 0x00049240, 0x00049241, 0x00049248, 0x00049249, 0x00200000,
 26 | 		0x00200001, 0x00200008, 0x00200009, 0x00200040, 0x00200041, 0x00200048, 0x00200049, 0x00200200,
 27 | 		0x00200201, 0x00200208, 0x00200209, 0x00200240, 0x00200241, 0x00200248, 0x00200249, 0x00201000,
 28 | 		0x00201001, 0x00201008, 0x00201009, 0x00201040, 0x00201041, 0x00201048, 0x00201049, 0x00201200,
 29 | 		0x00201201, 0x00201208, 0x00201209, 0x00201240, 0x00201241, 0x00201248, 0x00201249, 0x00208000,
 30 | 		0x00208001, 0x00208008, 0x00208009, 0x00208040, 0x00208041, 0x00208048, 0x00208049, 0x00208200,
 31 | 		0x00208201, 0x00208208, 0x00208209, 0x00208240, 0x00208241, 0x00208248, 0x00208249, 0x00209000,
 32 | 		0x00209001, 0x00209008, 0x00209009, 0x00209040, 0x00209041, 0x00209048, 0x00209049, 0x00209200,
 33 | 		0x00209201, 0x00209208, 0x00209209, 0x00209240, 0x00209241, 0x00209248, 0x00209249, 0x00240000,
 34 | 		0x00240001, 0x00240008, 0x00240009, 0x00240040, 0x00240041, 0x00240048, 0x00240049, 0x00240200,
 35 | 		0x00240201, 0x00240208, 0x00240209, 0x00240240, 0x00240241, 0x00240248, 0x00240249, 0x00241000,
 36 | 		0x00241001, 0x00241008, 0x00241009, 0x00241040, 0x00241041, 0x00241048, 0x00241049, 0x00241200,
 37 | 		0x00241201, 0x00241208, 0x00241209, 0x00241240, 0x00241241, 0x00241248, 0x00241249, 0x00248000,
 38 | 		0x00248001, 0x00248008, 0x00248009, 0x00248040, 0x00248041, 0x00248048, 0x00248049, 0x00248200,
 39 | 		0x00248201, 0x00248208, 0x00248209, 0x00248240, 0x00248241, 0x00248248, 0x00248249, 0x00249000,
 40 | 		0x00249001, 0x00249008, 0x00249009, 0x00249040, 0x00249041, 0x00249048, 0x00249049, 0x00249200,
 41 | 		0x00249201, 0x00249208, 0x00249209, 0x00249240, 0x00249241, 0x00249248, 0x00249249
 42 | 	};
 43 | 
 44 | 	// LUT for Morton3D encode Y
 45 | 	static const uint_fast32_t Morton3D_encode_y_256[256] = {
 46 | 		0x00000000,
 47 | 		0x00000002, 0x00000010, 0x00000012, 0x00000080, 0x00000082, 0x00000090, 0x00000092, 0x00000400,
 48 | 		0x00000402, 0x00000410, 0x00000412, 0x00000480, 0x00000482, 0x00000490, 0x00000492, 0x00002000,
 49 | 		0x00002002, 0x00002010, 0x00002012, 0x00002080, 0x00002082, 0x00002090, 0x00002092, 0x00002400,
 50 | 		0x00002402, 0x00002410, 0x00002412, 0x00002480, 0x00002482, 0x00002490, 0x00002492, 0x00010000,
 51 | 		0x00010002, 0x00010010, 0x00010012, 0x00010080, 0x00010082, 0x00010090, 0x00010092, 0x00010400,
 52 | 		0x00010402, 0x00010410, 0x00010412, 0x00010480, 0x00010482, 0x00010490, 0x00010492, 0x00012000,
 53 | 		0x00012002, 0x00012010, 0x00012012, 0x00012080, 0x00012082, 0x00012090, 0x00012092, 0x00012400,
 54 | 		0x00012402, 0x00012410, 0x00012412, 0x00012480, 0x00012482, 0x00012490, 0x00012492, 0x00080000,
 55 | 		0x00080002, 0x00080010, 0x00080012, 0x00080080, 0x00080082, 0x00080090, 0x00080092, 0x00080400,
 56 | 		0x00080402, 0x00080410, 0x00080412, 0x00080480, 0x00080482, 0x00080490, 0x00080492, 0x00082000,
 57 | 		0x00082002, 0x00082010, 0x00082012, 0x00082080, 0x00082082, 0x00082090, 0x00082092, 0x00082400,
 58 | 		0x00082402, 0x00082410, 0x00082412, 0x00082480, 0x00082482, 0x00082490, 0x00082492, 0x00090000,
 59 | 		0x00090002, 0x00090010, 0x00090012, 0x00090080, 0x00090082, 0x00090090, 0x00090092, 0x00090400,
 60 | 		0x00090402, 0x00090410, 0x00090412, 0x00090480, 0x00090482, 0x00090490, 0x00090492, 0x00092000,
 61 | 		0x00092002, 0x00092010, 0x00092012, 0x00092080, 0x00092082, 0x00092090, 0x00092092, 0x00092400,
 62 | 		0x00092402, 0x00092410, 0x00092412, 0x00092480, 0x00092482, 0x00092490, 0x00092492, 0x00400000,
 63 | 		0x00400002, 0x00400010, 0x00400012, 0x00400080, 0x00400082, 0x00400090, 0x00400092, 0x00400400,
 64 | 		0x00400402, 0x00400410, 0x00400412, 0x00400480, 0x00400482, 0x00400490, 0x00400492, 0x00402000,
 65 | 		0x00402002, 0x00402010, 0x00402012, 0x00402080, 0x00402082, 0x00402090, 0x00402092, 0x00402400,
 66 | 		0x00402402, 0x00402410, 0x00402412, 0x00402480, 0x00402482, 0x00402490, 0x00402492, 0x00410000,
 67 | 		0x00410002, 0x00410010, 0x00410012, 0x00410080, 0x00410082, 0x00410090, 0x00410092, 0x00410400,
 68 | 		0x00410402, 0x00410410, 0x00410412, 0x00410480, 0x00410482, 0x00410490, 0x00410492, 0x00412000,
 69 | 		0x00412002, 0x00412010, 0x00412012, 0x00412080, 0x00412082, 0x00412090, 0x00412092, 0x00412400,
 70 | 		0x00412402, 0x00412410, 0x00412412, 0x00412480, 0x00412482, 0x00412490, 0x00412492, 0x00480000,
 71 | 		0x00480002, 0x00480010, 0x00480012, 0x00480080, 0x00480082, 0x00480090, 0x00480092, 0x00480400,
 72 | 		0x00480402, 0x00480410, 0x00480412, 0x00480480, 0x00480482, 0x00480490, 0x00480492, 0x00482000,
 73 | 		0x00482002, 0x00482010, 0x00482012, 0x00482080, 0x00482082, 0x00482090, 0x00482092, 0x00482400,
 74 | 		0x00482402, 0x00482410, 0x00482412, 0x00482480, 0x00482482, 0x00482490, 0x00482492, 0x00490000,
 75 | 		0x00490002, 0x00490010, 0x00490012, 0x00490080, 0x00490082, 0x00490090, 0x00490092, 0x00490400,
 76 | 		0x00490402, 0x00490410, 0x00490412, 0x00490480, 0x00490482, 0x00490490, 0x00490492, 0x00492000,
 77 | 		0x00492002, 0x00492010, 0x00492012, 0x00492080, 0x00492082, 0x00492090, 0x00492092, 0x00492400,
 78 | 		0x00492402, 0x00492410, 0x00492412, 0x00492480, 0x00492482, 0x00492490, 0x00492492
 79 | 	};
 80 | 
 81 | 	// LUT for Morton3D encode Z
 82 | 	static const uint_fast32_t Morton3D_encode_z_256[256] = {
 83 | 		0x00000000,
 84 | 		0x00000004, 0x00000020, 0x00000024, 0x00000100, 0x00000104, 0x00000120, 0x00000124, 0x00000800,
 85 | 		0x00000804, 0x00000820, 0x00000824, 0x00000900, 0x00000904, 0x00000920, 0x00000924, 0x00004000,
 86 | 		0x00004004, 0x00004020, 0x00004024, 0x00004100, 0x00004104, 0x00004120, 0x00004124, 0x00004800,
 87 | 		0x00004804, 0x00004820, 0x00004824, 0x00004900, 0x00004904, 0x00004920, 0x00004924, 0x00020000,
 88 | 		0x00020004, 0x00020020, 0x00020024, 0x00020100, 0x00020104, 0x00020120, 0x00020124, 0x00020800,
 89 | 		0x00020804, 0x00020820, 0x00020824, 0x00020900, 0x00020904, 0x00020920, 0x00020924, 0x00024000,
 90 | 		0x00024004, 0x00024020, 0x00024024, 0x00024100, 0x00024104, 0x00024120, 0x00024124, 0x00024800,
 91 | 		0x00024804, 0x00024820, 0x00024824, 0x00024900, 0x00024904, 0x00024920, 0x00024924, 0x00100000,
 92 | 		0x00100004, 0x00100020, 0x00100024, 0x00100100, 0x00100104, 0x00100120, 0x00100124, 0x00100800,
 93 | 		0x00100804, 0x00100820, 0x00100824, 0x00100900, 0x00100904, 0x00100920, 0x00100924, 0x00104000,
 94 | 		0x00104004, 0x00104020, 0x00104024, 0x00104100, 0x00104104, 0x00104120, 0x00104124, 0x00104800,
 95 | 		0x00104804, 0x00104820, 0x00104824, 0x00104900, 0x00104904, 0x00104920, 0x00104924, 0x00120000,
 96 | 		0x00120004, 0x00120020, 0x00120024, 0x00120100, 0x00120104, 0x00120120, 0x00120124, 0x00120800,
 97 | 		0x00120804, 0x00120820, 0x00120824, 0x00120900, 0x00120904, 0x00120920, 0x00120924, 0x00124000,
 98 | 		0x00124004, 0x00124020, 0x00124024, 0x00124100, 0x00124104, 0x00124120, 0x00124124, 0x00124800,
 99 | 		0x00124804, 0x00124820, 0x00124824, 0x00124900, 0x00124904, 0x00124920, 0x00124924, 0x00800000,
100 | 		0x00800004, 0x00800020, 0x00800024, 0x00800100, 0x00800104, 0x00800120, 0x00800124, 0x00800800,
101 | 		0x00800804, 0x00800820, 0x00800824, 0x00800900, 0x00800904, 0x00800920, 0x00800924, 0x00804000,
102 | 		0x00804004, 0x00804020, 0x00804024, 0x00804100, 0x00804104, 0x00804120, 0x00804124, 0x00804800,
103 | 		0x00804804, 0x00804820, 0x00804824, 0x00804900, 0x00804904, 0x00804920, 0x00804924, 0x00820000,
104 | 		0x00820004, 0x00820020, 0x00820024, 0x00820100, 0x00820104, 0x00820120, 0x00820124, 0x00820800,
105 | 		0x00820804, 0x00820820, 0x00820824, 0x00820900, 0x00820904, 0x00820920, 0x00820924, 0x00824000,
106 | 		0x00824004, 0x00824020, 0x00824024, 0x00824100, 0x00824104, 0x00824120, 0x00824124, 0x00824800,
107 | 		0x00824804, 0x00824820, 0x00824824, 0x00824900, 0x00824904, 0x00824920, 0x00824924, 0x00900000,
108 | 		0x00900004, 0x00900020, 0x00900024, 0x00900100, 0x00900104, 0x00900120, 0x00900124, 0x00900800,
109 | 		0x00900804, 0x00900820, 0x00900824, 0x00900900, 0x00900904, 0x00900920, 0x00900924, 0x00904000,
110 | 		0x00904004, 0x00904020, 0x00904024, 0x00904100, 0x00904104, 0x00904120, 0x00904124, 0x00904800,
111 | 		0x00904804, 0x00904820, 0x00904824, 0x00904900, 0x00904904, 0x00904920, 0x00904924, 0x00920000,
112 | 		0x00920004, 0x00920020, 0x00920024, 0x00920100, 0x00920104, 0x00920120, 0x00920124, 0x00920800,
113 | 		0x00920804, 0x00920820, 0x00920824, 0x00920900, 0x00920904, 0x00920920, 0x00920924, 0x00924000,
114 | 		0x00924004, 0x00924020, 0x00924024, 0x00924100, 0x00924104, 0x00924120, 0x00924124, 0x00924800,
115 | 		0x00924804, 0x00924820, 0x00924824, 0x00924900, 0x00924904, 0x00924920, 0x00924924
116 | 	};
117 | 
118 | 	// LUT for Morton3D decode X
119 | 	static const uint_fast8_t Morton3D_decode_x_512[512] = {
120 | 		0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3,
121 | 		0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3,
122 | 		0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3,
123 | 		0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3,
124 | 		4, 5, 4, 5, 4, 5, 4, 5, 6, 7, 6, 7, 6, 7, 6, 7,
125 | 		4, 5, 4, 5, 4, 5, 4, 5, 6, 7, 6, 7, 6, 7, 6, 7,
126 | 		4, 5, 4, 5, 4, 5, 4, 5, 6, 7, 6, 7, 6, 7, 6, 7,
127 | 		4, 5, 4, 5, 4, 5, 4, 5, 6, 7, 6, 7, 6, 7, 6, 7,
128 | 		0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3,
129 | 		0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3,
130 | 		0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3,
131 | 		0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3,
132 | 		4, 5, 4, 5, 4, 5, 4, 5, 6, 7, 6, 7, 6, 7, 6, 7,
133 | 		4, 5, 4, 5, 4, 5, 4, 5, 6, 7, 6, 7, 6, 7, 6, 7,
134 | 		4, 5, 4, 5, 4, 5, 4, 5, 6, 7, 6, 7, 6, 7, 6, 7,
135 | 		4, 5, 4, 5, 4, 5, 4, 5, 6, 7, 6, 7, 6, 7, 6, 7,
136 | 		0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3,
137 | 		0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3,
138 | 		0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3,
139 | 		0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3,
140 | 		4, 5, 4, 5, 4, 5, 4, 5, 6, 7, 6, 7, 6, 7, 6, 7,
141 | 		4, 5, 4, 5, 4, 5, 4, 5, 6, 7, 6, 7, 6, 7, 6, 7,
142 | 		4, 5, 4, 5, 4, 5, 4, 5, 6, 7, 6, 7, 6, 7, 6, 7,
143 | 		4, 5, 4, 5, 4, 5, 4, 5, 6, 7, 6, 7, 6, 7, 6, 7,
144 | 		0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3,
145 | 		0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3,
146 | 		0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3,
147 | 		0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3,
148 | 		4, 5, 4, 5, 4, 5, 4, 5, 6, 7, 6, 7, 6, 7, 6, 7,
149 | 		4, 5, 4, 5, 4, 5, 4, 5, 6, 7, 6, 7, 6, 7, 6, 7,
150 | 		4, 5, 4, 5, 4, 5, 4, 5, 6, 7, 6, 7, 6, 7, 6, 7,
151 | 		4, 5, 4, 5, 4, 5, 4, 5, 6, 7, 6, 7, 6, 7, 6, 7
152 | 	};
153 | 
154 | 	// LUT for Morton3D decode Y
155 | 	static const uint_fast8_t Morton3D_decode_y_512[512] = {
156 | 		0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1,
157 | 		2, 2, 3, 3, 2, 2, 3, 3, 2, 2, 3, 3, 2, 2, 3, 3,
158 | 		0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1,
159 | 		2, 2, 3, 3, 2, 2, 3, 3, 2, 2, 3, 3, 2, 2, 3, 3,
160 | 		0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1,
161 | 		2, 2, 3, 3, 2, 2, 3, 3, 2, 2, 3, 3, 2, 2, 3, 3,
162 | 		0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1,
163 | 		2, 2, 3, 3, 2, 2, 3, 3, 2, 2, 3, 3, 2, 2, 3, 3,
164 | 		4, 4, 5, 5, 4, 4, 5, 5, 4, 4, 5, 5, 4, 4, 5, 5,
165 | 		6, 6, 7, 7, 6, 6, 7, 7, 6, 6, 7, 7, 6, 6, 7, 7,
166 | 		4, 4, 5, 5, 4, 4, 5, 5, 4, 4, 5, 5, 4, 4, 5, 5,
167 | 		6, 6, 7, 7, 6, 6, 7, 7, 6, 6, 7, 7, 6, 6, 7, 7,
168 | 		4, 4, 5, 5, 4, 4, 5, 5, 4, 4, 5, 5, 4, 4, 5, 5,
169 | 		6, 6, 7, 7, 6, 6, 7, 7, 6, 6, 7, 7, 6, 6, 7, 7,
170 | 		4, 4, 5, 5, 4, 4, 5, 5, 4, 4, 5, 5, 4, 4, 5, 5,
171 | 		6, 6, 7, 7, 6, 6, 7, 7, 6, 6, 7, 7, 6, 6, 7, 7,
172 | 		0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1,
173 | 		2, 2, 3, 3, 2, 2, 3, 3, 2, 2, 3, 3, 2, 2, 3, 3,
174 | 		0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1,
175 | 		2, 2, 3, 3, 2, 2, 3, 3, 2, 2, 3, 3, 2, 2, 3, 3,
176 | 		0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1,
177 | 		2, 2, 3, 3, 2, 2, 3, 3, 2, 2, 3, 3, 2, 2, 3, 3,
178 | 		0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1,
179 | 		2, 2, 3, 3, 2, 2, 3, 3, 2, 2, 3, 3, 2, 2, 3, 3,
180 | 		4, 4, 5, 5, 4, 4, 5, 5, 4, 4, 5, 5, 4, 4, 5, 5,
181 | 		6, 6, 7, 7, 6, 6, 7, 7, 6, 6, 7, 7, 6, 6, 7, 7,
182 | 		4, 4, 5, 5, 4, 4, 5, 5, 4, 4, 5, 5, 4, 4, 5, 5,
183 | 		6, 6, 7, 7, 6, 6, 7, 7, 6, 6, 7, 7, 6, 6, 7, 7,
184 | 		4, 4, 5, 5, 4, 4, 5, 5, 4, 4, 5, 5, 4, 4, 5, 5,
185 | 		6, 6, 7, 7, 6, 6, 7, 7, 6, 6, 7, 7, 6, 6, 7, 7,
186 | 		4, 4, 5, 5, 4, 4, 5, 5, 4, 4, 5, 5, 4, 4, 5, 5,
187 | 		6, 6, 7, 7, 6, 6, 7, 7, 6, 6, 7, 7, 6, 6, 7, 7
188 | 	};
189 | 
190 | 	// LUT for Morton3D decode Z
191 | 	static const uint_fast8_t Morton3D_decode_z_512[512] = {
192 | 		0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1,
193 | 		0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1,
194 | 		2, 2, 2, 2, 3, 3, 3, 3, 2, 2, 2, 2, 3, 3, 3, 3,
195 | 		2, 2, 2, 2, 3, 3, 3, 3, 2, 2, 2, 2, 3, 3, 3, 3,
196 | 		0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1,
197 | 		0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1,
198 | 		2, 2, 2, 2, 3, 3, 3, 3, 2, 2, 2, 2, 3, 3, 3, 3,
199 | 		2, 2, 2, 2, 3, 3, 3, 3, 2, 2, 2, 2, 3, 3, 3, 3,
200 | 		0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1,
201 | 		0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1,
202 | 		2, 2, 2, 2, 3, 3, 3, 3, 2, 2, 2, 2, 3, 3, 3, 3,
203 | 		2, 2, 2, 2, 3, 3, 3, 3, 2, 2, 2, 2, 3, 3, 3, 3,
204 | 		0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1,
205 | 		0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1,
206 | 		2, 2, 2, 2, 3, 3, 3, 3, 2, 2, 2, 2, 3, 3, 3, 3,
207 | 		2, 2, 2, 2, 3, 3, 3, 3, 2, 2, 2, 2, 3, 3, 3, 3,
208 | 		4, 4, 4, 4, 5, 5, 5, 5, 4, 4, 4, 4, 5, 5, 5, 5,
209 | 		4, 4, 4, 4, 5, 5, 5, 5, 4, 4, 4, 4, 5, 5, 5, 5,
210 | 		6, 6, 6, 6, 7, 7, 7, 7, 6, 6, 6, 6, 7, 7, 7, 7,
211 | 		6, 6, 6, 6, 7, 7, 7, 7, 6, 6, 6, 6, 7, 7, 7, 7,
212 | 		4, 4, 4, 4, 5, 5, 5, 5, 4, 4, 4, 4, 5, 5, 5, 5,
213 | 		4, 4, 4, 4, 5, 5, 5, 5, 4, 4, 4, 4, 5, 5, 5, 5,
214 | 		6, 6, 6, 6, 7, 7, 7, 7, 6, 6, 6, 6, 7, 7, 7, 7,
215 | 		6, 6, 6, 6, 7, 7, 7, 7, 6, 6, 6, 6, 7, 7, 7, 7,
216 | 		4, 4, 4, 4, 5, 5, 5, 5, 4, 4, 4, 4, 5, 5, 5, 5,
217 | 		4, 4, 4, 4, 5, 5, 5, 5, 4, 4, 4, 4, 5, 5, 5, 5,
218 | 		6, 6, 6, 6, 7, 7, 7, 7, 6, 6, 6, 6, 7, 7, 7, 7,
219 | 		6, 6, 6, 6, 7, 7, 7, 7, 6, 6, 6, 6, 7, 7, 7, 7,
220 | 		4, 4, 4, 4, 5, 5, 5, 5, 4, 4, 4, 4, 5, 5, 5, 5,
221 | 		4, 4, 4, 4, 5, 5, 5, 5, 4, 4, 4, 4, 5, 5, 5, 5,
222 | 		6, 6, 6, 6, 7, 7, 7, 7, 6, 6, 6, 6, 7, 7, 7, 7,
223 | 		6, 6, 6, 6, 7, 7, 7, 7, 6, 6, 6, 6, 7, 7, 7, 7
224 | 	};
225 | }


--------------------------------------------------------------------------------
/test/unittest_nodes.h:
--------------------------------------------------------------------------------
  1 | // Copyright (c) Microsoft Corporation.
  2 | // Licensed under the MIT license.
  3 | 
  4 | #include "doctest.h"
  5 | 
  6 | #include "alex_nodes.h"
  7 | 
  8 | using namespace alex;
  9 | 
 10 | TEST_SUITE("DataNode") {
 11 | 
 12 | /************************* Tests for Data Node *****************************/
 13 | 
 14 | TEST_CASE("TestBinarySearch") {
 15 |   AlexDataNode<int, int> node;
 16 | 
 17 |   AlexDataNode<int, int>::V values[100];
 18 |   for (int i = 0; i < 100; i++) {
 19 |     values[i].first = int(0.2 * i) * 2;
 20 |     values[i].second = 0;
 21 |   }
 22 | 
 23 |   std::vector<int> keys_to_search;
 24 |   for (int i = 0; i < 40; i++) {
 25 |     keys_to_search.push_back(i);
 26 |   }
 27 | 
 28 |   std::sort(values, values + 100);
 29 |   node.bulk_load(values, 100);
 30 | 
 31 |   for (int key : keys_to_search) {
 32 |     int lower_bound_pos =
 33 |         node.binary_search_lower_bound(0, node.data_capacity_, key);
 34 |     if (lower_bound_pos > 0) {
 35 |       CHECK_LT(node.get_key(lower_bound_pos - 1), key);
 36 |     }
 37 |     if (lower_bound_pos < node.data_capacity_) {
 38 |       CHECK_GE(node.get_key(lower_bound_pos), key);
 39 |     }
 40 | 
 41 |     int upper_bound_pos =
 42 |         node.binary_search_upper_bound(0, node.data_capacity_, key);
 43 |     if (upper_bound_pos > 0) {
 44 |       CHECK_LE(node.get_key(upper_bound_pos - 1), key);
 45 |     }
 46 |     if (upper_bound_pos < node.data_capacity_) {
 47 |       CHECK_GT(node.get_key(upper_bound_pos), key);
 48 |     }
 49 |   }
 50 | }
 51 | 
 52 | TEST_CASE("TestExponentialSearch") {
 53 |   AlexDataNode<int, int> node;
 54 | 
 55 |   AlexDataNode<int, int>::V values[100];
 56 |   for (int i = 0; i < 100; i++) {
 57 |     values[i].first = int(0.2 * i) * 2;
 58 |     values[i].second = 0;
 59 |   }
 60 | 
 61 |   std::vector<int> keys_to_search;
 62 |   for (int i = 0; i < 40; i++) {
 63 |     keys_to_search.push_back(i);
 64 |   }
 65 | 
 66 |   std::sort(values, values + 100);
 67 |   node.bulk_load(values, 100);
 68 | 
 69 |   for (int key : keys_to_search) {
 70 |     for (int m = 0; m < node.data_capacity_; m++) {
 71 |       int lower_bound_pos = node.exponential_search_lower_bound(m, key);
 72 |       if (lower_bound_pos > 0) {
 73 |         CHECK_LT(node.get_key(lower_bound_pos - 1), key);
 74 |       }
 75 |       if (lower_bound_pos < node.data_capacity_) {
 76 |         CHECK_GE(node.get_key(lower_bound_pos), key);
 77 |       }
 78 | 
 79 |       int upper_bound_pos = node.exponential_search_upper_bound(m, key);
 80 |       if (upper_bound_pos > 0) {
 81 |         CHECK_LE(node.get_key(upper_bound_pos - 1), key);
 82 |       }
 83 |       if (upper_bound_pos < node.data_capacity_) {
 84 |         CHECK_GT(node.get_key(upper_bound_pos), key);
 85 |       }
 86 |     }
 87 |   }
 88 | }
 89 | 
 90 | TEST_CASE("TestNumKeysInRange") {
 91 |   AlexDataNode<int, int> node;
 92 | 
 93 |   AlexDataNode<int, int>::V values[100];
 94 |   for (int i = 0; i < 100; i++) {
 95 |     values[i].first = 2 * i;
 96 |     values[i].second = rand();
 97 |   }
 98 | 
 99 |   std::sort(values, values + 100);
100 |   node.bulk_load(values, 100);
101 | 
102 |   int num_keys = node.num_keys_in_range(0, node.data_capacity_);
103 |   CHECK_EQ(num_keys, 100);
104 | 
105 |   int num_keys_first_half = node.num_keys_in_range(0, node.data_capacity_ / 2);
106 |   int num_keys_second_half =
107 |       node.num_keys_in_range(node.data_capacity_ / 2, node.data_capacity_);
108 |   CHECK_EQ(num_keys, num_keys_first_half + num_keys_second_half);
109 | }
110 | 
111 | TEST_CASE("TestNextFilledPosition") {
112 |   AlexDataNode<int, int> node;
113 | 
114 |   AlexDataNode<int, int>::V values[100];
115 |   for (int i = 0; i < 100; i++) {
116 |     values[i].first = 2 * i;
117 |     values[i].second = rand();
118 |   }
119 | 
120 |   std::sort(values, values + 100);
121 |   node.bulk_load(values, 100);
122 | 
123 |   for (int i = 0; i < node.data_capacity_; i++) {
124 |     int next_filled_pos = node.get_next_filled_position(i, true);
125 |     CHECK_LT(i, next_filled_pos);
126 |     if (next_filled_pos < node.data_capacity_) {
127 |       CHECK(node.check_exists(next_filled_pos));
128 |     }
129 |     for (int j = i + 1; j < next_filled_pos; j++) {
130 |       CHECK(!node.check_exists(j));
131 |     }
132 |   }
133 | }
134 | 
135 | TEST_CASE("TestClosestGap") {
136 |   auto node = new AlexDataNode<int, int>();
137 | 
138 | #if ALEX_DATA_NODE_SEP_ARRAYS
139 |   node->key_slots_ = new int[8]{1, 1, 2, 3, 4, 5, 6, 6};
140 |   node->payload_slots_ = new int[8]();
141 | #else
142 |   node->data_slots_ = new std::pair<int, int>[8]{
143 |       {1, 0}, {1, 0}, {2, 0}, {3, 0}, {4, 0}, {5, 0}, {6, 0}, {6, 0}};
144 | #endif
145 |   node->data_capacity_ = 8;
146 |   node->bitmap_ = new uint64_t[1]();
147 |   for (int i : {1, 2, 3, 4, 5, 7}) {
148 |     node->bitmap_[0] |= (1ULL << i);
149 |   }
150 | 
151 |   CHECK_EQ(0, node->closest_gap(1));
152 |   CHECK_EQ(0, node->closest_gap(2));
153 |   CHECK_EQ(0,
154 |             node->closest_gap(3));  // border case, by default choose left gap
155 |   CHECK_EQ(6, node->closest_gap(4));
156 |   CHECK_EQ(6, node->closest_gap(5));
157 |   CHECK_EQ(6, node->closest_gap(7));
158 |   delete node;
159 | 
160 |   // Test with 5 bitmap blocks
161 |   node = new AlexDataNode<int, int>();
162 | #if ALEX_DATA_NODE_SEP_ARRAYS
163 |   node->key_slots_ = new int[320];
164 |   node->payload_slots_ = new int[320]();
165 |   for (int i = 0; i < 50; i++) {
166 |     node->key_slots_[i] = 50;
167 |   }
168 |   for (int i = 50; i < 300; i++) {
169 |     node->key_slots_[i] = i;
170 |   }
171 |   for (int i = 300; i < 320; i++) {
172 |     node->key_slots_[i] = AlexDataNode<int, int>::kEndSentinel_;
173 |   }
174 | #else
175 |   node->data_slots_ = new std::pair<int, int>[192];
176 |   for (int i = 0; i < 50; i++) {
177 |     node->data_slots_[i] = std::pair<int, int>(50, 0);
178 |   }
179 |   for (int i = 50; i < 300; i++) {
180 |     node->data_slots_[i] = std::pair<int, int>(i, 0);
181 |   }
182 |   for (int i = 300; i < 320; i++) {
183 |     node->data_slots_[i] =
184 |         std::pair<int, int>(AlexDataNode<int, int>::kEndSentinel, 0);
185 |   }
186 | #endif
187 |   node->data_capacity_ = 320;
188 |   node->bitmap_ = new uint64_t[5]();
189 |   for (int i = 50; i < 300; i++) {
190 |     size_t bitmap_pos = i >> 6;
191 |     size_t bit_pos = i - (bitmap_pos << 6);
192 |     node->bitmap_[bitmap_pos] |= (1ULL << bit_pos);
193 |   }
194 | 
195 |   CHECK_EQ(49, node->closest_gap(75));    // pos in second block
196 |   CHECK_EQ(49, node->closest_gap(130));   // pos in third block
197 |   CHECK_EQ(300, node->closest_gap(180));  // pos in third block
198 |   CHECK_EQ(300, node->closest_gap(200));  // pos in fourth block
199 |   delete node;
200 | }
201 | 
202 | TEST_CASE("TestInsertUsingShifts") {
203 |   AlexDataNode<int, int> node;
204 | 
205 | #if ALEX_DATA_NODE_SEP_ARRAYS
206 |   node.key_slots_ = new int[8]{1, 1, 2, 3, 5, 6, 7, 7};
207 |   node.payload_slots_ = new int[8]();
208 | #else
209 |   node.data_slots_ = new std::pair<int, int>[8]{{1, 0}, {1, 0}, {2, 0}, {3, 0},
210 |                                                 {5, 0}, {6, 0}, {7, 0}, {7, 0}};
211 | #endif
212 |   node.data_capacity_ = 8;
213 |   node.bitmap_ = new uint64_t[1]();
214 |   for (int i : {1, 2, 3, 4, 5, 7}) {
215 |     node.bitmap_[0] |= (1ULL << i);
216 |   }
217 | 
218 |   node.insert_using_shifts(4, rand(), 4);
219 |   int expected[] = {1, 1, 2, 3, 4, 5, 6, 7};
220 |   for (int i = 0; i < 8; i++) {
221 |     CHECK_EQ(expected[i], node.get_key(i));
222 |   }
223 | }
224 | 
225 | TEST_CASE("TestCheckExists") {
226 |   AlexDataNode<int, int> node;
227 | 
228 | #if ALEX_DATA_NODE_SEP_ARRAYS
229 |   node.key_slots_ = new int[8]{1, 1, 2, 3, 4, 5, 6, 6};
230 |   node.payload_slots_ = new int[8]();
231 | #else
232 |   node.data_slots = new std::pair<int, int>[8]{{1, 0}, {1, 0}, {2, 0}, {3, 0},
233 |                                                {4, 0}, {5, 0}, {6, 0}, {6, 0}};
234 | #endif
235 |   node.data_capacity_ = 8;
236 |   node.bitmap_ = new uint64_t[1]();
237 |   bool exists[] = {false, true, true, true, true, true, false, true};
238 |   for (int i = 0; i < 8; i++) {
239 |     if (exists[i]) {
240 |       node.bitmap_[0] |= (1ULL << i);
241 |     }
242 |   }
243 | 
244 |   for (int i = 0; i < 8; i++) {
245 |     CHECK_EQ(exists[i], node.check_exists(i));
246 |   }
247 | }
248 | 
249 | TEST_CASE("TestExpansion") {
250 |   AlexDataNode<int, int> node;
251 | 
252 |   AlexDataNode<int, int>::V values[100];
253 |   for (int i = 0; i < 100; i++) {
254 |     values[i].first = rand() % 500;
255 |     values[i].second = rand();
256 |   }
257 | 
258 |   std::sort(values, values + 100);
259 |   node.bulk_load(values, 100);
260 | 
261 |   node.resize(0.5, true);
262 | 
263 |   for (int i = 0; i < 100; i++) {
264 |     int pos = node.find_key(values[i].first);
265 |     CHECK_EQ(values[i].first, node.get_key(pos));
266 |     CHECK(node.check_exists(pos));
267 |   }
268 | 
269 |   node.resize(0.9, true);
270 | 
271 |   for (int i = 0; i < 100; i++) {
272 |     int pos = node.find_key(values[i].first);
273 |     CHECK_EQ(values[i].first, node.get_key(pos));
274 |     CHECK(node.check_exists(pos));
275 |   }
276 | }
277 | 
278 | TEST_CASE("TestFindInsertPosition") {
279 |   AlexDataNode<int, int> node;
280 | 
281 |   AlexDataNode<int, int>::V values[100];
282 |   for (int i = 0; i < 100; i++) {
283 |     values[i].first = 2 * i;
284 |     values[i].second = rand();
285 |   }
286 | 
287 |   std::sort(values, values + 100);
288 |   node.bulk_load(values, 100);
289 | 
290 |   for (int key = 0; key < node.data_capacity_; key++) {
291 |     std::pair<int, int> insert_pos = node.find_insert_position(key);
292 |     int model_based_insert_pos = insert_pos.first;
293 |     if (model_based_insert_pos > 0) {
294 |       CHECK_GE(key, node.get_key(model_based_insert_pos - 1));
295 |     }
296 |     if (model_based_insert_pos < node.data_capacity_) {
297 |       CHECK_LE(key, node.get_key(model_based_insert_pos));
298 |     }
299 |   }
300 | }
301 | 
302 | TEST_CASE("TestIterator") {
303 |   AlexDataNode<int, int> node;
304 | 
305 |   AlexDataNode<int, int>::V values[100];
306 |   for (int i = 0; i < 100; i++) {
307 |     values[i].first = 2 * i;
308 |     values[i].second = rand();
309 |   }
310 | 
311 |   std::sort(values, values + 100);
312 |   node.bulk_load(values, 100);
313 | 
314 |   std::vector<int> results;
315 |   AlexDataNode<int, int>::const_iterator_type it(&node, 0);
316 |   for (; !it.is_end(); it++) {
317 |     results.push_back(it.key());
318 |   }
319 |   CHECK_EQ(100, results.size());
320 | }
321 | 
322 | TEST_CASE("TestIteratorWithDuplicates") {
323 |   AlexDataNode<int, int> node;
324 | 
325 |   AlexDataNode<int, int>::V values[100];
326 |   for (int i = 0; i < 100; i++) {
327 |     values[i].first = i / 2;
328 |     values[i].second = rand();
329 |   }
330 | 
331 |   std::sort(values, values + 100);
332 |   node.bulk_load(values, 100);
333 | 
334 |   CHECK_EQ(node.find_upper(10), node.find_lower(11));
335 | 
336 |   std::vector<int> results;
337 |   AlexDataNode<int, int>::const_iterator_type it(&node, node.find_lower(2));
338 |   AlexDataNode<int, int>::const_iterator_type end_it(&node, node.find_lower(4));
339 |   for (; it != end_it; it++) {
340 |     results.push_back(it.key());
341 |   }
342 |   CHECK_EQ(4, results.size());
343 | }
344 | 
345 | TEST_CASE("TestBulkLoadFromExisting") {
346 |   AlexDataNode<int, int> node;
347 | 
348 |   AlexDataNode<int, int>::V values[100];
349 |   for (int i = 0; i < 100; i++) {
350 |     values[i].first = 2 * i;
351 |     values[i].second = rand();
352 |   }
353 | 
354 |   std::sort(values, values + 100);
355 |   node.bulk_load(values, 100);
356 | 
357 |   AlexDataNode<int, int> new_node;
358 |   new_node.bulk_load_from_existing(&node, 0, node.data_capacity_);
359 | 
360 |   int key = 50;
361 |   for (int pos = 0; pos < new_node.data_capacity_; pos++) {
362 |     int actual_pos = new_node.exponential_search_upper_bound(pos, key) - 1;
363 |     CHECK_EQ(key, new_node.get_key(actual_pos));
364 |     CHECK(node.check_exists(actual_pos));
365 |   }
366 | }
367 | 
368 | TEST_CASE("TestInserts") {
369 |   AlexDataNode<int, int> node;
370 | 
371 |   AlexDataNode<int, int>::V values[200];
372 |   for (int i = 0; i < 200; i++) {
373 |     values[i].first = i;
374 |     values[i].second = i;
375 |   }
376 |   std::shuffle(values, values + 200, std::default_random_engine{});
377 | 
378 |   std::sort(values, values + 100);
379 |   node.bulk_load(values, 100);
380 | 
381 |   for (int i = 100; i < 200; i++) {
382 |     node.insert(values[i].first, values[i].second);
383 |   }
384 | 
385 |   for (int i = 0; i < 200; i++) {
386 |     int pos = node.find_key(values[i].first);
387 |     CHECK_EQ(values[i].first, node.get_key(pos));
388 |     CHECK(node.check_exists(pos));
389 |   }
390 | }
391 | 
392 | TEST_CASE("TestInsertsWithDuplicates") {
393 |   AlexDataNode<int, int> node;
394 | 
395 |   AlexDataNode<int, int>::V values[200];
396 |   for (int i = 0; i < 200; i++) {
397 |     values[i].first = i;
398 |     values[i].second = i;
399 |   }
400 | 
401 |   std::sort(values, values + 200);
402 |   node.bulk_load(values, 200);
403 | 
404 |   std::shuffle(values, values + 200, std::default_random_engine{});
405 |   for (int i = 0; i < 200; i++) {
406 |     node.insert(values[i].first, values[i].second);
407 |   }
408 | 
409 |   for (int i = 0; i < 200; i++) {
410 |     int pos = node.find_key(values[i].first);
411 |     CHECK_EQ(values[i].first, node.get_key(pos));
412 |     CHECK(node.check_exists(pos));
413 |   }
414 | }
415 | 
416 | TEST_CASE("TestEraseOne") {
417 |   AlexDataNode<int, int> node;
418 | 
419 |   AlexDataNode<int, int>::V values[200];
420 |   for (int i = 0; i < 200; i++) {
421 |     values[i].first = rand() % 500;
422 |     values[i].second = i;
423 |   }
424 | 
425 |   std::sort(values, values + 200);
426 |   node.bulk_load(values, 200);
427 | 
428 |   for (int i = 0; i < 150; i++) {
429 |     int num_erased = node.erase_one(values[i].first);
430 |     CHECK_EQ(num_erased, 1);
431 |   }
432 | 
433 |   for (int i = 150; i < 200; i++) {
434 |     int pos = node.find_key(values[i].first);
435 |     CHECK_EQ(values[i].first, node.get_key(pos));
436 |     CHECK(node.check_exists(pos));
437 |     node.erase_one(values[i].first);
438 |   }
439 | 
440 |   CHECK_EQ(node.num_keys_, 0);
441 | }
442 | 
443 | TEST_CASE("TestErase") {
444 |   AlexDataNode<int, int> node;
445 | 
446 |   AlexDataNode<int, int>::V values[200];
447 |   for (int i = 0; i < 200; i++) {
448 |     values[i].first = i / 2;
449 |     values[i].second = i;
450 |   }
451 | 
452 |   std::sort(values, values + 200);
453 |   node.bulk_load(values, 200);
454 | 
455 |   for (int i = 0; i < 75; i++) {
456 |     int num_erased = node.erase(i);
457 |     CHECK_EQ(num_erased, 2);
458 |   }
459 | 
460 |   for (int i = 75; i < 100; i++) {
461 |     int pos = node.find_key(i);
462 |     CHECK_EQ(i, node.get_key(pos));
463 |     CHECK(node.check_exists(pos));
464 |     node.erase(i);
465 |   }
466 | 
467 |   CHECK_EQ(node.num_keys_, 0);
468 | }
469 | 
470 | TEST_CASE("TestEraseRange") {
471 |   AlexDataNode<int, int> node;
472 | 
473 |   AlexDataNode<int, int>::V values[200];
474 |   for (int i = 0; i < 200; i++) {
475 |     values[i].first = i;
476 |     values[i].second = i;
477 |   }
478 | 
479 |   std::sort(values, values + 200);
480 |   node.bulk_load(values, 200);
481 | 
482 |   int num_erased = node.erase_range(50, 100);
483 |   CHECK_EQ(num_erased, 50);
484 | 
485 |   num_erased = node.erase_range(-50, 50);
486 |   CHECK_EQ(num_erased, 50);
487 | 
488 |   num_erased = node.erase_range(150, 300);
489 |   CHECK_EQ(num_erased, 50);
490 | 
491 |   for (int i = 100; i < 150; i++) {
492 |     int pos = node.find_key(i);
493 |     CHECK_EQ(i, node.get_key(pos));
494 |     CHECK(node.check_exists(pos));
495 |     node.erase_range(i, i + 1);
496 |   }
497 | 
498 |   CHECK_EQ(node.num_keys_, 0);
499 | }
500 | 
501 | TEST_CASE("TestBuildIndexWithSample") {
502 |   const int num_keys = 20000;
503 |   AlexDataNode<int, int>::V values[num_keys];
504 |   for (int i = 0; i < num_keys; i++) {
505 |     values[i].first = rand() % 50000 + 10000;
506 |   }
507 |   std::sort(values, values + num_keys);
508 | 
509 |   LinearModel<int> model;
510 |   LinearModel<int> model_using_sample;
511 | 
512 |   AlexDataNode<int, int>::build_model(values, num_keys, &model, false);
513 |   AlexDataNode<int, int>::build_model(values, num_keys, &model_using_sample,
514 |                                       true);
515 | 
516 |   double rel_diff_in_a =
517 |       std::abs((model.a_ - model_using_sample.a_) / model.a_);
518 |   double rel_diff_in_b =
519 |       std::abs((model.b_ - model_using_sample.b_) / model.b_);
520 |   CHECK_LT(rel_diff_in_a, 0.05);
521 |   CHECK_LT(rel_diff_in_b, 0.05);
522 | }
523 | 
524 | TEST_CASE("TestComputeCostWithSample") {
525 |   const int num_keys = 20000;
526 |   AlexDataNode<int, int>::V values[num_keys];
527 |   for (int i = 0; i < num_keys; i++) {
528 |     values[i].first = rand() % 50000;
529 |   }
530 |   std::sort(values, values + num_keys);
531 | 
532 |   LinearModel<int> model;
533 |   AlexDataNode<int, int>::build_model(values, num_keys, &model);
534 |   double density = 0.7;
535 |   double expected_insert_frac = 0.5;
536 |   ExpectedIterationsAndShiftsAccumulator ent;
537 |   ExpectedIterationsAndShiftsAccumulator ent_using_sample;
538 | 
539 |   DataNodeStats stats;
540 |   DataNodeStats stats_using_sample;
541 |   AlexDataNode<int, int>::compute_expected_cost(
542 |       values, num_keys, density, expected_insert_frac, &model, false, &stats);
543 |   AlexDataNode<int, int>::compute_expected_cost(values, num_keys, density,
544 |                                                 expected_insert_frac, &model,
545 |                                                 true, &stats_using_sample);
546 | 
547 |   double exp_iters = stats.num_search_iterations;
548 |   double exp_iters_sample = stats_using_sample.num_search_iterations;
549 |   double exp_shifts = stats.num_shifts;
550 |   double exp_shifts_sample = stats_using_sample.num_shifts;
551 |   double rel_diff_in_search_entropy =
552 |       std::abs((exp_iters - exp_iters_sample) / exp_iters);
553 |   double rel_diff_in_shifts_entropy =
554 |       std::abs((exp_shifts - exp_shifts_sample) / exp_shifts);
555 |   CHECK_LT(rel_diff_in_search_entropy, 0.5);
556 |   CHECK_LT(rel_diff_in_shifts_entropy, 0.5);
557 | }
558 | };


--------------------------------------------------------------------------------