├── .github └── workflows │ ├── BuildAndPublish.yml │ └── BuildAndTest.yml ├── .gitignore ├── .gitmodules ├── LICENSE ├── cpp ├── .clang-format ├── .gitignore ├── CMakeLists.txt ├── CMakePresets.json.sample ├── benchmark │ ├── CMakeLists.txt │ ├── include │ │ ├── benchmark.h │ │ └── stopwatch.h │ └── src │ │ ├── deglib_anns_bench.cpp │ │ ├── deglib_build_bench.cpp │ │ ├── deglib_explore_bench.cpp │ │ └── deglib_groundtruth.cpp ├── cmake-variants.yaml.sample ├── cmake_modules │ └── DetectCPUFeatures.cmake ├── deglib │ ├── CMakeLists.txt │ └── include │ │ ├── analysis.h │ │ ├── builder.h │ │ ├── concurrent.h │ │ ├── config.h │ │ ├── deglib.h │ │ ├── distances.h │ │ ├── filter.h │ │ ├── graph.h │ │ ├── graph │ │ ├── readonly_graph.h │ │ └── sizebounded_graph.h │ │ ├── memory.h │ │ ├── repository.h │ │ ├── search.h │ │ └── visited_list_pool.h └── readme.md ├── figures ├── anns_qps_vs_recall.jpg └── exploration_qps_vs_recall.jpg ├── java ├── LICENSE ├── buildSrc │ └── shared.gradle ├── deg-api │ ├── build.gradle │ ├── readme.md │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── vc │ │ └── deg │ │ ├── DynamicExplorationGraph.java │ │ ├── FeatureFactory.java │ │ ├── FeatureSpace.java │ │ ├── FeatureVector.java │ │ ├── GraphFactory.java │ │ ├── HierarchicalDynamicExplorationGraph.java │ │ ├── feature │ │ ├── BinaryFeature.java │ │ ├── ByteFeature.java │ │ ├── DoubleFeature.java │ │ ├── FloatFeature.java │ │ ├── IntFeature.java │ │ ├── LongFeature.java │ │ └── ShortFeature.java │ │ ├── graph │ │ ├── GraphDesigner.java │ │ ├── NeighborConsumer.java │ │ ├── VertexCursor.java │ │ ├── VertexFilter.java │ │ └── VertexFilterFactory.java │ │ └── io │ │ ├── LittleEndianDataInputStream.java │ │ └── LittleEndianDataOutputStream.java ├── deg-benchmark │ ├── build.gradle │ ├── readme.md │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── vc │ │ │ └── deg │ │ │ ├── anns │ │ │ ├── GraphConstructionBenchmark.java │ │ │ ├── GraphExploreBenchmark.java │ │ │ ├── GraphFilterBenchmark.java │ │ │ ├── GraphReductionBenchmark.java │ │ │ ├── GraphSearchBenchmark.java │ │ │ └── GraphStatsBenchmark.java │ │ │ ├── data │ │ │ ├── Describable.java │ │ │ ├── GloVe.java │ │ │ ├── Graph2DData.java │ │ │ ├── Identifiable.java │ │ │ └── Sift1M.java │ │ │ └── feature │ │ │ └── FloatL2Space.java │ │ └── test │ │ └── java │ │ └── com │ │ └── vc │ │ └── deg │ │ └── data │ │ └── Create2DGraph.java ├── deg-library │ ├── build.gradle │ ├── readme.md │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ ├── koloboke │ │ │ └── collect │ │ │ │ └── impl │ │ │ │ └── hash │ │ │ │ ├── ParallelIntKeyAdapter.java │ │ │ │ └── SeparateIntObjAdapter.java │ │ │ └── vc │ │ │ └── deg │ │ │ └── impl │ │ │ ├── DynamicExplorationGraph.java │ │ │ ├── GraphFactory.java │ │ │ ├── designer │ │ │ └── EvenRegularGraphDesigner.java │ │ │ ├── graph │ │ │ ├── VertexView.java │ │ │ ├── WeightedEdges.java │ │ │ └── WeightedUndirectedRegularGraph.java │ │ │ ├── kryo │ │ │ ├── HashIntFloatMapSerializer.java │ │ │ └── HashIntIntFloatMapSerializer.java │ │ │ └── search │ │ │ └── GraphSearch.java │ │ └── resources │ │ └── META-INF │ │ └── services │ │ └── com.vc.deg.GraphFactory ├── deg-reference │ ├── build.gradle │ ├── readme.md │ └── src │ │ ├── main │ │ ├── java │ │ │ ├── com │ │ │ │ └── vc │ │ │ │ │ └── deg │ │ │ │ │ └── ref │ │ │ │ │ ├── DynamicExplorationGraph.java │ │ │ │ │ ├── GraphFactory.java │ │ │ │ │ ├── HierarchicalDynamicExplorationGraph.java │ │ │ │ │ ├── designer │ │ │ │ │ └── EvenRegularGraphDesigner.java │ │ │ │ │ ├── feature │ │ │ │ │ └── PrimitiveFeatureFactories.java │ │ │ │ │ ├── graph │ │ │ │ │ ├── ArrayBasedWeightedUndirectedRegularGraph.java │ │ │ │ │ ├── MutableVertexFilter.java │ │ │ │ │ ├── QueryDistance.java │ │ │ │ │ ├── VertexCursor.java │ │ │ │ │ ├── VertexData.java │ │ │ │ │ └── VertexFilterFactory.java │ │ │ │ │ └── hierarchy │ │ │ │ │ └── HierarchicalGraphDesigner.java │ │ │ └── org │ │ │ │ └── roaringbitmap │ │ │ │ └── RoaringBitmapAdapter.java │ │ └── resources │ │ │ └── META-INF │ │ │ └── services │ │ │ ├── com.vc.deg.GraphFactory │ │ │ └── com.vc.deg.graph.VertexFilterFactory │ │ └── test │ │ └── java │ │ └── com │ │ └── vs │ │ └── deg │ │ └── ref │ │ └── graph │ │ └── Toy2DGraphTest.java ├── deg-visualization │ ├── build.gradle │ ├── readme.md │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── vc │ │ │ └── deg │ │ │ └── viz │ │ │ ├── MapDesigner.java │ │ │ ├── MapNavigator.java │ │ │ ├── model │ │ │ ├── GridMap.java │ │ │ ├── MotionVector.java │ │ │ └── WorldMap.java │ │ │ └── om │ │ │ ├── FLASnoMapSorter.java │ │ │ ├── FLASnoMapSorterAdapter.java │ │ │ └── JonkerVolgenantSolver.java │ │ └── test │ │ └── java │ │ └── com │ │ └── vc │ │ └── deg │ │ └── viz │ │ ├── MapNavigatorTest.java │ │ ├── OrganizingMapTest.java │ │ └── Visualize2DGraphTest.java ├── gradle │ └── wrapper │ │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── readme.md └── settings.gradle ├── python ├── .gitignore ├── CMakeLists.txt ├── LICENSE ├── MANIFEST.in ├── README.md ├── publish.sh ├── pyproject.toml ├── setup.py ├── src │ ├── deg_cpp │ │ └── deglib_cpp.cpp │ └── deglib │ │ ├── __init__.py │ │ ├── analysis.py │ │ ├── benchmark.py │ │ ├── builder.py │ │ ├── distances.py │ │ ├── graph.py │ │ ├── repository.py │ │ ├── search.py │ │ ├── std.py │ │ └── utils.py └── tests │ ├── .ipynb_checkpoints │ └── ECIR_Tuturial_Image_Search-checkpoint.ipynb │ ├── ECIR_Tuturial_Image_Search.ipynb │ ├── bench_anns.py │ ├── bench_build.py │ ├── bench_explore.py │ ├── experiments.py │ ├── test_builder.py │ └── test_graph.py └── readme.md /.github/workflows/BuildAndPublish.yml: -------------------------------------------------------------------------------- 1 | name: BuildAndPublish 2 | 3 | on: 4 | workflow_dispatch: 5 | pull_request: 6 | push: 7 | tags: 8 | - 'v*.*.*' 9 | 10 | defaults: 11 | run: 12 | working-directory: python 13 | 14 | jobs: 15 | build: 16 | strategy: 17 | fail-fast: false 18 | matrix: 19 | platform: [windows-latest, macos-latest, macos-13, ubuntu-latest] 20 | 21 | runs-on: ${{ matrix.platform }} 22 | 23 | steps: 24 | - uses: actions/checkout@v4 25 | with: 26 | submodules: false 27 | 28 | - uses: actions/setup-python@v5 29 | with: 30 | python-version: "3.12" 31 | 32 | - name: Add requirements 33 | run: python -m pip install --upgrade setuptools pybind11>=2.12.0 cibuildwheel 34 | 35 | - name: Copy Build Files 36 | run: python setup.py copy_build_files 37 | 38 | - name: Build Packages 39 | run: cibuildwheel --output-dir "dist" 40 | env: 41 | CIBW_ARCHS: auto64 42 | CIBW_BUILD_FRONTEND: "pip; args: --verbose" 43 | CIBW_ENVIRONMENT: "FORCE_AVX2=1" 44 | CIBW_ENVIRONMENT_MACOS: "FORCE_AVX2=1 MACOSX_DEPLOYMENT_TARGET=11.00" 45 | # CIBW_SKIP: "cp313-*" 46 | # CIBW_SKIP: "cp313-* pp*win*" 47 | CIBW_SKIP: "pp*win*" 48 | 49 | # upload artifacts 50 | - name: Upload Build Artifacts 51 | uses: actions/upload-artifact@v4 52 | with: 53 | name: release-dists-${{ matrix.platform }} 54 | path: python/dist 55 | 56 | build_sdist: 57 | runs-on: ubuntu-latest 58 | 59 | steps: 60 | - uses: actions/checkout@v4 61 | with: 62 | submodules: false 63 | 64 | - uses: actions/setup-python@v5 65 | with: 66 | python-version: "3.12" 67 | 68 | - name: Add requirements 69 | run: python -m pip install --upgrade setuptools pybind11>=2.12.0 70 | 71 | - name: Copy Build Files 72 | run: python setup.py copy_build_files 73 | 74 | - name: Build Source Distribution 75 | run: python setup.py sdist 76 | 77 | - name: Upload Build Artifacts 78 | uses: actions/upload-artifact@v4 79 | with: 80 | name: release-dists-sdist 81 | path: python/dist 82 | 83 | pypi-publish: 84 | runs-on: "ubuntu-latest" 85 | 86 | needs: 87 | - build 88 | - build_sdist 89 | 90 | permissions: 91 | # IMPORTANT: this permission is mandatory for trusted publishing 92 | id-token: write 93 | 94 | # Dedicated environments with protections for publishing are strongly recommended. 95 | environment: 96 | name: pypi 97 | # OPTIONAL: uncomment and update to include your PyPI project URL in the deployment status: 98 | # url: https://test.pypi.org/project/deglib/ 99 | 100 | steps: 101 | - name: Retrieve release distributions 102 | uses: actions/download-artifact@v4 103 | with: 104 | path: dist 105 | pattern: release-dists-* 106 | merge-multiple: true 107 | 108 | - name: Publish release distributions to PyPI 109 | uses: pypa/gh-action-pypi-publish@release/v1 110 | -------------------------------------------------------------------------------- /.github/workflows/BuildAndTest.yml: -------------------------------------------------------------------------------- 1 | name: BuildAndTest 2 | 3 | on: 4 | workflow_dispatch: 5 | 6 | defaults: 7 | run: 8 | working-directory: python 9 | 10 | jobs: 11 | build: 12 | strategy: 13 | fail-fast: false 14 | matrix: 15 | platform: [windows-latest, macos-latest, macos-13, ubuntu-latest] 16 | 17 | runs-on: ${{ matrix.platform }} 18 | 19 | steps: 20 | - uses: actions/checkout@v4 21 | with: 22 | submodules: false 23 | 24 | - uses: actions/setup-python@v5 25 | with: 26 | python-version: "3.12" 27 | 28 | - name: Add requirements 29 | run: python -m pip install --upgrade setuptools pybind11>=2.12.0 cibuildwheel 30 | 31 | - name: Copy Build Files 32 | run: python setup.py copy_build_files 33 | 34 | - name: Build Packages 35 | run: cibuildwheel --output-dir "dist" 36 | env: 37 | CIBW_ARCHS: auto64 38 | CIBW_BUILD_FRONTEND: "pip; args: --verbose" 39 | CIBW_ENVIRONMENT: "FORCE_AVX2=1" 40 | CIBW_ENVIRONMENT_MACOS: "FORCE_AVX2=1 MACOSX_DEPLOYMENT_TARGET=11.00" 41 | # CIBW_SKIP: "cp313-*" 42 | CIBW_SKIP: "pp*win*" 43 | 44 | build_sdist: 45 | runs-on: ubuntu-latest 46 | 47 | steps: 48 | - uses: actions/checkout@v4 49 | with: 50 | submodules: false 51 | 52 | - uses: actions/setup-python@v5 53 | with: 54 | python-version: "3.12" 55 | 56 | - name: Add requirements 57 | run: python -m pip install --upgrade setuptools pybind11>=2.12.0 58 | 59 | - name: Copy Build Files 60 | run: python setup.py copy_build_files 61 | 62 | - name: Build Source Distribution 63 | run: python setup.py sdist 64 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled class file 2 | *.class 3 | 4 | # Log file 5 | *.log 6 | 7 | # BlueJ files 8 | *.ctxt 9 | 10 | # Mobile Tools for Java (J2ME) 11 | .mtj.tmp/ 12 | 13 | # Package Files # 14 | *.jar 15 | *.war 16 | *.nar 17 | *.ear 18 | *.zip 19 | *.tar.gz 20 | *.rar 21 | 22 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 23 | hs_err_pid* 24 | 25 | # Ignore gradle files 26 | .gradle/ 27 | gradle.properties 28 | 29 | # generated files 30 | bin/ 31 | gen/ 32 | logs/ 33 | build/ 34 | buildSrc/build/ 35 | buildSrc/.gradle/ 36 | 37 | # open file 38 | ~$*.* 39 | 40 | # Windows thumbnail db 41 | Thumbs.db 42 | 43 | # OSX files 44 | .DS_Store 45 | 46 | # Eclipse project files 47 | .metadata/ 48 | .settings/ 49 | .classpath 50 | .project 51 | 52 | # IntelliJ IDEA files 53 | .idea 54 | *.iml 55 | *.iws 56 | *.ipr 57 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "external/fmt"] 2 | path = cpp/external/fmt 3 | url = https://github.com/fmtlib/fmt 4 | branch = 7.1.3 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Visual Computing Group 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /cpp/.clang-format: -------------------------------------------------------------------------------- 1 | --- 2 | Language: Cpp 3 | # BasedOnStyle: Google 4 | AccessModifierOffset: -2 5 | AlignAfterOpenBracket: Align 6 | AlignConsecutiveAssignments: false 7 | AlignConsecutiveDeclarations: false 8 | AlignEscapedNewlinesLeft: true 9 | AlignOperands: true 10 | AlignTrailingComments: true 11 | AllowAllParametersOfDeclarationOnNextLine: true 12 | AllowShortBlocksOnASingleLine: false 13 | AllowShortCaseLabelsOnASingleLine: false 14 | AllowShortFunctionsOnASingleLine: All 15 | AllowShortIfStatementsOnASingleLine: true 16 | AllowShortLoopsOnASingleLine: true 17 | AlwaysBreakAfterDefinitionReturnType: None 18 | AlwaysBreakAfterReturnType: None 19 | AlwaysBreakBeforeMultilineStrings: true 20 | AlwaysBreakTemplateDeclarations: true 21 | BinPackArguments: true 22 | BinPackParameters: true 23 | BraceWrapping: 24 | AfterClass: false 25 | AfterControlStatement: false 26 | AfterEnum: false 27 | AfterFunction: false 28 | AfterNamespace: false 29 | AfterObjCDeclaration: false 30 | AfterStruct: false 31 | AfterUnion: false 32 | BeforeCatch: false 33 | BeforeElse: false 34 | IndentBraces: false 35 | BreakBeforeBinaryOperators: None 36 | BreakBeforeBraces: Allman 37 | BreakBeforeTernaryOperators: true 38 | BreakConstructorInitializersBeforeComma: false 39 | ColumnLimit: 120 40 | CommentPragmas: '^ IWYU pragma:' 41 | ConstructorInitializerAllOnOneLineOrOnePerLine: true 42 | ConstructorInitializerIndentWidth: 4 43 | ContinuationIndentWidth: 4 44 | Cpp11BracedListStyle: true 45 | DerivePointerAlignment: false 46 | DisableFormat: false 47 | ExperimentalAutoDetectBinPacking: false 48 | ForEachMacros: [ foreach, Q_FOREACH, BOOST_FOREACH ] 49 | SortIncludes: true 50 | IncludeBlocks: Regroup 51 | IncludeCategories: 52 | # Headers in <> without extension. 53 | - Regex: '<([A-Za-z0-9\Q/-_\E])+>' 54 | Priority: 3 55 | # Headers in <> with extension. 56 | - Regex: '<([A-Za-z0-9.\Q/-_\E])+>' 57 | Priority: 2 58 | # Headers in "" with extension. 59 | - Regex: '"([A-Za-z0-9.\Q/-_\E])+"' 60 | Priority: 1 61 | - Regex: '.*' 62 | Priority: 0 63 | IndentCaseLabels: true 64 | IndentWidth: 4 65 | IndentWrappedFunctionNames: false 66 | KeepEmptyLinesAtTheStartOfBlocks: false 67 | MacroBlockBegin: '' 68 | MacroBlockEnd: '' 69 | MaxEmptyLinesToKeep: 1 70 | NamespaceIndentation: None 71 | ObjCBlockIndentWidth: 2 72 | ObjCSpaceAfterProperty: false 73 | ObjCSpaceBeforeProtocolList: false 74 | PenaltyBreakBeforeFirstCallParameter: 1 75 | PenaltyBreakComment: 300 76 | PenaltyBreakFirstLessLess: 120 77 | PenaltyBreakString: 1000 78 | PenaltyExcessCharacter: 1000000 79 | PenaltyReturnTypeOnItsOwnLine: 200 80 | PointerAlignment: Left 81 | ReflowComments: true 82 | SpaceAfterCStyleCast: false 83 | SpaceBeforeAssignmentOperators: true 84 | SpaceBeforeParens: ControlStatements 85 | SpaceInEmptyParentheses: false 86 | SpacesBeforeTrailingComments: 2 87 | SpacesInAngles: false 88 | SpacesInContainerLiterals: true 89 | SpacesInCStyleCastParentheses: false 90 | SpacesInParentheses: false 91 | SpacesInSquareBrackets: false 92 | Standard: Cpp11 93 | TabWidth: 8 94 | UseTab: Never 95 | --- 96 | Language: Proto 97 | -------------------------------------------------------------------------------- /cpp/.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | dist/ 3 | tmp/ 4 | var/ 5 | .vscode/ 6 | CMakePresets.json 7 | cmake-variants.yaml 8 | build-* 9 | cmake-build* 10 | data/ 11 | -------------------------------------------------------------------------------- /cpp/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.19) 2 | 3 | # Only interpret if() arguments as variables or keywords when unquoted. More 4 | # details run "cmake --help-policy CMP0054" 5 | cmake_policy(SET CMP0054 NEW) 6 | 7 | project( 8 | deg_lib 9 | HOMEPAGE_URL "https://github.com/Neiko2002/DEG" 10 | DESCRIPTION 11 | "Dynamic Exploration Graphs for fast approximate nearest neighbor search and navigation in large image datasets" 12 | VERSION 0.0.1 13 | LANGUAGES CXX 14 | ) 15 | 16 | # build options 17 | option(ENABLE_BENCHMARKS "compile benchmarks" ON) 18 | option(FORCE_AVX2 "compile for avx2 support. Otherwise compile for this machine." OFF) 19 | 20 | # https://cmake.org/cmake/help/latest/prop_tgt/CXX_STANDARD.html 21 | set(CMAKE_CXX_STANDARD 20) 22 | set(CMAKE_CXX_STANDARD_REQUIRED ON) 23 | set(CMAKE_CXX_EXTENSIONS OFF) 24 | 25 | add_library(compile-options INTERFACE) 26 | target_compile_features(compile-options INTERFACE cxx_std_20) 27 | target_compile_definitions(compile-options INTERFACE $<$:NOMINMAX>) 28 | 29 | # detecting CPU instruction support 30 | # https://github.com/lemire/FastPFor/blob/master/CMakeLists.txt 31 | if (APPLE AND CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64") 32 | message(WARNING "You are compiling on an Apple M chip. At the moment deglib does not support ARM optimizations, so using deglib will be slow!") 33 | endif() 34 | 35 | if (FORCE_AVX2) 36 | message(NOTICE "Compiling for avx2, ignoring native optimizations.") 37 | 38 | # setup compiler flags 39 | if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "GNU") 40 | target_compile_options( 41 | compile-options 42 | INTERFACE -O3 43 | -mavx2 44 | -mfma 45 | -fpic 46 | -w 47 | -fopenmp 48 | -pthread 49 | -ftree-vectorize 50 | -ftree-vectorizer-verbose=0 51 | ) 52 | elseif(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang") 53 | target_compile_options( 54 | compile-options 55 | INTERFACE -O2 56 | -mavx2 57 | -mfma 58 | -fpic) 59 | elseif(MSVC) 60 | target_compile_options( 61 | compile-options 62 | INTERFACE /O2 63 | /arch:AVX2 64 | /W1 65 | /openmp 66 | /EHsc) 67 | else() 68 | message(WARNING "Unknown compiler for AVX2 Build: ${CMAKE_CXX_COMPILER_ID}") 69 | endif() 70 | 71 | else() # Native build 72 | list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules") 73 | include(DetectCPUFeatures) 74 | 75 | # setup compiler flags 76 | if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "GNU") 77 | target_compile_options( 78 | compile-options 79 | INTERFACE -O3 80 | -march=native 81 | -mavx2 82 | -mfma 83 | -fpic 84 | -w 85 | -fopenmp 86 | -pthread 87 | -ftree-vectorize 88 | -ftree-vectorizer-verbose=0) 89 | elseif(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang") 90 | target_compile_options( 91 | compile-options 92 | INTERFACE -O2 93 | -DNDEBUG 94 | -march=native 95 | -fpic) 96 | elseif(MSVC) 97 | # warning level 4 (show all) 98 | target_compile_options(compile-options INTERFACE /W4) 99 | 100 | # detecting SUPPORT_AVX2 or SSE2 support 101 | cmake_host_system_information(RESULT SUPPORT_SSE2 QUERY HAS_SSE2) 102 | if(SUPPORT_AVX2) 103 | target_compile_options(compile-options INTERFACE /arch:AVX2) 104 | elseif(SUPPORT_SSE2) 105 | target_compile_options(compile-options INTERFACE /arch:SSE2) 106 | else() 107 | target_compile_options(compile-options INTERFACE /arch:native) 108 | endif() 109 | 110 | # disable string optimizations and function level linking 111 | # https://stackoverflow.com/questions/5063334/what-is-the-difference-between-the-ox-and-o2-compiler-options 112 | target_compile_options(compile-options INTERFACE $<$:/Ox>) 113 | else() 114 | message(WARNING "Unknown compiler: ${CMAKE_CXX_COMPILER_ID}") 115 | endif() 116 | endif() 117 | 118 | message("C++ compiler flags: ${CMAKE_CXX_FLAGS}") 119 | message("C++ compiler flags release: ${CMAKE_CXX_FLAGS_RELEASE}") 120 | 121 | add_subdirectory(deglib) 122 | 123 | if (ENABLE_BENCHMARKS) 124 | add_subdirectory(external/fmt) 125 | add_subdirectory(benchmark) 126 | endif() 127 | -------------------------------------------------------------------------------- /cpp/CMakePresets.json.sample: -------------------------------------------------------------------------------- 1 | { 2 | "version": 3, 3 | "cmakeMinimumRequired": { 4 | "major": 3, 5 | "minor": 19, 6 | "patch": 0 7 | }, 8 | "configurePresets": [ 9 | { 10 | "name": "default", 11 | "hidden": false, 12 | "description": "Preset that sets the DATA_PATH variable.", 13 | "cacheVariables": { 14 | "DATA_PATH": { 15 | "value": "", 16 | "type": "PATH" 17 | } 18 | } 19 | } 20 | ] 21 | } 22 | -------------------------------------------------------------------------------- /cpp/benchmark/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.19) 2 | 3 | project(benchmark) 4 | include_directories(${PROJECT_SOURCE_DIR}/include) 5 | set(DATA_PATH "" CACHE PATH "Path to data sets") 6 | 7 | if(DATA_PATH STREQUAL "") 8 | message(WARNING "DATA_PATH is empty") 9 | endif() 10 | 11 | # build DEG benchmark files 12 | find_package(OpenMP REQUIRED) 13 | 14 | add_executable(deglib_anns_bench ${PROJECT_SOURCE_DIR}/src/deglib_anns_bench.cpp) 15 | target_link_libraries(deglib_anns_bench PUBLIC DEG_LIB compile-options 16 | fmt::fmt-header-only) 17 | target_compile_definitions(deglib_anns_bench PUBLIC "DATA_PATH=\"${DATA_PATH}\"") 18 | 19 | add_executable(deglib_explore_bench ${PROJECT_SOURCE_DIR}/src/deglib_explore_bench.cpp) 20 | target_link_libraries(deglib_explore_bench PUBLIC DEG_LIB compile-options 21 | fmt::fmt-header-only) 22 | target_compile_definitions(deglib_explore_bench PUBLIC "DATA_PATH=\"${DATA_PATH}\"") 23 | 24 | add_executable(deglib_build_bench ${PROJECT_SOURCE_DIR}/src/deglib_build_bench.cpp) 25 | target_link_libraries(deglib_build_bench PUBLIC DEG_LIB compile-options 26 | fmt::fmt-header-only OpenMP::OpenMP_CXX) 27 | target_compile_definitions(deglib_build_bench PUBLIC "DATA_PATH=\"${DATA_PATH}\"") 28 | 29 | add_executable(deglib_groundtruth ${PROJECT_SOURCE_DIR}/src/deglib_groundtruth.cpp) 30 | target_link_libraries(deglib_groundtruth PUBLIC DEG_LIB compile-options 31 | fmt::fmt-header-only OpenMP::OpenMP_CXX) 32 | target_compile_definitions(deglib_groundtruth PUBLIC "DATA_PATH=\"${DATA_PATH}\"") 33 | -------------------------------------------------------------------------------- /cpp/benchmark/include/stopwatch.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | class StopW 6 | { 7 | std::chrono::steady_clock::time_point time_begin; 8 | 9 | public: 10 | StopW() 11 | { 12 | time_begin = std::chrono::steady_clock::now(); 13 | } 14 | 15 | long long getElapsedTimeMicro() 16 | { 17 | std::chrono::steady_clock::time_point time_end = std::chrono::steady_clock::now(); 18 | return (std::chrono::duration_cast(time_end - time_begin).count()); 19 | } 20 | 21 | void reset() 22 | { 23 | time_begin = std::chrono::steady_clock::now(); 24 | } 25 | }; 26 | 27 | /* 28 | * Author: David Robert Nadeau 29 | * Site: http://NadeauSoftware.com/ 30 | * License: Creative Commons Attribution 3.0 Unported License 31 | * http://creativecommons.org/licenses/by/3.0/deed.en_US 32 | */ 33 | 34 | #if defined(_WIN32) 35 | #include // max/min will be defined as macro, define std::max before including windows.h 36 | #include 37 | 38 | #elif defined(__unix__) || defined(__unix) || defined(unix) || (defined(__APPLE__) && defined(__MACH__)) 39 | 40 | #include 41 | #include 42 | 43 | #if defined(__APPLE__) && defined(__MACH__) 44 | #include 45 | 46 | #elif (defined(_AIX) || defined(__TOS__AIX__)) || (defined(__sun__) || defined(__sun) || defined(sun) && (defined(__SVR4) || defined(__svr4__))) 47 | #include 48 | #include 49 | 50 | #elif defined(__linux__) || defined(__linux) || defined(linux) || defined(__gnu_linux__) 51 | 52 | #endif 53 | 54 | #else 55 | #error "Cannot define getPeakRSS( ) or getCurrentRSS( ) for an unknown OS." 56 | #endif 57 | 58 | /** 59 | * Returns the peak (maximum so far) resident set size (physical 60 | * memory use) measured in bytes, or zero if the value cannot be 61 | * determined on this OS. 62 | */ 63 | static size_t getPeakRSS() 64 | { 65 | #if defined(_WIN32) 66 | /* Windows -------------------------------------------------- */ 67 | PROCESS_MEMORY_COUNTERS info; 68 | GetProcessMemoryInfo(GetCurrentProcess(), &info, sizeof(info)); 69 | return (size_t)info.PeakWorkingSetSize; 70 | 71 | #elif (defined(_AIX) || defined(__TOS__AIX__)) || (defined(__sun__) || defined(__sun) || defined(sun) && (defined(__SVR4) || defined(__svr4__))) 72 | /* AIX and Solaris ------------------------------------------ */ 73 | struct psinfo psinfo; 74 | int fd = -1; 75 | if ((fd = open("/proc/self/psinfo", O_RDONLY)) == -1) 76 | return (size_t)0L; /* Can't open? */ 77 | if (read(fd, &psinfo, sizeof(psinfo)) != sizeof(psinfo)) 78 | { 79 | close(fd); 80 | return (size_t)0L; /* Can't read? */ 81 | } 82 | close(fd); 83 | return (size_t)(psinfo.pr_rssize * 1024L); 84 | 85 | #elif defined(__unix__) || defined(__unix) || defined(unix) || (defined(__APPLE__) && defined(__MACH__)) 86 | /* BSD, Linux, and OSX -------------------------------------- */ 87 | struct rusage rusage; 88 | getrusage(RUSAGE_SELF, &rusage); 89 | #if defined(__APPLE__) && defined(__MACH__) 90 | return (size_t)rusage.ru_maxrss; 91 | #else 92 | return (size_t)(rusage.ru_maxrss * 1024L); 93 | #endif 94 | 95 | #else 96 | /* Unknown OS ----------------------------------------------- */ 97 | return (size_t)0L; /* Unsupported. */ 98 | #endif 99 | } 100 | 101 | /** 102 | * Returns the current resident set size (physical memory use) measured 103 | * in bytes, or zero if the value cannot be determined on this OS. 104 | */ 105 | static size_t getCurrentRSS() 106 | { 107 | #if defined(_WIN32) 108 | /* Windows -------------------------------------------------- */ 109 | PROCESS_MEMORY_COUNTERS info; 110 | GetProcessMemoryInfo(GetCurrentProcess(), &info, sizeof(info)); 111 | return (size_t)info.WorkingSetSize; 112 | 113 | #elif defined(__APPLE__) && defined(__MACH__) 114 | /* OSX ------------------------------------------------------ */ 115 | struct mach_task_basic_info info; 116 | mach_msg_type_number_t infoCount = MACH_TASK_BASIC_INFO_COUNT; 117 | if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, 118 | (task_info_t)&info, &infoCount) != KERN_SUCCESS) 119 | return (size_t)0L; /* Can't access? */ 120 | return (size_t)info.resident_size; 121 | 122 | #elif defined(__linux__) || defined(__linux) || defined(linux) || defined(__gnu_linux__) 123 | /* Linux ---------------------------------------------------- */ 124 | long rss = 0L; 125 | FILE *fp = NULL; 126 | if ((fp = fopen("/proc/self/statm", "r")) == NULL) 127 | return (size_t)0L; /* Can't open? */ 128 | if (fscanf(fp, "%*s%ld", &rss) != 1) 129 | { 130 | fclose(fp); 131 | return (size_t)0L; /* Can't read? */ 132 | } 133 | fclose(fp); 134 | return (size_t)rss * (size_t)sysconf(_SC_PAGESIZE); 135 | 136 | #else 137 | /* AIX, BSD, Solaris, and Unknown OS ------------------------ */ 138 | return (size_t)0L; /* Unsupported. */ 139 | #endif 140 | } 141 | -------------------------------------------------------------------------------- /cpp/benchmark/src/deglib_explore_bench.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | 5 | #include "benchmark.h" 6 | #include "deglib.h" 7 | 8 | int main(int argc, char *argv[]) { 9 | fmt::print("Testing ...\n"); 10 | 11 | #if defined(USE_AVX) 12 | fmt::print("use AVX2 ...\n"); 13 | #elif defined(USE_SSE) 14 | fmt::print("use SSE ...\n"); 15 | #else 16 | fmt::print("use arch ...\n"); 17 | #endif 18 | 19 | 20 | const uint32_t repeat_test = 1; 21 | const auto data_path = std::filesystem::path(DATA_PATH); 22 | 23 | const uint32_t k = 1000; 24 | 25 | // ------------------------------------------ SIFT1M --------------------------------------------- 26 | const auto graph_file = (data_path / "deg" / "best_distortion_decisions" / "128D_L2_K30_AddK60Eps0.2High_SwapK30-0StepEps0.001LowPath5Rnd0+0_improveEvery2ndNonPerfectEdge.deg").string(); 27 | const auto gt_file = (data_path / "SIFT1M" / "sift_explore_ground_truth.ivecs").string(); 28 | const auto query_file = (data_path / "SIFT1M" / "sift_explore_entry_vertex.ivecs").string(); 29 | 30 | // ------------------------------------------ Glove --------------------------------------------- 31 | // const auto graph_file = (data_path / "deg" / "100D_L2_K30_AddK30Eps0.2High_SwapK30-0StepEps0.001LowPath5Rnd0+0_improveEvery2ndNonPerfectEdge.deg").string(); 32 | // const auto gt_file = (data_path / "glove-100" / "glove-100_explore_ground_truth.ivecs").string(); 33 | // const auto query_file = (data_path / "glove-100" / "glove-100_explore_entry_vertex.ivecs").string(); 34 | 35 | // ------------------------------------------ Enron --------------------------------------------- 36 | // const auto graph_file = (data_path / "deg" / "1369D_L2_K30_AddK60Eps0.3High_SwapK30-0StepEps0.001LowPath5Rnd0+0_improveEvery2ndNonPerfectEdge.deg").string(); 37 | // const auto gt_file = (data_path / "enron" / "enron_explore_ground_truth.ivecs").string(); 38 | // const auto query_file = (data_path / "enron" / "enron_explore_entry_vertex.ivecs").string(); 39 | 40 | // ------------------------------------------ Audio --------------------------------------------- 41 | // const auto graph_file = (data_path / "deg" / "192D_L2_K20_AddK40Eps0.3High_SwapK20-0StepEps0.001LowPath5Rnd0+0_improveEvery2ndNonPerfectEdge.deg").string(); 42 | // const auto gt_file = (data_path / "audio" / "audio_explore_ground_truth.ivecs").string(); 43 | // const auto query_file = (data_path / "audio" / "audio_explore_entry_vertex.ivecs").string(); 44 | 45 | // load graph 46 | fmt::print("Load graph {} \n", graph_file); 47 | const auto graph = deglib::graph::load_readonly_graph(graph_file.c_str()); 48 | 49 | // load starting vertex data 50 | size_t entry_vertex_dims; 51 | size_t entry_vertex_count; 52 | const auto entry_vertex_f = deglib::fvecs_read(query_file.c_str(), entry_vertex_dims, entry_vertex_count); 53 | const auto entry_vertex = (uint32_t*)entry_vertex_f.get(); // not very clean, works as long as sizeof(int) == sizeof(float) 54 | fmt::print("{} entry vertex {} dimensions \n", entry_vertex_count, entry_vertex_dims); 55 | 56 | // load ground truth data (nearest neighbors of the starting vertices) 57 | size_t ground_truth_dims; 58 | size_t ground_truth_count; 59 | const auto ground_truth_f = deglib::fvecs_read(gt_file.c_str(), ground_truth_dims, ground_truth_count); 60 | const auto ground_truth = (uint32_t*)ground_truth_f.get(); // not very clean, works as long as sizeof(int) == sizeof(float) 61 | fmt::print("{} ground truth {} dimensions \n", ground_truth_count, ground_truth_dims); 62 | 63 | // explore the graph 64 | deglib::benchmark::test_graph_explore(graph, (uint32_t) ground_truth_count, ground_truth, (uint32_t) ground_truth_dims, entry_vertex, (uint32_t) entry_vertex_dims, repeat_test, k); 65 | 66 | fmt::print("Test OK\n"); 67 | return 0; 68 | } -------------------------------------------------------------------------------- /cpp/cmake-variants.yaml.sample: -------------------------------------------------------------------------------- 1 | buildType: 2 | default: Release 3 | choices: 4 | debug: 5 | short: Debug 6 | long: Unoptimized, with debug info, asserts enabled. 7 | buildType: Debug 8 | release: 9 | short: Release 10 | long: Highest optimization level, no debug info, code or asserts. 11 | buildType: Release 12 | releaseWithDebugInfo: 13 | short: RelWithDebInfo 14 | long: Optimized, with debug info but no debug (output) code or asserts. 15 | buildType: RelWithDebInfo 16 | 17 | main: 18 | default: deglib 19 | choices: 20 | deglib: 21 | short: deglib 22 | long: Build deglib 23 | settings: 24 | DATA_PATH: "" 25 | CMAKE_EXPORT_COMPILE_COMMAND: "on" -------------------------------------------------------------------------------- /cpp/cmake_modules/DetectCPUFeatures.cmake: -------------------------------------------------------------------------------- 1 | include(CheckCXXSourceCompiles) 2 | 3 | set(OLD_CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS}) 4 | 5 | set(SSE4PROG " 6 | 7 | #include 8 | int main(){ 9 | __m128 x=_mm_set1_ps(0.5); 10 | x=_mm_dp_ps(x,x,0x77); 11 | return _mm_movemask_ps(x); 12 | }") 13 | 14 | set(AVXPROG " 15 | 16 | #include 17 | int main(){ 18 | __m128 x=_mm_set1_ps(0.5); 19 | x=_mm_permute_ps(x,1); 20 | return _mm_movemask_ps(x); 21 | }") 22 | 23 | set(AVX2PROG " 24 | 25 | #include 26 | int main(){ 27 | __m256i x=_mm256_set1_epi32(5); 28 | x=_mm256_add_epi32(x,x); 29 | return _mm256_movemask_epi8(x); 30 | }") 31 | 32 | if(MSVC) 33 | set(CMAKE_REQUIRED_FLAGS "/EHsc /arch:SSE2") 34 | check_cxx_source_compiles("${SSE4PROG}" SUPPORT_SSE42) 35 | message(STATUS "SUPPORT_SSE42 ${SUPPORT_SSE42}") 36 | set(CMAKE_REQUIRED_FLAGS "/EHsc /arch:AVX") 37 | check_cxx_source_compiles("${AVXPROG}" SUPPORT_AVX) 38 | message(STATUS "SUPPORT_AVX ${SUPPORT_AVX}") 39 | set(CMAKE_REQUIRED_FLAGS "/EHsc /arch:AVX2") 40 | check_cxx_source_compiles("${AVX2PROG}" SUPPORT_AVX2) 41 | message(STATUS "SUPPORT_AVX2 ${SUPPORT_AVX2}") 42 | else() 43 | set(CMAKE_REQUIRED_FLAGS "-march=native -msse4.2") 44 | check_cxx_source_compiles("${SSE4PROG}" SUPPORT_SSE42) 45 | set(CMAKE_REQUIRED_FLAGS "-march=native -mavx") 46 | check_cxx_source_compiles("${AVXPROG}" SUPPORT_AVX) 47 | set(CMAKE_REQUIRED_FLAGS "-march=native -mavx2") 48 | check_cxx_source_compiles("${AVX2PROG}" SUPPORT_AVX2) 49 | endif() 50 | 51 | set(CMAKE_REQUIRED_FLAGS ${OLD_CMAKE_REQUIRED_FLAGS}) 52 | -------------------------------------------------------------------------------- /cpp/deglib/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 3.19) 2 | 3 | project(deglib) 4 | 5 | # setup the deg lib 6 | # http://mariobadr.com/creating-a-header-only-library-with-cmake.html 7 | add_library(DEG_LIB INTERFACE) 8 | target_include_directories(DEG_LIB INTERFACE ${PROJECT_SOURCE_DIR}/include) 9 | target_include_directories(DEG_LIB INTERFACE ${PROJECT_SOURCE_DIR}/include/graph) 10 | target_compile_features(DEG_LIB INTERFACE cxx_std_20) 11 | -------------------------------------------------------------------------------- /cpp/deglib/include/concurrent.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #include "config.h" 8 | 9 | namespace deglib::concurrent { 10 | 11 | // Multithreaded executor 12 | // The helper function copied from https://github.com/nmslib/hnswlib/blob/master/examples/cpp/example_mt_search.cpp (and that itself is copied from nmslib) 13 | // An alternative is using #pragme omp parallel for or any other C++ threading 14 | template 15 | inline void parallel_for(size_t start, size_t end, size_t numThreads, Function fn) { 16 | if (numThreads <= 0) { 17 | numThreads = std::thread::hardware_concurrency(); 18 | } 19 | 20 | if (numThreads == 1) { 21 | for (size_t id = start; id < end; id++) { 22 | fn(id, 0); 23 | } 24 | } else { 25 | std::vector threads; 26 | std::atomic current(start); 27 | 28 | // keep track of exceptions in threads 29 | // https://stackoverflow.com/a/32428427/1713196 30 | std::exception_ptr lastException = nullptr; 31 | std::mutex lastExceptMutex; 32 | 33 | for (size_t threadId = 0; threadId < numThreads; ++threadId) { 34 | threads.push_back(std::thread([&, threadId] { 35 | while (true) { 36 | size_t id = current.fetch_add(1); 37 | 38 | if (id >= end) { 39 | break; 40 | } 41 | 42 | try { 43 | fn(id, threadId); 44 | } catch (...) { 45 | std::unique_lock lastExcepLock(lastExceptMutex); 46 | lastException = std::current_exception(); 47 | /* 48 | * This will work even when current is the largest value that 49 | * size_t can fit, because fetch_add returns the previous value 50 | * before the increment (what will result in overflow 51 | * and produce 0 instead of current + 1). 52 | */ 53 | current = end; 54 | break; 55 | } 56 | } 57 | })); 58 | } 59 | for (auto &thread : threads) { 60 | thread.join(); 61 | } 62 | if (lastException) { 63 | std::rethrow_exception(lastException); 64 | } 65 | } 66 | } 67 | 68 | } // namespace deglib::memory 69 | -------------------------------------------------------------------------------- /cpp/deglib/include/config.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #ifndef NO_MANUAL_VECTORIZATION 4 | // Microsoft Visual C++ does not define __SSE__ or __SSE2__ but _M_IX86_FP instead 5 | // https://docs.microsoft.com/en-us/cpp/preprocessor/predefined-macros?view=msvc-170 6 | #ifdef _MSC_VER 7 | #if defined(_M_AMD64) || defined(_M_X64) || defined(_M_IX86_FP) == 2 8 | #define __SSE__ 9 | #define __SSE2__ 10 | #elif defined(_M_IX86_FP) == 1 11 | #define __SSE__ 12 | #endif 13 | #endif 14 | 15 | #if defined(__AVX512F__) 16 | #define USE_AVX512 17 | #endif 18 | 19 | #if defined(__AVX__) || defined(__AVX2__) 20 | #define USE_AVX 21 | #endif 22 | 23 | #if defined(__SSE__) || defined(__SSE2__) 24 | #define USE_SSE 25 | #endif 26 | 27 | #if !defined(USE_AVX) && !defined(USE_SSE) && !defined(USE_AVX512) 28 | #ifdef _MSC_VER 29 | #pragma message ( "warning: neither SSE, AVX nor AVX512 are defined" ) 30 | #else 31 | #warning "neither SSE, AVX nor AVX512 are defined" 32 | #endif 33 | #elif !defined(__FMA__) 34 | #ifdef _MSC_VER 35 | #pragma message ( "warning: no FMA support or compile flag is missing" ) 36 | #else 37 | #warning "no FMA support or compile flag is missing" 38 | #endif 39 | #endif 40 | 41 | // #undef USE_AVX512 // for testing arm processors 42 | // #undef USE_AVX 43 | // #undef USE_SSE 44 | #endif 45 | 46 | // TODO switch to only #include 47 | // https://stackoverflow.com/questions/11228855/header-files-for-x86-simd-intrinsics 48 | #if defined(USE_AVX) || defined(USE_SSE) || defined(USE_AVX512) 49 | #ifdef _MSC_VER 50 | #include 51 | #include 52 | #else 53 | #include 54 | #endif 55 | #endif 56 | -------------------------------------------------------------------------------- /cpp/deglib/include/deglib.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "config.h" 4 | #include "memory.h" 5 | #include "distances.h" 6 | #include "filter.h" 7 | #include "search.h" 8 | #include "repository.h" 9 | #include "graph.h" 10 | #include "analysis.h" 11 | #include "graph/readonly_graph.h" 12 | #include "graph/sizebounded_graph.h" 13 | #include "builder.h" -------------------------------------------------------------------------------- /cpp/deglib/include/filter.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | 9 | namespace deglib::graph { 10 | 11 | class Filter { 12 | private: 13 | std::vector bitset_; // Bitset as a vector of 64-bit integers 14 | size_t max_value_; // Maximum valid labels 15 | size_t max_label_count_; // Total possible valid labels 16 | size_t current_valid_count_; // Current number of valid labels 17 | 18 | // Helper function to determine the index and bit position in the bitset 19 | std::pair get_bit_position(int label) const { 20 | size_t index = label / 64; // Index in the vector 21 | uint64_t bit = 1ULL << (label % 64); // Position of the bit within the 64 bits 22 | return {index, bit}; 23 | } 24 | 25 | public: 26 | // Constructor: valid labels as int-pointer and size, max_value defines the maximum label value, max_label_count is the total possible labels 27 | Filter(const int* valid_labels, size_t size, size_t max_value, size_t max_label_count) 28 | : max_value_(max_value), max_label_count_(max_label_count), current_valid_count_(0) { 29 | size_t bitset_size = (max_value_ / 64) + 1; // Required size of the bitset 30 | bitset_.resize(bitset_size, 0); 31 | 32 | for (size_t i = 0; i < size; ++i) { 33 | if (valid_labels[i] >= 0 && static_cast(valid_labels[i]) <= max_value_) { 34 | auto [index, bit] = get_bit_position(valid_labels[i]); 35 | if ((bitset_[index] & bit) == 0) { // Only count unique valid labels 36 | bitset_[index] |= bit; 37 | current_valid_count_++; 38 | } 39 | } 40 | } 41 | } 42 | 43 | // Check if an label is valid 44 | bool is_valid(int label) const { 45 | if (label < 0 || static_cast(label) > max_value_) { 46 | return false; 47 | } 48 | auto [index, bit] = get_bit_position(label); 49 | return (bitset_[index] & bit) != 0; 50 | } 51 | 52 | // Get the number of valid labels 53 | size_t size() const { 54 | return current_valid_count_; 55 | } 56 | 57 | // Apply a function to each valid label 58 | template 59 | void for_each_valid_label(FUNC&& func) const { 60 | for (size_t i = 0; i < bitset_.size(); ++i) { 61 | uint64_t bits = bitset_[i]; 62 | for (int bit_pos = 0; bits != 0; ++bit_pos) { 63 | if (bits & 1ULL) { 64 | func(static_cast(i * 64 + bit_pos)); 65 | } 66 | bits >>= 1; 67 | } 68 | } 69 | } 70 | 71 | // Get the inclusion rate: ratio of valid labels to max possible labels 72 | double get_inclusion_rate() const { 73 | return static_cast(current_valid_count_) / max_label_count_; 74 | } 75 | }; 76 | 77 | } // namespace deglib::graph -------------------------------------------------------------------------------- /cpp/deglib/include/graph.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "search.h" 4 | 5 | namespace deglib::graph 6 | { 7 | 8 | class MutableGraph : public deglib::search::SearchGraph 9 | { 10 | public: 11 | 12 | /** 13 | * Add a new vertex. The neighbor indices will be prefilled with a self-loop, the weights will be 0. 14 | * 15 | * @return the internal index of the new vertex 16 | */ 17 | virtual uint32_t addVertex(const uint32_t external_label, const std::byte* feature_vector) = 0; 18 | 19 | /** 20 | * Remove an existing vertex and returns its list of neighbors 21 | */ 22 | virtual std::vector removeVertex(const uint32_t external_labelr) = 0; 23 | 24 | /** 25 | * Swap a neighbor with another neighbor and its weight. 26 | * 27 | * @param internal_index vertex index which neighbors should be changed 28 | * @param from_neighbor_index neighbor index to remove 29 | * @param to_neighbor_index neighbor index to add 30 | * @param to_neighbor_weight weight of the neighbor to add 31 | * @return true if the from_neighbor_index was found and changed 32 | */ 33 | virtual bool changeEdge(const uint32_t internal_index, const uint32_t from_neighbor_index, const uint32_t to_neighbor_index, const float to_neighbor_weight) = 0; 34 | 35 | 36 | /** 37 | * Change all edges of a vertex. 38 | * The neighbor indices/weights and feature vectors will be copied. 39 | * The neighbor array need to have enough neighbors to match the edge-per-vertex count of the graph. 40 | * The indices in the neighbor_indices array must be sorted. 41 | */ 42 | virtual void changeEdges(const uint32_t internal_index, const uint32_t* neighbor_indices, const float* neighbor_weights) = 0; 43 | 44 | 45 | /** 46 | * 47 | */ 48 | virtual const float* getNeighborWeights(const uint32_t internal_index) const = 0; 49 | 50 | virtual const float getEdgeWeight(const uint32_t from_neighbor_index, const uint32_t to_neighbor_index) const = 0; 51 | 52 | virtual const bool saveGraph(const char* path_to_graph) const = 0; 53 | }; 54 | 55 | } // end namespace deglib::graph 56 | -------------------------------------------------------------------------------- /cpp/deglib/include/memory.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "config.h" 4 | 5 | namespace deglib::memory { 6 | 7 | static const size_t L1_CACHE_LINE_SIZE = 64; 8 | 9 | inline static void prefetch(const char *ptr, const size_t size = 128) { 10 | #if defined(USE_AVX) || defined(USE_SSE) 11 | size_t pos = 0; 12 | while(pos < size) { 13 | _mm_prefetch(ptr+pos, _MM_HINT_T0); 14 | pos += L1_CACHE_LINE_SIZE; 15 | } 16 | #endif 17 | } 18 | 19 | } // namespace deglib::memory 20 | -------------------------------------------------------------------------------- /cpp/deglib/include/visited_list_pool.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | /** 10 | * Ref https://raw.githubusercontent.com/nmslib/hnswlib/master/hnswlib/visited_list_pool.h 11 | */ 12 | namespace deglib::graph { 13 | 14 | class VisitedList { 15 | private: 16 | uint16_t current_tag_{1}; 17 | std::unique_ptr slots_; 18 | unsigned int num_elements_; 19 | 20 | public: 21 | explicit VisitedList(int numelements1) : slots_(std::make_unique(numelements1)), num_elements_(numelements1) {} 22 | 23 | [[nodiscard]] auto* get_visited() const { 24 | return slots_.get(); 25 | } 26 | 27 | [[nodiscard]] auto get_tag() const { 28 | return current_tag_; 29 | } 30 | 31 | void reset() { 32 | ++current_tag_; 33 | if (current_tag_ == 0) { 34 | std::fill_n(slots_.get(), num_elements_, 0); 35 | ++current_tag_; 36 | } 37 | } 38 | }; 39 | 40 | class VisitedListPool { 41 | private: 42 | using ListPtr = std::unique_ptr; 43 | 44 | std::deque pool_; 45 | std::mutex pool_guard_; 46 | int num_elements_; 47 | 48 | public: 49 | VisitedListPool(int initmaxpools, int numelements) : num_elements_(numelements) { 50 | for (int i = 0; i < initmaxpools; i++) 51 | pool_.push_front(std::make_unique(numelements)); 52 | } 53 | 54 | class FreeVisitedList { 55 | private: 56 | friend VisitedListPool; 57 | 58 | VisitedListPool& pool_; 59 | ListPtr list_; 60 | 61 | FreeVisitedList(VisitedListPool& pool, ListPtr list) : pool_(pool), list_(std::move(list)) {} 62 | 63 | public: 64 | FreeVisitedList(const FreeVisitedList& other) = delete; 65 | FreeVisitedList(FreeVisitedList&& other) = delete; 66 | FreeVisitedList& operator=(const FreeVisitedList& other) = delete; 67 | FreeVisitedList& operator=(FreeVisitedList&& other) = delete; 68 | 69 | ~FreeVisitedList() noexcept { 70 | pool_.releaseVisitedList(std::move(list_)); 71 | } 72 | 73 | auto operator->() const { 74 | return list_.get(); 75 | } 76 | }; 77 | 78 | FreeVisitedList getFreeVisitedList() { 79 | ListPtr rez = popVisitedList(); 80 | if (rez) { 81 | rez->reset(); 82 | } else { 83 | rez = std::make_unique(num_elements_); 84 | } 85 | return {*this, std::move(rez)}; 86 | } 87 | 88 | private: 89 | void releaseVisitedList(ListPtr vl) { 90 | std::unique_lock lock(pool_guard_); 91 | pool_.push_back(std::move(vl)); 92 | } 93 | 94 | ListPtr popVisitedList() { 95 | ListPtr rez; 96 | std::unique_lock lock(pool_guard_); 97 | if (!pool_.empty()) { 98 | rez = std::move(pool_.front()); 99 | pool_.pop_front(); 100 | } 101 | return rez; 102 | } 103 | }; 104 | 105 | } // namespace deglib::graph -------------------------------------------------------------------------------- /cpp/readme.md: -------------------------------------------------------------------------------- 1 | # deglib: C++ library of the Dynamic Exploration Graph 2 | 3 | Header only C++ library of the Dynamic Exploration Graph (DEG) and its predecessor continuous refining Exploration Graph (crEG). 4 | 5 | ## How to use 6 | 7 | ### Prepare the data 8 | 9 | Download and extract the data set files from the main [readme](../readme.md) file. 10 | 11 | ### Prerequisites 12 | 13 | + GCC 10.0+ with OpenMP 14 | + CMake 3.19+ 15 | 16 | IMPORTANT NOTE: this code uses AVX-256 instructions for fast distance computation, so your machine will need to support AVX-256 instructions, this can be checked using cat /proc/cpuinfo | grep avx2. 17 | 18 | ### Compile 19 | 20 | 1. Install Dependencies: 21 | ``` 22 | $ sudo apt-get install gcc-10 g++-10 cmake libboost-dev libgoogle-perftools-dev 23 | ``` 24 | 25 | On older systems setup gcc10 as the default compiler version: 26 | ``` 27 | sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 1000 --slave /usr/bin/g++ g++ /usr/bin/g++-10 28 | sudo update-alternatives --config gcc 29 | ``` 30 | 31 | 2. Compile deglib 32 | 33 | After cloning the git repository, rename `CMakePresets.json.sample` to `CMakePresets.json` and change the `DATA_PATH` variable inside of the file to represent a directory where the dataset is located. 34 | 35 | ``` 36 | git clone --recurse-submodules https://github.com/Visual-Computing/DynamicExplorationGraph.git 37 | cd DynamicExplorationGraph/cpp/ 38 | mkdir build/ && cd build/ 39 | cmake -DCMAKE_BUILD_TYPE=Release --preset default .. 40 | make -j 41 | ``` 42 | 43 | ### Reproduce our results 44 | 45 | To create a new graph, modify and run the `/benchmark/src/deglib_build_benchmark.cpp` file. Existing graphs can be tested with `/benchmark/src/deglib_anns_benchmark.cpp` and `/benchmark/src/deglib_explore_benchmark.cpp`. There are some parameters which are used in older papers. Parameters with a value of 0 are ignored. 46 | 47 | Parameters: 48 | 49 | | Dataset | d | k_ext | eps_ext | k_opt | eps_opt | i_opt | 50 | |:---------:|:---:|:-----:|:-------:|:-----:|:-------:|:-----:| 51 | | SIFT1M | 30 | 60 | 0.1 | 0 | 0 | 0 | 52 | | DEEP1M | 30 | 60 | 0.1 | 0 | 0 | 0 | 53 | | GloVe-100 | 30 | 60 | 0.1 | 0 | 0 | 0 | 54 | 55 | ## Pre-build Graphs 56 | 57 | The provided Dynamic Exploration Graphs are used in the experiments section of our paper. 58 | 59 | | Dataset | DEG | 60 | |:------------:|:------------------------------------------------------------------------------------------------:| 61 | | SIFT1M | [sift_128D_L2_DEG30.deg](https://static.visual-computing.com/paper/DEG/sift_128D_L2_DEG30.zip) | 62 | | Deep1M | [deep1m_96D_L2_DEG30.deg](https://static.visual-computing.com/paper/DEG/deep1m_96D_L2_DEG30.zip) | 63 | | GloVe-100 | [glove_100D_L2_DEG30.deg](https://static.visual-computing.com/paper/DEG/glove_100D_L2_DEG30.zip) | 64 | 65 | -------------------------------------------------------------------------------- /figures/anns_qps_vs_recall.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Visual-Computing/DynamicExplorationGraph/3ebb0996a4cc192524fe66e57a48a7cd9a49321f/figures/anns_qps_vs_recall.jpg -------------------------------------------------------------------------------- /figures/exploration_qps_vs_recall.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Visual-Computing/DynamicExplorationGraph/3ebb0996a4cc192524fe66e57a48a7cd9a49321f/figures/exploration_qps_vs_recall.jpg -------------------------------------------------------------------------------- /java/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Visual Computing Group 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /java/buildSrc/shared.gradle: -------------------------------------------------------------------------------- 1 | // Apply any other configurations specific to your project 2 | 3 | repositories { 4 | mavenCentral() 5 | jcenter() 6 | } 7 | 8 | // use "apply plugin" here to allow "plugins { }" in the modules 9 | apply plugin: 'java-library' 10 | apply plugin: 'maven-publish' 11 | apply plugin: 'signing' 12 | 13 | sourceCompatibility = 1.8 14 | targetCompatibility = 1.8 15 | 16 | // group and version of the project 17 | version = '0.1.56' 18 | group = 'com.vc' 19 | 20 | ext { 21 | SLF4J_VERSION = '2.0.0' 22 | JUNIT_VERSION = '4.13.2' 23 | KOLOBOKE_VERSION = '1.0.0' 24 | KRYO_VERSION = '5.3.0' 25 | FASTUTIL_VERSION = '8.5.8' 26 | JBLAS_VERSION = '1.2.5' 27 | ROARING_VERSION = '1.0.6' 28 | } 29 | 30 | dependencies { 31 | testImplementation group: "junit", name: "junit", version: "${JUNIT_VERSION}" 32 | } -------------------------------------------------------------------------------- /java/deg-api/build.gradle: -------------------------------------------------------------------------------- 1 | // Use the build script defined in buildSrc 2 | apply from: rootProject.file('buildSrc/shared.gradle') 3 | 4 | dependencies { 5 | 6 | } 7 | 8 | archivesBaseName = "deg-api" 9 | 10 | 11 | //-------------------------------------------------------------------------------------------------- 12 | //------------------------------------- Publish to Maven ------------------------------------------- 13 | //-------------------------------------------------------------------------------------------------- 14 | 15 | java { 16 | withJavadocJar() 17 | withSourcesJar() 18 | } 19 | 20 | publishing { 21 | publications { 22 | mavenJava(MavenPublication) { 23 | from components.java 24 | 25 | pom { 26 | name = 'deg-api' 27 | description = 'API for constructing and using Dynamic Exploration Graph in approximate nearest neighbor search tasks.' 28 | url = 'http://visual-computing.com' 29 | 30 | licenses { 31 | license { 32 | name = 'MIT License' 33 | url = 'https://opensource.org/licenses/MIT' 34 | } 35 | } 36 | 37 | developers { 38 | developer { 39 | id = 'Neiko2002' 40 | name = 'Nico Hezel' 41 | email = 'hezel@visual-computing.com' 42 | organization = 'Visual Computing Group' 43 | organizationUrl = 'www.visual-computing.com' 44 | } 45 | } 46 | 47 | scm { 48 | connection = 'scm:git:git://github.com:Visual-Computing/DynamicExplorationGraph.git' 49 | developerConnection = 'scm:git:ssh://github.com:Visual-Computing/DynamicExplorationGraph.git' 50 | url = 'https://github.com/Visual-Computing/DynamicExplorationGraph/tree/master/' 51 | } 52 | } 53 | } 54 | } 55 | 56 | repositories { 57 | maven { 58 | url = layout.buildDirectory.dir("maven") 59 | } 60 | } 61 | } -------------------------------------------------------------------------------- /java/deg-api/readme.md: -------------------------------------------------------------------------------- 1 | API für DNG. Stellt alle Interfaces zur Verfügung die die Library und Referenz Implementierung einhalten. 2 | Außerdem können auf diese Weise die einzelnen Komponenten beider Implementierungen leicht erweitert oder ausgetauscht werden. 3 | 4 | Die API stellt keine fertigen Klassen bereit. -------------------------------------------------------------------------------- /java/deg-api/src/main/java/com/vc/deg/FeatureFactory.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg; 2 | 3 | import java.io.DataInput; 4 | import java.io.IOException; 5 | import java.util.HashSet; 6 | import java.util.ServiceLoader; 7 | import java.util.Set; 8 | 9 | /** 10 | * A {@link FeatureFactory} is needed when a graph is loaded from a drive and contains custom {@link FeatureVector}. 11 | * The factory must be registered before loading a graph via a ServiceLoader or the {@link FeatureFactory} class. 12 | * 13 | * 14 | * TODO rework or remove the following text 15 | * Can use native of on-heap memory depending on the implementation 16 | * Goal: single morph calls of the FeatureVector methods in the computeDistance-method of the FeatureSpace, native support and primitive-arrays for fast on-heap computations. 17 | * Adding a node or Queries could copy the feature first to native memory if needed. 18 | * FeatureFactory in reference-impl should always use column-based arrays when reading from file 19 | * 20 | * @author Nico Hezel 21 | */ 22 | public interface FeatureFactory { 23 | 24 | /** 25 | * Contains all {@link FeatureFactory}s registered either via a service loader or manually 26 | * 27 | * @author Nico Hezel 28 | */ 29 | public static class RegisteredFactoryHolder { 30 | 31 | private final static Set registeredFactories = serviceLoaderFactories(); 32 | 33 | /** 34 | * Get all factories registered via a service loader 35 | * 36 | * @return 37 | */ 38 | private static Set serviceLoaderFactories() { 39 | final Set serviceLoaderFactories = new HashSet<>(); 40 | for (FeatureFactory featureFactory : ServiceLoader.load(FeatureFactory.class)) 41 | serviceLoaderFactories.add(featureFactory); 42 | return serviceLoaderFactories; 43 | } 44 | 45 | /** 46 | * Register a new {@link FeatureFactory} manual 47 | * 48 | * @param factory 49 | */ 50 | private static void registerFactory(FeatureFactory factory) { 51 | if(registeredFactories.contains(factory) == false) 52 | registeredFactories.add(factory); 53 | } 54 | 55 | /** 56 | * Find a specific {@link FeatureFactory} based on the parameters 57 | * 58 | * @param componentType 59 | * @param dims 60 | * @return 61 | */ 62 | private static FeatureFactory findFactory(String componentType, int dims) { 63 | for (FeatureFactory registeredFactory : registeredFactories) 64 | if(componentType.equalsIgnoreCase(registeredFactory.getComponentType().getSimpleName()) && dims == registeredFactory.dims()) 65 | return registeredFactory; 66 | return null; 67 | } 68 | 69 | /** 70 | * Find a specific {@link FeatureFactory} based on the parameters 71 | * 72 | * @param componentType 73 | * @param dims 74 | * @return 75 | */ 76 | private static FeatureFactory findFactory(Class componentType, int dims) { 77 | for (FeatureFactory registeredFactory : registeredFactories) 78 | if(componentType == registeredFactory.getComponentType() && dims == registeredFactory.dims()) 79 | return registeredFactory; 80 | return null; 81 | } 82 | } 83 | 84 | /** 85 | * Register a new {@link FeatureFactory} manual 86 | * 87 | * @param factory 88 | */ 89 | public static void registerFactory(FeatureFactory factory) { 90 | RegisteredFactoryHolder.registerFactory(factory); 91 | } 92 | 93 | /** 94 | * Find a specific {@link FeatureFactory} based on the parameters 95 | * 96 | * @param componentType 97 | * @param dims 98 | * @return 99 | */ 100 | public static FeatureFactory findFactory(String componentType, int dims) { 101 | return RegisteredFactoryHolder.findFactory(componentType, dims); 102 | } 103 | 104 | /** 105 | * Find a specific {@link FeatureFactory} based on the parameters 106 | * 107 | * @param componentType 108 | * @param dims 109 | * @return 110 | */ 111 | public static FeatureFactory findFactory(Class componentType, int dims) { 112 | return RegisteredFactoryHolder.findFactory(componentType, dims); 113 | } 114 | 115 | /** 116 | * Either one of the primitives or an object 117 | * 118 | * @return 119 | */ 120 | public Class getComponentType(); 121 | 122 | /** 123 | * Size of the feature vectors in bytes 124 | * 125 | * @return 126 | */ 127 | public int featureSize(); 128 | 129 | /** 130 | * Dimensions of the feature 131 | * 132 | * @return 133 | */ 134 | public int dims(); 135 | 136 | // /** 137 | // * TODO should be (ByteOrder.LITTLE_ENDIAN) the method is the counterpart to FeatureVector#toBytes() 138 | // * TODO Having toBytes here requires the specification of a ByteOrder. 139 | // * The API should be free from such prefefined rules. 140 | // * Use read(...) instead 141 | // * 142 | // * Extract the feature from a pure byte array. 143 | // * There is not length check between the expected dimensions and the length of the given array. 144 | // * 145 | // * @param featureByte 146 | // * @return 147 | // */ 148 | // public FeatureVector of(byte[] featureByte); 149 | 150 | /** 151 | * Read the feature from a data input source 152 | * There is not length check between the expected dimensions and the end of the stream. 153 | * 154 | * @param is 155 | * @return 156 | */ 157 | public FeatureVector read(DataInput is) throws IOException; 158 | } -------------------------------------------------------------------------------- /java/deg-api/src/main/java/com/vc/deg/GraphFactory.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg; 2 | 3 | import java.io.IOException; 4 | import java.nio.file.Path; 5 | import java.util.ServiceLoader; 6 | 7 | 8 | /** 9 | * Factory to build or load graphs. 10 | * 11 | * Different implementation of the graph factory can be registered via a ServiceLoader. 12 | * Depending on the runtime implementation the {@link #getDefaultFactory()} presents more 13 | * readable or more efficient graph implementations. 14 | * 15 | * https://docs.oracle.com/javase/8/docs/api/java/util/ServiceLoader.html 16 | * https://riptutorial.com/java/example/19523/simple-serviceloader-example 17 | * 18 | * @author Nico Hezel 19 | */ 20 | public interface GraphFactory { 21 | 22 | public static class RegisteredFactoryHolder { 23 | private static final GraphFactory defaultFactory = ServiceLoader.load(GraphFactory.class).iterator().next(); 24 | } 25 | 26 | /** 27 | * Returns the default {@link GraphFactory} implementation 28 | * 29 | * @return the default {@link GraphFactory} implementation 30 | */ 31 | public static GraphFactory getDefaultFactory() { 32 | return RegisteredFactoryHolder.defaultFactory; 33 | } 34 | 35 | 36 | 37 | // -------------------------------------------------------------------------------------- 38 | // -------------------------------- Simple Graph ---------------------------------------- 39 | // -------------------------------------------------------------------------------------- 40 | 41 | /** 42 | * Create an empty new graph 43 | * 44 | * @param space 45 | * @param edgesPerNode 46 | * @return 47 | */ 48 | public DynamicExplorationGraph newGraph(FeatureSpace space, int edgesPerNode); 49 | 50 | /** 51 | * Load an existing graph. Read the feature type from the filename. 52 | * e.g. sift1m_k30.float.deg 53 | * 54 | * @param file 55 | * @return 56 | */ 57 | public DynamicExplorationGraph loadGraph(Path file) throws IOException; 58 | 59 | /** 60 | * Load an existing graph and expect a specific feature type 61 | * 62 | * @param file 63 | * @param featureType 64 | * @return 65 | * @throws IOException 66 | */ 67 | public DynamicExplorationGraph loadGraph(Path file, String featureType) throws IOException; 68 | 69 | 70 | 71 | // -------------------------------------------------------------------------------------- 72 | // ---------------------------- Hierarchical Graph -------------------------------------- 73 | // -------------------------------------------------------------------------------------- 74 | 75 | /** 76 | * Create an empty new graph 77 | * 78 | * @param space 79 | * @param edgesPerNode 80 | * @param topRankSize 81 | * @return 82 | */ 83 | public HierarchicalDynamicExplorationGraph newHierchicalGraph(FeatureSpace space, int edgesPerNode, int topRankSize); 84 | 85 | 86 | /** 87 | * Load an existing graph. Read the feature type from the filename. 88 | * e.g. sift1m_k30.float.deg 89 | * 90 | * @param file 91 | * @return 92 | */ 93 | public HierarchicalDynamicExplorationGraph loadHierchicalGraph(Path file) throws IOException; 94 | } 95 | -------------------------------------------------------------------------------- /java/deg-api/src/main/java/com/vc/deg/feature/BinaryFeature.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.feature; 2 | 3 | import java.io.DataInput; 4 | import java.io.DataOutput; 5 | import java.io.IOException; 6 | import java.nio.ByteBuffer; 7 | import java.util.Arrays; 8 | 9 | import com.vc.deg.FeatureVector; 10 | 11 | /** 12 | * The bits are represented in long, therefore is {@link #size()} in bytes always a multiple of 8. 13 | * The correct number of bits stored in this feature can be received by the {@link #dims()} method. 14 | * 15 | * @author Nico Hezel 16 | * 17 | */ 18 | public class BinaryFeature implements FeatureVector { 19 | 20 | public static final Class ComponentType = Binary.class; 21 | 22 | 23 | protected final long[] feature; 24 | protected final int dims; 25 | 26 | public BinaryFeature(int dims, long[] longFeatureVector) { 27 | this.feature = longFeatureVector; 28 | this.dims = dims; 29 | } 30 | 31 | /** 32 | * Encode the byte array into longs which are used internal to represent the bits 33 | * 34 | * @param byteFeatureVector 35 | */ 36 | public BinaryFeature(byte[] byteFeatureVector) { 37 | this.feature = encode(byteFeatureVector); 38 | this.dims = byteFeatureVector.length * 8; 39 | } 40 | 41 | /** 42 | * Encode the byte array into longs 43 | * 44 | * @param input with signed bytes 45 | */ 46 | protected long[] encode(byte[] input) { 47 | final int size = (int) Math.ceil((float)input.length / 8); 48 | final ByteBuffer bb = ByteBuffer.wrap(Arrays.copyOf(input, size * 8)); 49 | 50 | final long[] result = new long[size]; 51 | bb.asLongBuffer().get(result, 0, result.length); 52 | return result; 53 | } 54 | 55 | public long get(int index) { 56 | return feature[index]; 57 | } 58 | 59 | @Override 60 | public Class getComponentType() { 61 | return ComponentType; 62 | } 63 | 64 | @Override 65 | public int dims() { 66 | return dims; 67 | } 68 | 69 | @Override 70 | public int size() { 71 | return feature.length * Long.BYTES; 72 | } 73 | 74 | @Override 75 | public boolean isNative() { 76 | return false; 77 | } 78 | 79 | @Override 80 | public byte readByte(int byteOffset) { 81 | throw new UnsupportedOperationException(BinaryFeature.class.getSimpleName() + " does not support readByte"); 82 | } 83 | 84 | @Override 85 | public short readShort(int byteOffset) { 86 | throw new UnsupportedOperationException(BinaryFeature.class.getSimpleName() + " does not support readShort"); 87 | } 88 | 89 | @Override 90 | public int readInt(int byteOffset) { 91 | throw new UnsupportedOperationException(BinaryFeature.class.getSimpleName() + " does not support readInt"); 92 | } 93 | 94 | @Override 95 | public long readLong(int byteOffset) { 96 | return feature[byteOffset >> 3]; 97 | } 98 | 99 | @Override 100 | public float readFloat(int byteOffset) { 101 | throw new UnsupportedOperationException(BinaryFeature.class.getSimpleName() + " does not support readFloat"); 102 | } 103 | 104 | @Override 105 | public double readDouble(int byteOffset) { 106 | throw new UnsupportedOperationException(BinaryFeature.class.getSimpleName() + " does not support readDouble"); 107 | } 108 | 109 | @Override 110 | public FeatureVector copy() { 111 | return new BinaryFeature(dims, Arrays.copyOf(feature, feature.length)); 112 | } 113 | 114 | @Override 115 | public void writeObject(DataOutput out) throws IOException { 116 | for (long d : feature) 117 | out.writeLong(d); 118 | } 119 | 120 | @Override 121 | public void readObject(DataInput in) throws IOException { 122 | for (int i = 0; i < feature.length; i++) 123 | feature[i] = in.readLong(); 124 | } 125 | 126 | @Override 127 | public long nativeAddress() { 128 | throw new UnsupportedOperationException(BinaryFeature.class.getSimpleName() + " stores its values on-heap, using a native address is dangerous."); 129 | } 130 | 131 | 132 | 133 | // dummy class to represent an internal component of this feature vector 134 | public static class Binary {} 135 | } -------------------------------------------------------------------------------- /java/deg-api/src/main/java/com/vc/deg/feature/ByteFeature.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.feature; 2 | 3 | import java.io.DataInput; 4 | import java.io.DataOutput; 5 | import java.io.IOException; 6 | import java.util.Arrays; 7 | 8 | import com.vc.deg.FeatureVector; 9 | 10 | /** 11 | * Wraps an byte-array 12 | * 13 | * @author Nico Hezel 14 | */ 15 | public class ByteFeature implements FeatureVector { 16 | 17 | protected final byte[] feature; 18 | 19 | public ByteFeature(byte[] feature) { 20 | this.feature = feature; 21 | } 22 | 23 | public byte get(int index) { 24 | return feature[index]; 25 | } 26 | 27 | @Override 28 | public int dims() { 29 | return feature.length; 30 | } 31 | 32 | @Override 33 | public Class getComponentType() { 34 | return byte.class; 35 | } 36 | 37 | @Override 38 | public int size() { 39 | return feature.length; 40 | } 41 | 42 | @Override 43 | public byte readByte(int byteOffset) { 44 | return feature[byteOffset]; 45 | } 46 | 47 | @Override 48 | public short readShort(int byteOffset) { 49 | throw new UnsupportedOperationException(ByteFeature.class.getSimpleName() + " does not support readShort"); 50 | } 51 | 52 | @Override 53 | public int readInt(int byteOffset) { 54 | throw new UnsupportedOperationException(ByteFeature.class.getSimpleName() + " does not support readInt"); 55 | } 56 | 57 | @Override 58 | public long readLong(int byteOffset) { 59 | throw new UnsupportedOperationException(ByteFeature.class.getSimpleName() + " does not support readLong"); 60 | } 61 | 62 | @Override 63 | public float readFloat(int byteOffset) { 64 | throw new UnsupportedOperationException(ByteFeature.class.getSimpleName() + " does not support readFloat"); 65 | } 66 | 67 | @Override 68 | public double readDouble(int byteOffset) { 69 | throw new UnsupportedOperationException(ByteFeature.class.getSimpleName() + " does not support readDouble"); 70 | } 71 | 72 | // @Override 73 | // public byte[] toBytes() { 74 | // return Arrays.copyOf(feature, feature.length); 75 | // } 76 | 77 | @Override 78 | public FeatureVector copy() { 79 | return new ByteFeature(Arrays.copyOf(feature, feature.length)); 80 | } 81 | 82 | @Override 83 | public void writeObject(DataOutput out) throws IOException { 84 | out.write(feature); 85 | } 86 | 87 | @Override 88 | public void readObject(DataInput in) throws IOException { 89 | in.readFully(feature); 90 | } 91 | 92 | @Override 93 | public long nativeAddress() { 94 | throw new UnsupportedOperationException(ByteFeature.class.getSimpleName() + " stores its values on-heap, using a native address is dangerous."); 95 | } 96 | 97 | @Override 98 | public boolean isNative() { 99 | return false; 100 | } 101 | } -------------------------------------------------------------------------------- /java/deg-api/src/main/java/com/vc/deg/feature/DoubleFeature.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.feature; 2 | 3 | import java.io.DataInput; 4 | import java.io.DataOutput; 5 | import java.io.IOException; 6 | import java.util.Arrays; 7 | 8 | import com.vc.deg.FeatureVector; 9 | 10 | /** 11 | * Wraps an double-array 12 | * 13 | * @author Nico Hezel 14 | */ 15 | public class DoubleFeature implements FeatureVector { 16 | 17 | protected final double[] feature; 18 | 19 | public DoubleFeature(double[] feature) { 20 | this.feature = feature; 21 | } 22 | 23 | public double get(int index) { 24 | return feature[index]; 25 | } 26 | 27 | @Override 28 | public int dims() { 29 | return feature.length; 30 | } 31 | 32 | @Override 33 | public Class getComponentType() { 34 | return double.class; 35 | } 36 | 37 | @Override 38 | public int size() { 39 | return feature.length * Double.BYTES; 40 | } 41 | 42 | @Override 43 | public byte readByte(int byteOffset) { 44 | throw new UnsupportedOperationException(DoubleFeature.class.getSimpleName() + " does not support readByte"); 45 | } 46 | 47 | @Override 48 | public short readShort(int byteOffset) { 49 | throw new UnsupportedOperationException(DoubleFeature.class.getSimpleName() + " does not support readShort"); 50 | } 51 | 52 | @Override 53 | public int readInt(int byteOffset) { 54 | throw new UnsupportedOperationException(DoubleFeature.class.getSimpleName() + " does not support readInt"); 55 | } 56 | 57 | @Override 58 | public long readLong(int byteOffset) { 59 | throw new UnsupportedOperationException(DoubleFeature.class.getSimpleName() + " does not support readLong"); 60 | } 61 | 62 | @Override 63 | public float readFloat(int byteOffset) { 64 | throw new UnsupportedOperationException(DoubleFeature.class.getSimpleName() + " does not support readFloat"); 65 | } 66 | 67 | @Override 68 | public double readDouble(int byteOffset) { 69 | return feature[byteOffset >> 3]; 70 | } 71 | 72 | // @Override 73 | // public byte[] toBytes() { 74 | // final ByteBuffer bb = ByteBuffer.allocate(size()).order(ByteOrder.LITTLE_ENDIAN); 75 | // for (double value : feature) 76 | // bb.putDouble(value); 77 | // return bb.array(); 78 | // } 79 | 80 | @Override 81 | public FeatureVector copy() { 82 | return new DoubleFeature(Arrays.copyOf(feature, feature.length)); 83 | } 84 | 85 | @Override 86 | public void writeObject(DataOutput out) throws IOException { 87 | for (double d : feature) 88 | out.writeDouble(d); 89 | } 90 | 91 | @Override 92 | public void readObject(DataInput in) throws IOException { 93 | for (int i = 0; i < feature.length; i++) 94 | feature[i] = in.readDouble(); 95 | } 96 | 97 | @Override 98 | public long nativeAddress() { 99 | throw new UnsupportedOperationException(DoubleFeature.class.getSimpleName() + " stores its values on-heap, using a native address is dangerous."); 100 | } 101 | 102 | @Override 103 | public boolean isNative() { 104 | return false; 105 | } 106 | } -------------------------------------------------------------------------------- /java/deg-api/src/main/java/com/vc/deg/feature/FloatFeature.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.feature; 2 | 3 | import java.io.DataInput; 4 | import java.io.DataOutput; 5 | import java.io.IOException; 6 | import java.util.Arrays; 7 | 8 | import com.vc.deg.FeatureVector; 9 | 10 | /** 11 | * Wraps an float-array 12 | * 13 | * @author Nico Hezel 14 | */ 15 | public class FloatFeature implements FeatureVector { 16 | 17 | protected final float[] feature; 18 | 19 | public FloatFeature(float[] feature) { 20 | this.feature = feature; 21 | } 22 | 23 | public float get(int index) { 24 | return feature[index]; 25 | } 26 | 27 | @Override 28 | public int dims() { 29 | return feature.length; 30 | } 31 | 32 | @Override 33 | public Class getComponentType() { 34 | return float.class; 35 | } 36 | 37 | @Override 38 | public int size() { 39 | return feature.length * Float.BYTES; 40 | } 41 | 42 | @Override 43 | public byte readByte(int byteOffset) { 44 | throw new UnsupportedOperationException(FloatFeature.class.getSimpleName() + " does not support readByte"); 45 | } 46 | 47 | @Override 48 | public short readShort(int byteOffset) { 49 | throw new UnsupportedOperationException(FloatFeature.class.getSimpleName() + " does not support readShort"); 50 | } 51 | 52 | @Override 53 | public int readInt(int byteOffset) { 54 | throw new UnsupportedOperationException(FloatFeature.class.getSimpleName() + " does not support readInt"); 55 | } 56 | 57 | @Override 58 | public long readLong(int byteOffset) { 59 | throw new UnsupportedOperationException(FloatFeature.class.getSimpleName() + " does not support readLong"); 60 | } 61 | 62 | @Override 63 | public float readFloat(int byteOffset) { 64 | return feature[byteOffset >> 2]; 65 | } 66 | 67 | @Override 68 | public double readDouble(int byteOffset) { 69 | throw new UnsupportedOperationException(FloatFeature.class.getSimpleName() + " does not support readDouble"); 70 | } 71 | 72 | // @Override 73 | // public byte[] toBytes() { 74 | // final ByteBuffer bb = ByteBuffer.allocate(size()).order(ByteOrder.LITTLE_ENDIAN); 75 | // for (float value : feature) 76 | // bb.putFloat(value); 77 | // return bb.array(); 78 | // } 79 | 80 | @Override 81 | public FeatureVector copy() { 82 | return new FloatFeature(Arrays.copyOf(feature, feature.length)); 83 | } 84 | 85 | 86 | @Override 87 | public void writeObject(DataOutput out) throws IOException { 88 | for (float d : feature) 89 | out.writeFloat(d); 90 | } 91 | 92 | @Override 93 | public void readObject(DataInput in) throws IOException { 94 | for (int i = 0; i < feature.length; i++) 95 | feature[i] = in.readFloat(); 96 | } 97 | 98 | @Override 99 | public long nativeAddress() { 100 | throw new UnsupportedOperationException(FloatFeature.class.getSimpleName() + " stores its values on-heap, using a native address is dangerous."); 101 | } 102 | 103 | @Override 104 | public boolean isNative() { 105 | return false; 106 | } 107 | } -------------------------------------------------------------------------------- /java/deg-api/src/main/java/com/vc/deg/feature/IntFeature.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.feature; 2 | 3 | import java.io.DataInput; 4 | import java.io.DataOutput; 5 | import java.io.IOException; 6 | import java.util.Arrays; 7 | 8 | import com.vc.deg.FeatureVector; 9 | 10 | /** 11 | * Wraps an int-array 12 | * 13 | * @author Nico Hezel 14 | */ 15 | public class IntFeature implements FeatureVector { 16 | 17 | protected final int[] feature; 18 | 19 | public IntFeature(int[] feature) { 20 | this.feature = feature; 21 | } 22 | 23 | public int get(int index) { 24 | return feature[index]; 25 | } 26 | 27 | @Override 28 | public int dims() { 29 | return feature.length; 30 | } 31 | 32 | @Override 33 | public Class getComponentType() { 34 | return int.class; 35 | } 36 | 37 | @Override 38 | public int size() { 39 | return feature.length * Integer.BYTES; 40 | } 41 | 42 | @Override 43 | public byte readByte(int byteOffset) { 44 | throw new UnsupportedOperationException(IntFeature.class.getSimpleName() + " does not support readByte"); 45 | } 46 | 47 | @Override 48 | public short readShort(int byteOffset) { 49 | throw new UnsupportedOperationException(IntFeature.class.getSimpleName() + " does not support readShort"); 50 | } 51 | 52 | @Override 53 | public int readInt(int byteOffset) { 54 | return feature[byteOffset >> 2]; 55 | } 56 | 57 | @Override 58 | public long readLong(int byteOffset) { 59 | throw new UnsupportedOperationException(IntFeature.class.getSimpleName() + " does not support readLong"); 60 | } 61 | 62 | @Override 63 | public float readFloat(int byteOffset) { 64 | throw new UnsupportedOperationException(IntFeature.class.getSimpleName() + " does not support readFloat"); 65 | } 66 | 67 | @Override 68 | public double readDouble(int byteOffset) { 69 | throw new UnsupportedOperationException(IntFeature.class.getSimpleName() + " does not support readDouble"); 70 | } 71 | 72 | // @Override 73 | // public byte[] toBytes() { 74 | // final ByteBuffer bb = ByteBuffer.allocate(size()).order(ByteOrder.LITTLE_ENDIAN); 75 | // for (int value : feature) 76 | // bb.putInt(value); 77 | // return bb.array(); 78 | // } 79 | 80 | @Override 81 | public FeatureVector copy() { 82 | return new IntFeature(Arrays.copyOf(feature, feature.length)); 83 | } 84 | 85 | @Override 86 | public void writeObject(DataOutput out) throws IOException { 87 | for (int d : feature) 88 | out.writeInt(d); 89 | } 90 | 91 | @Override 92 | public void readObject(DataInput in) throws IOException { 93 | for (int i = 0; i < feature.length; i++) 94 | feature[i] = in.readInt(); 95 | } 96 | 97 | @Override 98 | public long nativeAddress() { 99 | throw new UnsupportedOperationException(IntFeature.class.getSimpleName() + " stores its values on-heap, using a native address is dangerous."); 100 | } 101 | 102 | @Override 103 | public boolean isNative() { 104 | return false; 105 | } 106 | } -------------------------------------------------------------------------------- /java/deg-api/src/main/java/com/vc/deg/feature/LongFeature.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.feature; 2 | 3 | import java.io.DataInput; 4 | import java.io.DataOutput; 5 | import java.io.IOException; 6 | import java.util.Arrays; 7 | 8 | import com.vc.deg.FeatureVector; 9 | 10 | /** 11 | * Wraps an long-array 12 | * 13 | * @author Nico Hezel 14 | */ 15 | public class LongFeature implements FeatureVector { 16 | 17 | protected final long[] feature; 18 | 19 | public LongFeature(long[] feature) { 20 | this.feature = feature; 21 | } 22 | 23 | public long get(int index) { 24 | return feature[index]; 25 | } 26 | 27 | @Override 28 | public int dims() { 29 | return feature.length; 30 | } 31 | 32 | @Override 33 | public Class getComponentType() { 34 | return long.class; 35 | } 36 | 37 | @Override 38 | public int size() { 39 | return feature.length * Long.BYTES; 40 | } 41 | 42 | @Override 43 | public byte readByte(int byteOffset) { 44 | throw new UnsupportedOperationException(LongFeature.class.getSimpleName() + " does not support readByte"); 45 | } 46 | 47 | @Override 48 | public short readShort(int byteOffset) { 49 | throw new UnsupportedOperationException(LongFeature.class.getSimpleName() + " does not support readShort"); 50 | } 51 | 52 | @Override 53 | public int readInt(int byteOffset) { 54 | throw new UnsupportedOperationException(LongFeature.class.getSimpleName() + " does not support readInt"); 55 | } 56 | 57 | @Override 58 | public long readLong(int byteOffset) { 59 | return feature[byteOffset >> 3]; 60 | } 61 | 62 | @Override 63 | public float readFloat(int byteOffset) { 64 | throw new UnsupportedOperationException(LongFeature.class.getSimpleName() + " does not support readFloat"); 65 | } 66 | 67 | @Override 68 | public double readDouble(int byteOffset) { 69 | throw new UnsupportedOperationException(LongFeature.class.getSimpleName() + " does not support readDouble"); 70 | } 71 | 72 | // @Override 73 | // public byte[] toBytes() { 74 | // final ByteBuffer bb = ByteBuffer.allocate(size()).order(ByteOrder.LITTLE_ENDIAN); 75 | // for (long value : feature) 76 | // bb.putLong(value); 77 | // return bb.array(); 78 | // } 79 | 80 | @Override 81 | public FeatureVector copy() { 82 | return new LongFeature(Arrays.copyOf(feature, feature.length)); 83 | } 84 | 85 | @Override 86 | public void writeObject(DataOutput out) throws IOException { 87 | for (long d : feature) 88 | out.writeLong(d); 89 | } 90 | 91 | @Override 92 | public void readObject(DataInput in) throws IOException { 93 | for (int i = 0; i < feature.length; i++) 94 | feature[i] = in.readLong(); 95 | } 96 | 97 | @Override 98 | public long nativeAddress() { 99 | throw new UnsupportedOperationException(LongFeature.class.getSimpleName() + " stores its values on-heap, using a native address is dangerous."); 100 | } 101 | 102 | @Override 103 | public boolean isNative() { 104 | return false; 105 | } 106 | } -------------------------------------------------------------------------------- /java/deg-api/src/main/java/com/vc/deg/feature/ShortFeature.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.feature; 2 | 3 | import java.io.DataInput; 4 | import java.io.DataOutput; 5 | import java.io.IOException; 6 | import java.util.Arrays; 7 | 8 | import com.vc.deg.FeatureVector; 9 | 10 | /** 11 | * Wraps an short-array 12 | * 13 | * @author Nico Hezel 14 | */ 15 | public class ShortFeature implements FeatureVector { 16 | 17 | protected final short[] feature; 18 | 19 | public ShortFeature(short[] feature) { 20 | this.feature = feature; 21 | } 22 | 23 | public short get(int index) { 24 | return feature[index]; 25 | } 26 | 27 | @Override 28 | public int dims() { 29 | return feature.length; 30 | } 31 | 32 | @Override 33 | public Class getComponentType() { 34 | return short.class; 35 | } 36 | 37 | @Override 38 | public int size() { 39 | return feature.length * Short.BYTES; 40 | } 41 | 42 | @Override 43 | public byte readByte(int byteOffset) { 44 | throw new UnsupportedOperationException(ShortFeature.class.getSimpleName() + " does not support readByte"); 45 | } 46 | 47 | @Override 48 | public short readShort(int byteOffset) { 49 | return feature[byteOffset >> 1]; 50 | } 51 | 52 | @Override 53 | public int readInt(int byteOffset) { 54 | throw new UnsupportedOperationException(ShortFeature.class.getSimpleName() + " does not support readInt"); 55 | } 56 | 57 | @Override 58 | public long readLong(int byteOffset) { 59 | throw new UnsupportedOperationException(ShortFeature.class.getSimpleName() + " does not support readLong"); 60 | } 61 | 62 | @Override 63 | public float readFloat(int byteOffset) { 64 | throw new UnsupportedOperationException(ShortFeature.class.getSimpleName() + " does not support readFloat"); 65 | } 66 | 67 | @Override 68 | public double readDouble(int byteOffset) { 69 | throw new UnsupportedOperationException(ShortFeature.class.getSimpleName() + " does not support readDouble"); 70 | } 71 | 72 | // @Override 73 | // public byte[] toBytes() { 74 | // final ByteBuffer bb = ByteBuffer.allocate(size()).order(ByteOrder.LITTLE_ENDIAN); 75 | // for (short value : feature) 76 | // bb.putShort(value); 77 | // return bb.array(); 78 | // } 79 | 80 | @Override 81 | public FeatureVector copy() { 82 | return new ShortFeature(Arrays.copyOf(feature, feature.length)); 83 | } 84 | 85 | @Override 86 | public void writeObject(DataOutput out) throws IOException { 87 | for (short d : feature) 88 | out.writeShort(d); 89 | } 90 | 91 | @Override 92 | public void readObject(DataInput in) throws IOException { 93 | for (int i = 0; i < feature.length; i++) 94 | feature[i] = in.readShort(); 95 | } 96 | 97 | @Override 98 | public long nativeAddress() { 99 | throw new UnsupportedOperationException(ShortFeature.class.getSimpleName() + " stores its values on-heap, using a native address is dangerous."); 100 | } 101 | 102 | @Override 103 | public boolean isNative() { 104 | return false; 105 | } 106 | } -------------------------------------------------------------------------------- /java/deg-api/src/main/java/com/vc/deg/graph/GraphDesigner.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.graph; 2 | 3 | import java.util.Random; 4 | import java.util.function.IntPredicate; 5 | 6 | import com.vc.deg.FeatureVector; 7 | 8 | /** 9 | * Adds or removed graph vertices and improves the connections. 10 | * 11 | * @author Nico Hezel 12 | * 13 | */ 14 | public interface GraphDesigner { 15 | 16 | /** 17 | * Set the random generator for all operations 18 | * 19 | * @param rnd 20 | */ 21 | public void setRandom(Random rnd); 22 | 23 | /** 24 | * Queue new data to add to the graph 25 | * 26 | * @param label 27 | * @param data 28 | */ 29 | public void add(int label, FeatureVector data); 30 | 31 | /** 32 | * Queue the removal of an existing vertex 33 | * 34 | * @param label 35 | */ 36 | public void remove(int label); 37 | 38 | /** 39 | * Queue the removal of all vertices which are in the graph but do not pass the filter 40 | * 41 | * @param labelFilter 42 | */ 43 | public void removeIf(IntPredicate labelFilter); 44 | 45 | /** 46 | * Builds and improve the graph. All new data points or removal requests will be processed first 47 | * afterwards the edges of the existing graph will be improved infinitely. 48 | * 49 | * @param listener will be informed about every change 50 | */ 51 | public void build(ChangeListener listener); 52 | 53 | /** 54 | * Stops the current building process after the next change. 55 | */ 56 | public void stop(); 57 | 58 | 59 | /** 60 | * The listener will be informed about every change 61 | * 62 | * @author Nico Hezel 63 | */ 64 | public static interface ChangeListener { 65 | 66 | /** 67 | * The method gets called after every change: 68 | * - adding a vertex 69 | * - removing a vertex 70 | * - improving a path of edges 71 | * 72 | * @param step number of graph manipulation steps 73 | * @param added number of added vertices 74 | * @param removed number of removed vertices 75 | * @param improved number of successful improvement 76 | * @param tries number of improvement tries 77 | * @param lastAdd label of the vertex added in this change 78 | * @param lastRemoved label of the vertex deleted in this change 79 | */ 80 | public void onChange(long step, long added, long removed, long improved, long tries, int lastAdd, int lastRemoved); 81 | } 82 | 83 | 84 | 85 | // ---------------------------------------------------------------------------------------------- 86 | // ----------------------------------- hyper parameters ----------------------------------------- 87 | // ---------------------------------------------------------------------------------------------- 88 | 89 | /** 90 | * Hyper parameter when adding new vertices 91 | * In order to find good candidates a search is performed, an this K is the search k. 92 | * 93 | * @param k 94 | */ 95 | public void setExtendK(int k); 96 | 97 | /** 98 | * Hyper parameter when adding new vertices 99 | * In order to find good candidates a search is performed, an this eps is the search eps. 100 | * 101 | * @param eps 102 | */ 103 | public void setExtendEps(float eps); 104 | 105 | /** 106 | * Hyper parameter when adding new vertices 107 | * When adding neighbors to the new vertex schema C or D can be used. 108 | * 109 | * @param useSchemaC 110 | */ 111 | public void setExtendSchema(boolean useSchemaC); 112 | 113 | /** 114 | * Hyper parameter when improving the edges 115 | * In order to find good candidates a search is performed, an this K is the search k. 116 | * 117 | * @param k 118 | */ 119 | public void setImproveK(int k); 120 | 121 | /** 122 | * Hyper parameter when improving the edges 123 | * In order to find good candidates a search is performed, an this eps is the search eps. 124 | * 125 | * @param eps 126 | */ 127 | public void setImproveEps(float eps); 128 | 129 | /** 130 | * Hyper parameter when improving the edges 131 | * 132 | * @param maxPathLength 133 | */ 134 | public void setMaxPathLength(int maxPathLength); 135 | 136 | 137 | 138 | // ---------------------------------------------------------------------------------------------- 139 | // ----------------------------------- evaluation methods --------------------------------------- 140 | // ---------------------------------------------------------------------------------------------- 141 | 142 | /** 143 | * Compute the average edge weight of all edges. 144 | * @return 145 | */ 146 | public float calcAvgEdgeWeight(); 147 | 148 | /** 149 | * Compute the average neighbor rank of all vertices. 150 | * @return 151 | */ 152 | public float calcAvgNeighborRank(); 153 | 154 | /** 155 | * Compute the average neighbor rank of all vertices. 156 | * Use the top list containing the best neighbors of each vertex if possible. 157 | * Otherwise compute the top list per vertex. 158 | * 159 | * 160 | * @param topList 161 | * @return 162 | */ 163 | public float calcAvgNeighborRank(int[][] topList); 164 | 165 | /** 166 | * Check if all vertices have a specific amount of neighbors and their edge weight corresponds to the 167 | * distance to the neighbor. 168 | * 169 | * @param expectedVertices 170 | * @param expectedNeighbors 171 | * @return 172 | */ 173 | public boolean checkGraphValidation(int expectedVertices, int expectedNeighbors); 174 | } 175 | -------------------------------------------------------------------------------- /java/deg-api/src/main/java/com/vc/deg/graph/NeighborConsumer.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.graph; 2 | 3 | /** 4 | * Iterates neighbors in a collection 5 | * 6 | * @author Nico Hezel 7 | */ 8 | public interface NeighborConsumer { 9 | 10 | /** 11 | * Performs this operation on the given neighbor. 12 | * 13 | * @param id 14 | * @param weight 15 | */ 16 | void accept(int id, float weight); 17 | } 18 | -------------------------------------------------------------------------------- /java/deg-api/src/main/java/com/vc/deg/graph/VertexCursor.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.graph; 2 | 3 | import com.vc.deg.FeatureVector; 4 | 5 | /** 6 | * Goal: All get methods (id, feature, neighbor count, neighbor id at index, neighbor weight at index) can be lazy. 7 | * The user of the API does not have problems with the lambda scopes. 8 | * 9 | * 10 | * TODO Add a position(), position(int newPos) and size() Method. 11 | * The reference implementation would need to create an array the first time position(int newPos) is called. 12 | * For a readonly graph the reference implementation should already have such an array. 13 | * 14 | * @author Nico Hezel 15 | * 16 | */ 17 | public interface VertexCursor { 18 | 19 | /** 20 | * Move to the next vertex. Need to be 21 | * 22 | * @return 23 | */ 24 | public boolean moveNext(); 25 | 26 | /** 27 | * Label of the current vertex 28 | * {@link #moveNext()} need to be called before 29 | * 30 | * @return 31 | */ 32 | public int getVertexLabel(); 33 | 34 | /** 35 | * Feature vector of the current vertex 36 | * {@link #moveNext()} need to be called before 37 | * 38 | * @return 39 | */ 40 | public FeatureVector getVertexFeature(); 41 | 42 | /** 43 | * Neighbor labels and weights of the current vertex 44 | * {@link #moveNext()} need to be called before 45 | * 46 | * @param neighborConsumer 47 | */ 48 | public void forEachNeighbor(NeighborConsumer neighborConsumer); 49 | } 50 | -------------------------------------------------------------------------------- /java/deg-api/src/main/java/com/vc/deg/graph/VertexFilter.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.graph; 2 | 3 | import java.util.function.IntConsumer; 4 | 5 | /** 6 | * A filter to check labels used in a search or exploration task 7 | * 8 | * @author Nico Hezel 9 | */ 10 | public interface VertexFilter { 11 | 12 | /** 13 | * Is the label valid 14 | * 15 | * @param label 16 | * @return 17 | */ 18 | public boolean isValid(int label); 19 | 20 | /** 21 | * Number of valid labels 22 | * 23 | * @return 24 | */ 25 | public int size(); 26 | 27 | /** 28 | * Include rate 29 | * 30 | * @return 31 | */ 32 | public float getInclusionRate(); 33 | 34 | /** 35 | * For each valid id in the filter the action called 36 | * 37 | * @param action 38 | */ 39 | public void forEachValidId(IntConsumer action); 40 | } -------------------------------------------------------------------------------- /java/deg-api/src/main/java/com/vc/deg/graph/VertexFilterFactory.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.graph; 2 | 3 | import java.util.ServiceLoader; 4 | import java.util.function.Consumer; 5 | import java.util.function.IntConsumer; 6 | 7 | 8 | 9 | /** 10 | * Factory to build graph filter. 11 | * 12 | * Different implementation of the graph filter factory can be registered via a ServiceLoader. 13 | * Depending on the runtime implementation the {@link #getDefaultFactory()} presents more 14 | * readable or more efficient graph filter implementations. 15 | * * 16 | * @author Nico Hezel 17 | */ 18 | public interface VertexFilterFactory { 19 | 20 | public static class RegisteredFactoryHolder { 21 | private static final VertexFilterFactory defaultFactory = ServiceLoader.load(VertexFilterFactory.class).iterator().next(); 22 | } 23 | 24 | /** 25 | * Returns the default {@link VertexFilterFactory} implementation 26 | * 27 | * @return the default {@link VertexFilterFactory} implementation 28 | */ 29 | public static VertexFilterFactory getDefaultFactory() { 30 | return RegisteredFactoryHolder.defaultFactory; 31 | } 32 | 33 | /** 34 | * Create a graph filter based on the given ids 35 | * 36 | * @param validIds 37 | * @param allElementCount 38 | * @return 39 | */ 40 | public VertexFilter of(int[] validIds, int allElementCount); 41 | 42 | /** 43 | * Create a graph filter based on the given ids 44 | * 45 | * @param validIds 46 | * @param allElementCount 47 | * @return 48 | */ 49 | public VertexFilter of(Consumer validIds, int allElementCount); 50 | 51 | 52 | /** 53 | * AND operation. 54 | * x1 get modified. 55 | * 56 | * @param x1 57 | * @param x2 58 | */ 59 | public void and(VertexFilter x1, VertexFilter x2); 60 | 61 | /** 62 | * ANDNOT operation. Is used to clear remove all the labels from x1 which are specified in x2. 63 | * x1 get modified. 64 | * 65 | * @param x1 66 | * @param x2 67 | */ 68 | public void andNot(VertexFilter x1, VertexFilter x2); 69 | 70 | /** 71 | * Add elements. 72 | * x1 get modified. 73 | * 74 | * @param x1 75 | * @param x2 76 | */ 77 | public void add(VertexFilter x1, Consumer x2); 78 | 79 | /** 80 | * Remove elements. 81 | * x1 get modified. 82 | * 83 | * @param x1 84 | * @param x2 85 | */ 86 | public void remove(VertexFilter x1, Consumer x2); 87 | } -------------------------------------------------------------------------------- /java/deg-benchmark/build.gradle: -------------------------------------------------------------------------------- 1 | // Use the build script defined in buildSrc 2 | apply from: rootProject.file('buildSrc/shared.gradle') 3 | 4 | dependencies { 5 | implementation project(':deg-api') 6 | implementation project(':deg-reference') 7 | 8 | implementation group: 'it.unimi.dsi', name: 'fastutil', version: "${FASTUTIL_VERSION}" 9 | 10 | implementation group: 'com.koloboke', name: 'koloboke-api-jdk8', version: "${KOLOBOKE_VERSION}" 11 | implementation group: 'com.koloboke', name: 'koloboke-impl-jdk8', version: "${KOLOBOKE_VERSION}" 12 | implementation group: 'com.koloboke', name: 'koloboke-impl-common-jdk8', version: "${KOLOBOKE_VERSION}" 13 | 14 | implementation group: 'org.roaringbitmap', name: 'RoaringBitmap', version: "${ROARING_VERSION}" 15 | } -------------------------------------------------------------------------------- /java/deg-benchmark/readme.md: -------------------------------------------------------------------------------- 1 | ANN Benchmarks für HNSW, DNG mithilfe des SIFT1M Datensets -------------------------------------------------------------------------------- /java/deg-benchmark/src/main/java/com/vc/deg/anns/GraphConstructionBenchmark.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.anns; 2 | 3 | import java.io.IOException; 4 | import java.nio.file.Path; 5 | import java.nio.file.Paths; 6 | import java.util.ArrayList; 7 | import java.util.Collections; 8 | import java.util.List; 9 | import java.util.Random; 10 | import java.util.concurrent.atomic.AtomicInteger; 11 | import java.util.concurrent.atomic.AtomicLong; 12 | import java.util.stream.Collectors; 13 | import java.util.stream.IntStream; 14 | 15 | import com.vc.deg.DynamicExplorationGraph; 16 | import com.vc.deg.FeatureSpace; 17 | import com.vc.deg.FeatureVector; 18 | import com.vc.deg.GraphFactory; 19 | import com.vc.deg.data.Sift1M; 20 | import com.vc.deg.feature.FloatFeature; 21 | import com.vc.deg.feature.FloatL2Space; 22 | import com.vc.deg.graph.GraphDesigner; 23 | 24 | /** 25 | * 26 | * 27 | * @author Nico Hezel 28 | */ 29 | public class GraphConstructionBenchmark { 30 | 31 | public static void main(String[] args) throws IOException, ClassNotFoundException { 32 | 33 | // load query data 34 | final Path siftBaseDir = Paths.get("C:/Data/Feature/SIFT1M/SIFT1M/"); 35 | System.out.println("Load the dataset "+siftBaseDir); 36 | final float[][] baseData = Sift1M.loadBaseData(siftBaseDir); 37 | 38 | // register the feature space needed in the graph 39 | final AtomicInteger distanceCalculationCount = new AtomicInteger(); 40 | FeatureSpace space = new FloatL2Space(128) { 41 | @Override 42 | public float computeDistance(FeatureVector f1, FeatureVector f2) { 43 | distanceCalculationCount.incrementAndGet(); 44 | return super.computeDistance(f1, f2); 45 | } 46 | }; 47 | 48 | // hierarchical 49 | // final int edgesPerNode = 30; 50 | // final int topRankSize = 400; 51 | // final Path graphFile = Paths.get("c:\\Data\\Feature\\SIFT1M\\deg\\best_distortion_decisions\\java\\128D_L2_K30_AddK60Eps0.2High_RNGMinimalAdd1\\"); 52 | // final DynamicExplorationGraph deg = GraphFactory.getDefaultFactory().newHierchicalGraph(space, edgesPerNode, topRankSize); 53 | 54 | // simple DEG 55 | final int edgesPerNode = 4; 56 | final Path graphFile = Paths.get("c:\\Data\\Feature\\SIFT1M\\deg\\best_distortion_decisions\\java\\128D_L2_K4_AddK60Eps0.2High_RNGMinimalAdd.java.float.deg"); 57 | final DynamicExplorationGraph deg = GraphFactory.getDefaultFactory().newGraph(space, edgesPerNode); 58 | 59 | // give all base data to the designer to be added to the graph during the build process 60 | int maxIndex = 0; 61 | final Random rnd = new Random(7); 62 | final GraphDesigner designer = deg.designer(); 63 | 64 | // random add 65 | if(maxIndex > 0) { 66 | final List availableIdx = IntStream.range(0, maxIndex).mapToObj(Integer::valueOf).collect(Collectors.toList()); 67 | final List unavailableIdx = new ArrayList<>(); 68 | for (int it = 0; it < 100; it++) { 69 | 70 | Collections.shuffle(availableIdx, rnd); 71 | final int add = rnd.nextInt(availableIdx.size()); 72 | for (int i = 0; i < add; i++) { 73 | final int index = availableIdx.remove(availableIdx.size()-1); 74 | designer.add(index, new FloatFeature(baseData[index])); 75 | unavailableIdx.add(index); 76 | } 77 | 78 | Collections.shuffle(unavailableIdx, rnd); 79 | final int remove = rnd.nextInt(unavailableIdx.size()); 80 | for (int i = 0; i < remove; i++) { 81 | final int index = unavailableIdx.remove(unavailableIdx.size()-1); 82 | designer.remove(index); 83 | availableIdx.add(index); 84 | } 85 | } 86 | for(int i : availableIdx) 87 | designer.add(i, new FloatFeature(baseData[i])); 88 | } 89 | 90 | // add all available data points to the graph 91 | for (int i = maxIndex; i < baseData.length; i++) 92 | designer.add(i, new FloatFeature(baseData[i])); 93 | 94 | 95 | // change designer settings 96 | designer.setRandom(rnd); 97 | designer.setExtendK(edgesPerNode*2); 98 | designer.setExtendEps(0.2f); 99 | designer.setImproveK(edgesPerNode); 100 | designer.setImproveEps(0.001f); 101 | designer.setMaxPathLength(5); 102 | 103 | // start the build process 104 | final AtomicLong start = new AtomicLong(System.currentTimeMillis()); 105 | final AtomicLong durationMs = new AtomicLong(0); 106 | designer.build((long step, long added, long removed, long improved, long tries, int lastAdd, int lastRemoved) -> { 107 | final int size = (int)added-(int)removed; 108 | if(step % 100 == 0) { 109 | durationMs.addAndGet(System.currentTimeMillis() - start.get()); 110 | final float avgEdgeWeight = designer.calcAvgEdgeWeight(); 111 | final boolean valid = designer.checkGraphValidation(size, edgesPerNode); 112 | final int duration = (int)(durationMs.get() / 1000); 113 | System.out.printf("Step %7d, %3ds, Q: %4.2f, Size %7d, Added %7d (last id %3d), Removed %7d (last id %3d), improved %3d, tries %3d, valid %s\n", step, duration, avgEdgeWeight, size, added, lastAdd, removed, lastRemoved, improved, tries, Boolean.toString(valid)); 114 | start.set(System.currentTimeMillis()); 115 | } 116 | 117 | if(step % 1000 == 0) { 118 | try { 119 | deg.writeToFile(graphFile); 120 | } catch (ClassNotFoundException | IOException e) { 121 | e.printStackTrace(); 122 | } 123 | } 124 | 125 | // stop the building process 126 | if(added == baseData.length) 127 | designer.stop(); 128 | }); 129 | 130 | // store and test the graph 131 | deg.writeToFile(graphFile); 132 | GraphSearchBenchmark.testGraph(deg, siftBaseDir); 133 | 134 | System.out.println("Finished"); 135 | } 136 | } -------------------------------------------------------------------------------- /java/deg-benchmark/src/main/java/com/vc/deg/anns/GraphReductionBenchmark.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.anns; 2 | 3 | import java.io.IOException; 4 | import java.nio.file.Path; 5 | import java.nio.file.Paths; 6 | import java.util.Random; 7 | import java.util.concurrent.atomic.AtomicInteger; 8 | import java.util.concurrent.atomic.AtomicLong; 9 | 10 | import com.vc.deg.DynamicExplorationGraph; 11 | import com.vc.deg.FeatureSpace; 12 | import com.vc.deg.FeatureVector; 13 | import com.vc.deg.GraphFactory; 14 | import com.vc.deg.feature.FloatL2Space; 15 | import com.vc.deg.graph.GraphDesigner; 16 | 17 | /** 18 | * 19 | * 20 | * @author Nico Hezel 21 | */ 22 | public class GraphReductionBenchmark { 23 | 24 | public static void main(String[] args) throws IOException, ClassNotFoundException { 25 | 26 | 27 | // register the feature space needed in the graph 28 | final AtomicInteger distanceCalculationCount = new AtomicInteger(); 29 | FeatureSpace space = new FloatL2Space(100) { 30 | @Override 31 | public float computeDistance(FeatureVector f1, FeatureVector f2) { 32 | distanceCalculationCount.incrementAndGet(); 33 | return super.computeDistance(f1, f2); 34 | } 35 | }; 36 | FeatureSpace.registerFeatureSpace(space); 37 | 38 | // load the graph 39 | final Path graphFile = Paths.get("e:\\Data\\Feature\\GloVe\\deg\\online\\100D_L2_K30_AddK30Eps0.2High_SwapK30-0StepEps0.001LowPath5Rnd0+0_improveEvery2ndNonPerfectEdge.deg"); 40 | System.out.println("Load the graph "+graphFile); 41 | final DynamicExplorationGraph deg = GraphFactory.getDefaultFactory().loadGraph(graphFile, "float"); 42 | System.out.println("Use DEG"+deg.edgePerVertex()+" with "+deg.size()+" vertices"); 43 | 44 | // remove the second half of the base data from the graph 45 | final GraphDesigner designer = deg.designer(); 46 | final int initialSize = deg.size(); 47 | final int finalSize = initialSize/2; 48 | for (int i = 0; i < finalSize; i++) 49 | designer.remove(finalSize + i); 50 | 51 | // change designer settings 52 | final Random rnd = new Random(); 53 | final int edgesPerNode = deg.edgePerVertex(); 54 | designer.setRandom(rnd); 55 | designer.setExtendK(edgesPerNode); 56 | designer.setExtendEps(0.2f); 57 | designer.setImproveK(edgesPerNode); 58 | designer.setImproveEps(0.001f); 59 | designer.setMaxPathLength(5); 60 | 61 | // start the build process 62 | final AtomicLong start = new AtomicLong(System.currentTimeMillis()); 63 | final AtomicLong durationMs = new AtomicLong(0); 64 | designer.build((long step, long added, long removed, long improved, long tries, int lastAdd, int lastRemoved) -> { 65 | final int size = (int)(initialSize+added)-(int)removed; 66 | if(step % 10000 == 0) { 67 | durationMs.addAndGet(System.currentTimeMillis() - start.get()); 68 | final float avgEdgeWeight = designer.calcAvgEdgeWeight() * 1000; // scaler 69 | final boolean valid = designer.checkGraphValidation(size, edgesPerNode); 70 | final int duration = (int)(durationMs.get() / 1000); 71 | System.out.printf("Step %7d, %3ds, Q: %4.2f, Size %7d, Added %7d (last id %3d), Removed %7d (last id %3d), improved %3d, tries %3d, valid %s\n", step, duration, avgEdgeWeight, size, added, lastAdd, removed, lastRemoved, improved, tries, Boolean.toString(valid)); 72 | start.set(System.currentTimeMillis()); 73 | } 74 | 75 | // stop the building process 76 | if(size == finalSize) 77 | designer.stop(); 78 | }); 79 | 80 | System.out.println("Finished"); 81 | } 82 | } -------------------------------------------------------------------------------- /java/deg-benchmark/src/main/java/com/vc/deg/anns/GraphSearchBenchmark.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.anns; 2 | 3 | import java.io.IOException; 4 | import java.nio.file.Path; 5 | import java.nio.file.Paths; 6 | import java.util.concurrent.atomic.AtomicInteger; 7 | import java.util.concurrent.atomic.AtomicLong; 8 | import java.util.stream.IntStream; 9 | 10 | import com.koloboke.collect.set.IntSet; 11 | import com.koloboke.collect.set.hash.HashIntSets; 12 | import com.vc.deg.DynamicExplorationGraph; 13 | import com.vc.deg.FeatureSpace; 14 | import com.vc.deg.GraphFactory; 15 | import com.vc.deg.data.Sift1M; 16 | import com.vc.deg.feature.FloatFeature; 17 | import com.vc.deg.feature.FloatL2Space; 18 | 19 | /** 20 | * Benchmark of the graph against the SIFT1M dataset 21 | * 22 | * @author Nico Hezel 23 | */ 24 | public class GraphSearchBenchmark { 25 | 26 | protected final static Path siftBaseDir = Paths.get("C:/Data/Feature/SIFT1M/SIFT1M/"); 27 | // protected final static Path graphFile = Paths.get("c:\\Data\\Feature\\SIFT1M\\deg\\best_distortion_decisions\\java\\128D_L2_K30_AddK60Eps0.2High_SwapK30-0StepEps0.001LowPath5Rnd15+15_improveTheBetterHalfOfTheNonPerfectEdges_RNGAddMinimalSwapAtStep0.float.deg"); 28 | protected final static Path graphFile = Paths.get("c:\\Data\\Feature\\SIFT1M\\deg\\best_distortion_decisions\\128D_L2_K30_AddK60Eps0.2High_SwapK30-0StepEps0.001LowPath5Rnd3+2_improveNonRNGAndSecondHalfOfNonPerfectEdges_RNGAddMinimalSwapAtStep0.add_rng_opt.deg"); 29 | 30 | public static void main(String[] args) throws IOException { 31 | testGraph(graphFile, siftBaseDir); 32 | } 33 | 34 | public static void testGraph(Path graphFile, Path siftBaseDir) throws IOException { 35 | 36 | // register the feature space needed in the graph 37 | FeatureSpace.registerFeatureSpace(new FloatL2Space(128)); 38 | 39 | // load graph 40 | DynamicExplorationGraph deg = GraphFactory.getDefaultFactory().loadGraph(graphFile, float.class.getSimpleName()); 41 | 42 | // test the graph 43 | testGraph(deg, siftBaseDir); 44 | } 45 | 46 | public static void testGraph(DynamicExplorationGraph deg, Path siftBaseDir) throws IOException { 47 | 48 | // load query data 49 | float[][] queryData = Sift1M.loadQueryData(siftBaseDir); 50 | int[][] groundtruthData = Sift1M.loadGroundtruthData(siftBaseDir); 51 | 52 | // test graph 53 | final int k = 100; 54 | for(float eps : new float[]{0.01f, 0.05f, 0.1f, 0.12f}) { 55 | final AtomicInteger precisionAtK = new AtomicInteger(); 56 | final AtomicLong elapsedMilliseconds = new AtomicLong(); 57 | IntStream.range(0, queryData.length).parallel().forEach(i -> { 58 | final float[] query = queryData[i]; 59 | final int[] groundtruth = groundtruthData[i]; 60 | 61 | // find nearest neighbors 62 | final long start = System.currentTimeMillis(); 63 | final int[] bestList = deg.search(new FloatFeature(query), k, eps); 64 | final long stop = System.currentTimeMillis(); 65 | elapsedMilliseconds.addAndGet(stop-start); 66 | 67 | // check ground truth against 68 | final IntSet bestIds = HashIntSets.newImmutableSet(bestList); 69 | for(int bestIndex : groundtruth) 70 | if(bestIds.contains(bestIndex)) 71 | precisionAtK.incrementAndGet(); 72 | }); 73 | 74 | // final score 75 | float meanElapsedTime = elapsedMilliseconds.floatValue() / queryData.length; 76 | float meanPrecisionAtK = precisionAtK.floatValue() / k / queryData.length; 77 | int queriesPerSecond = (int)(1000 / meanElapsedTime); 78 | System.out.printf("avg. precision@%3d: %.5f at %4d queries/sec (avg %6.3fms/query) \n", k, meanPrecisionAtK, queriesPerSecond, meanElapsedTime); 79 | } 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /java/deg-benchmark/src/main/java/com/vc/deg/anns/GraphStatsBenchmark.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.anns; 2 | 3 | import java.io.IOException; 4 | import java.nio.file.Path; 5 | import java.nio.file.Paths; 6 | 7 | import com.vc.deg.DynamicExplorationGraph; 8 | import com.vc.deg.FeatureSpace; 9 | import com.vc.deg.GraphFactory; 10 | import com.vc.deg.data.Sift1M; 11 | import com.vc.deg.feature.FloatL2Space; 12 | import com.vc.deg.graph.GraphDesigner; 13 | 14 | /** 15 | * Compute the stats of the graphs 16 | * 17 | * @author Nico Hezel 18 | */ 19 | public class GraphStatsBenchmark { 20 | 21 | protected final static Path siftBaseDir = Paths.get("C:/Data/Feature/SIFT1M/SIFT1M/"); 22 | // protected final static Path graphFile = Paths.get("c:\\Data\\Feature\\SIFT1M\\deg\\best_distortion_decisions\\java\\128D_L2_K30_AddK60Eps0.2High_SwapK30-0StepEps0.001LowPath5Rnd15+15_improveTheBetterHalfOfTheNonPerfectEdges_RNGAddMinimalSwapAtStep0.float.deg"); 23 | // protected final static Path graphFile = Paths.get("c:\\Data\\Feature\\SIFT1M\\deg\\best_distortion_decisions\\128D_L2_K30_AddK60Eps0.2High_SwapK30-0StepEps0.001LowPath5Rnd0+0_improveTheBetterHalfOfTheNonPerfectEdges_RNGAddMinimalSwapAtStep0.rng_optimized.deg"); 24 | 25 | // fast and good 1h 26 | // protected final static Path graphFile = Paths.get("c:\\Data\\Feature\\SIFT1M\\deg\\best_distortion_decisions\\128D_L2_K30_AddK60Eps0.2High_SwapK30-0StepEps0.001LowPath5Rnd3+2_improveNonRNGAndSecondHalfOfNonPerfectEdges_RNGAddMinimalSwapAtStep0.add_rng_opt.deg"); 27 | 28 | // DEG30 (fast opt rng high) 28min 29 | protected final static Path graphFile = Paths.get("c:\\Data\\Feature\\SIFT1M\\deg\\best_distortion_decisions\\128D_L2_K30_AddK60Eps0.2High_SwapK30-0StepEps0.001LowPath5Rnd0+0_improveTheBetterHalfOfTheNonPerfectEdges_RNGAddMinimalSwapAtStep0.test.deg"); 30 | 31 | // DEG30 (best opt rng) 5h 31min 32 | // protected final static Path graphFile = Paths.get("c:\\Data\\Feature\\SIFT1M\\deg\\best_distortion_decisions\\128D_L2_K30_AddK60Eps0.2High_SwapK30-0StepEps0.001LowPath5Rnd15+15_improveTheBetterHalfOfTheNonPerfectEdges_RNGIncompleteAddMinimalSwap.deg"); 33 | 34 | 35 | 36 | 37 | // protected final static Path graphFile = Paths.get("C:/Data/Feature/2DGraph/L2_K4_AddK10Eps0.2High_SwapK10-0StepEps0.001LowPath5Rnd100+0_improveNonRNGAndSecondHalfOfNonPerfectEdges_RNGAddMinimalSwapAtStep0.add_rng_opt.remove_non_rng_edges.deg"); 38 | // protected final static Path graphFile = Paths.get("C:/Data/Feature/2DGraph/L2_rng.deg"); 39 | // protected final static Path graphFile = Paths.get("C:/Data/Feature/2DGraph/L2_dg.deg"); 40 | // protected final static Path graphFile = Paths.get("C:/Data/Feature/2DGraph/L2_K3_knng.deg"); 41 | // protected final static Path graphFile = Paths.get("C:/Data/Feature/2DGraph/L2_K3_knnAproxRNG.deg"); 42 | 43 | 44 | public static void main(String[] args) throws IOException { 45 | 46 | int[][] top1000 = Sift1M.loadBaseTop1000(siftBaseDir); 47 | 48 | // register the feature space needed in the graph 49 | FeatureSpace.registerFeatureSpace(new FloatL2Space(128)); 50 | FeatureSpace.registerFeatureSpace(new FloatL2Space(2)); 51 | 52 | // load graph 53 | DynamicExplorationGraph deg = GraphFactory.getDefaultFactory().loadGraph(graphFile, float.class.getSimpleName()); 54 | GraphDesigner designer = deg.designer(); 55 | System.out.println("AEW of "+graphFile.getFileName()+" is "+designer.calcAvgEdgeWeight()); 56 | System.out.println("ANR of "+graphFile.getFileName()+" is "+designer.calcAvgNeighborRank(top1000)); 57 | System.out.println("ANR of "+graphFile.getFileName()+" is "+designer.calcAvgNeighborRank()); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /java/deg-benchmark/src/main/java/com/vc/deg/data/Describable.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.data; 2 | 3 | import com.vc.deg.FeatureVector; 4 | 5 | public interface Describable { 6 | public FeatureVector getFeature(); 7 | } 8 | 9 | -------------------------------------------------------------------------------- /java/deg-benchmark/src/main/java/com/vc/deg/data/GloVe.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.data; 2 | 3 | import java.io.IOException; 4 | import java.nio.file.Path; 5 | 6 | 7 | public class GloVe { 8 | 9 | 10 | public static final float[][] loadBaseData(Path baseDir) throws IOException { 11 | return Sift1M.fvecs_read(baseDir.resolve("glove-100_base.fvecs")); 12 | } 13 | 14 | public static final float[][] loadQueryData(Path baseDir) throws IOException { 15 | return Sift1M.fvecs_read(baseDir.resolve("glove-100_query.fvecs")); 16 | } 17 | 18 | public static final int[][] loadGroundtruthData(Path baseDir) throws IOException { 19 | return Sift1M.ivecs_read(baseDir.resolve("glove-100_groundtruth.ivecs")); 20 | } 21 | 22 | public static final int[][] loadGroundtruthDataBase591757(Path baseDir) throws IOException { 23 | return Sift1M.ivecs_read(baseDir.resolve("glove-100_groundtruth_base591757.ivecs")); 24 | } 25 | 26 | public static final int[][] loadExploreQueryData(Path baseDir) throws IOException { 27 | return Sift1M.ivecs_read(baseDir.resolve("glove-100_explore_entry_vertex.ivecs")); 28 | } 29 | 30 | public static final int[][] loadExploreGroundtruthData(Path baseDir) throws IOException { 31 | return Sift1M.ivecs_read(baseDir.resolve("sglove-100_explore_ground_truth.ivecs")); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /java/deg-benchmark/src/main/java/com/vc/deg/data/Graph2DData.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.data; 2 | 3 | import java.io.IOException; 4 | import java.nio.file.Path; 5 | 6 | 7 | public class Graph2DData { 8 | 9 | 10 | public static final float[][] loadBaseData(Path baseDir) throws IOException { 11 | return Sift1M.fvecs_read(baseDir.resolve("base.fvecs")); 12 | } 13 | 14 | public static final float[][] loadQueryData(Path baseDir) throws IOException { 15 | return Sift1M.fvecs_read(baseDir.resolve("query.fvecs")); 16 | } 17 | 18 | public static final int[][] loadGroundtruthData(Path baseDir) throws IOException { 19 | return Sift1M.ivecs_read(baseDir.resolve("query_gt.ivecs")); 20 | } 21 | 22 | public static final float[][] loadExploreData(Path baseDir) throws IOException { 23 | return Sift1M.fvecs_read(baseDir.resolve("explore.fvecs")); 24 | } 25 | 26 | public static final int[][] loadExploreQueryData(Path baseDir) throws IOException { 27 | return Sift1M.ivecs_read(baseDir.resolve("explore_entry_node.ivecs")); 28 | } 29 | 30 | public static final int[][] loadExploreGroundtruthData(Path baseDir) throws IOException { 31 | return Sift1M.ivecs_read(baseDir.resolve("explore_gt.ivecs")); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /java/deg-benchmark/src/main/java/com/vc/deg/data/Identifiable.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.data; 2 | 3 | public interface Identifiable { 4 | public int getId(); 5 | } 6 | -------------------------------------------------------------------------------- /java/deg-benchmark/src/main/java/com/vc/deg/data/Sift1M.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.data; 2 | 3 | import java.io.IOException; 4 | import java.nio.ByteBuffer; 5 | import java.nio.ByteOrder; 6 | import java.nio.channels.FileChannel; 7 | import java.nio.channels.SeekableByteChannel; 8 | import java.nio.file.Files; 9 | import java.nio.file.Path; 10 | import java.nio.file.StandardOpenOption; 11 | import java.util.ArrayList; 12 | import java.util.List; 13 | 14 | 15 | public class Sift1M { 16 | 17 | public static final float[][] loadLearnData(Path baseDir) throws IOException { 18 | return fvecs_read(baseDir.resolve("sift_learn.fvecs")); 19 | } 20 | 21 | public static final float[][] loadBaseData(Path baseDir) throws IOException { 22 | return fvecs_read(baseDir.resolve("sift_base.fvecs")); 23 | } 24 | 25 | public static final int[][] loadBaseTop1000(Path baseDir) throws IOException { 26 | return ivecs_read(baseDir.resolve("sift_base_top1000.ivecs")); 27 | } 28 | 29 | public static final float[][] loadQueryData(Path baseDir) throws IOException { 30 | return fvecs_read(baseDir.resolve("sift_query.fvecs")); 31 | } 32 | 33 | public static final int[][] loadGroundtruthData(Path baseDir) throws IOException { 34 | return ivecs_read(baseDir.resolve("sift_groundtruth.ivecs")); 35 | } 36 | 37 | public static final int[][] loadExploreQueryData(Path baseDir) throws IOException { 38 | return ivecs_read(baseDir.resolve("sift_explore_entry_node.ivecs")); 39 | } 40 | 41 | public static final int[][] loadExploreGroundtruthData(Path baseDir) throws IOException { 42 | return ivecs_read(baseDir.resolve("sift_explore_ground_truth.ivecs")); 43 | } 44 | 45 | /** 46 | * http://corpus-texmex.irisa.fr/fvecs_read.m 47 | * 48 | * @param feature_file 49 | * @return 50 | * @throws IOException 51 | */ 52 | public static float[][] fvecs_read(Path feature_file) throws IOException { 53 | List features = new ArrayList<>(); 54 | try(SeekableByteChannel ch = Files.newByteChannel(feature_file, StandardOpenOption.READ)) { 55 | while(ch.position() < ch.size()) { 56 | 57 | // read the size of the feature vector 58 | ByteBuffer countBuffer = ByteBuffer.allocate(4); 59 | countBuffer.order(ByteOrder.LITTLE_ENDIAN); 60 | ch.read(countBuffer); 61 | countBuffer.flip(); 62 | int c = countBuffer.getInt(); 63 | 64 | // read the values of the feature 65 | float[] feature = new float[c]; 66 | ByteBuffer valueBuffer = ByteBuffer.allocate(4 * c); 67 | ch.read(valueBuffer); 68 | valueBuffer.flip(); 69 | valueBuffer.order(ByteOrder.LITTLE_ENDIAN); 70 | valueBuffer.asFloatBuffer().get(feature); 71 | features.add(feature); 72 | } 73 | } 74 | 75 | return features.toArray(new float[features.size()][]); 76 | } 77 | 78 | public static void fvecs_write(float[][] data, Path feature_file) throws IOException { 79 | 80 | // count the bytes 81 | int byteCount = 0; 82 | for (float[] vector : data) { 83 | byteCount += Integer.BYTES; 84 | byteCount += vector.length * Float.BYTES; 85 | } 86 | 87 | // store the length and content of the data array 88 | try(FileChannel fileChannel = (FileChannel) Files.newByteChannel(feature_file, StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.CREATE)) { 89 | ByteBuffer byteBuffer = fileChannel.map(FileChannel.MapMode.READ_WRITE, 0, byteCount).order(ByteOrder.LITTLE_ENDIAN); 90 | 91 | for (float[] vector : data) { 92 | byteBuffer.putInt(vector.length); 93 | for (float value : vector) 94 | byteBuffer.putFloat(value); 95 | } 96 | } 97 | } 98 | 99 | /** 100 | * http://corpus-texmex.irisa.fr/ivecs_read.m 101 | * 102 | * @param feature_file 103 | * @return 104 | * @throws IOException 105 | */ 106 | public static int[][] ivecs_read(Path feature_file) throws IOException { 107 | List features = new ArrayList<>(); 108 | try(SeekableByteChannel ch = Files.newByteChannel(feature_file, StandardOpenOption.READ)) { 109 | while(ch.position() < ch.size()) { 110 | 111 | // read the size of the feature vector 112 | ByteBuffer countBuffer = ByteBuffer.allocate(4); 113 | countBuffer.order(ByteOrder.LITTLE_ENDIAN); 114 | ch.read(countBuffer); 115 | countBuffer.flip(); 116 | int c = countBuffer.getInt(); 117 | 118 | // read the values of the feature 119 | int[] feature = new int[c]; 120 | ByteBuffer valueBuffer = ByteBuffer.allocate(4 * c); 121 | ch.read(valueBuffer); 122 | valueBuffer.flip(); 123 | valueBuffer.order(ByteOrder.LITTLE_ENDIAN); 124 | valueBuffer.asIntBuffer().get(feature); 125 | features.add(feature); 126 | } 127 | } 128 | 129 | return features.toArray(new int[features.size()][]); 130 | } 131 | 132 | public static void ivecs_write(int[][] data, Path feature_file) throws IOException { 133 | 134 | // count the bytes 135 | int byteCount = 0; 136 | for (int[] vector : data) { 137 | byteCount += Integer.BYTES; 138 | byteCount += vector.length * Integer.BYTES; 139 | } 140 | 141 | // store the length and content of the data array 142 | try(FileChannel fileChannel = (FileChannel) Files.newByteChannel(feature_file, StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.CREATE)) { 143 | ByteBuffer byteBuffer = fileChannel.map(FileChannel.MapMode.READ_WRITE, 0, byteCount).order(ByteOrder.LITTLE_ENDIAN); 144 | 145 | for (int[] vector : data) { 146 | byteBuffer.putInt(vector.length); 147 | for (int value : vector) 148 | byteBuffer.putInt(value); 149 | } 150 | } 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /java/deg-benchmark/src/main/java/com/vc/deg/feature/FloatL2Space.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.feature; 2 | 3 | import com.vc.deg.FeatureSpace; 4 | import com.vc.deg.FeatureVector; 5 | 6 | public class FloatL2Space implements FeatureSpace { 7 | 8 | protected int dims; 9 | 10 | public FloatL2Space(int dims) { 11 | this.dims = dims; 12 | } 13 | 14 | @Override 15 | public int featureSize() { 16 | return Float.BYTES * dims(); 17 | } 18 | 19 | @Override 20 | public int dims() { 21 | return dims; 22 | } 23 | 24 | @Override 25 | public Class getComponentType() { 26 | return float.class; 27 | } 28 | 29 | 30 | @Override 31 | public int metric() { 32 | return Metric.L2.getId(); 33 | } 34 | 35 | @Override 36 | public boolean isNative() { 37 | return false; 38 | } 39 | 40 | @Override 41 | public float computeDistance(FeatureVector f1, FeatureVector f2) { 42 | final int byteSize = f1.size(); 43 | 44 | float result = 0; 45 | for (int i = 0; i < byteSize; i+=4) { 46 | float diff = f1.readFloat(i) - f2.readFloat(i); 47 | result += diff*diff; 48 | } 49 | 50 | return result; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /java/deg-benchmark/src/test/java/com/vc/deg/data/Create2DGraph.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.data; 2 | 3 | import java.io.IOException; 4 | import java.nio.file.Path; 5 | import java.nio.file.Paths; 6 | import java.util.Arrays; 7 | import java.util.Random; 8 | import java.util.stream.IntStream; 9 | 10 | public class Create2DGraph { 11 | 12 | public static void main(String[] args) throws IOException { 13 | Path baseDir = Paths.get("c:\\Data\\Feature\\2DGraph\\"); 14 | 15 | Path baseFile = baseDir.resolve("base.fvecs"); 16 | int seed = 80733299; //(int)(Math.random() * Integer.MAX_VALUE); 17 | System.out.println("Random seed "+seed); 18 | Random rnd = new Random(seed); 19 | float[][] baseData = new float[14][]; 20 | for (int i = 0; i < baseData.length; i++) { 21 | baseData[i] = new float[] { rnd.nextFloat()*13, rnd.nextFloat()*13 }; 22 | System.out.println(Arrays.toString(baseData[i])); 23 | } 24 | Sift1M.fvecs_write(baseData, baseFile); 25 | 26 | Path baseTopFile = baseDir.resolve("base_top13.ivecs"); 27 | int[][] baseTopData = new int[baseData.length][]; 28 | for (int i = 0; i < baseTopData.length; i++) { 29 | int baseIndex = i; 30 | baseTopData[i] = IntStream.range(0, baseData.length).filter(idx -> idx != baseIndex).boxed().sorted((idx1, idx2) -> Double.compare(distance(baseData[baseIndex], baseData[idx1]), distance(baseData[baseIndex], baseData[idx2]))).mapToInt(Integer::intValue).toArray(); 31 | } 32 | Sift1M.ivecs_write(baseTopData, baseTopFile); 33 | 34 | Path exploreFile = baseDir.resolve("explore.fvecs"); 35 | Sift1M.fvecs_write(baseData, exploreFile); 36 | 37 | Path exploreEntryFile = baseDir.resolve("explore_entry_node.ivecs"); 38 | int[][] exploreEntryNodeIdx = new int[baseData.length][]; 39 | for (int i = 0; i < exploreEntryNodeIdx.length; i++) 40 | exploreEntryNodeIdx[i] = new int[] {i}; 41 | Sift1M.ivecs_write(exploreEntryNodeIdx, exploreEntryFile); 42 | 43 | Path exploreGtFile = baseDir.resolve("explore_gt.ivecs"); 44 | int[][] exploreGtData = new int[baseData.length][]; 45 | for (int i = 0; i < exploreGtData.length; i++) 46 | exploreGtData[i] = Arrays.copyOf(baseTopData[i], 10); 47 | Sift1M.ivecs_write(exploreGtData, exploreGtFile); 48 | 49 | Path queryFile = baseDir.resolve("query.fvecs"); 50 | float[][] queryData = new float[][] { 51 | {10.23f, 8.33f} 52 | }; 53 | Sift1M.fvecs_write(queryData, queryFile); 54 | 55 | Path queryGtFile = baseDir.resolve("query_gt.ivecs"); 56 | int[][] queryGtData = new int[][] { 57 | IntStream.range(0, baseData.length).boxed().sorted((idx1, idx2) -> Double.compare(distance(queryData[0], baseData[idx1]), distance(queryData[0], baseData[idx2]))).mapToInt(Integer::intValue).limit(5).toArray() 58 | }; 59 | Sift1M.ivecs_write(queryGtData, queryGtFile); 60 | } 61 | 62 | /** 63 | * Compute euclidean distance 64 | * 65 | * @param vec1 66 | * @param vec2 67 | * @return 68 | */ 69 | public static double distance(float[] vec1, float[] vec2) { 70 | double result = 0; 71 | for (int i = 0; i < vec1.length; i++) { 72 | double diff = vec1[i] - vec2[i]; 73 | result += diff*diff; 74 | } 75 | return Math.sqrt(result); 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /java/deg-library/build.gradle: -------------------------------------------------------------------------------- 1 | // Use the build script defined in buildSrc 2 | apply from: rootProject.file('buildSrc/shared.gradle') 3 | 4 | dependencies { 5 | implementation project(':deg-api') 6 | 7 | implementation group: 'com.esotericsoftware', name: 'kryo', version: "${KRYO_VERSION}" 8 | 9 | implementation group: 'com.koloboke', name: 'koloboke-api-jdk8', version: "${KOLOBOKE_VERSION}" 10 | implementation group: 'com.koloboke', name: 'koloboke-impl-jdk8', version: "${KOLOBOKE_VERSION}" 11 | implementation group: 'com.koloboke', name: 'koloboke-impl-common-jdk8', version: "${KOLOBOKE_VERSION}" 12 | } -------------------------------------------------------------------------------- /java/deg-library/readme.md: -------------------------------------------------------------------------------- 1 | Effiziente Implementierung der Referenz Implementierung. Referenz Implementierung ist leichter zu verstehen. 2 | Enthält einige Klasse nur zur Optimierung von Rechengeschwindigkeit. 3 | 4 | Für alle Interfaces in der API gibt es hier eine Implementierung. Zusätzlich gibt es die DynamicNavigationGraph Klasse 5 | die alle Komponenten zusammenfasst und zeigt wie diese genutzt werden können. Diese Klasse kann auch genutzt werden 6 | wenn man sich keinen Kopf machen möchte und einfach nur einen Dynamischen Graphen brauch. -------------------------------------------------------------------------------- /java/deg-library/src/main/java/com/vc/deg/impl/DynamicExplorationGraph.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.impl; 2 | 3 | import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; 4 | import static java.nio.file.StandardOpenOption.CREATE; 5 | import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING; 6 | import static java.nio.file.StandardOpenOption.WRITE; 7 | 8 | import java.io.IOException; 9 | import java.nio.file.Files; 10 | import java.nio.file.Path; 11 | import java.nio.file.Paths; 12 | import java.util.Collection; 13 | import java.util.Random; 14 | 15 | import com.esotericsoftware.kryo.Kryo; 16 | import com.esotericsoftware.kryo.KryoException; 17 | import com.esotericsoftware.kryo.unsafe.UnsafeInput; 18 | import com.esotericsoftware.kryo.unsafe.UnsafeOutput; 19 | import com.vc.deg.FeatureSpace; 20 | import com.vc.deg.FeatureVector; 21 | import com.vc.deg.graph.GraphDesigner; 22 | import com.vc.deg.graph.VertexFilter; 23 | import com.vc.deg.graph.NeighborConsumer; 24 | import com.vc.deg.graph.VertexCursor; 25 | import com.vc.deg.impl.designer.EvenRegularGraphDesigner; 26 | import com.vc.deg.impl.graph.WeightedUndirectedRegularGraph; 27 | 28 | /** 29 | * This class wraps the main functions of the library 30 | * 31 | * @author Nico Hezel 32 | */ 33 | public class DynamicExplorationGraph implements com.vc.deg.DynamicExplorationGraph { 34 | 35 | protected WeightedUndirectedRegularGraph internalGraph; 36 | protected EvenRegularGraphDesigner designer; 37 | 38 | 39 | /** 40 | * Use a custom repository. This will be filled when the {@link #add(int, Object)} method is called. 41 | * Customs repos can have different storing and data access functions 42 | * 43 | * @param objectRepository 44 | */ 45 | public DynamicExplorationGraph(FeatureSpace space, int edgesPerNode) { 46 | this.internalGraph = new WeightedUndirectedRegularGraph(edgesPerNode, space); 47 | this.designer = new EvenRegularGraphDesigner(internalGraph); 48 | } 49 | 50 | 51 | @Override 52 | public GraphDesigner designer() { 53 | return designer; 54 | } 55 | 56 | @Override 57 | public void writeToFile(Path file) throws KryoException, IOException { 58 | Path tempFile = Paths.get(file.getParent().toString(), "~$" + file.getFileName().toString()); 59 | Kryo kryo = new Kryo(); 60 | try(UnsafeOutput output = new UnsafeOutput(Files.newOutputStream(tempFile, TRUNCATE_EXISTING, CREATE, WRITE))) { 61 | this.internalGraph.write(kryo, output); 62 | } 63 | Files.move(tempFile, file, REPLACE_EXISTING); 64 | } 65 | 66 | public void readFromFile(Path file) throws KryoException, IOException { 67 | Kryo kryo = new Kryo(); 68 | try(UnsafeInput input = new UnsafeInput(Files.newInputStream(file))) { 69 | this.internalGraph.read(kryo, input); 70 | } 71 | } 72 | 73 | @Override 74 | public DynamicExplorationGraph copy() { 75 | // TODO Auto-generated method stub 76 | return null; 77 | } 78 | 79 | 80 | @Override 81 | public int[] search(Collection queries, int k, float eps, VertexFilter filter) { 82 | // TODO Auto-generated method stub 83 | return null; 84 | } 85 | 86 | 87 | @Override 88 | public int[] explore(int[] entryLabel, int k, float eps, VertexFilter filter) { 89 | // TODO Auto-generated method stub 90 | return null; 91 | } 92 | 93 | 94 | @Override 95 | public boolean hasLabel(int label) { 96 | // TODO Auto-generated method stub 97 | return false; 98 | } 99 | 100 | 101 | @Override 102 | public FeatureSpace getFeatureSpace() { 103 | // TODO Auto-generated method stub 104 | return null; 105 | } 106 | 107 | 108 | @Override 109 | public FeatureVector getFeature(int label) { 110 | // TODO Auto-generated method stub 111 | return null; 112 | } 113 | 114 | 115 | @Override 116 | public int size() { 117 | // TODO Auto-generated method stub 118 | return 0; 119 | } 120 | 121 | 122 | @Override 123 | public VertexCursor vertexCursor() { 124 | // TODO Auto-generated method stub 125 | return null; 126 | } 127 | 128 | 129 | @Override 130 | public void forEachNeighbor(int label, NeighborConsumer neighborConsumer) { 131 | // TODO Auto-generated method stub 132 | 133 | } 134 | 135 | @Override 136 | public int getRandomLabel(Random random) { 137 | // TODO Auto-generated method stub 138 | return 0; 139 | } 140 | 141 | 142 | @Override 143 | public int getRandomLabel(Random random, VertexFilter filter) { 144 | // TODO Auto-generated method stub 145 | return 0; 146 | } 147 | 148 | 149 | @Override 150 | public int[] search(Collection queries, int k, float eps, VertexFilter filter, 151 | int[] seedVertexLabels) { 152 | // TODO Auto-generated method stub 153 | return null; 154 | } 155 | 156 | 157 | @Override 158 | public int edgePerVertex() { 159 | // TODO Auto-generated method stub 160 | return 0; 161 | } 162 | 163 | 164 | @Override 165 | public VertexFilter labelFilter() { 166 | // TODO Auto-generated method stub 167 | return null; 168 | } 169 | } 170 | -------------------------------------------------------------------------------- /java/deg-library/src/main/java/com/vc/deg/impl/GraphFactory.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.impl; 2 | 3 | import java.io.IOException; 4 | import java.nio.file.Path; 5 | 6 | import com.vc.deg.FeatureSpace; 7 | import com.vc.deg.HierarchicalDynamicExplorationGraph; 8 | 9 | public class GraphFactory implements com.vc.deg.GraphFactory { 10 | 11 | @Override 12 | public DynamicExplorationGraph newGraph(FeatureSpace space, int edgesPerNode) { 13 | return new DynamicExplorationGraph(space, edgesPerNode); 14 | } 15 | 16 | @Override 17 | public DynamicExplorationGraph loadGraph(Path file) { 18 | // TODO Auto-generated method stub 19 | return null; 20 | } 21 | 22 | @Override 23 | public com.vc.deg.DynamicExplorationGraph loadGraph(Path file, String featureType) throws IOException { 24 | // TODO Auto-generated method stub 25 | return null; 26 | } 27 | 28 | @Override 29 | public HierarchicalDynamicExplorationGraph newHierchicalGraph(FeatureSpace space, int edgesPerNode, 30 | int topRankSize) { 31 | // TODO Auto-generated method stub 32 | return null; 33 | } 34 | 35 | @Override 36 | public HierarchicalDynamicExplorationGraph loadHierchicalGraph(Path file) throws IOException { 37 | // TODO Auto-generated method stub 38 | return null; 39 | } 40 | } -------------------------------------------------------------------------------- /java/deg-library/src/main/java/com/vc/deg/impl/designer/EvenRegularGraphDesigner.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.impl.designer; 2 | 3 | import java.util.Random; 4 | import java.util.function.IntPredicate; 5 | 6 | import com.vc.deg.FeatureVector; 7 | import com.vc.deg.graph.GraphDesigner; 8 | import com.vc.deg.impl.graph.WeightedUndirectedRegularGraph; 9 | 10 | /** 11 | * Verbessert und erweitert den Graphen 12 | * 13 | * @author Neiko 14 | * 15 | */ 16 | public class EvenRegularGraphDesigner implements GraphDesigner { 17 | 18 | protected WeightedUndirectedRegularGraph graph; 19 | 20 | public EvenRegularGraphDesigner(WeightedUndirectedRegularGraph graph) { 21 | this.graph = graph; 22 | } 23 | 24 | /** 25 | * Add this new item to the graph 26 | * 27 | * @param id 28 | */ 29 | public void add(int id, FeatureVector data) { 30 | // TODO Auto-generated method stub 31 | } 32 | 33 | public void extendGraph(int id, boolean randomPosition) { 34 | if(randomPosition) 35 | randomGraphExtension(id); 36 | 37 | 38 | // TODO anns entry point should be always the same node 39 | 40 | } 41 | 42 | protected void randomGraphExtension(int id) { 43 | // TODO use worst edges instead of random edge 44 | } 45 | 46 | @Override 47 | public void remove(int label) { 48 | // TODO Auto-generated method stub 49 | } 50 | 51 | @Override 52 | public void removeIf(IntPredicate filter) { 53 | // TODO Auto-generated method stub 54 | } 55 | 56 | @Override 57 | public void build(ChangeListener listener) { 58 | // TODO Auto-generated method stub 59 | 60 | } 61 | 62 | @Override 63 | public void stop() { 64 | // TODO Auto-generated method stub 65 | 66 | } 67 | 68 | @Override 69 | public void setExtendK(int k) { 70 | // TODO Auto-generated method stub 71 | 72 | } 73 | 74 | @Override 75 | public void setExtendEps(float eps) { 76 | // TODO Auto-generated method stub 77 | 78 | } 79 | 80 | @Override 81 | public void setExtendSchema(boolean useSchemaC) { 82 | // TODO Auto-generated method stub 83 | 84 | } 85 | 86 | @Override 87 | public float calcAvgEdgeWeight() { 88 | // TODO Auto-generated method stub 89 | return 0; 90 | } 91 | 92 | @Override 93 | public void setRandom(Random rnd) { 94 | // TODO Auto-generated method stub 95 | 96 | } 97 | 98 | @Override 99 | public boolean checkGraphValidation(int expectedVertices, int expectedNeighbors) { 100 | // TODO Auto-generated method stub 101 | return false; 102 | } 103 | 104 | @Override 105 | public void setImproveK(int k) { 106 | // TODO Auto-generated method stub 107 | 108 | } 109 | 110 | @Override 111 | public void setImproveEps(float eps) { 112 | // TODO Auto-generated method stub 113 | 114 | } 115 | 116 | @Override 117 | public void setMaxPathLength(int maxPathLength) { 118 | // TODO Auto-generated method stub 119 | 120 | } 121 | 122 | @Override 123 | public float calcAvgNeighborRank() { 124 | // TODO Auto-generated method stub 125 | return 0; 126 | } 127 | 128 | @Override 129 | public float calcAvgNeighborRank(int[][] topList) { 130 | // TODO Auto-generated method stub 131 | return 0; 132 | } 133 | 134 | } 135 | -------------------------------------------------------------------------------- /java/deg-library/src/main/java/com/vc/deg/impl/graph/VertexView.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.impl.graph; 2 | 3 | import com.vc.deg.FeatureVector; 4 | 5 | public interface VertexView { 6 | 7 | public int getId(); 8 | public FeatureVector getFeature(); 9 | public WeightedEdges getNeighbors(); 10 | } 11 | -------------------------------------------------------------------------------- /java/deg-library/src/main/java/com/vc/deg/impl/graph/WeightedEdges.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.impl.graph; 2 | 3 | import java.util.function.IntConsumer; 4 | 5 | /** 6 | * An immutable collection of weighted edges for a specific node. 7 | * The edge weights and ids of the connected other nodes can be retrieved. 8 | * 9 | * @author Nico Hezel 10 | * 11 | */ 12 | public interface WeightedEdges { 13 | 14 | /** 15 | * The id of the node where the edges belong to. 16 | * 17 | * @return 18 | */ 19 | public int getNodeId(); 20 | 21 | /** 22 | * Number of edges in this collection 23 | * 24 | * @return 25 | */ 26 | public int size(); 27 | 28 | /** 29 | * Does this collection contain an edge to a specific node 30 | * 31 | * @param id 32 | * @return 33 | */ 34 | public boolean contains(int id); 35 | 36 | /** 37 | * For every weighed edge in this collection the consumer is called. 38 | * Just the ids of the connected nodes is presented. 39 | * See {@link #forEach(WeightedEdgeConsumer)} for more edge weights. 40 | * 41 | * @param consumer 42 | */ 43 | public void forEach(IntConsumer consumer); 44 | 45 | /** 46 | * For every weighed edge in this collection the consumer is called. 47 | * 48 | * @param consumer 49 | */ 50 | public void forEach(WeightedEdgeConsumer consumer); 51 | 52 | /** 53 | * Provides information about edges. 54 | * 55 | * @author Nico Hezel 56 | */ 57 | public interface WeightedEdgeConsumer { 58 | 59 | /** 60 | * Provides information about a single edge, 61 | * every time this method is called. 62 | * 63 | * @param id1 64 | * @param id2 65 | * @param weight 66 | */ 67 | void accept(int id1, int id2, float weight); 68 | 69 | } 70 | } -------------------------------------------------------------------------------- /java/deg-library/src/main/java/com/vc/deg/impl/graph/WeightedUndirectedRegularGraph.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.impl.graph; 2 | 3 | import com.esotericsoftware.kryo.Kryo; 4 | import com.esotericsoftware.kryo.KryoSerializable; 5 | import com.esotericsoftware.kryo.io.Input; 6 | import com.esotericsoftware.kryo.io.Output; 7 | import com.koloboke.collect.map.IntFloatMap; 8 | import com.koloboke.collect.map.IntObjMap; 9 | import com.koloboke.collect.map.hash.HashIntFloatMaps; 10 | import com.koloboke.collect.map.hash.HashIntObjMaps; 11 | import com.koloboke.collect.set.IntSet; 12 | import com.vc.deg.FeatureSpace; 13 | import com.vc.deg.impl.kryo.HashIntIntFloatMapSerializer; 14 | 15 | /** 16 | * Der Interne Graph mit seinen addNode,deleteNode,addEdge,deleteEdge,serialize Funktionen. 17 | * Stellt keine Such oder Navigationsmöglichen. 18 | * 19 | * @author Neiko 20 | * 21 | */ 22 | public class WeightedUndirectedRegularGraph implements KryoSerializable { 23 | 24 | protected final IntObjMap nodes; 25 | protected final FeatureSpace featureSpace; 26 | protected int edgesPerNode; 27 | 28 | public WeightedUndirectedRegularGraph(int edgesPerNode, FeatureSpace featureSpace) { 29 | this.edgesPerNode = edgesPerNode; 30 | this.nodes = HashIntObjMaps.newMutableMap(); 31 | this.featureSpace = featureSpace; 32 | 33 | } 34 | 35 | public WeightedUndirectedRegularGraph(int edgesPerNode, int expectedSize, FeatureSpace featureSpace) { 36 | this.edgesPerNode = edgesPerNode; 37 | this.nodes = HashIntObjMaps.newMutableMap(expectedSize); 38 | this.featureSpace = featureSpace; 39 | } 40 | 41 | public void addNode(int id) { 42 | nodes.put(id, HashIntFloatMaps.getDefaultFactory().withDefaultValue(Integer.MIN_VALUE).newMutableMap(edgesPerNode)); 43 | } 44 | 45 | public FeatureSpace getFeatureSpace() { 46 | return featureSpace; 47 | } 48 | 49 | public VertexView getNodeView(int id) { 50 | return null; 51 | } 52 | 53 | public IntSet getEdgeIds(int id) { 54 | return nodes.get(id).keySet(); 55 | } 56 | 57 | @Override 58 | public void write(Kryo kryo, Output output) { 59 | HashIntIntFloatMapSerializer.store(output, nodes); 60 | 61 | } 62 | 63 | @Override 64 | public void read(Kryo kryo, Input input) { 65 | HashIntIntFloatMapSerializer.loadMutable(input, nodes); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /java/deg-library/src/main/java/com/vc/deg/impl/kryo/HashIntFloatMapSerializer.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.impl.kryo; 2 | 3 | import com.esotericsoftware.kryo.Kryo; 4 | import com.esotericsoftware.kryo.Serializer; 5 | import com.esotericsoftware.kryo.io.Input; 6 | import com.esotericsoftware.kryo.io.Output; 7 | import com.koloboke.collect.impl.hash.MutableLHashParallelKVIntFloatMapGO; 8 | import com.koloboke.collect.impl.hash.ParallelIntKeyAdapter; 9 | import com.koloboke.collect.map.IntFloatMap; 10 | 11 | public class HashIntFloatMapSerializer extends Serializer { 12 | 13 | @Override 14 | public void write(Kryo kryo, Output output, IntFloatMap obj) { 15 | store(output, obj); 16 | } 17 | 18 | @Override 19 | public IntFloatMap read(Kryo kryo, Input input, Class type) { 20 | return load(input); 21 | } 22 | 23 | public static void store(Output output, IntFloatMap obj) { 24 | 25 | ParallelIntKeyAdapter adapter = new ParallelIntKeyAdapter(obj); 26 | output.writeInt(adapter.table().length); 27 | output.writeLongs(adapter.table(), 0, adapter.table().length); 28 | output.writeInt(adapter.freeValue()); 29 | output.writeInt(adapter.size()); 30 | output.writeInt(adapter.capacity()); 31 | output.writeInt(adapter.modCount()); 32 | } 33 | 34 | public static IntFloatMap load(Input input) { 35 | 36 | // create an adapter to fill everything in the final object 37 | ParallelIntKeyAdapter adapter = new ParallelIntKeyAdapter(); 38 | 39 | // read the data from file 40 | int tableSize = input.readInt(); 41 | adapter.setTable(input.readLongs(tableSize)); 42 | adapter.setFreeValue(input.readInt()); 43 | adapter.setSize(input.readInt()); 44 | adapter.setCapacity(input.readInt()); 45 | adapter.setModCount(input.readInt()); 46 | 47 | // create empty object and copy all data over 48 | MutableLHashParallelKVIntFloatMapGO result = new MutableLHashParallelKVIntFloatMapGO(); 49 | adapter.move(result); 50 | 51 | return result; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /java/deg-library/src/main/java/com/vc/deg/impl/kryo/HashIntIntFloatMapSerializer.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.impl.kryo; 2 | 3 | 4 | import com.esotericsoftware.kryo.Kryo; 5 | import com.esotericsoftware.kryo.Serializer; 6 | import com.esotericsoftware.kryo.io.Input; 7 | import com.esotericsoftware.kryo.io.Output; 8 | import com.koloboke.collect.impl.hash.MutableLHashSeparateKVIntObjMapGO; 9 | import com.koloboke.collect.impl.hash.SeparateIntObjAdapter; 10 | import com.koloboke.collect.map.IntFloatMap; 11 | import com.koloboke.collect.map.IntObjMap; 12 | import com.koloboke.collect.map.hash.HashIntFloatMap; 13 | 14 | public class HashIntIntFloatMapSerializer extends Serializer> { 15 | 16 | public static final byte NULL = 0; 17 | public static final byte NOT_NULL = 1; 18 | 19 | @Override 20 | public void write(Kryo kryo, Output output, IntObjMap object) { 21 | store(output, object); 22 | } 23 | 24 | @Override 25 | public IntObjMap read(Kryo kryo, Input input, Class> type) { 26 | return loadMutable(input); 27 | } 28 | 29 | // ------------------------------------------------------------------------------------------------------------------ 30 | // ------------------------------------------------------------------------------------------------------------------ 31 | // ------------------------------------------------------------------------------------------------------------------ 32 | 33 | 34 | public static void store(Output output, IntObjMap obj) { 35 | 36 | SeparateIntObjAdapter adapter = new SeparateIntObjAdapter(obj); 37 | output.writeInt(adapter.keys().length); 38 | output.writeInts(adapter.keys(), 0, adapter.keys().length); 39 | saveIntFloatMapArray(output, cast(adapter.valueArray())); 40 | output.writeInt(adapter.freeValue()); 41 | output.writeInt(adapter.size()); 42 | output.writeInt(adapter.capacity()); 43 | output.writeInt(adapter.modCount()); 44 | } 45 | 46 | private static HashIntFloatMap[] cast(Object[] values) { 47 | HashIntFloatMap[] result = new HashIntFloatMap[values.length]; 48 | for (int i = 0; i < values.length; i++) 49 | result[i] = (HashIntFloatMap)values[i]; 50 | return result; 51 | } 52 | 53 | private static void saveIntFloatMapArray(Output output, HashIntFloatMap[] arr) { 54 | 55 | output.writeInt(arr.length, true); 56 | for (HashIntFloatMap map : arr) { 57 | 58 | // überspringe null einträge 59 | output.writeByte((map == null) ? NULL : NOT_NULL); 60 | if(map == null) 61 | continue; 62 | 63 | HashIntFloatMapSerializer.store(output, map); 64 | } 65 | } 66 | 67 | 68 | // ------------------------------------------------------------------------------------------------------------------ 69 | // ------------------------------------------------------------------------------------------------------------------ 70 | // ------------------------------------------------------------------------------------------------------------------ 71 | 72 | public static IntObjMap loadMutable(Input input) { 73 | return load(input).mutableIntObjMap(); 74 | } 75 | 76 | public static IntObjMap loadMutable(Input input, IntObjMap map) { 77 | final MutableLHashSeparateKVIntObjMapGO res = (MutableLHashSeparateKVIntObjMapGO) map; 78 | return load(input).mutableIntObjMap(res); 79 | } 80 | 81 | private static SeparateIntObjAdapter load(Input input) { 82 | SeparateIntObjAdapter adapter = new SeparateIntObjAdapter(); 83 | 84 | // read the data from file 85 | int tableSize = input.readInt(); 86 | adapter.setKeys(input.readInts(tableSize)); 87 | adapter.setValueArray(loadIntFloatMapArray(input)); 88 | adapter.setFreeValue(input.readInt()); 89 | adapter.setSize(input.readInt()); 90 | adapter.setCapacity(input.readInt()); 91 | adapter.setModCount(input.readInt()); 92 | 93 | return adapter; 94 | } 95 | 96 | private static IntFloatMap[] loadIntFloatMapArray(Input input) { 97 | 98 | int length = input.readInt(true); 99 | if (length == 0) return null; 100 | 101 | IntFloatMap[] result = new IntFloatMap[length]; 102 | for (int i = 0; i < length; i++) { 103 | 104 | // wenn das object null ist 105 | if(input.readByte() == NULL) 106 | continue; 107 | 108 | result[i] = HashIntFloatMapSerializer.load(input); 109 | } 110 | 111 | return result; 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /java/deg-library/src/main/java/com/vc/deg/impl/search/GraphSearch.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.impl.search; 2 | 3 | import java.util.PriorityQueue; 4 | import java.util.TreeSet; 5 | 6 | import com.koloboke.collect.IntCursor; 7 | import com.koloboke.collect.set.IntSet; 8 | import com.koloboke.collect.set.hash.HashIntSets; 9 | import com.vc.deg.FeatureSpace; 10 | import com.vc.deg.FeatureVector; 11 | import com.vc.deg.impl.graph.VertexView; 12 | import com.vc.deg.impl.graph.WeightedUndirectedRegularGraph; 13 | 14 | /** 15 | * 16 | * @author Neiko 17 | * 18 | */ 19 | public class GraphSearch { 20 | 21 | public static TreeSet search(FeatureVector query, int k, float eps, int[] forbiddenIds, int[] entryPoints, WeightedUndirectedRegularGraph graph) { 22 | final FeatureSpace featureSpace = graph.getFeatureSpace(); 23 | 24 | // list of checked ids 25 | final IntSet C = HashIntSets.newMutableSet(forbiddenIds, forbiddenIds.length + entryPoints.length); 26 | for (int id : entryPoints) 27 | C.add(id); 28 | 29 | // items to traverse, start with the initial node 30 | final PriorityQueue S = new PriorityQueue<>(); 31 | for (int id : entryPoints) { 32 | final VertexView obj = graph.getNodeView(id); 33 | S.add(new ObjectDistance(id, obj, featureSpace.computeDistance(query, obj.getFeature()))); 34 | } 35 | 36 | // result set 37 | final TreeSet R = new TreeSet<>(S); 38 | 39 | // search radius 40 | float r = Float.MAX_VALUE; 41 | 42 | // iterate as long as good elements are in S 43 | while(S.size() > 0) { 44 | final ObjectDistance s = S.poll(); 45 | 46 | // max distance reached 47 | if(s.dist > r * (1 + eps)) 48 | break; 49 | 50 | // traverse never seen nodes 51 | final IntCursor topListCursor = graph.getEdgeIds(s.id).cursor(); 52 | while(topListCursor.moveNext()) { 53 | final int neighborId = topListCursor.elem(); 54 | 55 | if(C.add(neighborId) == false) { 56 | final VertexView n = graph.getNodeView(neighborId); 57 | final float nDist = featureSpace.computeDistance(query, n.getFeature()); 58 | 59 | // follow this node further 60 | if(nDist <= r * (1 + eps)) { 61 | final ObjectDistance candidate = new ObjectDistance(neighborId, n, nDist); 62 | S.add(candidate); 63 | 64 | // remember the node 65 | if(nDist < r) { 66 | R.add(candidate); 67 | if(R.size() > k) { 68 | R.pollLast(); 69 | r = R.last().dist; 70 | } 71 | } 72 | } 73 | } 74 | } 75 | } 76 | 77 | return R; 78 | } 79 | 80 | /** 81 | * 82 | * @author Neiko 83 | * 84 | * @param 85 | */ 86 | public static class ObjectDistance implements Comparable { 87 | 88 | public final int id; 89 | public final VertexView obj; 90 | public final float dist; 91 | 92 | public ObjectDistance(int id, VertexView obj, float dist) { 93 | this.id = id; 94 | this.obj = obj; 95 | this.dist = dist; 96 | } 97 | 98 | @Override 99 | public int compareTo(ObjectDistance o) { 100 | int cmp = Float.compare(-dist, -o.dist); 101 | if(cmp == 0) 102 | cmp = Integer.compare(obj.hashCode(), o.obj.hashCode()); 103 | return cmp; 104 | } 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /java/deg-library/src/main/resources/META-INF/services/com.vc.deg.GraphFactory: -------------------------------------------------------------------------------- 1 | com.vc.deg.impl.GraphFactory -------------------------------------------------------------------------------- /java/deg-reference/build.gradle: -------------------------------------------------------------------------------- 1 | // Use the build script defined in buildSrc 2 | apply from: rootProject.file('buildSrc/shared.gradle') 3 | 4 | dependencies { 5 | implementation project(':deg-api') 6 | 7 | implementation group: 'com.koloboke', name: 'koloboke-api-jdk8', version: "${KOLOBOKE_VERSION}" 8 | runtimeOnly group: 'com.koloboke', name: 'koloboke-impl-jdk8', version: "${KOLOBOKE_VERSION}" 9 | runtimeOnly group: 'com.koloboke', name: 'koloboke-impl-common-jdk8', version: "${KOLOBOKE_VERSION}" 10 | 11 | implementation group: 'org.roaringbitmap', name: 'RoaringBitmap', version: "${ROARING_VERSION}" 12 | 13 | implementation group: "org.slf4j", name: "slf4j-api", version: "${SLF4J_VERSION}" 14 | testImplementation group: "org.slf4j", name: "log4j-over-slf4j", version: "${SLF4J_VERSION}" 15 | } 16 | 17 | // group and version of the project 18 | archivesBaseName = "deg-reference" 19 | 20 | 21 | 22 | //-------------------------------------------------------------------------------------------------- 23 | //------------------------------------- Publish to Maven ------------------------------------------- 24 | //-------------------------------------------------------------------------------------------------- 25 | 26 | java { 27 | withJavadocJar() 28 | withSourcesJar() 29 | } 30 | 31 | publishing { 32 | publications { 33 | mavenJava(MavenPublication) { 34 | from components.java 35 | 36 | pom { 37 | name = 'deg-reference' 38 | description = 'Reference implementation for constructing and using Dynamic Exploration Graph in approximate nearest neighbor search tasks.' 39 | url = 'http://visual-computing.com' 40 | 41 | licenses { 42 | license { 43 | name = 'MIT License' 44 | url = 'https://opensource.org/licenses/MIT' 45 | } 46 | } 47 | 48 | developers { 49 | developer { 50 | id = 'Neiko2002' 51 | name = 'Nico Hezel' 52 | email = 'hezel@visual-computing.com' 53 | organization = 'Visual Computing Group' 54 | organizationUrl = 'www.visual-computing.com' 55 | } 56 | } 57 | 58 | scm { 59 | connection = 'scm:git:git://github.com:Visual-Computing/DynamicExplorationGraph.git' 60 | developerConnection = 'scm:git:ssh://github.com:Visual-Computing/DynamicExplorationGraph.git' 61 | url = 'https://github.com/Visual-Computing/DynamicExplorationGraph/tree/master/' 62 | } 63 | } 64 | } 65 | } 66 | 67 | repositories { 68 | maven { 69 | url = layout.buildDirectory.dir("maven") 70 | } 71 | } 72 | } -------------------------------------------------------------------------------- /java/deg-reference/readme.md: -------------------------------------------------------------------------------- 1 | Referenz Implementierung von DNG. Keine Abhängigkeiten 3rd Party. Langsamer als die Library Version. 2 | Es wird sich an die API von DNG gehalten um Komponenten mit der Library Version austauschen zu können. 3 | -------------------------------------------------------------------------------- /java/deg-reference/src/main/java/com/vc/deg/ref/GraphFactory.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.ref; 2 | 3 | import java.io.IOException; 4 | import java.nio.file.Path; 5 | 6 | import com.vc.deg.FeatureSpace; 7 | 8 | public class GraphFactory implements com.vc.deg.GraphFactory { 9 | 10 | @Override 11 | public DynamicExplorationGraph newGraph(FeatureSpace space, int edgesPerNode) { 12 | return new DynamicExplorationGraph(space, edgesPerNode); 13 | } 14 | 15 | @Override 16 | public DynamicExplorationGraph loadGraph(Path file) throws IOException { 17 | return DynamicExplorationGraph.readFromFile(file); 18 | } 19 | 20 | @Override 21 | public DynamicExplorationGraph loadGraph(Path file, String componentType) throws IOException { 22 | return DynamicExplorationGraph.readFromFile(file, componentType); 23 | } 24 | 25 | 26 | // -------------------------------------------------------------------------------------- 27 | // ---------------------------- Hierarchical Graph -------------------------------------- 28 | // -------------------------------------------------------------------------------------- 29 | 30 | @Override 31 | public HierarchicalDynamicExplorationGraph newHierchicalGraph(FeatureSpace space, int edgesPerNode, int topRankSize) { 32 | return new HierarchicalDynamicExplorationGraph(space, edgesPerNode, topRankSize); 33 | } 34 | 35 | @Override 36 | public HierarchicalDynamicExplorationGraph loadHierchicalGraph(Path file) throws IOException { 37 | return HierarchicalDynamicExplorationGraph.readFromFile(file); 38 | } 39 | } 40 | 41 | -------------------------------------------------------------------------------- /java/deg-reference/src/main/java/com/vc/deg/ref/graph/MutableVertexFilter.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.ref.graph; 2 | 3 | import java.util.function.Consumer; 4 | import java.util.function.IntConsumer; 5 | 6 | import org.roaringbitmap.RoaringBitmap; 7 | import org.roaringbitmap.RoaringBitmapAdapter; 8 | 9 | import com.vc.deg.graph.VertexFilter; 10 | 11 | /** 12 | * A prepared filter provides a list of valid ids. 13 | * 14 | * @author Nico Hezel 15 | */ 16 | public class MutableVertexFilter extends RoaringBitmap implements VertexFilter { 17 | 18 | protected int validIdCount; 19 | protected int allElementCount; 20 | 21 | public MutableVertexFilter() { 22 | validIdCount = 0; 23 | allElementCount = 0; 24 | } 25 | 26 | public MutableVertexFilter(Consumer validIds, int allElementCount) { 27 | add(validIds); 28 | this.allElementCount = allElementCount; 29 | } 30 | 31 | public MutableVertexFilter(int[] validIds, int allElementCount) { 32 | super(); 33 | RoaringBitmapAdapter.move(RoaringBitmap.bitmapOfUnordered(validIds), this); 34 | this.allElementCount = allElementCount; 35 | this.validIdCount = validIds.length; 36 | } 37 | 38 | public MutableVertexFilter(RoaringBitmap validIds, int allElementCount) { 39 | super(); 40 | RoaringBitmapAdapter.move(validIds, this); 41 | this.allElementCount = allElementCount; 42 | this.validIdCount = validIds.getCardinality(); 43 | } 44 | 45 | public void and(MutableVertexFilter x2) { 46 | super.and(x2); 47 | this.validIdCount = getCardinality(); 48 | } 49 | 50 | 51 | public void andNot(MutableVertexFilter x2) { 52 | super.andNot(x2); 53 | this.validIdCount = getCardinality(); 54 | } 55 | 56 | public void add(Consumer v) { 57 | v.accept(label -> add(label)); 58 | } 59 | 60 | public void remove(Consumer v) { 61 | v.accept(label -> remove(label)); 62 | } 63 | 64 | public void setAllElementCount(int allElementCount) { 65 | this.allElementCount = allElementCount; 66 | } 67 | 68 | @Override 69 | public void add(int x) { 70 | validIdCount++; 71 | super.add(x); 72 | } 73 | 74 | @Override 75 | public void remove(int x) { 76 | validIdCount--; 77 | super.remove(x); 78 | } 79 | 80 | @Override 81 | public void forEachValidId(IntConsumer action) { 82 | forEach((int i) -> action.accept(i)); 83 | } 84 | 85 | @Override 86 | public boolean isValid(int id) { 87 | return contains(id); 88 | } 89 | 90 | @Override 91 | public int size() { 92 | return validIdCount; 93 | } 94 | 95 | @Override 96 | public float getInclusionRate() { 97 | return Math.max(0, Math.min(1, ((float)size()) / allElementCount)); 98 | } 99 | 100 | @Override 101 | public String toString() { 102 | return this.getClass().getSimpleName() + " with "+size()+" valid ids."; 103 | } 104 | 105 | @Override 106 | public MutableVertexFilter clone() { 107 | return (MutableVertexFilter) super.clone(); 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /java/deg-reference/src/main/java/com/vc/deg/ref/graph/QueryDistance.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.ref.graph; 2 | 3 | import java.util.Comparator; 4 | 5 | import com.vc.deg.FeatureVector; 6 | 7 | /** 8 | * Distance between a query and a vertex. 9 | * Natural order is ascending by distance. 10 | * 11 | * @author Nico Hezel 12 | * 13 | */ 14 | public class QueryDistance implements Comparable { 15 | 16 | protected final int queryId; 17 | protected final FeatureVector queryFeature; 18 | 19 | protected final VertexData vertex; 20 | 21 | protected final float distance; 22 | 23 | public QueryDistance(int queryId, FeatureVector queryFeature, VertexData vertex, float distance) { 24 | this.queryId = queryId; 25 | this.queryFeature = queryFeature; 26 | this.vertex = vertex; 27 | this.distance = distance; 28 | } 29 | 30 | public VertexData getVertex() { 31 | return vertex; 32 | } 33 | 34 | public int getVertexLabel() { 35 | return vertex.getLabel(); 36 | } 37 | 38 | public int getVertexId() { 39 | return vertex.getId(); 40 | } 41 | 42 | public float getDistance() { 43 | return distance; 44 | } 45 | 46 | @Override 47 | public String toString() { 48 | return "label:"+getVertexLabel()+", distance:"+distance; 49 | } 50 | 51 | @Override 52 | public int hashCode() { 53 | return getVertexId(); 54 | } 55 | 56 | @Override 57 | public int compareTo(QueryDistance o) { 58 | int cmp = Float.compare(getDistance(), o.getDistance()); 59 | if (cmp == 0) 60 | cmp = Integer.compare(getVertexLabel(), o.getVertexLabel()); 61 | return cmp; 62 | } 63 | 64 | /** 65 | * Order in ascending order using the index 66 | * 67 | * @return 68 | */ 69 | public static Comparator ascByIndex() { 70 | return Comparator.comparingInt(QueryDistance::getVertexLabel).thenComparingDouble(QueryDistance::getDistance); 71 | } 72 | 73 | /** 74 | * Order in descending order using the index 75 | * 76 | * @return 77 | */ 78 | public static Comparator descByIndex() { 79 | return ascByIndex().reversed(); 80 | } 81 | } -------------------------------------------------------------------------------- /java/deg-reference/src/main/java/com/vc/deg/ref/graph/VertexCursor.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.ref.graph; 2 | 3 | import java.util.Iterator; 4 | 5 | import com.vc.deg.FeatureVector; 6 | import com.vc.deg.graph.NeighborConsumer; 7 | 8 | public class VertexCursor implements com.vc.deg.graph.VertexCursor { 9 | 10 | protected final ArrayBasedWeightedUndirectedRegularGraph graph; 11 | protected final Iterator it; 12 | protected VertexData currentElement; 13 | 14 | public VertexCursor(ArrayBasedWeightedUndirectedRegularGraph graph) { 15 | this.graph = graph; 16 | this.it = graph.getVertices().iterator(); 17 | } 18 | 19 | @Override 20 | public boolean moveNext() { 21 | if(it.hasNext()) { 22 | currentElement = it.next(); 23 | return true; 24 | } 25 | 26 | return false; 27 | } 28 | 29 | @Override 30 | public int getVertexLabel() { 31 | return currentElement.getLabel(); 32 | } 33 | 34 | @Override 35 | public FeatureVector getVertexFeature() { 36 | return currentElement.getFeature(); 37 | } 38 | 39 | @Override 40 | public void forEachNeighbor(NeighborConsumer neighborConsumer) { 41 | currentElement.getEdges().forEach((int neighborId, float weight) -> 42 | neighborConsumer.accept(graph.getVertexById(neighborId).getLabel(), weight) 43 | ); 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /java/deg-reference/src/main/java/com/vc/deg/ref/graph/VertexData.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.ref.graph; 2 | 3 | import com.koloboke.collect.map.IntFloatMap; 4 | import com.koloboke.collect.map.hash.HashIntFloatMapFactory; 5 | import com.koloboke.collect.map.hash.HashIntFloatMaps; 6 | import com.vc.deg.FeatureVector; 7 | 8 | /** 9 | * @author Nico Hezel 10 | */ 11 | public class VertexData { 12 | 13 | private static final HashIntFloatMapFactory mapFactory = HashIntFloatMaps.getDefaultFactory().withDefaultValue(Integer.MIN_VALUE); 14 | 15 | private final int label; 16 | private final int internalId; 17 | private final FeatureVector data; 18 | private final IntFloatMap edges; 19 | 20 | public VertexData(int label, int internalId, FeatureVector data, int edgesPerVertex) { 21 | this.label = label; 22 | this.internalId = internalId; 23 | this.data = data; 24 | this.edges = mapFactory.newMutableMap(edgesPerVertex); 25 | } 26 | 27 | public VertexData(int label, int internalId, FeatureVector data, IntFloatMap edges) { 28 | this.label = label; 29 | this.internalId = internalId; 30 | this.data = data; 31 | this.edges = edges; 32 | } 33 | 34 | public int getId() { 35 | return internalId; 36 | } 37 | 38 | public int getLabel() { 39 | return label; 40 | } 41 | 42 | public FeatureVector getFeature() { 43 | return data; 44 | } 45 | 46 | public IntFloatMap getEdges() { 47 | return edges; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /java/deg-reference/src/main/java/com/vc/deg/ref/graph/VertexFilterFactory.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.ref.graph; 2 | 3 | import java.util.function.Consumer; 4 | import java.util.function.IntConsumer; 5 | 6 | import com.vc.deg.graph.VertexFilter; 7 | 8 | public class VertexFilterFactory implements com.vc.deg.graph.VertexFilterFactory { 9 | 10 | @Override 11 | public VertexFilter of(int[] validIds, int allElementCount) { 12 | return new MutableVertexFilter(validIds, allElementCount); 13 | } 14 | 15 | @Override 16 | public VertexFilter of(Consumer validIds, int allElementCount) { 17 | return new MutableVertexFilter(validIds, allElementCount); 18 | } 19 | 20 | @Override 21 | public void and(VertexFilter x1, VertexFilter x2) { 22 | final MutableVertexFilter m1 = (MutableVertexFilter)x1; 23 | final MutableVertexFilter m2 = (MutableVertexFilter)x2; 24 | m1.and(m2); 25 | } 26 | 27 | @Override 28 | public void andNot(VertexFilter x1, VertexFilter x2) { 29 | final MutableVertexFilter m1 = (MutableVertexFilter)x1; 30 | final MutableVertexFilter m2 = (MutableVertexFilter)x2; 31 | m1.andNot(m2); 32 | } 33 | 34 | @Override 35 | public void add(VertexFilter x1, Consumer x2) { 36 | final MutableVertexFilter m1 = (MutableVertexFilter)x1; 37 | m1.add(x2); 38 | } 39 | 40 | @Override 41 | public void remove(VertexFilter x1, Consumer x2) { 42 | final MutableVertexFilter m1 = (MutableVertexFilter)x1; 43 | m1.remove(x2); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /java/deg-reference/src/main/java/org/roaringbitmap/RoaringBitmapAdapter.java: -------------------------------------------------------------------------------- 1 | package org.roaringbitmap; 2 | 3 | public class RoaringBitmapAdapter { 4 | 5 | public static RoaringArray getHighLowContainer(RoaringBitmap rm) { 6 | return rm.highLowContainer; 7 | } 8 | 9 | public static void move(RoaringArray obj, RoaringBitmap to) { 10 | to.highLowContainer = obj; 11 | } 12 | 13 | public static void move(RoaringBitmap from, RoaringBitmap to) { 14 | move(getHighLowContainer(from), to); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /java/deg-reference/src/main/resources/META-INF/services/com.vc.deg.GraphFactory: -------------------------------------------------------------------------------- 1 | com.vc.deg.ref.GraphFactory -------------------------------------------------------------------------------- /java/deg-reference/src/main/resources/META-INF/services/com.vc.deg.graph.VertexFilterFactory: -------------------------------------------------------------------------------- 1 | com.vc.deg.ref.graph.VertexFilterFactory -------------------------------------------------------------------------------- /java/deg-visualization/build.gradle: -------------------------------------------------------------------------------- 1 | // Use the build script defined in buildSrc 2 | apply from: rootProject.file('buildSrc/shared.gradle') 3 | 4 | dependencies { 5 | implementation project(':deg-api') 6 | runtimeOnly project(':deg-reference') 7 | 8 | implementation group: 'com.koloboke', name: 'koloboke-api-jdk8', version: "${KOLOBOKE_VERSION}" 9 | runtimeOnly group: 'com.koloboke', name: 'koloboke-impl-jdk8', version: "${KOLOBOKE_VERSION}" 10 | runtimeOnly group: 'com.koloboke', name: 'koloboke-impl-common-jdk8', version: "${KOLOBOKE_VERSION}" 11 | 12 | implementation group: 'org.roaringbitmap', name: 'RoaringBitmap', version: "${ROARING_VERSION}" 13 | 14 | implementation group: "org.slf4j", name: "slf4j-api", version: "${SLF4J_VERSION}" 15 | testImplementation group: "org.slf4j", name: "log4j-over-slf4j", version: "${SLF4J_VERSION}" 16 | } 17 | 18 | 19 | // group and version of the project 20 | archivesBaseName = "deg-visualization" 21 | 22 | 23 | 24 | //-------------------------------------------------------------------------------------------------- 25 | //------------------------------------- Publish to Maven ------------------------------------------- 26 | //-------------------------------------------------------------------------------------------------- 27 | 28 | java { 29 | withJavadocJar() 30 | withSourcesJar() 31 | } 32 | 33 | publishing { 34 | publications { 35 | mavenJava(MavenPublication) { 36 | from components.java 37 | 38 | pom { 39 | name = 'deg-visualization' 40 | description = '2D visualization and exploration of a Dynamic Exploration Graph.' 41 | url = 'http://visual-computing.com' 42 | 43 | licenses { 44 | license { 45 | name = 'MIT License' 46 | url = 'https://opensource.org/licenses/MIT' 47 | } 48 | } 49 | 50 | developers { 51 | developer { 52 | id = 'Neiko2002' 53 | name = 'Nico Hezel' 54 | email = 'hezel@visual-computing.com' 55 | organization = 'Visual Computing Group' 56 | organizationUrl = 'www.visual-computing.com' 57 | } 58 | } 59 | 60 | scm { 61 | connection = 'scm:git:git://github.com:Visual-Computing/DynamicExplorationGraph.git' 62 | developerConnection = 'scm:git:ssh://github.com:Visual-Computing/DynamicExplorationGraph.git' 63 | url = 'https://github.com/Visual-Computing/DynamicExplorationGraph/tree/master/' 64 | } 65 | } 66 | } 67 | } 68 | 69 | repositories { 70 | maven { 71 | url = layout.buildDirectory.dir("maven") 72 | } 73 | } 74 | } -------------------------------------------------------------------------------- /java/deg-visualization/readme.md: -------------------------------------------------------------------------------- 1 | Enthält die Exploration und Navigation von 2D Karten dennen ein Graph zur Grunde liegt. 2 | Im Testbereich gibt es eine JavaFX Visualisierung der Graphen Navigation und eine Debugging UI . -------------------------------------------------------------------------------- /java/deg-visualization/src/main/java/com/vc/deg/viz/MapNavigator.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.viz; 2 | 3 | import com.vc.deg.graph.VertexFilter; 4 | import com.vc.deg.viz.model.GridMap; 5 | import com.vc.deg.viz.model.MotionVector; 6 | import com.vc.deg.viz.model.WorldMap; 7 | 8 | /** 9 | * The MapNavigator is statefull in regards to the user calling it 10 | * 11 | * @author Nico Hezel 12 | */ 13 | public class MapNavigator { 14 | 15 | protected final WorldMap worldMap; 16 | protected final MapDesigner designer; 17 | 18 | // current level of the navigator 19 | protected int currentLevel; 20 | 21 | // current position on the world map, where the local map starts 22 | protected int worldPosX; 23 | protected int worldPosY; 24 | 25 | public MapNavigator(WorldMap worldMap, MapDesigner mapDesigner) { 26 | this.worldMap = worldMap; 27 | this.designer = mapDesigner; 28 | 29 | // start in the middle of the world 30 | reset(); 31 | } 32 | 33 | public int getLevel() { 34 | return currentLevel; 35 | } 36 | 37 | public int getWorldPosX() { 38 | return worldPosX; 39 | } 40 | 41 | public int getWorldPosY() { 42 | return worldPosY; 43 | } 44 | 45 | @Override 46 | public String toString() { 47 | return "Position "+worldPosX+"/"+worldPosY; 48 | } 49 | 50 | // ------------------------------------------------------------------------------------------- 51 | // ----------------------------- Navigation Methods ------------------------------------------ 52 | // ------------------------------------------------------------------------------------------- 53 | 54 | /** 55 | * Set the current world map position 56 | * 57 | * @param x 58 | * @param y 59 | */ 60 | protected void setWorldPosition(int x, int y) { 61 | this.worldPosX = x; 62 | this.worldPosY = y; 63 | } 64 | 65 | /** 66 | * Reset the entire world map and the position of the navigator 67 | */ 68 | public void reset() { 69 | this.worldPosX = 0; 70 | this.worldPosY = 0; 71 | this.worldMap.clear(); 72 | } 73 | 74 | /** 75 | * Move x/y steps on the world map and fill the local map with the resulting world map data. 76 | * A normalized direction vector (sum of components = 1) allows fine-grain definition of the direction. 77 | * 78 | * @param map 79 | * @param x 80 | * @param y 81 | * @param directionX 82 | * @param directionY 83 | * @param filter can be null 84 | * @return 85 | */ 86 | public boolean move(GridMap map, int x, int y, double directionX, double directionY, VertexFilter filter) { 87 | 88 | if(x != 0 || y != 0) { 89 | setWorldPosition(worldPosX - x, worldPosY - y); 90 | 91 | // Weltkarten aktualisieren 92 | MotionVector shiftVector = new MotionVector(x, y); 93 | MotionVector directionVector = new MotionVector(directionX, directionY); 94 | designer.move(worldMap, map, shiftVector, directionVector, worldPosX, worldPosY, this.currentLevel, filter); 95 | 96 | return true; 97 | } 98 | 99 | return false; 100 | } 101 | 102 | /** 103 | * Clear the world map and reset the position of the navigator to the center of the now empty world. 104 | * Jump in the graph to a position most similar to the content. Place the content at the target position in the map. 105 | * Arrange other similar images from the graph around the image on the map. Use the graph at given the level. 106 | * 107 | * @param map 108 | * @param content 109 | * @param posX 110 | * @param posY 111 | * @param toLevel 112 | * @param filter can be null 113 | */ 114 | public void jump(GridMap map, int content, int posX, int posY, int toLevel, VertexFilter filter) { 115 | this.currentLevel = toLevel; 116 | reset(); 117 | designer.jump(worldMap, map, content, posX, posY, worldPosX, worldPosY, toLevel, filter); 118 | } 119 | 120 | /** 121 | * Move to the position on the world map. Fill the local map with the elements from the world map. 122 | * If there are still holes fill them with the neighboring elements. 123 | * In the case there are no elements at all, an empty map will be returned. 124 | * 125 | * @param map 126 | * @param worldPosX 127 | * @param worldPosY 128 | * @param filter can be null 129 | */ 130 | public void explore(GridMap map, int worldPosX, int worldPosY, VertexFilter filter) { 131 | setWorldPosition(worldPosX, worldPosY); 132 | designer.fill(worldMap, map, worldPosX, worldPosY, this.currentLevel, filter); 133 | } 134 | 135 | } -------------------------------------------------------------------------------- /java/deg-visualization/src/main/java/com/vc/deg/viz/model/GridMap.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.viz.model; 2 | 3 | import java.util.Arrays; 4 | import java.util.Random; 5 | 6 | 7 | /** 8 | * A grid of ids. 9 | * All ids must be positive 10 | * 11 | * @author Nico Hezel 12 | */ 13 | public class GridMap { 14 | 15 | /** 16 | * Create a grid filled with -1 values 17 | * 18 | * @param rows must be greater than zero 19 | * @param columns must be greater than zero 20 | * @return 21 | */ 22 | protected static int[] emptyGrid(int rows, int columns) { 23 | final int[] cells = new int[rows * columns]; 24 | Arrays.fill(cells, -1); 25 | return cells; 26 | } 27 | 28 | // dimensions of the map 29 | protected final int rows; 30 | protected final int columns; 31 | protected final int[] cells; 32 | 33 | /** 34 | * 35 | * @param rows must be greater than zero 36 | * @param columns must be greater than zero 37 | */ 38 | public GridMap(int rows, int columns) { 39 | this(rows, columns, emptyGrid(rows, columns)); 40 | } 41 | 42 | /** 43 | * 44 | * @param rows must be greater than zero 45 | * @param columns must be greater than zero 46 | * @param cells can not be null 47 | */ 48 | public GridMap(int rows, int columns, int[] cells) { 49 | this.rows = rows; 50 | this.columns = columns; 51 | this.cells = cells; 52 | } 53 | 54 | public int rows() { 55 | return rows; 56 | } 57 | 58 | public int columns() { 59 | return columns; 60 | } 61 | 62 | /** 63 | * Count of all cells in the grid. 64 | * Count = Rows * Columns 65 | * 66 | * @return 67 | */ 68 | public int size() { 69 | return rows()*columns(); 70 | } 71 | 72 | 73 | /** 74 | * Get the content of a cell by x/y 75 | * 76 | * @param x 77 | * @param y 78 | * @return 79 | */ 80 | public int get(int x, int y) { 81 | return get(y*columns()+x); 82 | } 83 | 84 | /** 85 | * Get the content of a cell by using the index of the cell. 86 | * Indices go from 0 to size()-1. 87 | * 88 | * @param index 89 | * @return 90 | */ 91 | public int get(int index) { 92 | return cells[index]; 93 | } 94 | 95 | /** 96 | * Set the content of a cell 97 | * 98 | * @param x 99 | * @param y 100 | * @param content 101 | */ 102 | public void set(int x, int y, int content) { 103 | set(y*columns()+x, content); 104 | } 105 | 106 | /** 107 | * Set the content of a cell using a index. 108 | * Indices go from 0 to size()-1. 109 | * 110 | * @param index 111 | * @param content 112 | */ 113 | public void set(int index, int content) { 114 | cells[index] = content; 115 | } 116 | 117 | /** 118 | * Check if the entire map is empty 119 | * 120 | * @return 121 | */ 122 | public boolean isEmpty() { 123 | for (int i = 0; i < size(); i++) { 124 | if(get(i) != -1) 125 | return false; 126 | } 127 | return true; 128 | } 129 | 130 | /** 131 | * Check if the cell is empty 132 | * 133 | * @param x 134 | * @param y 135 | * @return 136 | */ 137 | public boolean isEmpty(int x, int y) { 138 | return isEmpty(y*columns()+x); 139 | } 140 | 141 | /** 142 | * Check if the cell is empty 143 | * 144 | * @param index 145 | * @return 146 | */ 147 | public boolean isEmpty(int index) { 148 | return get(index) == -1; 149 | } 150 | 151 | 152 | /** 153 | * Count how many map places are empty 154 | * 155 | * @return 156 | */ 157 | public int freeCount() { 158 | int freeCount = 0; 159 | for (int i = 0; i < size(); i++) 160 | if(get(i) == -1) 161 | freeCount++; 162 | return freeCount; 163 | } 164 | 165 | /** 166 | * Shuffle the map cells 167 | * 168 | * @param rand 169 | */ 170 | public void shuffle(Random rand) { 171 | for (int i = 0; i < cells.length; i++) { 172 | int randomIndexToSwap = rand.nextInt(cells.length); 173 | int temp = cells[randomIndexToSwap]; 174 | cells[randomIndexToSwap] = cells[i]; 175 | cells[i] = temp; 176 | } 177 | } 178 | 179 | /** 180 | * Remove the content of the entire map 181 | */ 182 | public void clear() { 183 | Arrays.fill(cells, -1); 184 | } 185 | 186 | /** 187 | * Create a copy of this map 188 | * @return 189 | */ 190 | public GridMap copy() { 191 | return new GridMap(rows, columns, Arrays.copyOf(cells, cells.length)); 192 | } 193 | 194 | @Override 195 | public String toString() { 196 | return "Grid with columns:"+columns()+", rows:"+rows(); 197 | } 198 | } 199 | -------------------------------------------------------------------------------- /java/deg-visualization/src/main/java/com/vc/deg/viz/model/MotionVector.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.viz.model; 2 | 3 | public class MotionVector { 4 | 5 | public static final MotionVector UP = new MotionVector(0, -1); 6 | public static final MotionVector DOWN = new MotionVector(0, 1); 7 | public static final MotionVector LEFT = new MotionVector(-1, 0); 8 | public static final MotionVector RIGHT = new MotionVector(1, 0); 9 | 10 | protected final double x; 11 | protected final double y; 12 | 13 | public MotionVector(double x, double y) { 14 | this.x = x; 15 | this.y = y; 16 | } 17 | 18 | public double getX() { 19 | return x; 20 | } 21 | 22 | public double getY() { 23 | return y; 24 | } 25 | 26 | public double length() { 27 | return Math.abs(x) + Math.abs(y); 28 | } 29 | 30 | public MotionVector normalize() { 31 | double max = Math.max(Math.abs(x), Math.abs(y)); 32 | return new MotionVector(x / max, y / max); 33 | } 34 | 35 | @Override 36 | public String toString() { 37 | return "x:"+x+", y:"+y; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /java/deg-visualization/src/main/java/com/vc/deg/viz/om/FLASnoMapSorterAdapter.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.viz.om; 2 | 3 | import java.util.Random; 4 | import java.util.function.IntFunction; 5 | 6 | import com.vc.deg.FeatureSpace; 7 | import com.vc.deg.FeatureVector; 8 | import com.vc.deg.viz.model.GridMap; 9 | import com.vc.deg.viz.om.FLASnoMapSorter.MapPlace; 10 | 11 | 12 | /** 13 | * Can handle holes and fixed cells 14 | * 15 | * @author barthel and hezel 16 | */ 17 | public class FLASnoMapSorterAdapter { 18 | 19 | private final IntFunction idToFloatFeature; 20 | private final FeatureSpace distFunc; 21 | 22 | public FLASnoMapSorterAdapter(IntFunction idToFloatFeature, FeatureSpace distFunc) { 23 | this.idToFloatFeature = idToFloatFeature; 24 | this.distFunc = distFunc; 25 | } 26 | 27 | /** 28 | * Start the swapping process 29 | * 30 | * @param map 31 | * @param inUse[y][x] 32 | */ 33 | public void arrangeWithHoles(GridMap map, boolean[][] inUse) { 34 | final int rows = map.rows(); 35 | final int columns = map.columns(); 36 | 37 | // copy the data 38 | final MapPlace[] mapPlaces = new MapPlace[columns * rows]; 39 | for (int y = 0; y < rows; y++) { 40 | for (int x = 0; x < columns; x++) { 41 | final int id = map.get(x, y); 42 | final FeatureVector feature = (id == -1) ? null : idToFloatFeature.apply(id); 43 | mapPlaces[x + y * columns] = new MapPlace(id, feature, inUse[y][x]); 44 | } 45 | } 46 | 47 | final Random rnd = new Random(7); 48 | final FLASnoMapSorter flas = new FLASnoMapSorter(rnd); 49 | flas.doSorting(mapPlaces, columns, rows, distFunc); 50 | 51 | // apply the new order to the map 52 | for (int y = 0; y < rows; y++) { 53 | for (int x = 0; x < columns; x++) { 54 | final MapPlace mapPlace = mapPlaces[x + y * columns]; 55 | map.set(x, y, mapPlace.getId()); 56 | } 57 | } 58 | } 59 | } -------------------------------------------------------------------------------- /java/deg-visualization/src/main/java/com/vc/deg/viz/om/JonkerVolgenantSolver.java: -------------------------------------------------------------------------------- 1 | package com.vc.deg.viz.om; 2 | 3 | 4 | public class JonkerVolgenantSolver { 5 | 6 | /** 7 | * https://github.com/heetch/lapjv/blob/master/solver.go 8 | * 9 | * @param matrix 10 | * @return 11 | */ 12 | public static int[] computeAssignment(int[][] matrix) { 13 | return computeAssignment(matrix, matrix.length); 14 | } 15 | 16 | public static int[] computeAssignment(int[][] matrix, int dim) { 17 | int i, imin, i0, freerow; 18 | int j, j1, j2=0, endofpath=0, last=0, min=0; 19 | 20 | int[] inRow = new int[dim]; 21 | int[] inCol = new int[dim]; 22 | 23 | //int[] u = new int[dim]; 24 | int[] v = new int[dim]; 25 | int[] free = new int[dim]; 26 | int[] collist = new int[dim]; 27 | int[] matches = new int[dim]; 28 | int[] pred = new int[dim]; 29 | 30 | int[] d = new int[dim]; 31 | 32 | // skipping L53-54 33 | for (j = dim - 1; j >= 0; j--) { 34 | min = matrix[0][j]; 35 | imin = 0; 36 | for (i = 1; i < dim; i++) { 37 | if (matrix[i][j] < min) { 38 | min = matrix[i][j]; 39 | imin = i; 40 | } 41 | } 42 | 43 | v[j] = min; 44 | matches[imin]++; 45 | if (matches[imin] == 1) { 46 | inRow[imin] = j; 47 | inCol[j] = imin; 48 | } else { 49 | inCol[j] = -1; 50 | } 51 | } 52 | 53 | int numfree=0; 54 | for (i = 0; i < dim; i++) { 55 | if (matches[i] == 0) { 56 | free[numfree] = i; 57 | numfree++; 58 | } else if (matches[i] == 1) { 59 | j1 = inRow[i]; 60 | min = Integer.MAX_VALUE; 61 | for (j = 0; j < dim; j++) { 62 | if (j != j1 && matrix[i][j]-v[j] < min) { 63 | min = matrix[i][j] - v[j]; 64 | } 65 | } 66 | v[j1] -= min; 67 | } 68 | } 69 | 70 | for (int loopcmt = 0; loopcmt < 2; loopcmt++) { 71 | int k = 0; 72 | int prvnumfree = numfree; 73 | numfree = 0; 74 | while (k < prvnumfree) { 75 | i = free[k]; 76 | k++; 77 | int umin = matrix[i][0] - v[0]; 78 | j1 = 0; 79 | int usubmin = Integer.MAX_VALUE; 80 | 81 | for (j = 1; j < dim; j++) { 82 | int h = matrix[i][j] - v[j]; 83 | 84 | if (h < usubmin) { 85 | if (h >= umin) { 86 | usubmin = h; 87 | j2 = j; 88 | } else { 89 | usubmin = umin; 90 | umin = h; 91 | j2 = j1; 92 | j1 = j; 93 | } 94 | } 95 | } 96 | 97 | i0 = inCol[j1]; 98 | if (umin < usubmin) { 99 | v[j1] = v[j1] - (usubmin - umin); 100 | } else if (i0 >= 0) { 101 | j1 = j2; 102 | i0 = inCol[j2]; 103 | } 104 | 105 | inRow[i] = j1; 106 | inCol[j1] = i; 107 | if (i0 >= 0) { 108 | if (umin < usubmin) { 109 | k--; 110 | free[k] = i0; 111 | } else { 112 | free[numfree] = i0; 113 | numfree++; 114 | } 115 | } 116 | } 117 | } 118 | 119 | for (int f = 0; f < numfree; f++) { 120 | freerow = free[f]; 121 | for (j = 0; j < dim; j++) { 122 | d[j] = matrix[freerow][j] - v[j]; 123 | pred[j] = freerow; 124 | collist[j] = j; 125 | } 126 | 127 | int low = 0; 128 | int up = 0; 129 | boolean unassignedfound = false; 130 | 131 | while (!unassignedfound) { 132 | if (up == low) { 133 | last = low - 1; 134 | min = d[collist[up]]; 135 | up++; 136 | 137 | for (int k = up; k < dim; k++) { 138 | j = collist[k]; 139 | int h = d[j]; 140 | if (h <= min) { 141 | if (h < min) { 142 | up = low; 143 | min = h; 144 | } 145 | collist[k] = collist[up]; 146 | collist[up] = j; 147 | up++; 148 | } 149 | } 150 | 151 | for (int k = low; k < up; k++) { 152 | if (inCol[collist[k]] < 0) { 153 | endofpath = collist[k]; 154 | unassignedfound = true; 155 | break; 156 | } 157 | } 158 | } 159 | 160 | if (!unassignedfound) { 161 | j1 = collist[low]; 162 | low++; 163 | i = inCol[j1]; 164 | int h = matrix[i][j1] - v[j1] - min; 165 | 166 | for (int k = up; k < dim; k++) { 167 | j = collist[k]; 168 | int v2 = matrix[i][j] - v[j] - h; 169 | 170 | if (v2 < d[j]) { 171 | pred[j] = i; 172 | 173 | if (v2 == min) { 174 | if (inCol[j] < 0) { 175 | endofpath = j; 176 | unassignedfound = true; 177 | break; 178 | } else { 179 | collist[k] = collist[up]; 180 | collist[up] = j; 181 | up++; 182 | } 183 | } 184 | 185 | d[j] = v2; 186 | } 187 | } 188 | } 189 | } 190 | 191 | for (int k = 0; k <= last; k++) { 192 | j1 = collist[k]; 193 | v[j1] += d[j1] - min; 194 | } 195 | 196 | i = freerow + 1; 197 | while (i != freerow) { 198 | i = pred[endofpath]; 199 | inCol[endofpath] = i; 200 | j1 = endofpath; 201 | endofpath = inRow[i]; 202 | inRow[i] = j1; 203 | } 204 | } 205 | 206 | return inRow; 207 | } 208 | } -------------------------------------------------------------------------------- /java/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /java/gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%" == "" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%" == "" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 34 | 35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 37 | 38 | @rem Find java.exe 39 | if defined JAVA_HOME goto findJavaFromJavaHome 40 | 41 | set JAVA_EXE=java.exe 42 | %JAVA_EXE% -version >NUL 2>&1 43 | if "%ERRORLEVEL%" == "0" goto init 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | 51 | goto fail 52 | 53 | :findJavaFromJavaHome 54 | set JAVA_HOME=%JAVA_HOME:"=% 55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 56 | 57 | if exist "%JAVA_EXE%" goto init 58 | 59 | echo. 60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 61 | echo. 62 | echo Please set the JAVA_HOME variable in your environment to match the 63 | echo location of your Java installation. 64 | 65 | goto fail 66 | 67 | :init 68 | @rem Get command-line arguments, handling Windows variants 69 | 70 | if not "%OS%" == "Windows_NT" goto win9xME_args 71 | 72 | :win9xME_args 73 | @rem Slurp the command line arguments. 74 | set CMD_LINE_ARGS= 75 | set _SKIP=2 76 | 77 | :win9xME_args_slurp 78 | if "x%~1" == "x" goto execute 79 | 80 | set CMD_LINE_ARGS=%* 81 | 82 | :execute 83 | @rem Setup the command line 84 | 85 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 86 | 87 | 88 | @rem Execute Gradle 89 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 90 | 91 | :end 92 | @rem End local scope for the variables with windows NT shell 93 | if "%ERRORLEVEL%"=="0" goto mainEnd 94 | 95 | :fail 96 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 97 | rem the _cmd.exe /c_ return code! 98 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 99 | exit /b 1 100 | 101 | :mainEnd 102 | if "%OS%"=="Windows_NT" endlocal 103 | 104 | :omega 105 | -------------------------------------------------------------------------------- /java/readme.md: -------------------------------------------------------------------------------- 1 | # DEG: Java library of the Dynamic Exploration Graph 2 | 3 | Java library of the Dynamic Exploration Graph (DEG) used in the paper: 4 | [Fast Approximate Nearest Neighbor Search with a Dynamic Exploration Graph using Continuous Refinement](https://arxiv.org/abs/2307.10479) 5 | 6 | 7 | -------------------------------------------------------------------------------- /java/settings.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * This file was generated by the Gradle 'init' task. 3 | * 4 | * The settings file is used to specify which projects to include in your build. 5 | * 6 | * Detailed information about configuring a multi-project build in Gradle can be found 7 | * in the user manual at https://docs.gradle.org/5.6.4/userguide/multi_project_builds.html 8 | */ 9 | 10 | rootProject.name = 'DynamicExplorationGraph' 11 | 12 | include "deg-api" 13 | include "deg-library" 14 | include "deg-reference" 15 | include "deg-benchmark" 16 | include "deg-visualization" -------------------------------------------------------------------------------- /python/.gitignore: -------------------------------------------------------------------------------- 1 | /deglib.egg-info 2 | __pycache__ 3 | /lib 4 | /dist 5 | /wheelhouse 6 | /cmake-build* 7 | *.egg-info 8 | *.so 9 | -------------------------------------------------------------------------------- /python/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # python-bindings/CMakeLists.txt 2 | cmake_minimum_required(VERSION 3.19) 3 | project(deglib_cpp) 4 | 5 | # Add python executable 6 | if(DEFINED PYTHON_EXECUTABLE) 7 | set(Python_EXECUTABLE ${PYTHON_EXECUTABLE}) 8 | endif() 9 | find_package(Python REQUIRED COMPONENTS Interpreter Development.Module) 10 | 11 | # Add pybind11 12 | find_package(pybind11 CONFIG REQUIRED) 13 | 14 | # disable benchmark building 15 | set(ENABLE_BENCHMARKS OFF CACHE BOOL "Disable benchmarks" FORCE) 16 | 17 | # Add cpp-deglib 18 | add_subdirectory(${CMAKE_SOURCE_DIR}/lib/) 19 | pybind11_add_module(deglib_cpp src/deg_cpp/deglib_cpp.cpp) 20 | 21 | # Specify the include directories 22 | include_directories(${CMAKE_SOURCE_DIR}/lib/deglib/include) 23 | 24 | target_link_libraries(deglib_cpp PRIVATE compile-options) 25 | -------------------------------------------------------------------------------- /python/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Visual Computing Group 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /python/MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include README.md 3 | recursive-include src *.cpp 4 | recursive-include lib *.h 5 | global-include CMakeLists.txt *.cmake 6 | -------------------------------------------------------------------------------- /python/publish.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | read -p "have you tested with BuildAndTest-Workflow? y/[n]: " answer 6 | if [ "$answer" != "y" ]; then 7 | exit 0 8 | fi 9 | 10 | # get new version from __init__.py 11 | new_version=$(grep --color=never -oP '__version__\s*=\s*"\K[0-9]+\.[0-9]+\.[0-9]+' "src/deglib/__init__.py") 12 | echo "detected new version: $new_version" 13 | 14 | # get old version from pypi 15 | old_version=$(curl -s "https://pypi.org/pypi/deglib/json" | jq -r '.info.version') 16 | 17 | # compare versions 18 | if [ "$old_version" = "$new_version" ]; then 19 | echo "ERROR: current version equals version on pypi" 20 | read -p "have you changed the version in \"src/deglib/__init__.py\"? y/[n]: " answer 21 | if [ "$answer" != "y" ]; then 22 | exit 0 23 | fi 24 | fi 25 | 26 | # publish package 27 | git add -A && git commit -m "v$new_version" 28 | git push 29 | 30 | # publish tag 31 | git tag -a "v$new_version" -m "v$new_version" 32 | git push origin --tags 33 | 34 | -------------------------------------------------------------------------------- /python/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = [ 3 | "setuptools", 4 | "pybind11==2.12.0", 5 | "wheel", 6 | "ninja", 7 | "cmake>=3.12" 8 | ] 9 | build-backend = "setuptools.build_meta" 10 | 11 | [project] 12 | name = "deglib" 13 | dynamic = ["version"] 14 | dependencies = [ 15 | "numpy>=2.2", 16 | "psutil", 17 | ] 18 | requires-python = ">= 3.11" 19 | authors = [ 20 | {name = "Bruno Schilling", email = "bruno.schilling@htw-berlin.de"}, 21 | {name = "Nico Hezel"}, 22 | ] 23 | description = "Python bindings for the Dynamic Exploration Graph library by Nico Hezel" 24 | readme = "README.md" 25 | license = {file = "LICENSE"} 26 | keywords = ["anns-search", "graph", "python"] 27 | 28 | classifiers = [ 29 | "Development Status :: 2 - Pre-Alpha", 30 | "Intended Audience :: Developers", 31 | "Programming Language :: Python :: 3", 32 | ] 33 | 34 | [project.urls] 35 | Homepage = "https://github.com/Visual-Computing/DynamicExplorationGraph" 36 | 37 | [project.optional-dependencies] 38 | test = ["pytest>=8.0"] 39 | 40 | [tool.pytest.ini_options] 41 | minversion = "6.0" 42 | addopts = ["-ra", "--showlocals", "--strict-markers", "--strict-config"] 43 | xfail_strict = true 44 | filterwarnings = [ 45 | "error", 46 | "ignore:(ast.Str|Attribute s|ast.NameConstant|ast.Num) is deprecated:DeprecationWarning:_pytest", 47 | ] 48 | testpaths = ["tests"] 49 | 50 | [tool.cibuildwheel] 51 | test-command = "pytest {project}" 52 | test-extras = ["test"] 53 | -------------------------------------------------------------------------------- /python/src/deglib/__init__.py: -------------------------------------------------------------------------------- 1 | from deglib_cpp import avx_usable, sse_usable 2 | 3 | from . import analysis 4 | from . import benchmark 5 | from . import builder 6 | from . import graph 7 | from . import repository 8 | from . import search 9 | from .std import Mt19937 10 | from .distances import Metric, FloatSpace 11 | 12 | 13 | __version__ = "0.1.3" 14 | 15 | 16 | __all__ = [ 17 | 'avx_usable', 'sse_usable', 'graph', 'benchmark', 'Metric', 'FloatSpace', 18 | 'builder', 'Mt19937', 'analysis', 'repository', 'search' 19 | ] 20 | -------------------------------------------------------------------------------- /python/src/deglib/analysis.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from deglib_cpp import (calc_avg_edge_weight as calc_avg_edge_weight_cpp, 4 | calc_edge_weight_histogram as calc_edge_weight_histogram_cpp, 5 | check_graph_weights as check_graph_weights_cpp, 6 | check_graph_regularity as check_graph_regularity_cpp, 7 | check_graph_connectivity as check_graph_connectivity_cpp, 8 | calc_non_rng_edges as calc_non_rng_edges_cpp 9 | ) 10 | from .graph import MutableGraph, SearchGraph 11 | 12 | 13 | def calc_avg_edge_weight(graph: MutableGraph, scale: int = 1) -> float: 14 | """ 15 | Compute the average weight of all edges in the input graph. 16 | Weights are scaled by the specified scale factor. 17 | 18 | This function uses the C++ version (calc_avg_edge_weight_cpp) in deglib_cpp package as a backend. 19 | 20 | 21 | :param graph: The graph for which the average edge weight is calculated. 22 | :param scale: The scale factor by which edge weights are multiplied (default is 1). 23 | 24 | :returns: The average edge weight in the graph after scaling. 25 | """ 26 | if not isinstance(graph, MutableGraph): 27 | raise TypeError('Expected type of graph to be MutableGraph, but got: {}'.format(type(graph))) 28 | return calc_avg_edge_weight_cpp(graph.to_cpp(), scale) 29 | 30 | 31 | def calc_edge_weight_histogram(graph: MutableGraph, sort: bool, scale: int = 1) -> List[float]: 32 | """ 33 | The function calculates a histogram of edge weights for a given graph by: 34 | 35 | - Collecting all non-zero edge weights. 36 | - Optionally sorting these weights. 37 | - Dividing the weights into 10 bins. 38 | - Computing and scaling the average weight for each bin. 39 | 40 | The result is a vector containing the scaled average weights of the edge weights in each bin. 41 | 42 | :param graph: The graph to calculate the average edge weight for 43 | :param sort: sort edge weights before creating histogram 44 | :param scale: scale factor for each edge weight 45 | 46 | :returns: A list of 10 float values representing the scaled average weights 47 | """ 48 | return calc_edge_weight_histogram_cpp(graph.to_cpp(), sort, scale) 49 | 50 | 51 | def check_graph_weights(graph: MutableGraph) -> bool: 52 | """ 53 | Check if the weights of the graph are still the same to the distance of the vertices 54 | 55 | :param graph: The graph to calculate the average edge weight for 56 | 57 | :returns: True, if the graph are still the same to the distance of the vertices otherwise False 58 | """ 59 | return check_graph_weights_cpp(graph.to_cpp()) 60 | 61 | 62 | def check_graph_regularity(graph: SearchGraph, expected_vertices: int, check_back_link: bool = False) -> bool: 63 | """ 64 | TODO: rework documentation 65 | Is the vertex_index an RNG conform neighbor if it gets connected to target_index? 66 | 67 | Does vertex_index has a neighbor which is connected to the target_index and has a lower weight? 68 | """ 69 | return check_graph_regularity_cpp(graph.to_cpp(), expected_vertices, check_back_link) 70 | 71 | 72 | def check_graph_connectivity(graph: SearchGraph) -> bool: 73 | """ 74 | Check if the graph is connected and contains only one graph component. 75 | 76 | :param graph: The graph to check connectivity for 77 | """ 78 | return check_graph_connectivity_cpp(graph.to_cpp()) 79 | 80 | 81 | def calc_non_rng_edges(graph: MutableGraph) -> int: 82 | """ 83 | TODO: rework documentation 84 | """ 85 | return calc_non_rng_edges_cpp(graph.to_cpp()) 86 | -------------------------------------------------------------------------------- /python/src/deglib/distances.py: -------------------------------------------------------------------------------- 1 | import enum 2 | from abc import ABC, abstractmethod 3 | from typing import Self 4 | 5 | import numpy as np 6 | 7 | import deglib_cpp 8 | 9 | 10 | class Metric(enum.IntEnum): 11 | L2 = deglib_cpp.Metric.L2 12 | InnerProduct = deglib_cpp.Metric.InnerProduct 13 | L2_Uint8 = deglib_cpp.Metric.L2_Uint8 14 | 15 | def to_cpp(self) -> deglib_cpp.Metric: 16 | if self == Metric.L2: 17 | return deglib_cpp.Metric.L2 18 | elif self == Metric.InnerProduct: 19 | return deglib_cpp.Metric.InnerProduct 20 | elif self == Metric.L2_Uint8: 21 | return deglib_cpp.Metric.L2_Uint8 22 | 23 | def get_dtype(self): 24 | if self in (Metric.L2, Metric.InnerProduct): 25 | dtype = np.float32 26 | elif self == Metric.L2_Uint8: 27 | dtype = np.uint8 28 | else: 29 | raise ValueError('unknown metric: {}'.format(self)) 30 | return dtype 31 | 32 | 33 | class SpaceInterface(ABC): 34 | @abstractmethod 35 | def dim(self) -> int: 36 | return NotImplemented() 37 | 38 | @abstractmethod 39 | def metric(self) -> Metric: 40 | return NotImplemented() 41 | 42 | @abstractmethod 43 | def get_data_size(self) -> int: 44 | return NotImplemented() 45 | 46 | 47 | class FloatSpace(SpaceInterface): 48 | def __init__(self, float_space_cpp: deglib_cpp.FloatSpace): 49 | """ 50 | Create a FloatSpace. 51 | 52 | :param float_space_cpp: The cpp implementation of a float space 53 | """ 54 | self.float_space_cpp = float_space_cpp 55 | 56 | @classmethod 57 | def create(cls, dim: int, metric: Metric) -> Self: 58 | """ 59 | Create a FloatSpace. 60 | 61 | :param dim: The dimension of the space 62 | :param metric: Metric to calculate distances between features 63 | """ 64 | return FloatSpace(deglib_cpp.FloatSpace(dim, metric.to_cpp())) 65 | 66 | def dim(self) -> int: 67 | """ 68 | :return: the dimensionality of the space 69 | """ 70 | return self.float_space_cpp.dim() 71 | 72 | def metric(self) -> deglib_cpp.Metric: 73 | """ 74 | :return: the metric that can be used to calculate distances between features 75 | """ 76 | return Metric(int(self.float_space_cpp.metric())) 77 | 78 | def get_data_size(self) -> int: 79 | """ 80 | :returns: number of features. 81 | """ 82 | return self.float_space_cpp.get_data_size() 83 | 84 | def to_cpp(self) -> deglib_cpp.FloatSpace: 85 | return self.float_space_cpp 86 | 87 | def __repr__(self): 88 | return f'FloatSpace(size={self.get_data_size()} dim={self.dim()}, metric={self.metric()})' 89 | -------------------------------------------------------------------------------- /python/src/deglib/repository.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | import numpy as np 3 | 4 | 5 | def ivecs_read(filename: str | Path) -> np.ndarray: 6 | """ 7 | Taken from https://github.com/facebookresearch/faiss/blob/main/benchs/datasets.py#L12. 8 | The loaded dataset should be in the format described here: http://corpus-texmex.irisa.fr/ 9 | """ 10 | a = np.fromfile(filename, dtype=np.int32) 11 | d = a[0] 12 | return a.reshape(-1, d + 1)[:, 1:].copy() 13 | 14 | 15 | def u8vecs_read(filename: str | Path) -> np.ndarray: 16 | """ 17 | The loaded dataset should be in the format described here: http://corpus-texmex.irisa.fr/ 18 | """ 19 | a = np.fromfile(filename, dtype=np.int32) 20 | b = a.view(np.uint8).reshape(-1, a[0] + 4) 21 | return b[:, 4:].copy() 22 | 23 | 24 | def fvecs_read(filename: str | Path) -> np.ndarray: 25 | """ 26 | Taken from https://github.com/facebookresearch/faiss/blob/main/benchs/datasets.py#L12 27 | The loaded dataset should be in the format described here: http://corpus-texmex.irisa.fr/ 28 | """ 29 | return ivecs_read(filename).view(np.int32) 30 | -------------------------------------------------------------------------------- /python/src/deglib/std.py: -------------------------------------------------------------------------------- 1 | from deglib_cpp import Mt19937 as Mt19937cpp 2 | 3 | 4 | class Mt19937: 5 | def __init__(self, seed: int = 5489): 6 | """ 7 | Create a mersenne twister engine for pseudo random number generation. 8 | """ 9 | self.mt19937_cpp = Mt19937cpp(seed) 10 | 11 | def to_cpp(self) -> Mt19937cpp: 12 | return self.mt19937_cpp 13 | -------------------------------------------------------------------------------- /python/src/deglib/utils.py: -------------------------------------------------------------------------------- 1 | import time 2 | import warnings 3 | import numpy as np 4 | import psutil 5 | 6 | 7 | class NonContiguousWarning(Warning): 8 | pass 9 | 10 | 11 | def assure_contiguous(arr: np.ndarray, name: str) -> np.ndarray: 12 | """ 13 | Assures that the returned array is c contiguous. If it is not, a new c-contiguous array will be created. 14 | Furthermore, the user is warned about poor performance. 15 | """ 16 | if not arr.flags['C_CONTIGUOUS']: 17 | warnings.warn('{} is not c contiguous. This will lead to poor performance.'.format(name), NonContiguousWarning) 18 | arr = np.ascontiguousarray(arr) # copy to create c-contiguous array 19 | return arr 20 | 21 | 22 | class WrongDtypeException(Exception): 23 | pass 24 | 25 | 26 | class InvalidShapeException(Exception): 27 | pass 28 | 29 | 30 | def assure_dtype(arr: np.ndarray, name: str, dtype: np.dtype): 31 | """ 32 | Raise an exception, if the given numpy array has the wrong dtype 33 | """ 34 | if arr.dtype != dtype: 35 | raise WrongDtypeException('{} has wrong dtype \"{}\", expected was \"{}\"'.format(name, arr.dtype, dtype)) 36 | 37 | 38 | def assure_array(arr: np.ndarray, name: str, dtype: np.dtype) -> np.ndarray: 39 | """ 40 | Makes sure, that the given array fulfills the requirements for c++ api. 41 | """ 42 | assure_dtype(arr, name, dtype) 43 | return assure_contiguous(arr, name) 44 | 45 | 46 | class StopWatch: 47 | def __init__(self): 48 | self.start_time = time.perf_counter_ns() 49 | 50 | def get_elapsed_time_micro(self) -> int: 51 | end_time = time.perf_counter_ns() 52 | return (end_time - self.start_time) // 1000 53 | 54 | 55 | def get_current_rss_mb(): 56 | return psutil.Process().memory_info().rss // 1000_000 57 | -------------------------------------------------------------------------------- /python/tests/bench_anns.py: -------------------------------------------------------------------------------- 1 | """ 2 | Mirror of the c++ benchmark "cpp/benchmark/src/deglib_anns_bench.cpp". 3 | """ 4 | import sys 5 | import pathlib 6 | 7 | import deglib 8 | from deglib.utils import StopWatch, get_current_rss_mb 9 | 10 | 11 | # TODO: check slow performance 12 | 13 | 14 | def test_deglib_anns_bench(): 15 | print("Testing ...") 16 | 17 | if deglib.avx_usable(): 18 | print("use AVX2 ...") 19 | elif deglib.sse_usable(): 20 | print("use SSE ...") 21 | else: 22 | print("use arch ...") 23 | 24 | data_path: pathlib.Path = pathlib.Path(sys.argv[1]) 25 | repeat_test = 1 26 | k = 100 27 | 28 | query_file: pathlib.Path = data_path / "SIFT1M" / "sift_query.fvecs" 29 | graph_file: pathlib.Path = (data_path / "deg" / "best_distortion_decisions" / 30 | "128D_L2_K30_AddK60Eps0.2High_SwapK30-0StepEps0.001LowPath5Rnd0" 31 | "+0_improveEvery2ndNonPerfectEdge.deg") 32 | gt_file: pathlib.Path = data_path / "SIFT1M" / "sift_groundtruth.ivecs" 33 | 34 | assert query_file.is_file(), 'Could not find query file: {}'.format(query_file) 35 | assert graph_file.is_file(), 'Could not find graph file: {}'.format(graph_file) 36 | assert gt_file.is_file(), 'Could not find ground truth file: {}'.format(gt_file) 37 | 38 | print("Load graph {}".format(graph_file)) 39 | print("Actual memory usage: {} Mb".format(get_current_rss_mb())) 40 | stop_watch = StopWatch() 41 | graph = deglib.graph.load_readonly_graph(graph_file) 42 | elapsed_us = stop_watch.get_elapsed_time_micro() 43 | print("Graph with {} vertices".format(graph.size())) 44 | print("Actual memory usage: {} Mb".format(get_current_rss_mb())) 45 | print("Loading Graph took {} us".format(elapsed_us)) 46 | 47 | query_repository = deglib.repository.fvecs_read(query_file) 48 | print("{} Query Features with {} dimensions".format(query_repository.shape[0], query_repository.shape[1])) 49 | 50 | ground_truth = deglib.repository.ivecs_read(gt_file) 51 | print("{} ground truth {} dimensions".format(ground_truth.shape[0], ground_truth.shape[1])) 52 | 53 | print("Test with k={} and repeat_test={}".format(k, repeat_test)) 54 | 55 | deglib.benchmark.test_graph_anns(graph, query_repository, ground_truth, repeat_test, k) 56 | 57 | 58 | if __name__ == '__main__': 59 | test_deglib_anns_bench() # TODO: replace with test framework 60 | -------------------------------------------------------------------------------- /python/tests/bench_explore.py: -------------------------------------------------------------------------------- 1 | import enum 2 | import pathlib 3 | import sys 4 | 5 | import deglib 6 | 7 | 8 | class DatasetName(enum.Enum): 9 | Audio = enum.auto() 10 | Enron = enum.auto() 11 | Sift1m = enum.auto() 12 | Glove = enum.auto() 13 | 14 | 15 | def main(): 16 | print("Testing ...") 17 | 18 | if deglib.avx_usable(): 19 | print("use AVX2 ...") 20 | elif deglib.sse_usable(): 21 | print("use SSE ...") 22 | else: 23 | print("use arch ...") 24 | 25 | repeat_test = 1 26 | 27 | data_path: pathlib.Path = pathlib.Path(sys.argv[1]) 28 | 29 | k = 1000 30 | 31 | dataset_name = DatasetName.Audio 32 | 33 | if dataset_name == DatasetName.Sift1m: 34 | graph_file = data_path / "deg" / "best_distortion_decisions" / "128D_L2_K30_AddK60Eps0.2High_SwapK30-0StepEps0.001LowPath5Rnd0+0_improveEvery2ndNonPerfectEdge.deg" 35 | gt_file = data_path / "SIFT1M" / "sift_explore_ground_truth.ivecs" 36 | query_file = data_path / "SIFT1M" / "sift_explore_entry_vertex.ivecs" 37 | elif dataset_name == DatasetName.Glove: 38 | graph_file = data_path / "deg" / "100D_L2_K30_AddK30Eps0.2High_SwapK30-0StepEps0.001LowPath5Rnd0+0_improveEvery2ndNonPerfectEdge.deg" 39 | gt_file = data_path / "glove-100" / "glove-100_explore_ground_truth.ivecs" 40 | query_file = data_path / "glove-100" / "glove-100_explore_entry_vertex.ivecs" 41 | elif dataset_name == DatasetName.Glove: 42 | graph_file = data_path / "deg" / "1369D_L2_K30_AddK60Eps0.3High_SwapK30-0StepEps0.001LowPath5Rnd0+0_improveEvery2ndNonPerfectEdge.deg" 43 | gt_file = data_path / "enron" / "enron_explore_ground_truth.ivecs" 44 | query_file = data_path / "enron" / "enron_explore_entry_vertex.ivecs" 45 | elif dataset_name == DatasetName.Audio: 46 | graph_file = data_path / "deg" / "neighbor_choice" / "192D_L2_K20_AddK40Eps0.3Low_schemeA.deg" 47 | # graph_file = data_path / "deg" / "192D_L2_K20_AddK40Eps0.3High_SwapK20-0StepEps0.001LowPath5Rnd0+0_improveEvery2ndNonPerfectEdge.deg" 48 | gt_file = data_path / "audio" / "audio_explore_ground_truth.ivecs" 49 | query_file = data_path / "audio" / "audio_explore_entry_vertex.ivecs" 50 | else: 51 | raise ValueError(f"Unknown dataset name: {dataset_name}") 52 | 53 | # load graph 54 | print("Load graph {}".format(graph_file)) 55 | graph = deglib.graph.load_readonly_graph(graph_file) 56 | 57 | # load starting vertex data 58 | entry_vertex = deglib.repository.ivecs_read(query_file) 59 | print("{} entry vertex {} dimensions".format(entry_vertex.shape[0], entry_vertex.shape[1])) 60 | 61 | # load ground truth data (nearest neighbors of the starting vertices) 62 | ground_truth = deglib.repository.ivecs_read(gt_file) 63 | print("{} ground truth {} dimensions".format(ground_truth.shape[0], ground_truth.shape[1])) 64 | 65 | # explore the graph 66 | deglib.benchmark.test_graph_explore(graph, ground_truth, entry_vertex, repeat_test, k) 67 | 68 | print("Test OK") 69 | 70 | 71 | if __name__ == '__main__': 72 | main() 73 | -------------------------------------------------------------------------------- /python/tests/experiments.py: -------------------------------------------------------------------------------- 1 | from concurrent.futures import ThreadPoolExecutor, wait 2 | 3 | import numpy as np 4 | import deglib 5 | from deglib.builder import EvenRegularGraphBuilder 6 | 7 | 8 | def main(): 9 | samples = 1000 10 | dims = 8 11 | 12 | data = np.random.random((samples, dims)).astype(np.float32) 13 | # data = data / np.linalg.norm(data, axis=1)[:, None] # L2 normalize 14 | 15 | graph = deglib.graph.SizeBoundedGraph.create_empty(data.shape[0], data.shape[1], 16, deglib.Metric.L2) 16 | builder = deglib.builder.EvenRegularGraphBuilder(graph, extend_k=32, extend_eps=0.01, improve_k=0) 17 | 18 | for i, vec in enumerate(data): 19 | vec: np.ndarray 20 | builder.add_entry(i, vec) 21 | 22 | builder.build(callback='progress') 23 | 24 | valid_labels = np.random.choice(graph.size(), size=5, replace=False) 25 | 26 | query = np.random.random(dims).astype(np.float32) 27 | 28 | results, dists = graph.search(query, filter_labels=valid_labels, eps=0.0, k=8) 29 | 30 | print(results) 31 | 32 | print('indices:', results.shape, results.dtype) 33 | print('valid:', valid_labels.shape) 34 | print('all results in labels:', np.all(np.isin(results, valid_labels))) 35 | 36 | 37 | def main2(): 38 | # constants 39 | samples, dims = 10_000, 256 40 | 41 | # build index 42 | data = np.random.random((samples, dims)).astype(np.float32) 43 | index = deglib.builder.build_from_data(data, extend_eps=0.1, callback='progress') 44 | 45 | # search 46 | query = np.random.random(dims).astype(np.float32) 47 | result_indices, dists = index.search(query, eps=0.1, k=16) 48 | 49 | print(result_indices) 50 | print(dists) 51 | 52 | 53 | def dump_data(seed): 54 | np.random.seed(seed) 55 | 56 | samples = 100 57 | dims = 128 58 | data = np.random.random((samples, dims)).astype(np.float32) 59 | 60 | dim_row = np.zeros((samples, 1), dtype=np.int32) + dims 61 | 62 | print(data.shape, dim_row.shape) 63 | data_to_dump = np.concatenate((dim_row, data.view(np.int32)), axis=1) 64 | 65 | print(data_to_dump.shape, data_to_dump.dtype) 66 | 67 | data_to_dump.tofile('crash_data.fvecs') 68 | print('data dumped to crash_data.fvecs') 69 | 70 | 71 | def do_build_with_remove(seed, edges_per_vertex): 72 | np.random.seed(seed) 73 | 74 | samples = 100 75 | dims = 128 76 | # edges_per_vertex = 2 # samples // 10 77 | data = np.random.random((samples, dims)).astype(np.float32) 78 | 79 | # data = deglib.repository.fvecs_read('crash_data.fvecs') 80 | graph = deglib.graph.SizeBoundedGraph.create_empty( 81 | data.shape[0], data.shape[1], edges_per_vertex, deglib.Metric.L2 82 | ) 83 | builder = deglib.builder.EvenRegularGraphBuilder(graph, extend_k=30, extend_eps=0.2, improve_k=30) 84 | 85 | for label, vec in enumerate(data): 86 | vec: np.ndarray 87 | builder.add_entry(label, vec) 88 | 89 | # remove half of the vertices 90 | for label in range(0, data.shape[0], 2): 91 | builder.remove_entry(label) 92 | 93 | builder.build(callback='progress') 94 | 95 | 96 | KNOWN_CRASHES = { 97 | (1, 10) 98 | } 99 | 100 | 101 | def do_all(): 102 | for i in range(100): 103 | for epv in range(2, 34, 2): 104 | if (i, epv) in KNOWN_CRASHES: 105 | print('skipping seed:', i, ' epv:', epv) 106 | else: 107 | print('seed:', i, ' epv:', epv) 108 | do_build_with_remove(i, epv) 109 | 110 | 111 | def build_graph(jobname, data, dim): 112 | print('starting', jobname) 113 | graph = deglib.graph.SizeBoundedGraph.create_empty(1_000_000, dim, edges_per_vertex=8) 114 | print(graph) 115 | 116 | builder = EvenRegularGraphBuilder(graph, improve_k=0, extend_eps=0, extend_k=8) 117 | print(builder) 118 | 119 | builder.add_entry(range(data.shape[0]), data) 120 | 121 | builder.build() 122 | 123 | 124 | class FinishPrinter: 125 | def __init__(self, jobname: str): 126 | self.jobname = jobname 127 | 128 | def __call__(self, fut): 129 | print('finish', self.jobname) 130 | 131 | 132 | def test_free_memory(): 133 | dim = 512 134 | data = np.random.random((100_000, dim)).astype(np.float32) 135 | 136 | jobs = 2 137 | 138 | with ThreadPoolExecutor(max_workers=jobs) as executor: 139 | futures = [] 140 | for i in range(10): 141 | jobname = 'job {}'.format(i) 142 | print('start: {}'.format(jobname)) 143 | future = executor.submit(build_graph, jobname, data, dim) 144 | 145 | future.add_done_callback(FinishPrinter(jobname)) 146 | futures.append(future) 147 | wait(futures) 148 | 149 | 150 | if __name__ == '__main__': 151 | main() 152 | # main() 153 | # do_build_with_remove(1, 10) 154 | # do_all() 155 | # dump_data(1) 156 | # test_free_memory() 157 | -------------------------------------------------------------------------------- /python/tests/test_builder.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pytest 3 | 4 | import deglib 5 | import deglib_cpp 6 | 7 | 8 | class CallbackTester: 9 | def __init__(self): 10 | self.num_callbacks = 0 11 | self.last_status = None 12 | 13 | def __call__(self, status: deglib_cpp.BuilderStatus): 14 | assert isinstance(status, deglib_cpp.BuilderStatus), \ 15 | 'Got instance of type \"{}\" for builder_status in callback'.format(type(status)) 16 | self.last_status = status 17 | self.num_callbacks += 1 18 | 19 | 20 | class TestGraphs: 21 | def setup_method(self): 22 | self.samples = 100 23 | self.dims = 128 24 | self.edges_per_vertex = self.samples // 10 25 | 26 | self.data = np.random.random((self.samples, self.dims)).astype(np.float32) 27 | 28 | @pytest.mark.parametrize('batch', [True, False]) 29 | def test_add_entry(self, batch): 30 | graph = deglib.graph.SizeBoundedGraph.create_empty( 31 | self.data.shape[0], self.data.shape[1], self.edges_per_vertex, deglib.Metric.L2 32 | ) 33 | builder = deglib.builder.EvenRegularGraphBuilder(graph, extend_k=30, extend_eps=0.2, improve_k=30) 34 | 35 | if batch: 36 | builder.add_entry(range(self.data.shape[0]), self.data) 37 | else: 38 | for i, vec in enumerate(self.data): 39 | vec: np.ndarray 40 | builder.add_entry(i, vec) 41 | 42 | @pytest.mark.parametrize('lid', list(deglib.builder.LID)) 43 | def test_build_simple(self, lid): 44 | graph = deglib.graph.SizeBoundedGraph.create_empty( 45 | self.data.shape[0], self.data.shape[1], self.edges_per_vertex, deglib.Metric.L2 46 | ) 47 | builder = deglib.builder.EvenRegularGraphBuilder(graph, extend_k=30, extend_eps=0.2, improve_k=30, lid=lid) 48 | for i, vec in enumerate(self.data): 49 | vec: np.ndarray 50 | builder.add_entry(i, vec) 51 | 52 | builder.build() 53 | 54 | @pytest.mark.parametrize('lid', list(deglib.builder.LID)) 55 | def test_build_batch(self, lid): 56 | graph = deglib.graph.SizeBoundedGraph.create_empty( 57 | self.data.shape[0], self.data.shape[1], self.edges_per_vertex, deglib.Metric.L2 58 | ) 59 | builder = deglib.builder.EvenRegularGraphBuilder(graph, extend_k=30, extend_eps=0.2, improve_k=30, lid=lid) 60 | builder.add_entry(range(self.data.shape[0]), self.data) 61 | 62 | builder.build() 63 | 64 | def test_build_with_remove(self): 65 | graph = deglib.graph.SizeBoundedGraph.create_empty( 66 | self.data.shape[0], self.data.shape[1], self.edges_per_vertex, deglib.Metric.L2 67 | ) 68 | builder = deglib.builder.EvenRegularGraphBuilder(graph, extend_k=30, extend_eps=0.2, improve_k=30) 69 | 70 | for label, vec in enumerate(self.data): 71 | vec: np.ndarray 72 | builder.add_entry(label, vec) 73 | 74 | # remove half of the vertices 75 | for label in range(0, self.data.shape[0], 2): 76 | builder.remove_entry(label) 77 | 78 | builder.build() 79 | 80 | def test_get_num_entries(self): 81 | graph = deglib.graph.SizeBoundedGraph.create_empty( 82 | self.data.shape[0], self.data.shape[1], self.edges_per_vertex, deglib.Metric.L2 83 | ) 84 | builder = deglib.builder.EvenRegularGraphBuilder(graph, extend_k=30, extend_eps=0.2, improve_k=30) 85 | 86 | def _check_entries(expected: int, action: str): 87 | assert action in ('new', 'remove') 88 | 89 | actual = builder.get_num_new_entries() if action == 'new' else builder.get_num_remove_entries() 90 | assert actual == expected, \ 91 | 'Added {} {} entries, but get_num_{}_entries() returned {}'.format( 92 | expected, action, action, builder.get_num_new_entries() 93 | ) 94 | 95 | _check_entries(0, 'new') 96 | _check_entries(0, 'remove') 97 | 98 | for label, vec in enumerate(self.data): 99 | vec: np.ndarray 100 | builder.add_entry(label, vec) 101 | 102 | _check_entries(self.data.shape[0], 'new') 103 | _check_entries(0, 'remove') 104 | 105 | # remove half of the vertices 106 | for label in range(0, self.data.shape[0], 2): 107 | builder.remove_entry(label) 108 | 109 | _check_entries(self.data.shape[0] // 2, 'remove') 110 | 111 | # def test_callback(self): 112 | # graph = deglib.graph.SizeBoundedGraph.create_empty( 113 | # self.data.shape[0], self.data.shape[1], self.edges_per_vertex, deglib.Metric.L2 114 | # ) 115 | # builder = deglib.builder.EvenRegularGraphBuilder(graph, extend_k=30, extend_eps=0.2, improve_k=30) 116 | # for i, vec in enumerate(self.data): 117 | # vec: np.ndarray 118 | # builder.add_entry(i, vec) 119 | # 120 | # tester = CallbackTester() 121 | # builder.build(callback=tester) 122 | # assert tester.num_callbacks == self.data.shape[0], 'Got {} callbacks, but expected {}'.format( 123 | # tester.num_callbacks, self.data.shape[0] 124 | # ) 125 | # assert tester.last_status.step == self.data.shape[0], 'Got {} steps, but expected {}'.format( 126 | # tester.last_status.step, self.data.shape[0] 127 | # ) 128 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # DEG: Fast Approximate Nearest Neighbor Search 2 | 3 | The Dynamic Exploration Graph (DEG) is a graph-based algorithm for approximate nearest neighbor search (ANNS). It indexes both static and dynamic datasets using three algorithms: incremental extension, continuous edge optimization, and vertex deletion. The resulting graph demonstrates high efficiency in terms of queries per second relative to the achieved recall rate. DEG provides state-of-the-art performance for both indexed and unindexed queries (where the query is not part of the index). 4 | 5 | ## Usage 6 | For a short introduction on how to use deglib for vector search, see our [Python Examples](python/README.md#examples). 7 | 8 | ## Release 9 | 10 | - [2025/01/09] The latest iteration of DEG uses a more efficient way of removing and adding a vertex. More details can be found in our new paper [Dynamic Exploration Graph: A Novel Approach for Efficient Nearest Neighbor Search in Evolving Multimedia Datasets](https://link.springer.com/chapter/10.1007/978-981-96-2054-8_25). 11 | - [2024/05/01] Our paper [An Exploration Graph with Continuous Refinement for Efficient Multimedia Retrieval](https://doi.org/10.1145/3652583.3658117) is accepted by ICMR2024 as **oral presentation** 12 | - [2023/12/02] The new continuous refining Exploration Graph (crEG) containing a more efficient and thread-safe way to extend DEG. Currently found in the [crEG branch](https://github.com/Visual-Computing/DynamicExplorationGraph/tree/crEG) of this repository. 13 | - [2023/07/19] First version of Dynamic Exploration Graph is out! For more details please refere to our paper: 14 | [Fast Approximate nearest neighbor search with the Dynamic Exploration Graph using continuous refinement](https://arxiv.org/abs/2307.10479) 15 | 16 | ## Reproduction 17 | 18 | The following files contain the datasets used in our paper. Including exploration queries and ground truth data. 19 | 20 | | Data set | Download | dimension | nb base vectors | nb query vectors | original website | 21 | |-----------|------------------------------------------------------------------------------------|-----------|-----------------|------------------|----------------------------------------------------------------| 22 | | Audio | [audio.tar.gz](https://static.visual-computing.com/paper/DEG/audio.tar.gz) | 192 | 53,387 | 200 | [original website](https://www.cs.princeton.edu/cass/) | 23 | | Enron | [enron.tar.gz](https://static.visual-computing.com/paper/DEG/enron.tar.gz) | 1369 | 94,987 | 200 | [original website](https://www.cs.cmu.edu/~enron/) | 24 | | SIFT1M | [sift.tar.gz](https://static.visual-computing.com/paper/DEG/sift.tar.gz) | 128 | 1,000,000 | 10,000 | [original website](http://corpus-texmex.irisa.fr/) | 25 | | DEEP1M | [deep1m.tar.gz](https://static.visual-computing.com/paper/DEG/deep1m.tar.gz) | 96 | 1,000,000 | 10,000 | [original website](https://github.com/facebookresearch/ppuda) | 26 | | GloVe-100 | [glove-100.tar.gz](https://static.visual-computing.com/paper/DEG/glove-100.tar.gz) | 100 | 1,183,514 | 10,000 | [original website](https://nlp.stanford.edu/projects/glove/) | 27 | 28 | In order to reproduce the results in our older papers, please visit the corresponding github branch. 29 | - [main branch](https://github.com/Visual-Computing/DynamicExplorationGraph/tree/main) for "Dynamic Exploration Graph: A Novel Approach for Efficient Nearest Neighbor Search in Evolving Multimedia Datasets" 30 | - [crEG branch](https://github.com/Visual-Computing/DynamicExplorationGraph/tree/crEG) for "An Exploration Graph with Continuous Refinement for Efficient Multimedia Retrieval" 31 | - [arxiv branch](https://github.com/Visual-Computing/DynamicExplorationGraph/tree/arxiv) for "Fast Approximate nearest neighbor search with the Dynamic Exploration Graph using continuous refinement" 32 | 33 | 34 | 35 | 36 | ## Performance 37 | 38 | ***NOTE:** All experiments where conduced single threaded on a Ryzen 2700x CPU, operating at a constant core clock speed of 4GHz, and 64GB of DDR4 memory running at 2133MHz. 39 | 40 | **Approximate Nearest Neighbor Search** 41 | ![ANNS](figures/anns_qps_vs_recall.jpg) 42 | 43 | **Exploratory Search (indexed queries)** 44 | ![Exploration](figures/exploration_qps_vs_recall.jpg) 45 | 46 | ## Reference 47 | 48 | Please cite our work in your publications if it helps your research: 49 | 50 | Dynamic Exploration Graph 51 | ``` 52 | @article{Hezel2025, 53 | author = {Hezel, Nico and Barthel, Uwe Kai and Schilling, Bruno and Schall, Konstantin and Jung, Klaus}, 54 | title = {Dynamic Exploration Graph: A Novel Approach for Efficient Nearest Neighbor Search in Evolving Multimedia Datasets}, 55 | booktitle={MultiMedia Modeling}, 56 | publisher={Springer Nature}, 57 | pages={333--347}, 58 | isbn={978-981-96-2054-8}, 59 | year = 2025 60 | } 61 | ``` 62 | 63 | continuous refining Exploration Graph 64 | ``` 65 | @article{Hezel2024, 66 | author = {Hezel, Nico and Barthel, Uwe Kai and Schall, Konstantin and Jung, Klaus}, 67 | title = {An Exploration Graph with Continuous Refinement for Efficient Multimedia Retrieval}, 68 | booktitle = {Proceedings of the 2024 International Conference on Multimedia Retrieval}, 69 | year = {2024}, 70 | isbn = {9798400706196}, 71 | publisher = {Association for Computing Machinery}, 72 | doi = {10.1145/3652583.3658117}, 73 | pages = {657–665}, 74 | series = {ICMR '24} 75 | } 76 | ``` 77 | --------------------------------------------------------------------------------