├── .clang-format ├── .clang-tidy ├── .devcontainer ├── Dockerfile └── devcontainer.json ├── .editorconfig ├── .github └── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── .gitignore ├── .gitmodules ├── CHANGELOG.md ├── CMakeLists.txt ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── README.md ├── benchmark-graph ├── random-performance-barchart.svg └── seq-performance-barchart.svg ├── benchmark ├── README.md ├── rand_utils.hpp ├── random_read_benchmark.cpp ├── read_s3_object.cpp └── sequential_read_benchmark.cpp ├── doc ├── common_problem_troubleshooting.md ├── consistency_spec.md ├── example_usage.md ├── performance_troubleshooting_notes.md └── resource_consumption.md ├── extension_config.cmake ├── src ├── cache_entry_info.cpp ├── cache_filesystem.cpp ├── cache_filesystem_config.cpp ├── cache_filesystem_ref_registry.cpp ├── cache_httpfs_extension.cpp ├── cache_reader_manager.cpp ├── cache_status_query_function.cpp ├── disk_cache_reader.cpp ├── histogram.cpp ├── in_memory_cache_reader.cpp ├── include │ ├── base_cache_reader.hpp │ ├── base_profile_collector.hpp │ ├── cache_entry_info.hpp │ ├── cache_filesystem.hpp │ ├── cache_filesystem_config.hpp │ ├── cache_filesystem_ref_registry.hpp │ ├── cache_httpfs_extension.hpp │ ├── cache_reader_manager.hpp │ ├── cache_status_query_function.hpp │ ├── disk_cache_reader.hpp │ ├── histogram.hpp │ ├── in_mem_cache_block.hpp │ ├── in_memory_cache_reader.hpp │ ├── noop_cache_reader.hpp │ ├── scope_guard.hpp │ └── temp_profile_collector.hpp ├── noop_cache_reader.cpp ├── temp_profile_collector.cpp └── utils │ ├── fake_filesystem.cpp │ ├── filesystem_utils.cpp │ ├── include │ ├── copiable_value_lru_cache.hpp │ ├── exclusive_lru_cache.hpp │ ├── exclusive_multi_lru_cache.hpp │ ├── fake_filesystem.hpp │ ├── filesystem_utils.hpp │ ├── map_utils.hpp │ ├── mock_filesystem.hpp │ ├── no_destructor.hpp │ ├── resize_uninitialized.hpp │ ├── shared_lru_cache.hpp │ ├── size_literals.hpp │ ├── thread_pool.hpp │ ├── thread_utils.hpp │ ├── time_utils.hpp │ └── type_traits.hpp │ ├── mock_filesystem.cpp │ ├── thread_pool.cpp │ └── thread_utils.cpp ├── test ├── data │ ├── README.md │ ├── attach_test.db │ └── stock-exchanges.csv └── sql │ ├── cache_access_info.test │ ├── clear_cache.test │ ├── disk_cache_filesystem.test │ ├── extension.test │ ├── glob_read.test │ ├── inmem_cache_filesystem.test │ ├── insufficient_disk_space.test │ ├── max_subrequest_fanout.test │ ├── noop_cache_filesystem.test │ ├── wrap_cache_filesystem.test │ └── wrapped_filesystems_info.test ├── unit ├── test_base_cache_filesystem.cpp ├── test_cache_filesystem.cpp ├── test_cache_filesystem_with_mock.cpp ├── test_copiable_value_lru_cache.cpp ├── test_disk_cache_filesystem.cpp ├── test_exclusive_lru_cache.cpp ├── test_exclusive_multi_lru_cache.cpp ├── test_filesystem_config.cpp ├── test_filesystem_utils.cpp ├── test_histogram.cpp ├── test_in_memory_cache_filesystem.cpp ├── test_large_file_disk_reader.cpp ├── test_large_file_inmem_reader.cpp ├── test_no_destructor.cpp ├── test_noop_cache_reader.cpp ├── test_set_extension_config.cpp ├── test_shared_lru_cache.cpp ├── test_size_literals.cpp └── test_thread_pool.cpp └── vcpkg.json /.clang-format: -------------------------------------------------------------------------------- 1 | duckdb/.clang-format -------------------------------------------------------------------------------- /.clang-tidy: -------------------------------------------------------------------------------- 1 | duckdb/.clang-tidy -------------------------------------------------------------------------------- /.devcontainer/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:22.04 2 | 3 | RUN yes | unminimize 4 | 5 | RUN apt update \ 6 | && apt install -y \ 7 | bison \ 8 | clang-format \ 9 | cmake \ 10 | curl \ 11 | dstat \ 12 | flex \ 13 | g++ \ 14 | gdb \ 15 | git \ 16 | htop \ 17 | libicu-dev \ 18 | libreadline-dev \ 19 | libssl-dev \ 20 | locales \ 21 | man \ 22 | pkg-config \ 23 | sudo \ 24 | vim \ 25 | zlib1g-dev \ 26 | && rm -rf /var/lib/apt/lists/* 27 | 28 | RUN locale-gen en_US.UTF-8 29 | 30 | -------------------------------------------------------------------------------- /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | { 2 | "build": { 3 | "dockerfile": "Dockerfile" 4 | }, 5 | "customizations": { 6 | "vscode": { 7 | "extensions": [ 8 | "ms-vscode.cpptools", 9 | "eamodio.gitlens", 10 | ] 11 | } 12 | }, 13 | "runArgs": [ 14 | "--cap-add=SYS_PTRACE" 15 | ], 16 | "features": { 17 | "ghcr.io/devcontainers/features/sshd:1": {} 18 | } 19 | } 20 | 21 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | duckdb/.editorconfig -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Desktop (please complete the following information):** 27 | - OS: [e.g. iOS] 28 | - Browser [e.g. chrome, safari] 29 | - Version [e.g. 22] 30 | 31 | **Smartphone (please complete the following information):** 32 | - Device: [e.g. iPhone6] 33 | - OS: [e.g. iOS8.1] 34 | - Browser [e.g. stock browser, safari] 35 | - Version [e.g. 22] 36 | 37 | **Additional context** 38 | Add any other context about the problem here. 39 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | #==============================================================================# 2 | # This file specifies intentionally untracked files that git should ignore. 3 | #==============================================================================# 4 | 5 | #==============================================================================# 6 | # File extensions to be ignored anywhere in the tree. 7 | #==============================================================================# 8 | # Temp files created by most text editors. 9 | *~ 10 | # Merge files created by git. 11 | *.orig 12 | # Java bytecode 13 | *.class 14 | # Byte compiled python modules. 15 | *.pyc 16 | # vim swap files 17 | .*.sw? 18 | .sw? 19 | # vscode settings directory 20 | .vscode 21 | #OS X specific files. 22 | .DS_store 23 | 24 | #==============================================================================# 25 | # Explicit files to ignore (only matches one). 26 | #==============================================================================# 27 | # Various tag programs 28 | /tags 29 | /TAGS 30 | /GPATH 31 | /GRTAGS 32 | /GSYMS 33 | /GTAGS 34 | .gitusers 35 | autom4te.cache 36 | cscope.files 37 | cscope.out 38 | autoconf/aclocal.m4 39 | autoconf/autom4te.cache 40 | /compile_commands.json 41 | 42 | #==============================================================================# 43 | # Build artifacts 44 | #==============================================================================# 45 | #m4/ 46 | build/ 47 | #*.m4 48 | *.o 49 | *.lo 50 | *.la 51 | *~ 52 | *.pdf 53 | *.swp 54 | a.out 55 | 56 | #==============================================================================# 57 | # Kate Swap Files 58 | #==============================================================================# 59 | *.kate-swp 60 | .#kate-* 61 | 62 | #==============================================================================# 63 | # Backup artifacts 64 | #==============================================================================# 65 | ~* 66 | *~ 67 | tmp/ 68 | 69 | #==============================================================================# 70 | # KDevelop files 71 | #==============================================================================# 72 | .kdev4 73 | *.kdev4 74 | .dirstamp 75 | .deps 76 | .libs 77 | 78 | #==============================================================================# 79 | # Eclipse files 80 | #==============================================================================# 81 | .wtpmodules 82 | .classpath 83 | .project 84 | .cproject 85 | .pydevproject 86 | .settings 87 | .autotools 88 | 89 | /Debug/ 90 | 91 | #==============================================================================# 92 | # Intellij files 93 | #==============================================================================# 94 | .idea 95 | *.iml 96 | 97 | #==============================================================================# 98 | # Code Coverage files 99 | #==============================================================================# 100 | *.gcno 101 | *.gcda 102 | 103 | 104 | #==============================================================================# 105 | # Eclipse 106 | #==============================================================================# 107 | 108 | .metadata 109 | bin/ 110 | tmp/ 111 | *.tmp 112 | *.bak 113 | *.swp 114 | *~.nib 115 | local.properties 116 | .settings/ 117 | .loadpath 118 | .recommenders 119 | 120 | # Eclipse Core 121 | .project 122 | 123 | # External tool builders 124 | .externalToolBuilders/ 125 | 126 | # Locally stored "Eclipse launch configurations" 127 | *.launch 128 | 129 | # PyDev specific (Python IDE for Eclipse) 130 | *.pydevproject 131 | 132 | # CDT-specific (C/C++ Development Tooling) 133 | .cproject 134 | 135 | # JDT-specific (Eclipse Java Development Tools) 136 | .classpath 137 | 138 | # Java annotation processor (APT) 139 | .factorypath 140 | 141 | # PDT-specific (PHP Development Tools) 142 | .buildpath 143 | 144 | # sbteclipse plugin 145 | .target 146 | 147 | # Tern plugin 148 | .tern-project 149 | 150 | # TeXlipse plugin 151 | .texlipse 152 | 153 | # STS (Spring Tool Suite) 154 | .springBeans 155 | 156 | # Code Recommenders 157 | .recommenders/ 158 | io_file 159 | 160 | ## General 161 | 162 | # Compiled Object files 163 | *.slo 164 | *.lo 165 | *.o 166 | *.cuo 167 | 168 | # Compiled Dynamic libraries 169 | *.so 170 | *.dylib 171 | 172 | # Compiled Static libraries 173 | *.lai 174 | *.la 175 | *.a 176 | 177 | # Compiled python 178 | *.pyc 179 | 180 | # Compiled MATLAB 181 | *.mex* 182 | 183 | # IPython notebook checkpoints 184 | .ipynb_checkpoints 185 | 186 | # Editor temporaries 187 | *.swp 188 | *~ 189 | 190 | # Sublime Text settings 191 | *.sublime-workspace 192 | *.sublime-project 193 | 194 | # Eclipse Project settings 195 | *.*project 196 | .settings 197 | 198 | # Visual Studio 199 | .vs 200 | 201 | # QtCreator files 202 | *.user 203 | 204 | # PyCharm files 205 | .idea 206 | 207 | # OSX dir files 208 | .DS_Store 209 | 210 | # User's build configuration 211 | Makefile.config 212 | 213 | # build, distribute, and bins (+ python proto bindings) 214 | build 215 | .build_debug/* 216 | .build_release/* 217 | distribute/* 218 | *.testbin 219 | *.bin 220 | cmake_build 221 | .cmake_build 222 | cmake-build-* 223 | 224 | # Generated documentation 225 | apidoc/doc 226 | docs/_site 227 | docs/gathered 228 | _site 229 | doxygen 230 | docs/dev 231 | 232 | # Config files 233 | *.conf 234 | 235 | # Vagrant 236 | .vagrant 237 | 238 | # Clangd cache index 239 | .cache 240 | 241 | # Submission zip files 242 | *.zip 243 | 244 | # Ignore bazel output folders 245 | bazel-* 246 | 247 | # Added by cargo 248 | /target 249 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "duckdb"] 2 | path = duckdb 3 | url = https://github.com/duckdb/duckdb 4 | branch = main 5 | [submodule "extension-ci-tools"] 6 | path = extension-ci-tools 7 | url = https://github.com/duckdb/extension-ci-tools 8 | branch = main 9 | [submodule "duckdb-httpfs"] 10 | path = duckdb-httpfs 11 | url = https://github.com/duckdb/duckdb-httpfs 12 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # 0.3.0 2 | 3 | ## Changed 4 | 5 | - Upgrade duckdb v1.3.0 and httpfs ([#198]) 6 | 7 | [#198]: https://github.com/dentiny/duck-read-cache-fs/pull/198 8 | 9 | - Re-enable filesystem wrap ([#199]) 10 | 11 | [#199]: https://github.com/dentiny/duck-read-cache-fs/pull/199 12 | 13 | # 0.2.1 14 | 15 | ## Fixed 16 | 17 | - Fix extension compilation with musl libc. ([#174]) 18 | 19 | [#174]: https://github.com/dentiny/duck-read-cache-fs/pull/174 20 | 21 | - Update (aka, revert) duckdb to stable release v1.2.1. ([#176]) 22 | 23 | [#176]: https://github.com/dentiny/duck-read-cache-fs/pull/176 24 | 25 | ## Changed 26 | 27 | - Temporarily disable filesystem wrap SQL query until a later DuckDB release is available. ([#175]) 28 | 29 | [#175]: https://github.com/dentiny/duck-read-cache-fs/pull/175 30 | 31 | # 0.2.0 32 | 33 | ## Added 34 | 35 | - Allow users to configure min required disk space for disk cache. ([#106]) 36 | 37 | [#106]: https://github.com/dentiny/duck-read-cache-fs/pull/106 38 | 39 | - Cache httpfs extension is able to wrap all duckdb-compatible filesystems. ([#110]) 40 | 41 | [#110]: https://github.com/dentiny/duck-read-cache-fs/pull/110 42 | 43 | - Add cache for file open and glob. ([#133], [#145]) 44 | 45 | [#133]: https://github.com/dentiny/duck-read-cache-fs/pull/133 46 | [#145]: https://github.com/dentiny/duck-read-cache-fs/pull/145 47 | 48 | - Provide SQL function to query cache status. ([#107], [#109]) 49 | 50 | [#107]: https://github.com/dentiny/duck-read-cache-fs/pull/107 51 | [#109]: https://github.com/dentiny/duck-read-cache-fs/pull/109 52 | 53 | - Add stats observability for open and glob operations. ([#126]) 54 | 55 | [#126]: https://github.com/dentiny/duck-read-cache-fs/pull/126 56 | 57 | ## Fixed 58 | 59 | - Fix data race between open, read and delete on-disk cache files. ([#113]) 60 | 61 | [#113]: https://github.com/dentiny/duck-read-cache-fs/pull/113 62 | 63 | - Fix max thread number for parallel read subrequests. ([#151]) 64 | 65 | [#151]: https://github.com/dentiny/duck-read-cache-fs/pull/151 66 | 67 | - Fix file offset update from httpfs extension upstream change ([#158]) 68 | 69 | [#158]: https://github.com/dentiny/duck-read-cache-fs/pull/158 70 | 71 | ## Improved 72 | 73 | - Avoid unnecessary string creation for on-disk cache reader. ([#114]) 74 | 75 | [#114]: https://github.com/dentiny/duck-read-cache-fs/pull/114 76 | 77 | ## Changed 78 | 79 | - Change SQl function to get on-disk cache size from `cache_httpfs_get_cache_size` to `cache_httpfs_get_ondisk_data_cache_size`. ([#153]) 80 | 81 | [#153]: https://github.com/dentiny/duck-read-cache-fs/pull/153 82 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.5) 2 | 3 | # TODO(hjiang): Upgrade to higher version. 4 | set(CMAKE_CXX_STANDARD 14) 5 | 6 | set(TARGET_NAME cache_httpfs) 7 | 8 | # Suppress warnings. 9 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-write-strings") 10 | 11 | set(EXTENSION_NAME ${TARGET_NAME}_extension) 12 | set(LOADABLE_EXTENSION_NAME ${TARGET_NAME}_loadable_extension) 13 | 14 | project(${TARGET_NAME}) 15 | include_directories(src/utils/include) 16 | include_directories(src/include) 17 | include_directories(duckdb-httpfs/extension/httpfs/include) 18 | include_directories(duckdb/third_party/httplib) 19 | 20 | set(EXTENSION_SOURCES 21 | src/cache_entry_info.cpp 22 | src/cache_filesystem.cpp 23 | src/cache_filesystem_config.cpp 24 | src/cache_filesystem_ref_registry.cpp 25 | src/cache_reader_manager.cpp 26 | src/cache_status_query_function.cpp 27 | src/disk_cache_reader.cpp 28 | src/in_memory_cache_reader.cpp 29 | src/histogram.cpp 30 | src/noop_cache_reader.cpp 31 | src/cache_httpfs_extension.cpp 32 | src/temp_profile_collector.cpp 33 | src/utils/fake_filesystem.cpp 34 | src/utils/filesystem_utils.cpp 35 | src/utils/mock_filesystem.cpp 36 | src/utils/thread_pool.cpp 37 | src/utils/thread_utils.cpp 38 | duckdb-httpfs/extension/httpfs/create_secret_functions.cpp 39 | duckdb-httpfs/extension/httpfs/crypto.cpp 40 | duckdb-httpfs/extension/httpfs/hffs.cpp 41 | duckdb-httpfs/extension/httpfs/httpfs_client.cpp 42 | duckdb-httpfs/extension/httpfs/httpfs.cpp 43 | duckdb-httpfs/extension/httpfs/httpfs_extension.cpp 44 | duckdb-httpfs/extension/httpfs/http_state.cpp 45 | duckdb-httpfs/extension/httpfs/s3fs.cpp) 46 | 47 | # Avoid building tooling we won't need for release. 48 | set(BUILD_BENCHMARKS 49 | OFF 50 | CACHE BOOL "" FORCE) 51 | set(BUILD_FILTERS 52 | OFF 53 | CACHE BOOL "" FORCE) 54 | set(BUILD_GENERATORS 55 | OFF 56 | CACHE BOOL "" FORCE) 57 | set(BUILD_TESTING 58 | OFF 59 | CACHE BOOL "" FORCE) 60 | set(BUILD_FUZZERS 61 | OFF 62 | CACHE BOOL "" FORCE) 63 | set(ENABLE_DOCS 64 | OFF 65 | CACHE BOOL "" FORCE) 66 | set(ENABLE_TESTING 67 | OFF 68 | CACHE BOOL "" FORCE) 69 | set(ENABLE_LINTING 70 | OFF 71 | CACHE BOOL "" FORCE) 72 | set(ENABLE_FORMAT 73 | OFF 74 | CACHE BOOL "" FORCE) 75 | 76 | # Build as shared library. 77 | set(BUILD_SHARED_LIBS ON) 78 | 79 | build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES}) 80 | build_loadable_extension(${TARGET_NAME} " " ${EXTENSION_SOURCES}) 81 | 82 | find_package(OpenSSL REQUIRED) 83 | include_directories(${OPENSSL_INCLUDE_DIR}) 84 | target_link_libraries(${EXTENSION_NAME} ${OPENSSL_LIBRARIES}) 85 | target_link_libraries(${LOADABLE_EXTENSION_NAME} ${OPENSSL_LIBRARIES}) 86 | 87 | install( 88 | TARGETS ${EXTENSION_NAME} 89 | EXPORT "${DUCKDB_EXPORT_SET}" 90 | LIBRARY DESTINATION "${INSTALL_LIB_DIR}") 91 | 92 | # Test cases. 93 | include_directories(duckdb/third_party/catch) 94 | 95 | add_executable(test_noop_cache_reader unit/test_noop_cache_reader.cpp) 96 | target_link_libraries(test_noop_cache_reader ${EXTENSION_NAME}) 97 | 98 | add_executable(test_disk_cache_filesystem unit/test_disk_cache_filesystem.cpp) 99 | target_link_libraries(test_disk_cache_filesystem ${EXTENSION_NAME}) 100 | 101 | add_executable(test_large_file_disk_reader unit/test_large_file_disk_reader.cpp) 102 | target_link_libraries(test_large_file_disk_reader ${EXTENSION_NAME}) 103 | 104 | add_executable(test_in_memory_cache_filesystem 105 | unit/test_in_memory_cache_filesystem.cpp) 106 | target_link_libraries(test_in_memory_cache_filesystem ${EXTENSION_NAME}) 107 | 108 | add_executable(test_large_file_inmem_reader 109 | unit/test_large_file_inmem_reader.cpp) 110 | target_link_libraries(test_large_file_inmem_reader ${EXTENSION_NAME}) 111 | 112 | add_executable(test_filesystem_utils unit/test_filesystem_utils.cpp) 113 | target_link_libraries(test_filesystem_utils ${EXTENSION_NAME}) 114 | 115 | add_executable(test_cache_filesystem unit/test_cache_filesystem.cpp) 116 | target_link_libraries(test_cache_filesystem ${EXTENSION_NAME}) 117 | 118 | add_executable(test_histogram unit/test_histogram.cpp) 119 | target_link_libraries(test_histogram ${EXTENSION_NAME}) 120 | 121 | add_executable(test_thread_pool unit/test_thread_pool.cpp) 122 | target_link_libraries(test_thread_pool ${EXTENSION_NAME}) 123 | 124 | add_executable(test_shared_lru_cache unit/test_shared_lru_cache.cpp) 125 | target_link_libraries(test_shared_lru_cache ${EXTENSION_NAME}) 126 | 127 | add_executable(test_exclusive_lru_cache unit/test_exclusive_lru_cache.cpp) 128 | target_link_libraries(test_exclusive_lru_cache ${EXTENSION_NAME}) 129 | 130 | add_executable(test_exclusive_multi_lru_cache 131 | unit/test_exclusive_multi_lru_cache.cpp) 132 | target_link_libraries(test_exclusive_multi_lru_cache ${EXTENSION_NAME}) 133 | 134 | add_executable(test_copiable_value_lru_cache 135 | unit/test_copiable_value_lru_cache.cpp) 136 | target_link_libraries(test_copiable_value_lru_cache ${EXTENSION_NAME}) 137 | 138 | add_executable(test_size_literals unit/test_size_literals.cpp) 139 | target_link_libraries(test_size_literals ${EXTENSION_NAME}) 140 | 141 | add_executable(test_filesystem_config unit/test_filesystem_config.cpp) 142 | target_link_libraries(test_filesystem_config ${EXTENSION_NAME}) 143 | 144 | add_executable(test_set_extension_config unit/test_set_extension_config.cpp) 145 | target_link_libraries(test_set_extension_config ${EXTENSION_NAME}) 146 | 147 | add_executable(test_base_cache_filesystem unit/test_base_cache_filesystem.cpp) 148 | target_link_libraries(test_base_cache_filesystem ${EXTENSION_NAME}) 149 | 150 | add_executable(test_cache_filesystem_with_mock 151 | unit/test_cache_filesystem_with_mock.cpp) 152 | target_link_libraries(test_cache_filesystem_with_mock ${EXTENSION_NAME}) 153 | 154 | add_executable(test_no_destructor unit/test_no_destructor.cpp) 155 | target_link_libraries(test_no_destructor ${EXTENSION_NAME}) 156 | 157 | # Benchmark 158 | add_executable(read_s3_object benchmark/read_s3_object.cpp) 159 | target_link_libraries(read_s3_object ${EXTENSION_NAME}) 160 | 161 | add_executable(sequential_read_benchmark 162 | benchmark/sequential_read_benchmark.cpp) 163 | target_link_libraries(sequential_read_benchmark ${EXTENSION_NAME}) 164 | 165 | add_executable(random_read_benchmark benchmark/random_read_benchmark.cpp) 166 | target_link_libraries(random_read_benchmark ${EXTENSION_NAME}) 167 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | ## Did you find a bug? 4 | 5 | * **Ensure the bug was not already reported** by searching on GitHub under [Issues](https://github.com/dentiny/duck-read-cache-fs/issues). 6 | * If you're unable to find an open issue addressing the problem, [open a new one](https://github.com/dentiny/duck-read-cache-fs/issues/new/choose). Be sure to include a **title and clear description**, as much relevant information as possible, and a **code sample** or an **executable test case** demonstrating the expected behavior that is not occurring. 7 | 8 | ## Did you write a patch that fixes a bug? 9 | 10 | * Great! 11 | * If possible, add a unit test case to make sure the issue does not occur again. 12 | * Make sure you run the code formatter (`make format-all`). 13 | * Open a new GitHub pull request with the patch. 14 | * Ensure the PR description clearly describes the problem and solution. Include the relevant issue number if applicable. 15 | 16 | ## Pull Requests 17 | 18 | * Do not commit/push directly to the main branch. Instead, create a fork and file a pull request. 19 | * When maintaining a branch, merge frequently with the main. 20 | * When maintaining a branch, submit pull requests to the main frequently. 21 | * If you are working on a bigger issue try to split it up into several smaller issues. 22 | * Please do not open "Draft" pull requests. Rather, use issues or discussion topics to discuss whatever needs discussing. 23 | * We reserve full and final discretion over whether or not we will merge a pull request. Adhering to these guidelines is not a complete guarantee that your pull request will be merged. 24 | 25 | ## Building 26 | 27 | * Install `ccache` to improve compilation speed. 28 | * To pull latest dependencies, run `git submodule update --init --recursive`. 29 | * To build the project, run `CMAKE_BUILD_PARALLEL_LEVEL=$(nproc) make `. 30 | 31 | ## Testing 32 | 33 | * To run all the SQL tests, run `make test` (or `make test_debug` for debug build binaries). 34 | * To run all C++ tests, run `make test_unit` (or `test_debug_unit` for debug build binaries). 35 | 36 | ## Formatting 37 | 38 | * Use tabs for indentation, spaces for alignment. 39 | * Lines should not exceed 120 columns. 40 | * `clang-format` enforce these rules automatically, use `make format-all` to run the formatter. 41 | 42 | ### DuckDB C++ Guidelines 43 | 44 | * Do not use `malloc`, prefer the use of smart pointers. Keywords `new` and `delete` are a code smell. 45 | * Strongly prefer the use of `unique_ptr` over `shared_ptr`, only use `shared_ptr` if you **absolutely** have to. 46 | * Use `const` whenever possible. 47 | * Do **not** import namespaces (e.g. `using std`). 48 | * All functions in source files in the core (`src` directory) should be part of the `pgduckdb` namespace. 49 | * When overriding a virtual method, avoid repeating virtual and always use `override` or `final`. 50 | * Use `[u]int(8|16|32|64)_t` instead of `int`, `long`, `uint` etc. Use `idx_t` instead of `size_t` for offsets/indices/counts of any kind. 51 | * Prefer using references over pointers as arguments. 52 | * Use `const` references for arguments of non-trivial objects (e.g. `std::vector`, ...). 53 | * Use C++11 for loops when possible: `for (const auto& item : items) {...}` 54 | * Use braces for indenting `if` statements and loops. Avoid single-line if statements and loops, especially nested ones. 55 | * **Class Layout:** Start out with a `public` block containing the constructor and public variables, followed by a `public` block containing public methods of the class. After that follow any private functions and private variables. For example: 56 | ```cpp 57 | class MyClass { 58 | public: 59 | MyClass(); 60 | 61 | int my_public_variable; 62 | 63 | public: 64 | void MyFunction(); 65 | 66 | private: 67 | void MyPrivateFunction(); 68 | 69 | private: 70 | int my_private_variable; 71 | }; 72 | ``` 73 | * Avoid [unnamed magic numbers](https://en.wikipedia.org/wiki/Magic_number_(programming)). Instead, use named variables that are stored in a `constexpr`. 74 | * [Return early](https://medium.com/swlh/return-early-pattern-3d18a41bba8). Avoid deep nested branches. 75 | * Do not include commented out code blocks in pull requests. 76 | 77 | ## Error Handling 78 | 79 | * Use exceptions **only** when an error is encountered that terminates a query (e.g. parser error, table not found). Exceptions should only be used for **exceptional** situations. For regular errors that do not break the execution flow (e.g. errors you **expect** might occur) use a return value instead. 80 | * Try to add test cases that trigger exceptions. If an exception cannot be easily triggered using a test case then it should probably be an assertion. This is not always true (e.g. out of memory errors are exceptions, but are very hard to trigger). 81 | * Use `D_ASSERT` to assert. Use **assert** only when failing the assert means a programmer error. Assert should never be triggered by user input. Avoid code like `D_ASSERT(a > b + 3);` without comments or context. 82 | * Assert liberally, but make it clear with comments next to the assert what went wrong when the assert is triggered. 83 | 84 | ## Naming Conventions 85 | 86 | * Choose descriptive names. Avoid single-letter variable names. 87 | * Files: lowercase separated by underscores, e.g., abstract_operator.cpp 88 | * Types (classes, structs, enums, typedefs, using): CamelCase starting with uppercase letter, e.g., BaseColumn 89 | * Variables: lowercase separated by underscores, e.g., chunk_size 90 | * Functions: CamelCase starting with uppercase letter, e.g., GetChunk 91 | * Avoid `i`, `j`, etc. in **nested** loops. Prefer to use e.g. **column_idx**, **check_idx**. In a **non-nested** loop it is permissible to use **i** as iterator index. 92 | * These rules are partially enforced by `clang-tidy`. 93 | 94 | ## Release process 95 | 96 | Extension is released to [duckdb community extension](https://github.com/duckdb/community-extensions) periodically to pick up latest changes. 97 | 98 | Before submitting a PR to the repo, we need to make sure extension builds and runs well in **ALL** platforms. 99 | 100 | Use Linux (operating system), amd64 (ISA) and musl (lib) as an example, 101 | ```sh 102 | ubuntu@hjiang-devbox-pg$ git clone git@github.com:duckdb/extension-ci-tools.git 103 | ubuntu@hjiang-devbox-pg$ cd extension-ci-tools/docker/linux_amd64_musl 104 | # Build docker image with the Dockerfile of a specific platform. 105 | ubuntu@hjiang-devbox-pg$ docker build -t duckdb-ci-linux-amd64-musl . 106 | # Start docker container and build the extension. 107 | ubuntu@hjiang-devbox-pg$ docker run -it duckdb-ci-linux-amd64-musl 108 | # Inside of the container. 109 | /duckdb_build_dir # git clone https://github.com/dentiny/duck-read-cache-fs.git && cd duck-read-cache-fs 110 | /duckdb_build_dir/duck-read-cache-fs # git submodule update --init --recursive 111 | /duckdb_build_dir/duck-read-cache-fs # CMAKE_BUILD_PARALLEL_LEVEL=$(nproc) make 112 | ``` 113 | See [link](https://github.com/duckdb/extension-ci-tools/tree/main/docker) for all required environments and docker files. 114 | 115 | ## Update duckdb version 116 | 117 | Community extension should use latest released duckdb, see [thread](https://github.com/duckdb/community-extensions/pull/346#issuecomment-2780398504) for details. 118 | 119 | Steps to update duckdb commit and version: 120 | ```sh 121 | # Switch to the desired version 122 | ubuntu@hjiang-devbox-pg$ cd duckdb && git checkout tags/v1.2.1 123 | # Commit updated duckdb. 124 | ubuntu@hjiang-devbox-pg$ cd - && git add duckdb 125 | ``` 126 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025 dentiny 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PROJ_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) 2 | 3 | # Configuration of extension 4 | EXT_NAME=cache_httpfs 5 | EXT_CONFIG=${PROJ_DIR}extension_config.cmake 6 | 7 | # Include the Makefile from extension-ci-tools 8 | include extension-ci-tools/makefiles/duckdb_extension.Makefile 9 | 10 | format-all: format 11 | find unit/ -iname *.hpp -o -iname *.cpp | xargs clang-format --sort-includes=0 -style=file -i 12 | find benchmark/ -iname *.hpp -o -iname *.cpp | xargs clang-format --sort-includes=0 -style=file -i 13 | cmake-format -i CMakeLists.txt 14 | 15 | test_unit: all 16 | find build/release/extension/cache_httpfs/ -type f -name "test*" -not -name "*.o" -not -name "*.cpp" -not -name "*.d" -exec {} \; 17 | 18 | test_reldebug_unit: all 19 | find build/reldebug/extension/cache_httpfs/ -type f -name "test*" -not -name "*.o" -not -name "*.cpp" -not -name "*.d" -exec {} \; 20 | 21 | test_debug_unit: debug 22 | find build/debug/extension/cache_httpfs/ -type f -name "test*" -not -name "*.o" -not -name "*.cpp" -not -name "*.d" -exec {} \; 23 | 24 | PHONY: format-all test_unit test_debug_unit 25 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # duck-read-cache-fs 2 | 3 | A DuckDB extension for remote filesystem access cache. 4 | 5 | ## Loading cache httpfs 6 | Since DuckDB v1.0.0, cache httpfs can be loaded as a community extension without requiring the `unsigned` flag. From any DuckDB instance, the following two commands will allow you to install and load the extension: 7 | ```sql 8 | INSTALL cache_httpfs from community; 9 | -- Or upgrade to latest version with `FORCE INSTALL cache_httpfs from community;` 10 | LOAD cache_httpfs; 11 | ``` 12 | See the [cache httpfs community extension page](https://community-extensions.duckdb.org/extensions/cache_httpfs.html) for more information. 13 | 14 | ## Introduction 15 | 16 | This repository is made as read-only filesystem for remote access, which serves as cache layer above duckdb [httpfs](https://github.com/duckdb/duckdb-httpfs). 17 | 18 | Key features: 19 | - Caching for data, which adds support for remote file access to improve IO performance and reduce egress cost; several caching options and entities are supported 20 | + in-memory, cache fetched file content into blocks and leverages a LRU cache to evict stale blocks 21 | + on-disk (default), already read blocks are stored to load filesystem, and evicted on insufficient disk space based on their access timestamp 22 | + no cache, it's allowed to disable cache and fallback to httpfs without any side effects 23 | - Parallel read, read operations are split into size-tunable chunks to increase cache hit rate and improve performance 24 | - Apart from data blocks, the extension also supports cache file handle, file metadata and glob operation 25 | + The cache for these entities are enabled by default. 26 | - Profiling helps us to understand system better, key metrics measured include cache access stats, and IO operation latency, we plan to support multiple types of profile result access; as of now there're three types of profiling 27 | + temp, all access stats are stored in memory, which could be retrieved via `SELECT cache_httpfs_get_profile();` 28 | + duckdb (under work), stats are stored in duckdb so we could leverage its rich feature for analysis purpose (i.e. use histogram to understant latency distribution) 29 | + profiling is by default disabled 30 | - 100% Compatibility with duckdb `httpfs` 31 | + Extension is built upon `httpfs` extension and automatically load it beforehand, so it's fully compatible with it; we provide option `SET cache_httpfs_type='noop';` to fallback to and behave exactly as httpfs. 32 | - Able to wrap **ALL** duckdb-compatible filesystem with one simple SQL `SELECT cache_httpfs_wrap_cache_filesystem()`, and get all the benefit of caching, parallel read, IO performance stats, you name it. 33 | 34 | Caveat: 35 | - The extension is implemented for object storage, which is expected to be read-heavy workload and (mostly) immutable, so it only supports read cache (at the moment), cache won't be cleared on write operation for the same object. 36 | + We provide workaround for overwrite -- user could call `cache_httpfs_clear_cache` to delete all cache content, and `cache_httpfs_clear_cache_for_file` for a certain object. 37 | + All types of cache provides eventual consistency guarantee, which gets evicted after a tunable timeout. 38 | - Filesystem requests are split into multiple sub-requests and aligned with block size for parallel IO requests and cache efficiency, so for small requests (i.e. read 1 byte) could suffer read amplification. 39 | A workaround for reducing amplification is to tune down block size via `cache_httpfs_cache_block_size` or fallback to native httpfs. 40 | 41 | ## Example usage 42 | ```sql 43 | -- No need to load httpfs. 44 | D LOAD cache_httpfs; 45 | -- Create S3 secret to access objects. 46 | D CREATE SECRET my_secret ( TYPE S3, KEY_ID '', SECRET '', REGION 'us-east-1', ENDPOINT 's3express-use1-az6.us-east-1.amazonaws.com'); 47 | ┌─────────┐ 48 | │ Success │ 49 | │ boolean │ 50 | ├─────────┤ 51 | │ true │ 52 | └─────────┘ 53 | 54 | -- Set cache type to in-memory. 55 | D SET cache_httpfs_type='in_mem'; 56 | 57 | -- Access remote file. 58 | D SELECT * FROM 's3://s3-bucket-user-2skzy8zuigonczyfiofztl0zbug--use1-az6--x-s3/t.parquet'; 59 | ┌───────┬───────┐ 60 | │ i │ j │ 61 | │ int64 │ int64 │ 62 | ├───────┼───────┤ 63 | │ 0 │ 1 │ 64 | │ 1 │ 2 │ 65 | │ 2 │ 3 │ 66 | │ 3 │ 4 │ 67 | │ 4 │ 5 │ 68 | ├───────┴───────┤ 69 | │ 5 rows │ 70 | └───────────────┘ 71 | ``` 72 | 73 | For more example usage, checkout [example usage](/doc/example_usage.md) 74 | 75 | ## [More About Benchmark](/benchmark/README.md) 76 | 77 | ![sequential-read.cpp](benchmark-graph/seq-performance-barchart.svg) 78 | 79 | ![random-read.cpp](benchmark-graph/random-performance-barchart.svg) 80 | 81 | ## Platform support 82 | 83 | At the moment macOS and Linux are supported, shoot us a [feature request](https://github.com/dentiny/duck-read-cache-fs/issues/new?template=feature_request.md) if you would like to run extension on other platforms. 84 | 85 | ## Development 86 | 87 | For development, the extension requires [CMake](https://cmake.org), and a `C++14` compliant compiler. Run `make` in the root directory to compile the sources. For development, use `make debug` to build a non-optimized debug version. You should run `make unit`. 88 | 89 | Please also refer to our [Contribution Guide](https://github.com/dentiny/duck-read-cache-fs/blob/main/CONTRIBUTING.md). 90 | -------------------------------------------------------------------------------- /benchmark-graph/random-performance-barchart.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Random Read Performance: S3 lineitem.parquet (25 runs × 10 bytes) 8 | 9 | 10 | 11 | 12 | 13 | 14 | Time (milliseconds) 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | DuckDB Native 25 | 26 | 27 | httpfs 28 | 29 | 30 | 3,263 ms 31 | 32 | 33 | 34 | 35 | 36 | Cache FS Extension 37 | 38 | 39 | (uncached read) 40 | 41 | 42 | 3,602 ms 43 | 44 | 45 | 46 | 0 47 | 1,000 48 | 2,000 49 | 3,000 50 | 4,000 51 | 52 | 53 | 54 | 55 | 56 | 57 | -------------------------------------------------------------------------------- /benchmark-graph/seq-performance-barchart.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Sequential Read Performance: S3 lineitem.parquet (256 MB) 8 | 9 | 10 | 11 | 12 | 13 | 14 | Time (milliseconds) 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | DuckDB Native 25 | 26 | 27 | httpfs 28 | 29 | 30 | 10,681 ms 31 | 32 | 33 | 34 | 35 | 36 | Cache FS Extension 37 | 38 | 39 | (uncached read) 40 | 41 | 42 | 3,934 ms 43 | 44 | 45 | 46 | 47 | 48 | Cache FS Extension 49 | 50 | 51 | (cached read) 52 | 53 | 54 | 31 ms 55 | 56 | 57 | 58 | 0 59 | 2,500 60 | 5,000 61 | 7,500 62 | 10,000 63 | 64 | 65 | 66 | 67 | 68 | 69 | -------------------------------------------------------------------------------- /benchmark/README.md: -------------------------------------------------------------------------------- 1 | # Duck-Read-Cache-FS Performance Benchmark Suite 2 | 3 | ## Overview 4 | This benchmark suite evaluates the performance of our DuckDB fs extension compared to the standard DuckDB httpfs. 5 | 6 | The TLDR here is: 7 | - If request size larger than request block size, our extension's performance is much better than httpfs due to parallelism and cache hit 8 | - If request size smaller than request block size, its performance is similar to httpfs but slightly worse, because extension read more bytes due to alignment 9 | - Overall, the extension provides no worse performance, meanwhile providing a few extra features 10 | 11 | ## Configuration 12 | 13 | ### AWS Credentials 14 | Set up your AWS credentials in your environment: 15 | ```bash 16 | export AWS_ACCESS_KEY_ID='your-key-id' 17 | export AWS_SECRET_ACCESS_KEY='your-secret-key' 18 | export AWS_DEFAULT_REGION='your-region' 19 | ``` 20 | 21 | ### Available Benchmark Suites 22 | ```bash 23 | build/release/extension/cache_httpfs/read_s3_object 24 | build/release/extension/cache_httpfs/sequential_read_benchmark 25 | build/release/extension/cache_httpfs/random_read_benchmark 26 | ``` 27 | 28 | ## Benchmark Methodology 29 | 30 | ### Environment Setup 31 | 32 | #### Location Details 33 | - Benchmark Machine Region: us-west1 34 | - S3 Storage Bucket Location: ap-northeast-1 35 | 36 | #### Hardware Specifications 37 | 38 | ```sh 39 | ubuntu$ lscpu 40 | Architecture: x86_64 41 | CPU op-mode(s): 32-bit, 64-bit 42 | Address sizes: 46 bits physical, 48 bits virtual 43 | Byte Order: Little Endian 44 | CPU(s): 32 45 | On-line CPU(s) list: 0-31 46 | Caches (sum of all): 47 | L1d: 512 KiB (16 instances) 48 | L1i: 512 KiB (16 instances) 49 | L2: 16 MiB (16 instances) 50 | L3: 35.8 MiB (1 instance) 51 | 52 | ubuntu$ lsmem 53 | RANGE SIZE STATE REMOVABLE BLOCK 54 | 0x0000000000000000-0x00000000bfffffff 3G online yes 0-23 55 | 0x0000000100000000-0x0000001fe7ffffff 123.6G online yes 32-1020 56 | 57 | Memory block size: 128M 58 | Total online memory: 126.6G 59 | Total offline memory: 0B 60 | ``` 61 | 62 | ### Test Categories 63 | 64 | - [Sequential read operations](https://github.com/dentiny/duck-read-cache-fs/blob/main/benchmark/random_read_benchmark.cpp) 65 | - [Random read operations](https://github.com/dentiny/duck-read-cache-fs/blob/main/benchmark/random_read_benchmark.cpp) 66 | -------------------------------------------------------------------------------- /benchmark/rand_utils.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | namespace duckdb { 6 | 7 | // Generate a random value base on [min, max) range using in uniform distribution feature. 8 | template 9 | T GetRandomValueInRange(T min, T max) { 10 | thread_local std::random_device rand_dev; 11 | std::mt19937 rand_engine(rand_dev()); 12 | std::uniform_real_distribution unif(min, max); 13 | return unif(rand_engine); 14 | } 15 | 16 | } // namespace duckdb 17 | -------------------------------------------------------------------------------- /benchmark/random_read_benchmark.cpp: -------------------------------------------------------------------------------- 1 | // Benchmark setup: start from a random offset, and read 10 bytes every time (which is much less than block size). 2 | 3 | #include "disk_cache_reader.hpp" 4 | #include "duckdb/storage/standard_buffer_manager.hpp" 5 | #include "duckdb/main/client_context_file_opener.hpp" 6 | #include "rand_utils.hpp" 7 | #include "s3fs.hpp" 8 | #include "scope_guard.hpp" 9 | #include "time_utils.hpp" 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | namespace { 16 | const std::string BENCHMARK_DISK_CACHE_DIRECTORY = "/tmp/benchmark_cache"; 17 | constexpr uint64_t BYTES_TO_READ = 10; 18 | constexpr uint64_t BENCHMARK_RUNS = 25; 19 | } // namespace 20 | 21 | namespace duckdb { 22 | 23 | namespace { 24 | 25 | struct BenchmarkSetup { 26 | std::string cache_type; 27 | std::string profile_type; 28 | std::string disk_cache_directory; 29 | uint64_t block_size = DEFAULT_CACHE_BLOCK_SIZE; 30 | }; 31 | 32 | void SetConfig(case_insensitive_map_t &setting, char *env_key, char *secret_key) { 33 | const char *env_val = getenv(env_key); 34 | if (env_val == nullptr) { 35 | return; 36 | } 37 | setting[secret_key] = Value(env_val); 38 | } 39 | 40 | void SetOpenerConfig(shared_ptr ctx, const BenchmarkSetup &benchmark_setup) { 41 | auto &set_vars = ctx->config.set_variables; 42 | SetConfig(set_vars, "AWS_DEFAULT_REGION", "s3_region"); 43 | SetConfig(set_vars, "AWS_ACCESS_KEY_ID", "s3_access_key_id"); 44 | SetConfig(set_vars, "AWS_SECRET_ACCESS_KEY", "s3_secret_access_key"); 45 | set_vars["cache_httpfs_profile_type"] = Value(benchmark_setup.profile_type); 46 | set_vars["cache_httpfs_type"] = Value(benchmark_setup.cache_type); 47 | set_vars["cache_httpfs_cache_directory"] = Value(benchmark_setup.disk_cache_directory); 48 | set_vars["cache_httpfs_cache_block_size"] = Value::UBIGINT(benchmark_setup.block_size); 49 | } 50 | 51 | void TestSequentialRead(const BenchmarkSetup &benchmark_setup) { 52 | DuckDB db {}; 53 | StandardBufferManager buffer_manager {*db.instance, "/tmp/cache_httpfs_fs_benchmark"}; 54 | auto s3fs = make_uniq(buffer_manager); 55 | auto cache_fs = make_uniq(std::move(s3fs)); 56 | auto client_context = make_shared_ptr(db.instance); 57 | 58 | SetOpenerConfig(client_context, benchmark_setup); 59 | ClientContextFileOpener file_opener {*client_context}; 60 | client_context->transaction.BeginTransaction(); 61 | 62 | auto file_handle = 63 | cache_fs->OpenFile("s3://duckdb-cache-fs/lineitem.parquet", FileOpenFlags::FILE_FLAGS_READ, &file_opener); 64 | const uint64_t file_size = cache_fs->GetFileSize(*file_handle); 65 | std::string buffer(BYTES_TO_READ, '\0'); 66 | 67 | const auto start = GetSteadyNowMilliSecSinceEpoch(); 68 | 69 | for (uint16_t ii = 0; ii < BENCHMARK_RUNS; ++ii) { 70 | const uint64_t start_offset = GetRandomValueInRange(0, file_size); 71 | const uint64_t cur_bytes_to_read = MinValue(BYTES_TO_READ, file_size - start_offset); 72 | cache_fs->Read(*file_handle, const_cast(buffer.data()), /*nr_bytes=*/cur_bytes_to_read, 73 | /*location=*/start_offset); 74 | } 75 | 76 | const auto end = GetSteadyNowMilliSecSinceEpoch(); 77 | const auto duration_millisec = end - start; 78 | std::cout << BENCHMARK_RUNS << " runs of random read of " << BYTES_TO_READ << " bytes takes " << duration_millisec 79 | << " milliseconds" << std::endl; 80 | } 81 | 82 | } // namespace 83 | 84 | } // namespace duckdb 85 | 86 | int main(int argc, char **argv) { 87 | // Ignore SIGPIPE, reference: https://blog.erratasec.com/2018/10/tcpip-sockets-and-sigpipe.html 88 | std::signal(SIGPIPE, SIG_IGN); 89 | 90 | // Warm up system resource (i.e. httpfs metadata cache, TCP congestion window, etc). 91 | std::cout << "Starts to warmup read" << std::endl; 92 | duckdb::FileSystem::CreateLocal()->RemoveDirectory(BENCHMARK_DISK_CACHE_DIRECTORY); 93 | duckdb::BenchmarkSetup benchmark_setup; 94 | benchmark_setup.cache_type = *duckdb::NOOP_CACHE_TYPE; 95 | benchmark_setup.profile_type = *duckdb::NOOP_PROFILE_TYPE; 96 | duckdb::TestSequentialRead(benchmark_setup); 97 | 98 | // Benchmark httpfs (with no cache reader). 99 | std::cout << "Starts with httpfs read with no cache" << std::endl; 100 | duckdb::FileSystem::CreateLocal()->RemoveDirectory(BENCHMARK_DISK_CACHE_DIRECTORY); 101 | benchmark_setup.cache_type = *duckdb::NOOP_CACHE_TYPE; 102 | benchmark_setup.profile_type = *duckdb::TEMP_PROFILE_TYPE; 103 | benchmark_setup.disk_cache_directory = BENCHMARK_DISK_CACHE_DIRECTORY; 104 | duckdb::TestSequentialRead(benchmark_setup); 105 | 106 | // Benchmark on-disk cache reader. 107 | std::cout << "Starts on-disk cache read with no existing cache" << std::endl; 108 | duckdb::FileSystem::CreateLocal()->RemoveDirectory(BENCHMARK_DISK_CACHE_DIRECTORY); 109 | benchmark_setup.cache_type = *duckdb::ON_DISK_CACHE_TYPE; 110 | benchmark_setup.profile_type = *duckdb::TEMP_PROFILE_TYPE; 111 | benchmark_setup.disk_cache_directory = BENCHMARK_DISK_CACHE_DIRECTORY; 112 | duckdb::TestSequentialRead(benchmark_setup); 113 | 114 | // Cleanup on-disk cache after benchmark. 115 | duckdb::FileSystem::CreateLocal()->RemoveDirectory(BENCHMARK_DISK_CACHE_DIRECTORY); 116 | 117 | return 0; 118 | } 119 | -------------------------------------------------------------------------------- /benchmark/read_s3_object.cpp: -------------------------------------------------------------------------------- 1 | // This file serves as a benchmark to read a whole S3 objects; it only tests 2 | // uncached read. 3 | 4 | #include "disk_cache_reader.hpp" 5 | #include "duckdb/storage/standard_buffer_manager.hpp" 6 | #include "duckdb/main/client_context_file_opener.hpp" 7 | #include "s3fs.hpp" 8 | #include "scope_guard.hpp" 9 | 10 | #include 11 | #include 12 | #include 13 | 14 | namespace duckdb { 15 | 16 | namespace { 17 | 18 | // Set configuration in client context from env variables. 19 | void SetConfig(case_insensitive_map_t &setting, char *env_key, char *secret_key) { 20 | const char *env_val = getenv(env_key); 21 | if (env_val == nullptr) { 22 | return; 23 | } 24 | setting[secret_key] = Value(env_val); 25 | } 26 | 27 | // Baseline benchmark, which reads the whole file with no parallelism and 28 | // caching. 29 | void BaseLineRead() { 30 | DuckDB db {}; 31 | StandardBufferManager buffer_manager {*db.instance, "/tmp/cache_httpfs_fs_benchmark"}; 32 | auto s3fs = make_uniq(buffer_manager); 33 | 34 | auto client_context = make_shared_ptr(db.instance); 35 | auto &set_vars = client_context->config.set_variables; 36 | SetConfig(set_vars, "AWS_DEFAULT_REGION", "s3_region"); 37 | SetConfig(set_vars, "AWS_ACCESS_KEY_ID", "s3_access_key_id"); 38 | SetConfig(set_vars, "AWS_SECRET_ACCESS_KEY", "s3_secret_access_key"); 39 | SetConfig(set_vars, "DUCKDB_S3_ENDPOINT", "s3_endpoint"); 40 | 41 | ClientContextFileOpener file_opener {*client_context}; 42 | client_context->transaction.BeginTransaction(); 43 | auto file_handle = s3fs->OpenFile("s3://s3-bucket-user-2skzy8zuigonczyfiofztl0zbug--use1-az6--x-s3/" 44 | "large-csv.csv", 45 | FileOpenFlags::FILE_FLAGS_READ, &file_opener); 46 | const uint64_t file_size = s3fs->GetFileSize(*file_handle); 47 | std::string content(file_size, '\0'); 48 | 49 | const auto now = std::chrono::steady_clock::now(); 50 | s3fs->Read(*file_handle, const_cast(content.data()), file_size, 51 | /*location=*/0); 52 | const auto end = std::chrono::steady_clock::now(); 53 | const auto duration_sec = std::chrono::duration_cast>(end - now).count(); 54 | std::cout << "Baseline S3 filesystem reads " << file_size << " bytes takes " << duration_sec << " seconds" 55 | << std::endl; 56 | } 57 | 58 | void ReadUncachedWholeFile(uint64_t block_size) { 59 | g_cache_block_size = block_size; 60 | *g_cache_type = *DEFAULT_ON_DISK_CACHE_DIRECTORY; 61 | SCOPE_EXIT { 62 | ResetGlobalConfig(); 63 | }; 64 | 65 | DuckDB db {}; 66 | StandardBufferManager buffer_manager {*db.instance, "/tmp/cache_httpfs_fs_benchmark"}; 67 | auto s3fs = make_uniq(buffer_manager); 68 | 69 | FileSystem::CreateLocal()->RemoveDirectory(*g_on_disk_cache_directory); 70 | auto disk_cache_fs = make_uniq(std::move(s3fs)); 71 | 72 | auto client_context = make_shared_ptr(db.instance); 73 | auto &set_vars = client_context->config.set_variables; 74 | SetConfig(set_vars, "AWS_DEFAULT_REGION", "s3_region"); 75 | SetConfig(set_vars, "AWS_ACCESS_KEY_ID", "s3_access_key_id"); 76 | SetConfig(set_vars, "AWS_SECRET_ACCESS_KEY", "s3_secret_access_key"); 77 | SetConfig(set_vars, "DUCKDB_S3_ENDPOINT", "s3_endpoint"); 78 | ClientContextFileOpener file_opener {*client_context}; 79 | client_context->transaction.BeginTransaction(); 80 | 81 | auto file_handle = disk_cache_fs->OpenFile("s3://s3-bucket-user-2skzy8zuigonczyfiofztl0zbug--use1-az6--x-s3/" 82 | "large-csv.csv", 83 | FileOpenFlags::FILE_FLAGS_READ, &file_opener); 84 | const uint64_t file_size = disk_cache_fs->GetFileSize(*file_handle); 85 | std::string content(file_size, '\0'); 86 | 87 | auto read_whole_file = [&]() { 88 | const auto now = std::chrono::steady_clock::now(); 89 | disk_cache_fs->Read(*file_handle, const_cast(content.data()), file_size, /*location=*/0); 90 | const auto end = std::chrono::steady_clock::now(); 91 | const auto duration_sec = std::chrono::duration_cast>(end - now).count(); 92 | std::cout << "Cached http filesystem reads " << file_size << " bytes with block size " << block_size 93 | << " takes " << duration_sec << " seconds" << std::endl; 94 | }; 95 | 96 | // Uncached but parallel read. 97 | read_whole_file(); 98 | // Cached and parallel read. 99 | read_whole_file(); 100 | } 101 | 102 | } // namespace 103 | 104 | } // namespace duckdb 105 | 106 | int main(int argc, char **argv) { 107 | // Ignore SIGPIPE, reference: https://blog.erratasec.com/2018/10/tcpip-sockets-and-sigpipe.html 108 | std::signal(SIGPIPE, SIG_IGN); 109 | 110 | constexpr std::array BLOCK_SIZE_ARR { 111 | 64ULL * 1024, // 64KiB 112 | 256ULL * 1024, // 256KiB 113 | 1ULL * 1024 * 1024, // 1MiB 114 | 4ULL * 1024 * 1024, // 4MiB 115 | 16ULL * 1024 * 1024, // 16MiB 116 | }; 117 | 118 | duckdb::BaseLineRead(); 119 | for (uint64_t cur_block_size : BLOCK_SIZE_ARR) { 120 | duckdb::ReadUncachedWholeFile(cur_block_size); 121 | } 122 | return 0; 123 | } 124 | -------------------------------------------------------------------------------- /benchmark/sequential_read_benchmark.cpp: -------------------------------------------------------------------------------- 1 | // Benchmark setup: 2 | // - Request number of bytes larger than cache block size; 3 | // - Sequentially read forward until the end of remote file. 4 | 5 | #include "disk_cache_reader.hpp" 6 | #include "duckdb/storage/standard_buffer_manager.hpp" 7 | #include "duckdb/main/client_context_file_opener.hpp" 8 | #include "s3fs.hpp" 9 | #include "scope_guard.hpp" 10 | #include "time_utils.hpp" 11 | 12 | #include 13 | #include 14 | #include 15 | 16 | namespace duckdb { 17 | 18 | namespace { 19 | 20 | struct BenchmarkSetup { 21 | std::string cache_type; 22 | std::string profile_type; 23 | std::string disk_cache_directory; 24 | uint64_t block_size = DEFAULT_CACHE_BLOCK_SIZE; 25 | }; 26 | 27 | void SetConfig(case_insensitive_map_t &setting, char *env_key, char *secret_key) { 28 | const char *env_val = getenv(env_key); 29 | if (env_val == nullptr) { 30 | return; 31 | } 32 | setting[secret_key] = Value(env_val); 33 | } 34 | 35 | void SetOpenerConfig(shared_ptr ctx, const BenchmarkSetup &benchmark_setup) { 36 | auto &set_vars = ctx->config.set_variables; 37 | SetConfig(set_vars, "AWS_DEFAULT_REGION", "s3_region"); 38 | SetConfig(set_vars, "AWS_ACCESS_KEY_ID", "s3_access_key_id"); 39 | SetConfig(set_vars, "AWS_SECRET_ACCESS_KEY", "s3_secret_access_key"); 40 | set_vars["cache_httpfs_profile_type"] = Value(benchmark_setup.profile_type); 41 | set_vars["cache_httpfs_type"] = Value(benchmark_setup.cache_type); 42 | set_vars["cache_httpfs_cache_directory"] = Value(benchmark_setup.disk_cache_directory); 43 | set_vars["cache_httpfs_cache_block_size"] = Value::UBIGINT(benchmark_setup.block_size); 44 | } 45 | 46 | void TestSequentialRead(const BenchmarkSetup &benchmark_setup) { 47 | DuckDB db {}; 48 | StandardBufferManager buffer_manager {*db.instance, "/tmp/cache_httpfs_fs_benchmark"}; 49 | auto s3fs = make_uniq(buffer_manager); 50 | auto cache_fs = make_uniq(std::move(s3fs)); 51 | auto client_context = make_shared_ptr(db.instance); 52 | 53 | SetOpenerConfig(client_context, benchmark_setup); 54 | ClientContextFileOpener file_opener {*client_context}; 55 | client_context->transaction.BeginTransaction(); 56 | 57 | auto file_handle = 58 | cache_fs->OpenFile("s3://duckdb-cache-fs/lineitem.parquet", FileOpenFlags::FILE_FLAGS_READ, &file_opener); 59 | const uint64_t file_size = cache_fs->GetFileSize(*file_handle); 60 | 61 | const size_t chunk_size = 32_MiB; 62 | std::string buffer(chunk_size, '\0'); 63 | 64 | uint64_t bytes_read = 0; 65 | const auto start = GetSteadyNowMilliSecSinceEpoch(); 66 | 67 | while (bytes_read < file_size) { 68 | size_t current_chunk = std::min(chunk_size, file_size - bytes_read); 69 | cache_fs->Read(*file_handle, const_cast(buffer.data()), current_chunk, bytes_read); 70 | bytes_read += current_chunk; 71 | } 72 | 73 | const auto end = GetSteadyNowMilliSecSinceEpoch(); 74 | const auto duration_millisec = end - start; 75 | std::cout << "Sequential read of " << file_size << " bytes in " << chunk_size << "-byte chunks takes " 76 | << duration_millisec << " milliseconds" << std::endl; 77 | } 78 | 79 | } // namespace 80 | 81 | } // namespace duckdb 82 | 83 | int main(int argc, char **argv) { 84 | // Ignore SIGPIPE, reference: https://blog.erratasec.com/2018/10/tcpip-sockets-and-sigpipe.html 85 | std::signal(SIGPIPE, SIG_IGN); 86 | 87 | const std::string disk_cache_directory = "/tmp/benchmark_cache"; 88 | 89 | // Warm up system resource (i.e. httpfs metadata cache, TCP congestion window, etc). 90 | std::cout << "Starts to warmup read" << std::endl; 91 | duckdb::FileSystem::CreateLocal()->RemoveDirectory(disk_cache_directory); 92 | duckdb::BenchmarkSetup benchmark_setup; 93 | benchmark_setup.cache_type = *duckdb::NOOP_CACHE_TYPE; 94 | benchmark_setup.profile_type = *duckdb::NOOP_PROFILE_TYPE; 95 | duckdb::TestSequentialRead(benchmark_setup); 96 | 97 | // Benchmark httpfs (with no cache reader). 98 | std::cout << "Starts with httpfs read with no cache" << std::endl; 99 | duckdb::FileSystem::CreateLocal()->RemoveDirectory(disk_cache_directory); 100 | benchmark_setup.cache_type = *duckdb::NOOP_CACHE_TYPE; 101 | benchmark_setup.profile_type = *duckdb::TEMP_PROFILE_TYPE; 102 | benchmark_setup.disk_cache_directory = disk_cache_directory; 103 | duckdb::TestSequentialRead(benchmark_setup); 104 | 105 | // Benchmark on-disk cache reader. 106 | std::cout << "Starts on-disk cache read with no existing cache" << std::endl; 107 | duckdb::FileSystem::CreateLocal()->RemoveDirectory(disk_cache_directory); 108 | benchmark_setup.cache_type = *duckdb::ON_DISK_CACHE_TYPE; 109 | benchmark_setup.profile_type = *duckdb::TEMP_PROFILE_TYPE; 110 | benchmark_setup.disk_cache_directory = disk_cache_directory; 111 | benchmark_setup.block_size = 2_MiB; 112 | duckdb::TestSequentialRead(benchmark_setup); 113 | 114 | // No delete cache directory to verify cache read. 115 | std::cout << "Starts on-disk cache read with local cache" << std::endl; 116 | benchmark_setup.cache_type = *duckdb::ON_DISK_CACHE_TYPE; 117 | benchmark_setup.profile_type = *duckdb::TEMP_PROFILE_TYPE; 118 | benchmark_setup.disk_cache_directory = disk_cache_directory; 119 | duckdb::TestSequentialRead(benchmark_setup); 120 | 121 | // Cleanup on-disk cache after benchmark. 122 | duckdb::FileSystem::CreateLocal()->RemoveDirectory(disk_cache_directory); 123 | 124 | return 0; 125 | } 126 | -------------------------------------------------------------------------------- /doc/common_problem_troubleshooting.md: -------------------------------------------------------------------------------- 1 | ### Common problem troubleshooting 2 | 3 | #### Problem-1: program crashes due to SIGPIPE 4 | 5 | `SIGPIPE` is notorious in network programming, which **could** be caused by improper handling to a closed socket. A common solution is directly ignore certain the signal and trigger TCP reconnection and rewrite. 6 | 7 | cache httpfs provides such option (by default off), 8 | ```sql 9 | D SET cache_httpfs_ignore_sigpipe=true; 10 | ``` 11 | 12 | It's by default off since it's a process-wise signal handling, rather than an extension-wise configuration. 13 | 14 | Reference: https://blog.erratasec.com/2018/10/tcpip-sockets-and-sigpipe.html 15 | -------------------------------------------------------------------------------- /doc/consistency_spec.md: -------------------------------------------------------------------------------- 1 | ### Consistency specification 2 | 3 | All types of filesystem caches provides **eventual consistency** guarantee, which means they don't reflect the latest write result. 4 | Cache entries will become invalid and get evicted after a tunable timeout. 5 | 6 | We provide several workarounds, so users are able to configure based on their workload. 7 | 8 | #### Users could clear cache based on different cache type. 9 | ```sql 10 | -- Clear all types of cache. 11 | D SELECT cache_httpfs_clear_cache(); 12 | -- Clear cache based on filepath. 13 | D SELECT cache_httpfs_clear_cache_for_file(); 14 | ``` 15 | 16 | > [!CAUTION] 17 | > It's worth noting due to the filepath format for on-disk cache, it's not possible to delete cache entries by filepath (we will try to implement if it's requested). 18 | 19 | #### Configure cache timeout, so staleness become more tolerable. 20 | ```sql 21 | -- Tune in-memory data block cache timeout. 22 | D SET cache_httpfs_in_mem_cache_block_timeout_millisec=30000; 23 | -- Tune metadata cache timeout. 24 | D SET cache_httpfs_metadata_cache_entry_timeout_millisec=30000; 25 | -- Tune file handle cache timeout. 26 | D SET cache_httpfs_file_handle_cache_entry_timeout_millisec=30000; 27 | -- Tune glob cache timeout. 28 | D SET cache_httpfs_glob_cache_entry_timeout_millisec=30000; 29 | ``` 30 | 31 | #### Disable cache for certain types of cache 32 | 33 | Users are able to disable a certain type of cache. 34 | ```sql 35 | -- Disable data block cache. 36 | D SET cache_httpfs_type='noop'; 37 | -- Disable metdata cache. 38 | D SET cache_httpfs_enable_metadata_cache=false; 39 | -- Disable file handle cache. 40 | D SET cache_httpfs_enable_file_handle_cache=false; 41 | -- Disable glob cache. 42 | D SET cache_httpfs_enable_glob_cache=false; 43 | ``` 44 | -------------------------------------------------------------------------------- /doc/example_usage.md: -------------------------------------------------------------------------------- 1 | ### Example usage for cache httpfs. 2 | 3 | - Cache httpfs is 100% compatible with native httpfs; we provide an option to fallback to httpfs extension. 4 | ```sql 5 | -- Later access won't be cached, but existing cache (whether it's in-memory or on-disk) will be kept. 6 | D SET cache_httpfs_profile_type='noop'; 7 | ``` 8 | 9 | - Extension allows user to set on-disk cache file directory, it could be local filesystem or remote storage (via mount), which provides much flexibility. 10 | ```sql 11 | D SET cache_httpfs_profile_type='on_disk'; 12 | -- By default cache files will be found under `/tmp/duckdb_cache_httpfs_cache`. 13 | D SET cache_httpfs_cache_directory='/tmp/mounted_cache_directory'; 14 | -- Update min required disk space to enable on-disk cache; by default 5% of disk space is required. 15 | -- Here we set 1GB as the min requried disk size. 16 | D SET cache_httpfs_min_disk_bytes_for_cache=1000000000; 17 | ``` 18 | 19 | - For the extension, filesystem requests are split into multiple sub-requests and aligned with block size for parallel IO requests and cache efficiency. 20 | We provide options to tune block size. 21 | ```sql 22 | -- By default block size is 64KiB, here we update it to 4KiB. 23 | D SET cache_httpfs_cache_block_size=4096; 24 | ``` 25 | 26 | - Parallel read feature mentioned above is achieved by spawning multiple threads, with users allowed to adjust thread number. 27 | ```sql 28 | -- By default we don't set any limit for subrequest number, with the new setting 10 requests will be performed at the same time. 29 | D SET cache_httpfs_max_fanout_subrequest=10; 30 | ``` 31 | 32 | - User could understand IO characteristics by enabling profiling; currently the extension exposes cache access and IO latency distribution. 33 | ```sql 34 | D SET cache_httpfs_profile_type='temp'; 35 | -- When profiling enabled, dump to local filesystem for better display. 36 | D COPY (SELECT cache_httpfs_get_profile()) TO '/tmp/output.txt'; 37 | ``` 38 | 39 | - A rich set of parameters and util functions are provided for the above features, including but not limited to type of caching, IO request size, etc. 40 | Checkout by 41 | ```sql 42 | -- Get all extension configs. 43 | D SELECT * FROM duckdb_settings() WHERE name LIKE 'cache_httpfs%'; 44 | -- Get all extension util functions. 45 | D SELECT * FROM duckdb_functions() WHERE function_name LIKE 'cache_httpfs%'; 46 | ``` 47 | 48 | - Users could clear cache, whether it's in-memory or on-disk with 49 | ```sql 50 | D SELECT cache_httpfs_clear_cache(); 51 | ``` 52 | or clear cache for a particular file with 53 | ```sql 54 | D SELECT cache_httpfs_clear_cache_for_file('filename'); 55 | ``` 56 | Notice the query could be slow. 57 | 58 | - The extension supports not only httpfs, but also ALL filesystems compatible with duckdb. 59 | ```sql 60 | D SELECT cache_httpfs_wrap_cache_filesystem('filesystem-name'); 61 | ``` 62 | 63 | - Apart from data block cache, the extension also supports caching other entities, including file handle, file metadata and glob operations. The cache options are turned on by default, users are able to opt off. 64 | ```sql 65 | D SET cache_httpfs_enable_metadata_cache=false; 66 | D SET cache_httpfs_enable_glob_cache=false; 67 | D SET cache_httpfs_enable_file_handle_cache=false; 68 | 69 | -- Users are able to check cache access information. 70 | D SELECT * FROM cache_httpfs_cache_access_info_query(); 71 | 72 | ┌─────────────┬─────────────────┬──────────────────┐ 73 | │ cache_type │ cache_hit_count │ cache_miss_count │ 74 | │ varchar │ uint64 │ uint64 │ 75 | ├─────────────┼─────────────────┼──────────────────┤ 76 | │ metadata │ 0 │ 0 │ 77 | │ data │ 0 │ 0 │ 78 | │ file handle │ 0 │ 0 │ 79 | │ glob │ 0 │ 0 │ 80 | └─────────────┴─────────────────┴──────────────────┘ 81 | ``` 82 | -------------------------------------------------------------------------------- /doc/performance_troubleshooting_notes.md: -------------------------------------------------------------------------------- 1 | ### Performance troubleshooting notes 2 | 3 | cache httpfs extension bakes observability from the beginning of the design: users are able to enable stats for IO operations and dump reports for better display. 4 | 5 | For a slow query, it's usually not easy to tell it's CPU bottlenecked or IO bottlenecked, and how much time does your query spend on IO operations, etc. 6 | With cache httpfs extension, one thing you could do to understand the system and query better is: 7 | 8 | First, clear the cache. 9 | ```sql 10 | D SELECT cache_httpfs_clear_cache(); 11 | ``` 12 | And run the query with profiling enabled. 13 | ```sql 14 | D SET cache_httpfs_profile_type='temp'; 15 | D 16 | -- When profiling enabled, dump to local filesystem for better display. 17 | D COPY (SELECT cache_httpfs_get_profile()) TO '/tmp/uncached_io_stats.txt'; 18 | ``` 19 | Now you could check the IO profile results in the file. 20 | It's worth noting that cache httpfs split every request into multiple subrequests in block size, so the IO latency in the record represents a subrequest. 21 | 22 | For example, assumne (1) cache filesystem is requested to read from the first byte and read for 8 KiB, and (2) cache block size is 4KiB, filesystem will issue two subrequests, with each request performing a 4KiB read, the latency recorded represents each 4KiB read. 23 | 24 | Till this point, we should have a basic understand how IO characteristics look like (i.e. IO latency for a uncached block read). 25 | 26 | Second, clear the profile records and perform a cached read with the same query. 27 | ```sql 28 | D SELECT cache_httpfs_clear_profile(); 29 | -- It should be same query as previous. 30 | D 31 | -- Now we get a new profile for a cached access. 32 | D COPY (SELECT cache_httpfs_get_profile()) TO '/tmp/cached_io_stats.txt'; 33 | ``` 34 | 35 | Third, in theory, as long as the requested file is not too big (so that cache space gets exhausted too quickly), we shouldn't suffer any cache miss. 36 | Now it's good to elimintate the possibility of slow IO operations. 37 | -------------------------------------------------------------------------------- /doc/resource_consumption.md: -------------------------------------------------------------------------------- 1 | ### Resource consumption 2 | 3 | For this extension, users are able to have control of its resource consumption. For each type of cache, we provide certain parameter for users to tune based on their requirement. 4 | 5 | ```sql 6 | -- Reserve disk space, to avoid on-disk data cache taking too much space. 7 | -- By default the 5% of disk space will be reserved, but it's allowed to override. Eg, the following sql will reserve 5GB space. 8 | D SET cache_httpfs_min_disk_bytes_for_cache=5000000; 9 | 10 | -- Control the maximum memory usage for in-memory data cache. 11 | -- The maximum memory consumption is calculated as [cache_httpfs_cache_block_size] * [cache_httpfs_max_in_mem_cache_block_count]. 12 | D SET cache_httpfs_max_in_mem_cache_block_count=10; 13 | 14 | -- Control the number of metadata cache entries. 15 | D SET cache_httpfs_metadata_cache_entry_size=10; 16 | 17 | -- Control the number of file handle entries. 18 | D SET cache_httpfs_file_handle_cache_entry_size=10; 19 | 20 | -- Control the number of glob cache entries. 21 | D SET cache_httpfs_glob_cache_entry_size=10; 22 | ``` 23 | 24 | In extreme cases when resource become critically important, users are able to (1) cleanup cache entries; or (2) disable certain cache types. 25 | -------------------------------------------------------------------------------- /extension_config.cmake: -------------------------------------------------------------------------------- 1 | # This file is included by DuckDB's build system. It specifies which extension to load 2 | 3 | # Extension from this repo 4 | duckdb_extension_load(cache_httpfs 5 | SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} 6 | LOAD_TESTS 7 | ) 8 | 9 | # Any extra extensions that should be built 10 | # e.g.: duckdb_extension_load(json) 11 | -------------------------------------------------------------------------------- /src/cache_entry_info.cpp: -------------------------------------------------------------------------------- 1 | #include "cache_entry_info.hpp" 2 | 3 | #include 4 | 5 | namespace duckdb { 6 | 7 | bool operator<(const DataCacheEntryInfo &lhs, const DataCacheEntryInfo &rhs) { 8 | return std::tie(lhs.cache_filepath, lhs.remote_filename, lhs.start_offset, lhs.end_offset, lhs.cache_type) < 9 | std::tie(rhs.cache_filepath, rhs.remote_filename, rhs.start_offset, rhs.end_offset, rhs.cache_type); 10 | } 11 | 12 | } // namespace duckdb 13 | -------------------------------------------------------------------------------- /src/cache_filesystem_ref_registry.cpp: -------------------------------------------------------------------------------- 1 | #include "cache_filesystem_ref_registry.hpp" 2 | 3 | #include "cache_filesystem.hpp" 4 | 5 | namespace duckdb { 6 | 7 | /*static*/ CacheFsRefRegistry &CacheFsRefRegistry::Get() { 8 | static auto *registry = new CacheFsRefRegistry(); 9 | return *registry; 10 | } 11 | 12 | void CacheFsRefRegistry::Register(CacheFileSystem *fs) { 13 | cache_filesystems.emplace_back(fs); 14 | } 15 | 16 | void CacheFsRefRegistry::Reset() { 17 | cache_filesystems.clear(); 18 | } 19 | 20 | const vector &CacheFsRefRegistry::GetAllCacheFs() const { 21 | return cache_filesystems; 22 | } 23 | 24 | } // namespace duckdb 25 | -------------------------------------------------------------------------------- /src/cache_reader_manager.cpp: -------------------------------------------------------------------------------- 1 | #include "cache_reader_manager.hpp" 2 | 3 | #include "base_cache_reader.hpp" 4 | #include "disk_cache_reader.hpp" 5 | #include "in_memory_cache_reader.hpp" 6 | #include "noop_cache_reader.hpp" 7 | 8 | namespace duckdb { 9 | 10 | /*static*/ CacheReaderManager &CacheReaderManager::Get() { 11 | static auto *cache_reader_manager = new CacheReaderManager(); 12 | return *cache_reader_manager; 13 | } 14 | 15 | void CacheReaderManager::InitializeDiskCacheReader() { 16 | if (on_disk_cache_reader == nullptr) { 17 | on_disk_cache_reader = make_uniq(); 18 | } 19 | } 20 | 21 | void CacheReaderManager::SetCacheReader() { 22 | if (*g_cache_type == *NOOP_CACHE_TYPE) { 23 | if (noop_cache_reader == nullptr) { 24 | noop_cache_reader = make_uniq(); 25 | } 26 | internal_cache_reader = noop_cache_reader.get(); 27 | return; 28 | } 29 | 30 | if (*g_cache_type == *ON_DISK_CACHE_TYPE) { 31 | if (on_disk_cache_reader == nullptr) { 32 | on_disk_cache_reader = make_uniq(); 33 | } 34 | internal_cache_reader = on_disk_cache_reader.get(); 35 | return; 36 | } 37 | 38 | if (*g_cache_type == *IN_MEM_CACHE_TYPE) { 39 | if (in_mem_cache_reader == nullptr) { 40 | in_mem_cache_reader = make_uniq(); 41 | } 42 | internal_cache_reader = in_mem_cache_reader.get(); 43 | return; 44 | } 45 | } 46 | 47 | BaseCacheReader *CacheReaderManager::GetCacheReader() const { 48 | return internal_cache_reader; 49 | } 50 | 51 | vector CacheReaderManager::GetCacheReaders() const { 52 | vector cache_readers; 53 | if (in_mem_cache_reader != nullptr) { 54 | cache_readers.emplace_back(in_mem_cache_reader.get()); 55 | } 56 | if (on_disk_cache_reader != nullptr) { 57 | cache_readers.emplace_back(on_disk_cache_reader.get()); 58 | } 59 | return cache_readers; 60 | } 61 | 62 | void CacheReaderManager::ClearCache() { 63 | if (noop_cache_reader != nullptr) { 64 | noop_cache_reader->ClearCache(); 65 | } 66 | if (in_mem_cache_reader != nullptr) { 67 | in_mem_cache_reader->ClearCache(); 68 | } 69 | if (on_disk_cache_reader != nullptr) { 70 | on_disk_cache_reader->ClearCache(); 71 | } 72 | } 73 | 74 | void CacheReaderManager::ClearCache(const string &fname) { 75 | if (noop_cache_reader != nullptr) { 76 | noop_cache_reader->ClearCache(fname); 77 | } 78 | if (in_mem_cache_reader != nullptr) { 79 | in_mem_cache_reader->ClearCache(fname); 80 | } 81 | if (on_disk_cache_reader != nullptr) { 82 | on_disk_cache_reader->ClearCache(fname); 83 | } 84 | } 85 | 86 | void CacheReaderManager::Reset() { 87 | noop_cache_reader.reset(); 88 | in_mem_cache_reader.reset(); 89 | on_disk_cache_reader.reset(); 90 | internal_cache_reader = nullptr; 91 | } 92 | 93 | } // namespace duckdb 94 | -------------------------------------------------------------------------------- /src/histogram.cpp: -------------------------------------------------------------------------------- 1 | #include "histogram.hpp" 2 | 3 | #include 4 | #include 5 | 6 | #include "duckdb/common/assert.hpp" 7 | #include "duckdb/common/helper.hpp" 8 | #include "duckdb/common/string_util.hpp" 9 | 10 | namespace duckdb { 11 | 12 | Histogram::Histogram(double min_val, double max_val, int num_bkt) 13 | : min_val_(min_val), max_val_(max_val), num_bkt_(num_bkt) { 14 | D_ASSERT(min_val_ < max_val_); 15 | D_ASSERT(num_bkt > 0); 16 | Reset(); 17 | } 18 | 19 | void Histogram::SetStatsDistribution(std::string name, std::string unit) { 20 | distribution_name_ = std::move(name); 21 | distribution_unit_ = std::move(unit); 22 | } 23 | 24 | void Histogram::Reset() { 25 | min_encountered_ = max_val_; 26 | max_encountered_ = min_val_; 27 | total_counts_ = 0; 28 | sum_ = 0; 29 | hist_ = std::vector(num_bkt_, 0); 30 | outliers_.clear(); 31 | } 32 | 33 | size_t Histogram::Bucket(double val) const { 34 | D_ASSERT(val >= min_val_); 35 | D_ASSERT(val < max_val_); 36 | 37 | if (val == max_val_) { 38 | return hist_.size() - 1; 39 | } 40 | const double idx = (val - min_val_) / (max_val_ - min_val_); 41 | return static_cast(std::floor(idx * hist_.size())); 42 | } 43 | 44 | void Histogram::Add(double val) { 45 | if (val < min_val_ || val >= max_val_) { 46 | outliers_.emplace_back(val); 47 | return; 48 | } 49 | ++hist_[Bucket(val)]; 50 | min_encountered_ = std::min(min_encountered_, val); 51 | max_encountered_ = std::max(max_encountered_, val); 52 | ++total_counts_; 53 | sum_ += val; 54 | } 55 | 56 | double Histogram::mean() const { 57 | if (total_counts_ == 0) { 58 | return 0.0; 59 | } 60 | return sum_ / total_counts_; 61 | } 62 | 63 | std::string Histogram::FormatString() const { 64 | std::string res; 65 | 66 | // Format outliers. 67 | if (!outliers_.empty()) { 68 | auto double_to_string = [](double v) -> string { 69 | return StringUtil::Format("%lf", v); 70 | }; 71 | res = StringUtil::Format("Outliers %s with unit %s: %s\n", distribution_name_, distribution_unit_, 72 | StringUtil::Join(outliers_, outliers_.size(), ", ", double_to_string)); 73 | } 74 | 75 | // Format aggregated stats. 76 | res += StringUtil::Format("Max %s = %lf %s\n", distribution_name_, max(), distribution_unit_); 77 | res += StringUtil::Format("Min %s = %lf %s\n", distribution_name_, min(), distribution_unit_); 78 | res += StringUtil::Format("Mean %s = %lf %s\n", distribution_name_, mean(), distribution_unit_); 79 | 80 | // Format stats distribution. 81 | const double interval = (max_val_ - min_val_) / num_bkt_; 82 | for (size_t idx = 0; idx < hist_.size(); ++idx) { 83 | // Skip empty bucket. 84 | if (hist_[idx] == 0) { 85 | continue; 86 | } 87 | const double cur_min_val = min_val_ + interval * idx; 88 | const double cur_max_val = MinValue(cur_min_val + interval, max_val_); 89 | const double percentage = hist_[idx] * 1.0 / total_counts_ * 100; 90 | res += StringUtil::Format("Distribution %s [%lf, %lf) %s: %lf %%\n", distribution_name_, cur_min_val, 91 | cur_max_val, distribution_unit_, percentage); 92 | } 93 | 94 | return res; 95 | } 96 | 97 | } // namespace duckdb 98 | -------------------------------------------------------------------------------- /src/include/base_cache_reader.hpp: -------------------------------------------------------------------------------- 1 | // This class is the base class for reader implementation. 2 | // 3 | // All cache-related resource and operations are delegated to the corresponding cache reader. 4 | // For example, access local cache files should go through on-disk cache reader. 5 | 6 | #pragma once 7 | 8 | #include "base_cache_reader.hpp" 9 | #include "base_profile_collector.hpp" 10 | #include "cache_entry_info.hpp" 11 | #include "duckdb/common/exception.hpp" 12 | #include "duckdb/common/file_system.hpp" 13 | #include "duckdb/common/vector.hpp" 14 | 15 | namespace duckdb { 16 | 17 | class BaseCacheReader { 18 | public: 19 | BaseCacheReader() = default; 20 | virtual ~BaseCacheReader() = default; 21 | BaseCacheReader(const BaseCacheReader &) = delete; 22 | BaseCacheReader &operator=(const BaseCacheReader &) = delete; 23 | 24 | // Read from [handle] for an block-size aligned chunk into [start_addr]; cache to local filesystem and return to 25 | // user. 26 | virtual void ReadAndCache(FileHandle &handle, char *buffer, idx_t requested_start_offset, 27 | idx_t requested_bytes_to_read, idx_t file_size) = 0; 28 | 29 | // Get status information for all cache entries for the current cache reader. Entries are returned in a random 30 | // order. 31 | virtual vector GetCacheEntriesInfo() const = 0; 32 | 33 | // Clear all cache. 34 | virtual void ClearCache() = 0; 35 | 36 | // Clear cache for the given [fname]. 37 | virtual void ClearCache(const string &fname) = 0; 38 | 39 | // Get name for cache reader. 40 | virtual std::string GetName() const { 41 | throw NotImplementedException("Base cache reader doesn't implement GetName."); 42 | } 43 | 44 | void SetProfileCollector(BaseProfileCollector *profile_collector_p) { 45 | profile_collector = profile_collector_p; 46 | profile_collector->SetCacheReaderType(GetName()); 47 | } 48 | 49 | BaseProfileCollector *GetProfileCollector() const { 50 | return profile_collector; 51 | } 52 | 53 | template 54 | TARGET &Cast() { 55 | DynamicCastCheck(this); 56 | return reinterpret_cast(*this); 57 | } 58 | template 59 | const TARGET &Cast() const { 60 | DynamicCastCheck(this); 61 | return reinterpret_cast(*this); 62 | } 63 | 64 | protected: 65 | // Ownership lies in cache filesystem. 66 | BaseProfileCollector *profile_collector = nullptr; 67 | }; 68 | 69 | } // namespace duckdb 70 | -------------------------------------------------------------------------------- /src/include/base_profile_collector.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | #include "cache_entry_info.hpp" 7 | #include "cache_filesystem_config.hpp" 8 | #include "duckdb/common/vector.hpp" 9 | 10 | namespace duckdb { 11 | 12 | // A commonly seen way to lay filesystem features is decorator pattern, with each feature as a new class and layer. 13 | // In the ideal world, profiler should be achieved as another layer, just like how we implement cache filesystem; but 14 | // that requires us to implant more config settings and global variables. For simplicity (since we only target cache 15 | // filesystem in the extension), profiler collector is used as a data member for cache filesystem. 16 | class BaseProfileCollector { 17 | public: 18 | enum class CacheEntity { 19 | kMetadata, // File metadata. 20 | kData, // File data block. 21 | kFileHandle, // File handle. 22 | kGlob, // Glob. 23 | kUnknown, 24 | }; 25 | enum class CacheAccess { 26 | kCacheHit, 27 | kCacheMiss, 28 | }; 29 | enum class IoOperation { 30 | kOpen, 31 | kRead, 32 | kGlob, 33 | kUnknown, 34 | }; 35 | static constexpr auto kCacheEntityCount = static_cast(CacheEntity::kUnknown); 36 | static constexpr auto kIoOperationCount = static_cast(IoOperation::kUnknown); 37 | 38 | BaseProfileCollector() = default; 39 | virtual ~BaseProfileCollector() = default; 40 | BaseProfileCollector(const BaseProfileCollector &) = delete; 41 | BaseProfileCollector &operator=(const BaseProfileCollector &) = delete; 42 | 43 | // Get an ID which uniquely identifies current operation. 44 | virtual std::string GenerateOperId() const = 0; 45 | // Record the start of operation [io_oper] with operation identifier [oper_id]. 46 | virtual void RecordOperationStart(IoOperation io_oper, const std::string &oper_id) = 0; 47 | // Record the finish of operation [io_oper] with operation identifier [oper_id]. 48 | virtual void RecordOperationEnd(IoOperation io_oper, const std::string &oper_id) = 0; 49 | // Record cache access condition. 50 | virtual void RecordCacheAccess(CacheEntity cache_entity, CacheAccess cache_access) = 0; 51 | // Get profiler type. 52 | virtual std::string GetProfilerType() = 0; 53 | // Get cache access information. 54 | // It's guaranteed that access info are returned in the order of and are size of [CacheEntity]. 55 | virtual vector GetCacheAccessInfo() const = 0; 56 | // Set cache reader type. 57 | void SetCacheReaderType(std::string cache_reader_type_p) { 58 | cache_reader_type = std::move(cache_reader_type_p); 59 | } 60 | // Reset profile stats. 61 | virtual void Reset() = 0; 62 | // Get human-readable aggregated profile collection, and its latest completed IO operation timestamp. 63 | virtual std::pair GetHumanReadableStats() = 0; 64 | 65 | template 66 | TARGET &Cast() { 67 | DynamicCastCheck(this); 68 | return reinterpret_cast(*this); 69 | } 70 | template 71 | const TARGET &Cast() const { 72 | DynamicCastCheck(this); 73 | return reinterpret_cast(*this); 74 | } 75 | 76 | // Some platforms supports musl libc, which doesn't support operator[], so we use `vector<>` instead of `constexpr 77 | // static std::array<>` here. 78 | // 79 | // Cache entity name, indexed by cache entity enum. 80 | inline static const vector CACHE_ENTITY_NAMES = []() { 81 | vector cache_entity_names; 82 | cache_entity_names.reserve(kIoOperationCount); 83 | cache_entity_names.emplace_back("metadata"); 84 | cache_entity_names.emplace_back("data"); 85 | cache_entity_names.emplace_back("file handle"); 86 | cache_entity_names.emplace_back("glob"); 87 | return cache_entity_names; 88 | }(); 89 | // Operation names, indexed by operation enums. 90 | inline static const vector OPER_NAMES = []() { 91 | vector oper_names; 92 | oper_names.reserve(kIoOperationCount); 93 | oper_names.emplace_back("open"); 94 | oper_names.emplace_back("read"); 95 | oper_names.emplace_back("glob"); 96 | return oper_names; 97 | }(); 98 | 99 | protected: 100 | std::string cache_reader_type = ""; 101 | }; 102 | 103 | class NoopProfileCollector final : public BaseProfileCollector { 104 | public: 105 | NoopProfileCollector() = default; 106 | ~NoopProfileCollector() override = default; 107 | 108 | std::string GenerateOperId() const override { 109 | return ""; 110 | } 111 | void RecordOperationStart(IoOperation io_oper, const std::string &oper_id) override { 112 | } 113 | void RecordOperationEnd(IoOperation io_oper, const std::string &oper_id) override { 114 | } 115 | void RecordCacheAccess(CacheEntity cache_entity, CacheAccess cache_access) override { 116 | } 117 | std::string GetProfilerType() override { 118 | return *NOOP_PROFILE_TYPE; 119 | } 120 | vector GetCacheAccessInfo() const override { 121 | vector cache_access_info; 122 | cache_access_info.resize(kCacheEntityCount); 123 | for (size_t idx = 0; idx < kCacheEntityCount; ++idx) { 124 | cache_access_info[idx].cache_type = CACHE_ENTITY_NAMES[idx]; 125 | } 126 | return cache_access_info; 127 | } 128 | void Reset() override {}; 129 | std::pair GetHumanReadableStats() override { 130 | return std::make_pair("(noop profile collector)", /*timestamp=*/0); 131 | } 132 | }; 133 | 134 | } // namespace duckdb 135 | -------------------------------------------------------------------------------- /src/include/cache_entry_info.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | namespace duckdb { 7 | 8 | // Entry information for data cache, which applies to both in-memory cache and on-disk cache. 9 | struct DataCacheEntryInfo { 10 | std::string cache_filepath; 11 | std::string remote_filename; 12 | uint64_t start_offset = 0; // Inclusive. 13 | uint64_t end_offset = 0; // Exclusive. 14 | std::string cache_type; // Either in-memory or on-disk. 15 | }; 16 | 17 | bool operator<(const DataCacheEntryInfo &lhs, const DataCacheEntryInfo &rhs); 18 | 19 | // Cache access information, which applies to metadata and file handle cache. 20 | struct CacheAccessInfo { 21 | std::string cache_type; 22 | uint64_t cache_hit_count = 0; 23 | uint64_t cache_miss_count = 0; 24 | }; 25 | 26 | bool operator<(const CacheAccessInfo &lhs, const CacheAccessInfo &rhs); 27 | 28 | } // namespace duckdb 29 | -------------------------------------------------------------------------------- /src/include/cache_filesystem_config.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "duckdb/common/file_opener.hpp" 9 | #include "duckdb/common/typedefs.hpp" 10 | #include "no_destructor.hpp" 11 | #include "size_literals.hpp" 12 | 13 | namespace duckdb { 14 | 15 | //===--------------------------------------------------------------------===// 16 | // Config constant 17 | //===--------------------------------------------------------------------===// 18 | inline const NoDestructor NOOP_CACHE_TYPE {"noop"}; 19 | inline const NoDestructor ON_DISK_CACHE_TYPE {"on_disk"}; 20 | inline const NoDestructor IN_MEM_CACHE_TYPE {"in_mem"}; 21 | inline const std::unordered_set ALL_CACHE_TYPES {*NOOP_CACHE_TYPE, *ON_DISK_CACHE_TYPE, 22 | *IN_MEM_CACHE_TYPE}; 23 | 24 | // Default profile option, which performs no-op. 25 | inline const NoDestructor NOOP_PROFILE_TYPE {"noop"}; 26 | // Store the latest IO operation profiling result, which potentially suffers concurrent updates. 27 | inline const NoDestructor TEMP_PROFILE_TYPE {"temp"}; 28 | // Store the IO operation profiling results into duckdb table, which unblocks advanced analysis. 29 | inline const NoDestructor PERSISTENT_PROFILE_TYPE {"duckdb"}; 30 | inline const NoDestructor> ALL_PROFILE_TYPES {*NOOP_PROFILE_TYPE, *TEMP_PROFILE_TYPE, 31 | *PERSISTENT_PROFILE_TYPE}; 32 | 33 | //===--------------------------------------------------------------------===// 34 | // Default configuration 35 | //===--------------------------------------------------------------------===// 36 | inline const idx_t DEFAULT_CACHE_BLOCK_SIZE = 64_KiB; 37 | inline const NoDestructor DEFAULT_ON_DISK_CACHE_DIRECTORY {"/tmp/duckdb_cache_httpfs_cache"}; 38 | 39 | // Default to use on-disk cache filesystem. 40 | inline NoDestructor DEFAULT_CACHE_TYPE {*ON_DISK_CACHE_TYPE}; 41 | 42 | // To prevent go out of disk space, we set a threshold to disallow local caching if insufficient. It applies to all 43 | // filesystems. The value here is the decimal representation for percentage value; for example, 0.05 means 5%. 44 | inline constexpr double MIN_DISK_SPACE_PERCENTAGE_FOR_CACHE = 0.05; 45 | 46 | // Maximum in-memory cache block number, which caps the overall memory consumption as (block size * max block count). 47 | inline constexpr idx_t DEFAULT_MAX_IN_MEM_CACHE_BLOCK_COUNT = 256; 48 | 49 | // Default timeout in seconds for in-memory block cache entries. 50 | inline constexpr idx_t DEFAULT_IN_MEM_BLOCK_CACHE_TIMEOUT_MILLISEC = 3600ULL * 1000 /*1hour*/; 51 | 52 | // Max number of cache entries for file metadata cache. 53 | inline static constexpr size_t DEFAULT_MAX_METADATA_CACHE_ENTRY = 125; 54 | 55 | // Timeout in milliseconds of cache entries for file metadata cache. 56 | inline static constexpr uint64_t DEFAULT_METADATA_CACHE_ENTRY_TIMEOUT_MILLISEC = 3600ULL * 1000 /*1hour*/; 57 | 58 | // Number of seconds which we define as the threshold of staleness for metadata entries. 59 | inline constexpr idx_t CACHE_FILE_STALENESS_SECOND = 24 * 3600; // 1 day 60 | 61 | // Max number of cache entries for file handle cache. 62 | inline static constexpr size_t DEFAULT_MAX_FILE_HANDLE_CACHE_ENTRY = 125; 63 | 64 | // Timeout in milliseconds of cache entries for file handle cache. 65 | inline static constexpr uint64_t DEFAULT_FILE_HANDLE_CACHE_ENTRY_TIMEOUT_MILLISEC = 3600ULL * 1000 /*1hour*/; 66 | 67 | // Max number of cache entries for glob cache. 68 | inline static constexpr size_t DEFAULT_MAX_GLOB_CACHE_ENTRY = 64; 69 | 70 | // Timeout in milliseconds of cache entries for file handle cache. 71 | inline static constexpr uint64_t DEFAULT_GLOB_CACHE_ENTRY_TIMEOUT_MILLISEC = 1800ULL * 1000 /*30min*/; 72 | 73 | // Default option for profile type. 74 | inline NoDestructor DEFAULT_PROFILE_TYPE {*NOOP_PROFILE_TYPE}; 75 | 76 | // Default max number of parallel subrequest for a single filesystem read request. 0 means no limit. 77 | inline uint64_t DEFAULT_MAX_SUBREQUEST_COUNT = 0; 78 | 79 | // Default enable metadata cache. 80 | inline bool DEFAULT_ENABLE_METADATA_CACHE = true; 81 | 82 | // Default enable file handle cache. 83 | inline bool DEFAULT_ENABLE_FILE_HANDLE_CACHE = true; 84 | 85 | // Default enable glob cache. 86 | inline bool DEFAULT_ENABLE_GLOB_CACHE = true; 87 | 88 | // Default not ignore SIGPIPE in the extension. 89 | inline bool DEFAULT_IGNORE_SIGPIPE = false; 90 | 91 | // Default min disk bytes required for on-disk cache; by default 0 which user doesn't specify and override, and default 92 | // value will be considered. 93 | inline idx_t DEFAULT_MIN_DISK_BYTES_FOR_CACHE = 0; 94 | 95 | //===--------------------------------------------------------------------===// 96 | // Global configuration 97 | //===--------------------------------------------------------------------===// 98 | 99 | // Global configuration. 100 | inline idx_t g_cache_block_size = DEFAULT_CACHE_BLOCK_SIZE; 101 | inline bool g_ignore_sigpipe = DEFAULT_IGNORE_SIGPIPE; 102 | inline NoDestructor g_cache_type {*DEFAULT_CACHE_TYPE}; 103 | inline NoDestructor g_profile_type {*DEFAULT_PROFILE_TYPE}; 104 | inline uint64_t g_max_subrequest_count = DEFAULT_MAX_SUBREQUEST_COUNT; 105 | 106 | // On-disk cache configuration. 107 | inline NoDestructor g_on_disk_cache_directory {*DEFAULT_ON_DISK_CACHE_DIRECTORY}; 108 | inline idx_t g_min_disk_bytes_for_cache = DEFAULT_MIN_DISK_BYTES_FOR_CACHE; 109 | 110 | // In-memory cache configuration. 111 | inline idx_t g_max_in_mem_cache_block_count = DEFAULT_MAX_IN_MEM_CACHE_BLOCK_COUNT; 112 | inline idx_t g_in_mem_cache_block_timeout_millisec = DEFAULT_IN_MEM_BLOCK_CACHE_TIMEOUT_MILLISEC; 113 | 114 | // Metadata cache configuration. 115 | inline bool g_enable_metadata_cache = DEFAULT_ENABLE_METADATA_CACHE; 116 | inline idx_t g_max_metadata_cache_entry = DEFAULT_MAX_METADATA_CACHE_ENTRY; 117 | inline idx_t g_metadata_cache_entry_timeout_millisec = DEFAULT_METADATA_CACHE_ENTRY_TIMEOUT_MILLISEC; 118 | 119 | // File handle cache configuration. 120 | inline bool g_enable_file_handle_cache = DEFAULT_ENABLE_FILE_HANDLE_CACHE; 121 | inline idx_t g_max_file_handle_cache_entry = DEFAULT_MAX_FILE_HANDLE_CACHE_ENTRY; 122 | inline idx_t g_file_handle_cache_entry_timeout_millisec = DEFAULT_FILE_HANDLE_CACHE_ENTRY_TIMEOUT_MILLISEC; 123 | 124 | // File glob configuration. 125 | inline bool g_enable_glob_cache = DEFAULT_ENABLE_GLOB_CACHE; 126 | inline idx_t g_max_glob_cache_entry = DEFAULT_MAX_GLOB_CACHE_ENTRY; 127 | inline idx_t g_glob_cache_entry_timeout_millisec = DEFAULT_GLOB_CACHE_ENTRY_TIMEOUT_MILLISEC; 128 | 129 | // Used for testing purpose, which has a higher priority over [g_cache_type], and won't be reset. 130 | // TODO(hjiang): A better is bake configuration into `FileOpener`. 131 | inline NoDestructor g_test_cache_type {""}; 132 | 133 | // Used for testing purpose, which disable on-disk cache if true. 134 | inline bool g_test_insufficient_disk_space = false; 135 | 136 | //===--------------------------------------------------------------------===// 137 | // Util function for filesystem configurations. 138 | //===--------------------------------------------------------------------===// 139 | 140 | // Set global cache filesystem configuration. 141 | void SetGlobalConfig(optional_ptr opener); 142 | 143 | // Reset all global cache filesystem configuration. 144 | void ResetGlobalConfig(); 145 | 146 | // Get concurrent IO sub-request count. 147 | uint64_t GetThreadCountForSubrequests(uint64_t io_request_count); 148 | 149 | } // namespace duckdb 150 | -------------------------------------------------------------------------------- /src/include/cache_filesystem_ref_registry.hpp: -------------------------------------------------------------------------------- 1 | // CacheFsRefRegistry is a singleton registry which stores references for all cache filesystems. 2 | // The class is not thread-safe. 3 | 4 | #pragma once 5 | 6 | #include "duckdb/common/vector.hpp" 7 | 8 | namespace duckdb { 9 | 10 | // Forward declaration. 11 | class CacheFileSystem; 12 | 13 | class CacheFsRefRegistry { 14 | public: 15 | // Get the singleton instance for the registry. 16 | static CacheFsRefRegistry &Get(); 17 | 18 | // Register the cache filesystem to the registry. 19 | void Register(CacheFileSystem *fs); 20 | 21 | // Reset the registry. 22 | void Reset(); 23 | 24 | // Get all cache filesystems. 25 | const vector &GetAllCacheFs() const; 26 | 27 | private: 28 | CacheFsRefRegistry() = default; 29 | 30 | // The ownership lies in db instance. 31 | vector cache_filesystems; 32 | }; 33 | 34 | } // namespace duckdb 35 | -------------------------------------------------------------------------------- /src/include/cache_httpfs_extension.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "duckdb.hpp" 4 | 5 | #include 6 | 7 | namespace duckdb { 8 | 9 | class CacheHttpfsExtension : public Extension { 10 | public: 11 | void Load(DuckDB &db) override; 12 | std::string Name() override; 13 | std::string Version() const override; 14 | 15 | private: 16 | unique_ptr httpfs_extension; 17 | }; 18 | 19 | } // namespace duckdb 20 | -------------------------------------------------------------------------------- /src/include/cache_reader_manager.hpp: -------------------------------------------------------------------------------- 1 | // A class which manages all cache readers and is shared among all cache filesystems. 2 | // It's designed as singleton instead of shared pointer passed to everywhere, because it's accessed in quite a few 3 | // places, and it's not easy to pass around shared pointer in all cases. 4 | 5 | #pragma once 6 | 7 | #include "base_cache_reader.hpp" 8 | #include "duckdb/common/unique_ptr.hpp" 9 | #include "duckdb/common/vector.hpp" 10 | 11 | namespace duckdb { 12 | 13 | class CacheReaderManager { 14 | public: 15 | static CacheReaderManager &Get(); 16 | 17 | // Set cache reader if uninitialized. 18 | void SetCacheReader(); 19 | 20 | // Get current cache reader. 21 | BaseCacheReader *GetCacheReader() const; 22 | 23 | // Get all cache readers if they're initialized. 24 | vector GetCacheReaders() const; 25 | 26 | // Initialize disk cache reader if uninitialized. 27 | void InitializeDiskCacheReader(); 28 | 29 | // Clear cache for all cache readers. 30 | void ClearCache(); 31 | 32 | // Clear cache for all cache readers on the given [fname]. 33 | void ClearCache(const string &fname); 34 | 35 | // Reset all cache readers. 36 | void Reset(); 37 | 38 | private: 39 | CacheReaderManager() = default; 40 | 41 | // Noop, in-memory and on-disk cache reader. 42 | unique_ptr noop_cache_reader; 43 | unique_ptr in_mem_cache_reader; 44 | unique_ptr on_disk_cache_reader; 45 | // Either in-memory or on-disk cache reader, whichever is actively being used, ownership lies the above cache 46 | // reader. 47 | BaseCacheReader *internal_cache_reader = nullptr; 48 | }; 49 | 50 | } // namespace duckdb 51 | -------------------------------------------------------------------------------- /src/include/cache_status_query_function.hpp: -------------------------------------------------------------------------------- 1 | // Function which queries cache status. 2 | 3 | #pragma once 4 | 5 | #include "duckdb/function/table_function.hpp" 6 | 7 | namespace duckdb { 8 | 9 | // Get the table function to query cache status. 10 | TableFunction GetDataCacheStatusQueryFunc(); 11 | 12 | // Get the table function to query cache access status. 13 | TableFunction GetCacheAccessInfoQueryFunc(); 14 | 15 | // Get the table function to query wrapped cache filesystems. 16 | TableFunction GetWrappedCacheFileSystemsFunc(); 17 | 18 | } // namespace duckdb 19 | -------------------------------------------------------------------------------- /src/include/disk_cache_reader.hpp: -------------------------------------------------------------------------------- 1 | // A filesystem wrapper, which performs on-disk cache for read operations. 2 | 3 | #pragma once 4 | 5 | #include "base_cache_reader.hpp" 6 | #include "duckdb/common/file_system.hpp" 7 | #include "duckdb/common/helper.hpp" 8 | #include "duckdb/common/local_file_system.hpp" 9 | #include "duckdb/common/unique_ptr.hpp" 10 | #include "cache_filesystem.hpp" 11 | #include "cache_filesystem_config.hpp" 12 | 13 | namespace duckdb { 14 | 15 | class DiskCacheReader final : public BaseCacheReader { 16 | public: 17 | DiskCacheReader(); 18 | ~DiskCacheReader() override = default; 19 | 20 | std::string GetName() const override { 21 | return "on_disk_cache_reader"; 22 | } 23 | 24 | void ClearCache() override; 25 | void ClearCache(const string &fname) override; 26 | 27 | void ReadAndCache(FileHandle &handle, char *buffer, idx_t requested_start_offset, idx_t requested_bytes_to_read, 28 | idx_t file_size) override; 29 | 30 | vector GetCacheEntriesInfo() const override; 31 | 32 | private: 33 | // Used to access local cache files. 34 | unique_ptr local_filesystem; 35 | }; 36 | 37 | } // namespace duckdb 38 | -------------------------------------------------------------------------------- /src/include/histogram.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | namespace duckdb { 8 | 9 | // Historgram supports two types of records 10 | // - For values within the given range, all the stats functions (i.e. min and max) only considers in-range values; 11 | // - For values out of range, we provide extra functions to retrieve. 12 | // 13 | // The reason why outliers are not considered as statistic is they disturb statistical value a lot. 14 | class Histogram { 15 | public: 16 | // [min_val] is inclusive, and [max_val] is exclusive. 17 | Histogram(double min_val, double max_val, int num_bkt); 18 | 19 | Histogram(const Histogram &) = delete; 20 | Histogram &operator=(const Histogram &) = delete; 21 | Histogram(Histogram &&) = delete; 22 | Histogram &operator=(Histogram &&) = delete; 23 | 24 | // Set the distribution stats name and unit, used for formatting purpose. 25 | void SetStatsDistribution(std::string name, std::string unit); 26 | 27 | // Add [val] into the histogram. 28 | // Return whether [val] is valid. 29 | void Add(double val); 30 | 31 | // Get bucket index for the given [val]. 32 | size_t Bucket(double val) const; 33 | 34 | // Stats data. 35 | size_t counts() const { 36 | return total_counts_; 37 | } 38 | double sum() const { 39 | return sum_; 40 | } 41 | double mean() const; 42 | // Precondition: there's at least one value inserted. 43 | double min() const { 44 | return min_encountered_; 45 | } 46 | double max() const { 47 | return max_encountered_; 48 | } 49 | 50 | // Get outliers for stat records. 51 | const std::vector outliers() const { 52 | return outliers_; 53 | } 54 | 55 | // Display histogram into string format. 56 | std::string FormatString() const; 57 | 58 | // Reset histogram. 59 | void Reset(); 60 | 61 | private: 62 | const double min_val_; 63 | const double max_val_; 64 | const int num_bkt_; 65 | // Max and min value encountered. 66 | double min_encountered_; 67 | double max_encountered_; 68 | // Total number of values. 69 | size_t total_counts_ = 0; 70 | // Accumulated sum. 71 | double sum_ = 0.0; 72 | // List of bucket counts. 73 | std::vector hist_; 74 | // List of outliers. 75 | std::vector outliers_; 76 | // Item name and unit for stats distribution. 77 | std::string distribution_name_; 78 | std::string distribution_unit_; 79 | }; 80 | 81 | } // namespace duckdb 82 | -------------------------------------------------------------------------------- /src/include/in_mem_cache_block.hpp: -------------------------------------------------------------------------------- 1 | // In-memory cache cache block key. 2 | 3 | #pragma once 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "duckdb/common/string_util.hpp" 12 | 13 | namespace duckdb { 14 | 15 | struct InMemCacheBlock { 16 | std::string fname; 17 | idx_t start_off = 0; 18 | idx_t blk_size = 0; 19 | }; 20 | 21 | struct InMemCacheBlockEqual { 22 | bool operator()(const InMemCacheBlock &lhs, const InMemCacheBlock &rhs) const { 23 | return std::tie(lhs.fname, lhs.start_off, lhs.blk_size) == std::tie(rhs.fname, rhs.start_off, rhs.blk_size); 24 | } 25 | }; 26 | struct InMemCacheBlockHash { 27 | std::size_t operator()(const InMemCacheBlock &key) const { 28 | return std::hash {}(key.fname) ^ std::hash {}(key.start_off) ^ 29 | std::hash {}(key.blk_size); 30 | } 31 | }; 32 | 33 | } // namespace duckdb 34 | -------------------------------------------------------------------------------- /src/include/in_memory_cache_reader.hpp: -------------------------------------------------------------------------------- 1 | // A filesystem wrapper, which performs in-memory cache for read operations. 2 | 3 | #pragma once 4 | 5 | #include "base_cache_reader.hpp" 6 | #include "cache_filesystem.hpp" 7 | #include "cache_filesystem_config.hpp" 8 | #include "copiable_value_lru_cache.hpp" 9 | #include "duckdb/common/file_opener.hpp" 10 | #include "duckdb/common/file_system.hpp" 11 | #include "duckdb/common/local_file_system.hpp" 12 | #include "duckdb/common/unique_ptr.hpp" 13 | #include "in_mem_cache_block.hpp" 14 | #include "shared_lru_cache.hpp" 15 | 16 | namespace duckdb { 17 | 18 | class InMemoryCacheReader final : public BaseCacheReader { 19 | public: 20 | InMemoryCacheReader() = default; 21 | ~InMemoryCacheReader() override = default; 22 | 23 | std::string GetName() const override { 24 | return "in_mem_cache_reader"; 25 | } 26 | 27 | void ClearCache() override; 28 | void ClearCache(const string &fname) override; 29 | void ReadAndCache(FileHandle &handle, char *buffer, uint64_t requested_start_offset, 30 | uint64_t requested_bytes_to_read, uint64_t file_size) override; 31 | vector GetCacheEntriesInfo() const override; 32 | 33 | private: 34 | using InMemCache = ThreadSafeSharedLruCache; 35 | 36 | // Once flag to guard against cache's initialization. 37 | std::once_flag cache_init_flag; 38 | // LRU cache to store blocks; late initialized after first access. 39 | unique_ptr cache; 40 | }; 41 | 42 | } // namespace duckdb 43 | -------------------------------------------------------------------------------- /src/include/noop_cache_reader.hpp: -------------------------------------------------------------------------------- 1 | // A noop cache reader, which simply delegates IO request to filesystem API calls. 2 | // - It provides an option for users to disable caching and parallel reads; 3 | // - It eases performance comparison benchmarks. 4 | 5 | #pragma once 6 | 7 | #include "base_cache_reader.hpp" 8 | #include "base_profile_collector.hpp" 9 | #include "duckdb/common/file_system.hpp" 10 | 11 | namespace duckdb { 12 | 13 | class NoopCacheReader : public BaseCacheReader { 14 | public: 15 | NoopCacheReader() = default; 16 | virtual ~NoopCacheReader() = default; 17 | 18 | void ClearCache() override { 19 | } 20 | void ClearCache(const string &fname) override { 21 | } 22 | void ReadAndCache(FileHandle &handle, char *buffer, idx_t requested_start_offset, idx_t requested_bytes_to_read, 23 | idx_t file_size) override; 24 | 25 | vector GetCacheEntriesInfo() const override { 26 | return {}; 27 | } 28 | 29 | virtual std::string GetName() const { 30 | return "noop_cache_reader"; 31 | } 32 | }; 33 | 34 | } // namespace duckdb 35 | -------------------------------------------------------------------------------- /src/include/scope_guard.hpp: -------------------------------------------------------------------------------- 1 | // SCOPE_EXIT is used to execute a series of registered functions when it goes 2 | // out of scope. 3 | // For details, refer to Andrei Alexandrescu's CppCon 2015 talk "Declarative 4 | // Control Flow" 5 | // 6 | // Examples: 7 | // void Function() { 8 | // FILE* fp = fopen("my_file.txt", "r"); 9 | // SCOPE_EXIT { fclose(fp); }; 10 | // // Do something. 11 | // } // fp will be closed at exit. 12 | // 13 | 14 | #pragma once 15 | 16 | #include 17 | #include 18 | 19 | // Need to have two macro invocation to allow [x] and [y] to be replaced. 20 | #define __DUCKDB_CONCAT(x, y) x##y 21 | 22 | #define DUCKDB_CONCAT(x, y) __DUCKDB_CONCAT(x, y) 23 | 24 | // Macros which gets unique variable name. 25 | #define DUCKDB_UNIQUE_VARIABLE(base) DUCKDB_CONCAT(base, __LINE__) 26 | 27 | namespace duckdb { 28 | 29 | class ScopeGuard { 30 | private: 31 | using Func = std::function; 32 | 33 | public: 34 | ScopeGuard() : func_([]() {}) { 35 | } 36 | explicit ScopeGuard(Func &&func) : func_(std::forward(func)) { 37 | } 38 | ~ScopeGuard() noexcept { 39 | func_(); 40 | } 41 | 42 | // Register a new function to be invoked at destruction. 43 | // Execution will be performed at the reversed order they're registered. 44 | ScopeGuard &operator+=(Func &&another_func) { 45 | Func cur_func = std::move(func_); 46 | func_ = [cur_func = std::move(cur_func), another_func = std::move(another_func)]() { 47 | // Executed in the reverse order functions are registered. 48 | another_func(); 49 | cur_func(); 50 | }; 51 | return *this; 52 | } 53 | 54 | private: 55 | Func func_; 56 | }; 57 | 58 | namespace internal { 59 | 60 | using ScopeGuardFunc = std::function; 61 | 62 | // Constructs a scope guard that calls 'fn' when it exits. 63 | enum class ScopeGuardOnExit {}; 64 | inline auto operator+(ScopeGuardOnExit /*unused*/, ScopeGuardFunc fn) { 65 | return ScopeGuard {std::move(fn)}; 66 | } 67 | 68 | } // namespace internal 69 | 70 | } // namespace duckdb 71 | 72 | #define SCOPE_EXIT auto DUCKDB_UNIQUE_VARIABLE(SCOPE_EXIT_TEMP_EXIT) = duckdb::internal::ScopeGuardOnExit {} + [&]() 73 | -------------------------------------------------------------------------------- /src/include/temp_profile_collector.hpp: -------------------------------------------------------------------------------- 1 | // Profile collector, which profiles and stores the latest result in memory. 2 | 3 | #pragma once 4 | 5 | #include "base_profile_collector.hpp" 6 | #include "duckdb/common/helper.hpp" 7 | #include "duckdb/common/profiler.hpp" 8 | #include "histogram.hpp" 9 | 10 | #include 11 | #include 12 | 13 | namespace duckdb { 14 | 15 | class TempProfileCollector final : public BaseProfileCollector { 16 | public: 17 | TempProfileCollector(); 18 | ~TempProfileCollector() override = default; 19 | 20 | std::string GenerateOperId() const override; 21 | void RecordOperationStart(IoOperation io_oper, const std::string &oper) override; 22 | void RecordOperationEnd(IoOperation io_oper, const std::string &oper) override; 23 | void RecordCacheAccess(CacheEntity cache_entity, CacheAccess cache_access) override; 24 | std::string GetProfilerType() override { 25 | return *TEMP_PROFILE_TYPE; 26 | } 27 | vector GetCacheAccessInfo() const override; 28 | void Reset() override; 29 | std::pair GetHumanReadableStats() override; 30 | 31 | private: 32 | struct OperationStats { 33 | // Accounted as time elapsed since unix epoch in milliseconds. 34 | int64_t start_timestamp = 0; 35 | }; 36 | 37 | using OperationStatsMap = unordered_map; 38 | std::array operation_events; 39 | // Only records finished operations, which maps from io operation to histogram. 40 | std::array, kIoOperationCount> histograms; 41 | // Aggregated cache access condition. 42 | std::array cache_access_count {}; 43 | // Latest access timestamp in milliseconds since unix epoch. 44 | uint64_t latest_timestamp = 0; 45 | 46 | mutable std::mutex stats_mutex; 47 | }; 48 | } // namespace duckdb 49 | -------------------------------------------------------------------------------- /src/noop_cache_reader.cpp: -------------------------------------------------------------------------------- 1 | #include "noop_cache_reader.hpp" 2 | 3 | #include "cache_filesystem.hpp" 4 | 5 | namespace duckdb { 6 | 7 | void NoopCacheReader::ReadAndCache(FileHandle &handle, char *buffer, idx_t requested_start_offset, 8 | idx_t requested_bytes_to_read, idx_t file_size) { 9 | auto &disk_cache_handle = handle.Cast(); 10 | auto *internal_filesystem = disk_cache_handle.GetInternalFileSystem(); 11 | const string oper_id = profile_collector->GenerateOperId(); 12 | profile_collector->RecordOperationStart(BaseProfileCollector::IoOperation::kRead, oper_id); 13 | internal_filesystem->Read(*disk_cache_handle.internal_file_handle, buffer, requested_bytes_to_read, 14 | requested_start_offset); 15 | profile_collector->RecordOperationEnd(BaseProfileCollector::IoOperation::kRead, oper_id); 16 | } 17 | 18 | } // namespace duckdb 19 | -------------------------------------------------------------------------------- /src/temp_profile_collector.cpp: -------------------------------------------------------------------------------- 1 | #include "temp_profile_collector.hpp" 2 | 3 | #include "duckdb/common/types/uuid.hpp" 4 | #include "utils/include/no_destructor.hpp" 5 | #include "utils/include/time_utils.hpp" 6 | 7 | namespace duckdb { 8 | 9 | namespace { 10 | // Heuristic estimation of single IO request latency, out of which range are classified as outliers. 11 | constexpr double MIN_READ_LATENCY_MILLISEC = 0; 12 | constexpr double MAX_READ_LATENCY_MILLISEC = 1000; 13 | constexpr int READ_LATENCY_NUM_BKT = 100; 14 | 15 | constexpr double MIN_OPEN_LATENCY_MILLISEC = 0; 16 | constexpr double MAX_OPEN_LATENCY_MILLISEC = 1000; 17 | constexpr int OPEN_LATENCY_NUM_BKT = 100; 18 | 19 | constexpr double MIN_GLOB_LATENCY_MILLISEC = 0; 20 | constexpr double MAX_GLOB_LATENCY_MILLISEC = 1000; 21 | constexpr int GLOB_LATENCY_NUM_BKT = 100; 22 | 23 | const NoDestructor LATENCY_HISTOGRAM_ITEM {"latency"}; 24 | const NoDestructor LATENCY_HISTOGRAM_UNIT {"millisec"}; 25 | } // namespace 26 | 27 | TempProfileCollector::TempProfileCollector() { 28 | histograms[static_cast(IoOperation::kRead)] = 29 | make_uniq(MIN_READ_LATENCY_MILLISEC, MAX_READ_LATENCY_MILLISEC, READ_LATENCY_NUM_BKT); 30 | histograms[static_cast(IoOperation::kRead)]->SetStatsDistribution(*LATENCY_HISTOGRAM_ITEM, 31 | *LATENCY_HISTOGRAM_UNIT); 32 | operation_events[static_cast(IoOperation::kRead)] = OperationStatsMap {}; 33 | 34 | histograms[static_cast(IoOperation::kOpen)] = 35 | make_uniq(MIN_OPEN_LATENCY_MILLISEC, MAX_OPEN_LATENCY_MILLISEC, OPEN_LATENCY_NUM_BKT); 36 | histograms[static_cast(IoOperation::kOpen)]->SetStatsDistribution(*LATENCY_HISTOGRAM_ITEM, 37 | *LATENCY_HISTOGRAM_UNIT); 38 | operation_events[static_cast(IoOperation::kOpen)] = OperationStatsMap {}; 39 | 40 | histograms[static_cast(IoOperation::kGlob)] = 41 | make_uniq(MIN_GLOB_LATENCY_MILLISEC, MAX_GLOB_LATENCY_MILLISEC, GLOB_LATENCY_NUM_BKT); 42 | histograms[static_cast(IoOperation::kGlob)]->SetStatsDistribution(*LATENCY_HISTOGRAM_ITEM, 43 | *LATENCY_HISTOGRAM_UNIT); 44 | operation_events[static_cast(IoOperation::kGlob)] = OperationStatsMap {}; 45 | } 46 | 47 | std::string TempProfileCollector::GenerateOperId() const { 48 | return UUID::ToString(UUID::GenerateRandomUUID()); 49 | } 50 | 51 | void TempProfileCollector::RecordOperationStart(IoOperation io_oper, const std::string &oper_id) { 52 | std::lock_guard lck(stats_mutex); 53 | auto &cur_oper_event = operation_events[static_cast(io_oper)]; 54 | const bool is_new = cur_oper_event 55 | .emplace(oper_id, 56 | OperationStats { 57 | .start_timestamp = GetSteadyNowMilliSecSinceEpoch(), 58 | }) 59 | .second; 60 | D_ASSERT(is_new); 61 | } 62 | 63 | void TempProfileCollector::RecordOperationEnd(IoOperation io_oper, const std::string &oper_id) { 64 | const auto now = GetSteadyNowMilliSecSinceEpoch(); 65 | 66 | std::lock_guard lck(stats_mutex); 67 | auto &cur_oper_event = operation_events[static_cast(io_oper)]; 68 | auto iter = cur_oper_event.find(oper_id); 69 | D_ASSERT(iter != cur_oper_event.end()); 70 | 71 | auto &cur_histogram = histograms[static_cast(io_oper)]; 72 | cur_histogram->Add(now - iter->second.start_timestamp); 73 | cur_oper_event.erase(iter); 74 | latest_timestamp = now; 75 | } 76 | 77 | void TempProfileCollector::RecordCacheAccess(CacheEntity cache_entity, CacheAccess cache_access) { 78 | std::lock_guard lck(stats_mutex); 79 | const size_t arr_idx = static_cast(cache_entity) * 2 + static_cast(cache_access); 80 | ++cache_access_count[arr_idx]; 81 | } 82 | 83 | void TempProfileCollector::Reset() { 84 | std::lock_guard lck(stats_mutex); 85 | for (auto &cur_oper_event : operation_events) { 86 | cur_oper_event.clear(); 87 | } 88 | for (auto &cur_histogram : histograms) { 89 | cur_histogram->Reset(); 90 | } 91 | cache_access_count.fill(0); 92 | latest_timestamp = 0; 93 | } 94 | 95 | vector TempProfileCollector::GetCacheAccessInfo() const { 96 | std::lock_guard lck(stats_mutex); 97 | vector cache_access_info; 98 | cache_access_info.reserve(kCacheEntityCount); 99 | for (idx_t idx = 0; idx < kCacheEntityCount; ++idx) { 100 | cache_access_info.emplace_back(CacheAccessInfo { 101 | .cache_type = CACHE_ENTITY_NAMES[idx], 102 | .cache_hit_count = cache_access_count[idx * 2], 103 | .cache_miss_count = cache_access_count[idx * 2 + 1], 104 | }); 105 | } 106 | return cache_access_info; 107 | } 108 | 109 | std::pair TempProfileCollector::GetHumanReadableStats() { 110 | std::lock_guard lck(stats_mutex); 111 | 112 | string stats = 113 | StringUtil::Format("For temp profile collector and stats for %s (unit in milliseconds)\n", cache_reader_type); 114 | 115 | // Record cache miss and cache hit count. 116 | for (idx_t cur_entity_idx = 0; cur_entity_idx < kCacheEntityCount; ++cur_entity_idx) { 117 | stats = StringUtil::Format("%s\n" 118 | "%s cache hit count = %d\n" 119 | "%s cache miss count = %d\n", 120 | stats, CACHE_ENTITY_NAMES[cur_entity_idx], cache_access_count[cur_entity_idx * 2], 121 | CACHE_ENTITY_NAMES[cur_entity_idx], cache_access_count[cur_entity_idx * 2 + 1]); 122 | } 123 | 124 | // Record IO operation latency. 125 | for (idx_t cur_oper_idx = 0; cur_oper_idx < kIoOperationCount; ++cur_oper_idx) { 126 | const auto &cur_histogram = histograms[cur_oper_idx]; 127 | if (cur_histogram->counts() == 0) { 128 | continue; 129 | } 130 | stats = StringUtil::Format("%s\n" 131 | "%s operation latency is %s", 132 | stats, OPER_NAMES[cur_oper_idx], cur_histogram->FormatString()); 133 | } 134 | 135 | return std::make_pair(std::move(stats), latest_timestamp); 136 | } 137 | 138 | } // namespace duckdb 139 | -------------------------------------------------------------------------------- /src/utils/fake_filesystem.cpp: -------------------------------------------------------------------------------- 1 | #include "fake_filesystem.hpp" 2 | 3 | #include "duckdb/common/string_util.hpp" 4 | #include "no_destructor.hpp" 5 | 6 | namespace duckdb { 7 | 8 | namespace { 9 | const NoDestructor FAKE_FILESYSTEM_PREFIX {"/tmp/cache_httpfs_fake_filesystem"}; 10 | } // namespace 11 | 12 | CacheHttpfsFakeFsHandle::CacheHttpfsFakeFsHandle(string path, unique_ptr internal_file_handle_p, 13 | CacheHttpfsFakeFileSystem &fs) 14 | : FileHandle(fs, std::move(path), internal_file_handle_p->GetFlags()), 15 | internal_file_handle(std::move(internal_file_handle_p)) { 16 | } 17 | CacheHttpfsFakeFileSystem::CacheHttpfsFakeFileSystem() : local_filesystem(LocalFileSystem::CreateLocal()) { 18 | local_filesystem->CreateDirectory(*FAKE_FILESYSTEM_PREFIX); 19 | } 20 | bool CacheHttpfsFakeFileSystem::CanHandleFile(const string &path) { 21 | return StringUtil::StartsWith(path, *FAKE_FILESYSTEM_PREFIX); 22 | } 23 | 24 | unique_ptr CacheHttpfsFakeFileSystem::OpenFile(const string &path, FileOpenFlags flags, 25 | optional_ptr opener) { 26 | auto file_handle = local_filesystem->OpenFile(path, flags, opener); 27 | return make_uniq(path, std::move(file_handle), *this); 28 | } 29 | void CacheHttpfsFakeFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) { 30 | auto &local_filesystem_handle = handle.Cast().internal_file_handle; 31 | local_filesystem->Read(*local_filesystem_handle, buffer, nr_bytes, location); 32 | } 33 | int64_t CacheHttpfsFakeFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes) { 34 | auto &local_filesystem_handle = handle.Cast().internal_file_handle; 35 | return local_filesystem->Read(*local_filesystem_handle, buffer, nr_bytes); 36 | } 37 | 38 | void CacheHttpfsFakeFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) { 39 | auto &local_filesystem_handle = handle.Cast().internal_file_handle; 40 | local_filesystem->Write(*local_filesystem_handle, buffer, nr_bytes, location); 41 | } 42 | int64_t CacheHttpfsFakeFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_bytes) { 43 | auto &local_filesystem_handle = handle.Cast().internal_file_handle; 44 | return local_filesystem->Write(*local_filesystem_handle, buffer, nr_bytes); 45 | } 46 | int64_t CacheHttpfsFakeFileSystem::GetFileSize(FileHandle &handle) { 47 | auto &local_filesystem_handle = handle.Cast().internal_file_handle; 48 | return local_filesystem->GetFileSize(*local_filesystem_handle); 49 | } 50 | void CacheHttpfsFakeFileSystem::FileSync(FileHandle &handle) { 51 | auto &local_filesystem_handle = handle.Cast().internal_file_handle; 52 | local_filesystem->FileSync(*local_filesystem_handle); 53 | } 54 | 55 | void CacheHttpfsFakeFileSystem::Seek(FileHandle &handle, idx_t location) { 56 | auto &local_filesystem_handle = handle.Cast().internal_file_handle; 57 | local_filesystem->Seek(*local_filesystem_handle, location); 58 | } 59 | idx_t CacheHttpfsFakeFileSystem::SeekPosition(FileHandle &handle) { 60 | auto &local_filesystem_handle = handle.Cast().internal_file_handle; 61 | return local_filesystem->SeekPosition(*local_filesystem_handle); 62 | } 63 | bool CacheHttpfsFakeFileSystem::Trim(FileHandle &handle, idx_t offset_bytes, idx_t length_bytes) { 64 | auto &local_filesystem_handle = handle.Cast().internal_file_handle; 65 | return local_filesystem->Trim(*local_filesystem_handle, offset_bytes, length_bytes); 66 | } 67 | time_t CacheHttpfsFakeFileSystem::GetLastModifiedTime(FileHandle &handle) { 68 | auto &local_filesystem_handle = handle.Cast().internal_file_handle; 69 | return local_filesystem->GetLastModifiedTime(*local_filesystem_handle); 70 | } 71 | FileType CacheHttpfsFakeFileSystem::GetFileType(FileHandle &handle) { 72 | auto &local_filesystem_handle = handle.Cast().internal_file_handle; 73 | return local_filesystem->GetFileType(*local_filesystem_handle); 74 | } 75 | void CacheHttpfsFakeFileSystem::Truncate(FileHandle &handle, int64_t new_size) { 76 | auto &local_filesystem_handle = handle.Cast().internal_file_handle; 77 | local_filesystem->Truncate(*local_filesystem_handle, new_size); 78 | } 79 | bool CacheHttpfsFakeFileSystem::OnDiskFile(FileHandle &handle) { 80 | auto &local_filesystem_handle = handle.Cast().internal_file_handle; 81 | return local_filesystem->OnDiskFile(*local_filesystem_handle); 82 | } 83 | 84 | } // namespace duckdb 85 | -------------------------------------------------------------------------------- /src/utils/filesystem_utils.cpp: -------------------------------------------------------------------------------- 1 | #include "filesystem_utils.hpp" 2 | 3 | #include 4 | #include 5 | 6 | #include 7 | 8 | #include "cache_filesystem_config.hpp" 9 | #include "duckdb/common/file_system.hpp" 10 | #include "duckdb/common/string_util.hpp" 11 | #include "duckdb/common/local_file_system.hpp" 12 | 13 | namespace duckdb { 14 | 15 | void EvictStaleCacheFiles(FileSystem &local_filesystem, const string &cache_directory) { 16 | const time_t now = std::time(nullptr); 17 | local_filesystem.ListFiles( 18 | cache_directory, [&local_filesystem, &cache_directory, now](const string &fname, bool /*unused*/) { 19 | // Multiple threads could attempt to access and delete stale files, tolerate non-existent file. 20 | const string full_name = StringUtil::Format("%s/%s", cache_directory, fname); 21 | auto file_handle = local_filesystem.OpenFile(full_name, FileOpenFlags::FILE_FLAGS_READ | 22 | FileOpenFlags::FILE_FLAGS_NULL_IF_NOT_EXISTS); 23 | if (file_handle == nullptr) { 24 | return; 25 | } 26 | 27 | const time_t last_mod_time = local_filesystem.GetLastModifiedTime(*file_handle); 28 | const double diff = std::difftime(/*time_end=*/now, /*time_beg=*/last_mod_time); 29 | if (static_cast(diff) >= CACHE_FILE_STALENESS_SECOND) { 30 | if (std::remove(full_name.data()) < -1 && errno != EEXIST) { 31 | throw IOException("Fails to delete stale cache file %s", full_name); 32 | } 33 | } 34 | }); 35 | } 36 | 37 | int GetFileCountUnder(const std::string &folder) { 38 | int file_count = 0; 39 | LocalFileSystem::CreateLocal()->ListFiles( 40 | folder, [&file_count](const string & /*unused*/, bool /*unused*/) { ++file_count; }); 41 | return file_count; 42 | } 43 | 44 | vector GetSortedFilesUnder(const std::string &folder) { 45 | vector file_names; 46 | LocalFileSystem::CreateLocal()->ListFiles( 47 | folder, [&file_names](const string &fname, bool /*unused*/) { file_names.emplace_back(fname); }); 48 | std::sort(file_names.begin(), file_names.end()); 49 | return file_names; 50 | } 51 | 52 | idx_t GetOverallFileSystemDiskSpace(const std::string &path) { 53 | struct statvfs vfs; 54 | 55 | const auto ret = statvfs(path.c_str(), &vfs); 56 | D_ASSERT(ret == 0); 57 | 58 | auto total_blocks = vfs.f_blocks; 59 | auto block_size = vfs.f_frsize; 60 | return static_cast(total_blocks) * static_cast(block_size); 61 | } 62 | 63 | bool CanCacheOnDisk(const std::string &path) { 64 | if (g_test_insufficient_disk_space) { 65 | return false; 66 | } 67 | 68 | const auto avai_fs_bytes = FileSystem::GetAvailableDiskSpace(path); 69 | if (!avai_fs_bytes.IsValid()) { 70 | return false; 71 | } 72 | 73 | // If the left disk space is smaller than a cache block, there's no need to do on-disk cache. 74 | if (avai_fs_bytes.GetIndex() <= g_cache_block_size) { 75 | return false; 76 | } 77 | 78 | // Check user override configurations if specified. 79 | if (g_min_disk_bytes_for_cache != DEFAULT_MIN_DISK_BYTES_FOR_CACHE) { 80 | return g_min_disk_bytes_for_cache <= avai_fs_bytes.GetIndex(); 81 | } 82 | 83 | // Check default reserved disk space. 84 | // The function if frequently called on critical path, but filesystem metadata is highly cache-able, so the overhead 85 | // is just syscall. 86 | const idx_t overall_fs_bytes = GetOverallFileSystemDiskSpace(path); 87 | return overall_fs_bytes * MIN_DISK_SPACE_PERCENTAGE_FOR_CACHE <= avai_fs_bytes.GetIndex(); 88 | } 89 | 90 | } // namespace duckdb 91 | -------------------------------------------------------------------------------- /src/utils/include/fake_filesystem.hpp: -------------------------------------------------------------------------------- 1 | // A fake filesystem for cache httpfs extension testing purpose. 2 | 3 | #pragma once 4 | 5 | #include "duckdb/common/file_system.hpp" 6 | #include "duckdb/common/local_file_system.hpp" 7 | 8 | namespace duckdb { 9 | 10 | // Forward declaration. 11 | class CacheHttpfsFakeFileSystem; 12 | 13 | class CacheHttpfsFakeFsHandle : public FileHandle { 14 | public: 15 | CacheHttpfsFakeFsHandle(string path, unique_ptr internal_file_handle_p, CacheHttpfsFakeFileSystem &fs); 16 | ~CacheHttpfsFakeFsHandle() override = default; 17 | void Close() override { 18 | internal_file_handle->Close(); 19 | } 20 | 21 | unique_ptr internal_file_handle; 22 | }; 23 | 24 | // WARNING: fake filesystem is used for testing purpose and shouldn't be used in production. 25 | class CacheHttpfsFakeFileSystem : public LocalFileSystem { 26 | public: 27 | CacheHttpfsFakeFileSystem(); 28 | bool CanHandleFile(const string &path) override; 29 | std::string GetName() const override { 30 | return "cache_httpfs_fake_filesystem"; 31 | } 32 | 33 | // Delegate to local filesystem. 34 | unique_ptr OpenFile(const string &path, FileOpenFlags flags, optional_ptr opener) override; 35 | void Read(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) override; 36 | int64_t Read(FileHandle &handle, void *buffer, int64_t nr_bytes) override; 37 | void Write(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) override; 38 | int64_t Write(FileHandle &handle, void *buffer, int64_t nr_bytes) override; 39 | int64_t GetFileSize(FileHandle &handle) override; 40 | void FileSync(FileHandle &handle) override; 41 | void Seek(FileHandle &handle, idx_t location) override; 42 | idx_t SeekPosition(FileHandle &handle) override; 43 | bool Trim(FileHandle &handle, idx_t offset_bytes, idx_t length_bytes) override; 44 | time_t GetLastModifiedTime(FileHandle &handle) override; 45 | FileType GetFileType(FileHandle &handle) override; 46 | void Truncate(FileHandle &handle, int64_t new_size) override; 47 | bool OnDiskFile(FileHandle &handle) override; 48 | 49 | private: 50 | unique_ptr local_filesystem; 51 | }; 52 | 53 | } // namespace duckdb 54 | -------------------------------------------------------------------------------- /src/utils/include/filesystem_utils.hpp: -------------------------------------------------------------------------------- 1 | // Utils on filesystem operations. 2 | 3 | #pragma once 4 | 5 | #include "duckdb/common/file_system.hpp" 6 | #include "duckdb/common/typedefs.hpp" 7 | 8 | namespace duckdb { 9 | 10 | // Evict stale cache files. 11 | // 12 | // The function iterates all cache files under the directory, and performs a 13 | // stat call on each of them, could be a performance bottleneck. But as the 14 | // initial implementation, cache eviction only happens when insufficient disk 15 | // space detected, which happens rarely thus not a big concern. 16 | void EvictStaleCacheFiles(FileSystem &local_filesystem, const string &cache_directory); 17 | 18 | // Get the number of files under the given local filesystem [folder]. 19 | int GetFileCountUnder(const std::string &folder); 20 | 21 | // Get all files under the given local filesystem [folder] in alphabetically 22 | // ascending order. 23 | vector GetSortedFilesUnder(const std::string &folder); 24 | 25 | // Get all disk space in bytes for the filesystem indicated by the given [path]. 26 | idx_t GetOverallFileSystemDiskSpace(const std::string &path); 27 | 28 | // Return whether we could cache content in the filesystem specified by the given [path]. 29 | bool CanCacheOnDisk(const std::string &path); 30 | 31 | } // namespace duckdb 32 | -------------------------------------------------------------------------------- /src/utils/include/map_utils.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | namespace duckdb { 7 | 8 | // A hash wrapper made for `std::reference_wrapper`. 9 | template 10 | struct RefHash : Hash { 11 | RefHash() = default; 12 | template 13 | RefHash(H &&h) : Hash(std::forward(h)) { 14 | } // NOLINT 15 | 16 | RefHash(const RefHash &) = default; 17 | RefHash(RefHash &&) noexcept = default; 18 | RefHash &operator=(const RefHash &) = default; 19 | RefHash &operator=(RefHash &&) noexcept = default; 20 | 21 | template 22 | size_t operator()(std::reference_wrapper val) const { 23 | return Hash::operator()(val.get()); 24 | } 25 | template 26 | size_t operator()(const T &val) const { 27 | return Hash::operator()(val); 28 | } 29 | }; 30 | 31 | // A hash equal wrapper made for `std::reference_wrapper`. 32 | template 33 | struct RefEq : Equal { 34 | RefEq() = default; 35 | template 36 | RefEq(Eq &&eq) : Equal(std::forward(eq)) { 37 | } // NOLINT 38 | 39 | RefEq(const RefEq &) = default; 40 | RefEq(RefEq &&) noexcept = default; 41 | RefEq &operator=(const RefEq &) = default; 42 | RefEq &operator=(RefEq &&) noexcept = default; 43 | 44 | template 45 | bool operator()(std::reference_wrapper lhs, std::reference_wrapper rhs) const { 46 | return Equal::operator()(lhs.get(), rhs.get()); 47 | } 48 | template 49 | bool operator()(const T1 &lhs, std::reference_wrapper rhs) const { 50 | return Equal::operator()(lhs, rhs.get()); 51 | } 52 | template 53 | bool operator()(std::reference_wrapper lhs, const T2 &rhs) const { 54 | return Equal::operator()(lhs.get(), rhs); 55 | } 56 | template 57 | bool operator()(const T1 &lhs, const T2 &rhs) const { 58 | return Equal::operator()(lhs, rhs); 59 | } 60 | }; 61 | 62 | } // namespace duckdb 63 | -------------------------------------------------------------------------------- /src/utils/include/mock_filesystem.hpp: -------------------------------------------------------------------------------- 1 | // This file defines mock filesystem, which is used for testing. 2 | // It checks a few things: 3 | // 1. Whether bytes to read to correct (whether request is correctly chunked and cached). 4 | // 2. Whether file handles are properly closed and destructed. 5 | 6 | #pragma once 7 | 8 | #include "duckdb/common/file_system.hpp" 9 | #include "duckdb/common/open_file_info.hpp" 10 | #include "duckdb/common/vector.hpp" 11 | 12 | #include 13 | #include 14 | #include 15 | 16 | namespace duckdb { 17 | 18 | class MockFileHandle : public FileHandle { 19 | public: 20 | MockFileHandle(FileSystem &file_system, string path, FileOpenFlags flags, std::function close_callback_p, 21 | std::function dtor_callback_p); 22 | ~MockFileHandle() override { 23 | D_ASSERT(dtor_callback); 24 | dtor_callback(); 25 | } 26 | void Close() override { 27 | D_ASSERT(close_callback); 28 | close_callback(); 29 | } 30 | 31 | private: 32 | std::function close_callback; 33 | std::function dtor_callback; 34 | }; 35 | 36 | class MockFileSystem : public FileSystem { 37 | public: 38 | struct ReadOper { 39 | uint64_t start_offset = 0; 40 | int64_t bytes_to_read = 0; 41 | }; 42 | 43 | MockFileSystem(std::function close_callback_p, std::function dtor_callback_p) 44 | : close_callback(std::move(close_callback_p)), dtor_callback(std::move(dtor_callback_p)) { 45 | } 46 | ~MockFileSystem() override = default; 47 | 48 | unique_ptr OpenFile(const string &path, FileOpenFlags flags, optional_ptr opener) override; 49 | void Read(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) override; 50 | vector Glob(const string &path, FileOpener *opener = nullptr) override { 51 | ++glob_invocation; 52 | return {}; 53 | } 54 | int64_t GetFileSize(FileHandle &handle) override { 55 | ++get_file_size_invocation; 56 | return file_size; 57 | } 58 | void Seek(FileHandle &handle, idx_t location) override { 59 | } 60 | std::string GetName() const override { 61 | return "mock filesystem"; 62 | } 63 | 64 | void SetFileSize(int64_t file_size_p) { 65 | file_size = file_size_p; 66 | } 67 | vector GetSortedReadOperations(); 68 | uint64_t GetFileOpenInvocation() const { 69 | return file_open_invocation; 70 | } 71 | uint64_t GetGlobInvocation() const { 72 | return glob_invocation; 73 | } 74 | void ClearReadOperations() { 75 | read_operations.clear(); 76 | } 77 | 78 | private: 79 | int64_t file_size = 0; 80 | std::function close_callback; 81 | std::function dtor_callback; 82 | 83 | uint64_t file_open_invocation = 0; // Number of `FileOpen` gets called. 84 | uint64_t glob_invocation = 0; // Number of `Glob` gets called. 85 | uint64_t get_file_size_invocation = 0; // Number of `GetFileSize` get called. 86 | vector read_operations; 87 | std::mutex mtx; 88 | }; 89 | 90 | bool operator<(const MockFileSystem::ReadOper &lhs, const MockFileSystem::ReadOper &rhs); 91 | bool operator>(const MockFileSystem::ReadOper &lhs, const MockFileSystem::ReadOper &rhs); 92 | bool operator<=(const MockFileSystem::ReadOper &lhs, const MockFileSystem::ReadOper &rhs); 93 | bool operator>=(const MockFileSystem::ReadOper &lhs, const MockFileSystem::ReadOper &rhs); 94 | bool operator==(const MockFileSystem::ReadOper &lhs, const MockFileSystem::ReadOper &rhs); 95 | bool operator!=(const MockFileSystem::ReadOper &lhs, const MockFileSystem::ReadOper &rhs); 96 | 97 | } // namespace duckdb 98 | -------------------------------------------------------------------------------- /src/utils/include/no_destructor.hpp: -------------------------------------------------------------------------------- 1 | // A wrapper class which defines a static type that does not need to be destructed upon program exit. Instead, 2 | // such an object survives during program exit (and can be safely accessed at any time). 3 | // 4 | // In theory, the best implementation is `absl::NoDestructor`. 5 | // Reference: https://github.com/abseil/abseil-cpp/blob/master/absl/base/no_destructor.h 6 | // But C++11 doesn't support `std::launder` so we have to switch `new`-allocation method, instead of `placement new`. 7 | // 8 | // Example usage: 9 | // - Initialization: NoDestructor obj{...}; 10 | // - Re-assignment: *obj = T{...}; 11 | 12 | #pragma once 13 | 14 | #include 15 | 16 | namespace duckdb { 17 | 18 | template 19 | class NoDestructor { 20 | public: 21 | NoDestructor() : obj(*new T {}) { 22 | } 23 | 24 | explicit NoDestructor(const T &data) : obj(*new T {data}) { 25 | } 26 | 27 | explicit NoDestructor(T &&data) : obj(*new T {std::move(data)}) { 28 | } 29 | 30 | template 31 | explicit NoDestructor(Args &&...args) : obj(*new T {std::forward(args)...}) { 32 | } 33 | 34 | NoDestructor(const NoDestructor &) = delete; 35 | NoDestructor &operator=(const NoDestructor &) = delete; 36 | NoDestructor(NoDestructor &&) = delete; 37 | NoDestructor &operator=(NoDestructor &&) = delete; 38 | 39 | // Intentionally no destruct. 40 | ~NoDestructor() = default; 41 | 42 | T &Get() & { 43 | return obj; 44 | } 45 | const T &Get() const & { 46 | return obj; 47 | } 48 | 49 | T &operator*() & { 50 | return Get(); 51 | } 52 | const T &operator*() const & { 53 | return Get(); 54 | } 55 | 56 | T *operator->() & { 57 | return &Get(); 58 | } 59 | const T *operator->() const & { 60 | return &Get(); 61 | } 62 | 63 | private: 64 | T &obj; 65 | }; 66 | 67 | } // namespace duckdb 68 | -------------------------------------------------------------------------------- /src/utils/include/resize_uninitialized.hpp: -------------------------------------------------------------------------------- 1 | // Copyright 2017 The Abseil Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Apapted from abseil `resize_uninitialized` implementation. 16 | 17 | #pragma once 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | #include "type_traits.hpp" 24 | 25 | namespace duckdb { 26 | 27 | namespace internal { 28 | 29 | // In this type trait, we look for a __resize_default_init member function, and 30 | // we use it if available, otherwise, we use resize. We provide HasMember to 31 | // indicate whether __resize_default_init is present. 32 | template 33 | struct ResizeUninitializedTraits { 34 | using HasMember = std::false_type; 35 | static void Resize(string_type *s, size_t new_size) { 36 | s->resize(new_size); 37 | } 38 | }; 39 | 40 | // __resize_default_init is provided by libc++ >= 8.0 41 | template 42 | struct ResizeUninitializedTraits().__resize_default_init(237))>> { 44 | using HasMember = std::true_type; 45 | static void Resize(string_type *s, size_t new_size) { 46 | s->__resize_default_init(new_size); 47 | } 48 | }; 49 | 50 | } // namespace internal 51 | 52 | // Like str->resize(new_size), except any new characters added to "*str" as a 53 | // result of resizing may be left uninitialized, rather than being filled with 54 | // '0' bytes. Typically used when code is then going to overwrite the backing 55 | // store of the std::string with known data. 56 | template 57 | inline void STLStringResizeUninitialized(string_type *s, size_t new_size) { 58 | internal::ResizeUninitializedTraits::Resize(s, new_size); 59 | } 60 | 61 | // Create a string with the given size, with all bytes uninitialized. Useful to 62 | // use as a buffer. 63 | inline std::string CreateResizeUninitializedString(size_t size) { 64 | std::string content; 65 | STLStringResizeUninitialized(&content, size); 66 | return content; 67 | } 68 | 69 | } // namespace duckdb 70 | -------------------------------------------------------------------------------- /src/utils/include/size_literals.hpp: -------------------------------------------------------------------------------- 1 | // These user-defined literals makes sizes. 2 | 3 | #pragma once 4 | 5 | #include 6 | 7 | inline unsigned long long operator""_PiB(unsigned long long sz) { 8 | return sz * 1024ULL * 1024ULL * 1024ULL * 1024ULL * 1024ULL; 9 | } 10 | inline unsigned long long operator""_PB(unsigned long long sz) { 11 | return sz * 1000ULL * 1000ULL * 1000ULL * 1000ULL * 1000ULL; 12 | } 13 | 14 | inline unsigned long long operator""_TiB(unsigned long long sz) { 15 | return sz * 1024ULL * 1024ULL * 1024ULL * 1024ULL; 16 | } 17 | inline unsigned long long operator""_TB(unsigned long long sz) { 18 | return sz * 1000ULL * 1000ULL * 1000ULL * 1000ULL; 19 | } 20 | 21 | inline unsigned long long operator""_GiB(unsigned long long sz) { 22 | return sz * 1024ULL * 1024ULL * 1024ULL; 23 | } 24 | inline unsigned long long operator""_GB(unsigned long long sz) { 25 | return sz * 1000ULL * 1000ULL * 1000ULL; 26 | } 27 | 28 | inline unsigned long long operator""_MiB(unsigned long long sz) { 29 | return sz * 1024ULL * 1024ULL; 30 | } 31 | inline unsigned long long operator""_MB(unsigned long long sz) { 32 | return sz * 1000ULL * 1000ULL; 33 | } 34 | 35 | inline unsigned long long operator""_KiB(unsigned long long sz) { 36 | return sz * 1024ULL; 37 | } 38 | inline unsigned long long operator""_KB(unsigned long long sz) { 39 | return sz * 1000ULL; 40 | } 41 | 42 | inline unsigned long long operator""_PiB(long double sz) { 43 | const long double res = sz * 1024ULL * 1024ULL * 1024ULL * 1024ULL * 1024ULL; 44 | return static_cast(res); 45 | } 46 | inline unsigned long long operator""_PB(long double sz) { 47 | const long double res = sz * 1000ULL * 1000ULL * 1000ULL * 1000ULL * 1000ULL; 48 | return static_cast(res); 49 | } 50 | 51 | inline unsigned long long operator""_TiB(long double sz) { 52 | const long double res = sz * 1024ULL * 1024ULL * 1024ULL * 1024ULL; 53 | return static_cast(res); 54 | } 55 | inline unsigned long long operator""_TB(long double sz) { 56 | const long double res = sz * 1000ULL * 1000ULL * 1000ULL * 1000ULL; 57 | return static_cast(res); 58 | } 59 | 60 | inline unsigned long long operator""_GiB(long double sz) { 61 | const long double res = sz * 1024ULL * 1024ULL * 1024ULL; 62 | return static_cast(res); 63 | } 64 | inline unsigned long long operator""_GB(long double sz) { 65 | const long double res = sz * 1000ULL * 1000ULL * 1000ULL; 66 | return static_cast(res); 67 | } 68 | 69 | inline unsigned long long operator""_MiB(long double sz) { 70 | const long double res = sz * 1024ULL * 1024ULL; 71 | return static_cast(res); 72 | } 73 | inline unsigned long long operator""_MB(long double sz) { 74 | const long double res = sz * 1000ULL * 1000ULL; 75 | return static_cast(res); 76 | } 77 | 78 | inline unsigned long long operator""_KiB(long double sz) { 79 | const long double res = sz * 1024ULL; 80 | return static_cast(res); 81 | } 82 | inline unsigned long long operator""_KB(long double sz) { 83 | const long double res = sz * 1000ULL; 84 | return static_cast(res); 85 | } 86 | 87 | inline unsigned long long operator""_B(unsigned long long sz) { 88 | return sz; 89 | } 90 | -------------------------------------------------------------------------------- /src/utils/include/thread_pool.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | namespace duckdb { 14 | 15 | class ThreadPool { 16 | public: 17 | ThreadPool(); 18 | explicit ThreadPool(size_t thread_num); 19 | 20 | ThreadPool(const ThreadPool &) = delete; 21 | ThreadPool &operator=(const ThreadPool &) = delete; 22 | 23 | ~ThreadPool() noexcept; 24 | 25 | // @return future for synchronization. 26 | template 27 | auto Push(Fn &&fn, Args &&...args) -> std::future>; 28 | 29 | // Block until the threadpool is dead, or all enqueued tasks finish. 30 | void Wait(); 31 | 32 | private: 33 | using Job = std::function; 34 | 35 | size_t idle_num_ = 0; 36 | bool stopped_ = false; 37 | std::mutex mutex_; 38 | std::condition_variable new_job_cv_; 39 | std::condition_variable job_completion_cv_; 40 | std::queue jobs_; 41 | std::vector workers_; 42 | }; 43 | 44 | template 45 | auto ThreadPool::Push(Fn &&fn, Args &&...args) -> std::future> { 46 | using Ret = typename std::result_of_t; 47 | 48 | auto job = 49 | std::make_shared>(std::bind(std::forward(fn), std::forward(args)...)); 50 | std::future result = job->get_future(); 51 | { 52 | std::lock_guard lck(mutex_); 53 | jobs_.emplace([job = std::move(job)]() mutable { (*job)(); }); 54 | new_job_cv_.notify_one(); 55 | } 56 | return result; 57 | } 58 | 59 | } // namespace duckdb 60 | -------------------------------------------------------------------------------- /src/utils/include/thread_utils.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | namespace duckdb { 6 | 7 | // Set thread name to current thread; fail silently if an error happens. 8 | void SetThreadName(const std::string &thread_name); 9 | 10 | // Get the number of cores available to the system. 11 | // On linux platform, this function not only gets physical core number for CPU, but also considers available core number 12 | // within kubernetes pod, and container cgroup resource. 13 | int GetCpuCoreCount(); 14 | 15 | } // namespace duckdb 16 | -------------------------------------------------------------------------------- /src/utils/include/time_utils.hpp: -------------------------------------------------------------------------------- 1 | // Time related utils. 2 | 3 | #pragma once 4 | 5 | #include 6 | #include 7 | 8 | namespace duckdb { 9 | 10 | inline constexpr uint64_t kMicrosToNanos = 1000ULL; 11 | inline constexpr uint64_t kSecondsToMicros = 1000ULL * 1000ULL; 12 | inline constexpr uint64_t kSecondsToNanos = 1000ULL * 1000ULL * 1000ULL; 13 | inline constexpr uint64_t kMilliToNanos = 1000ULL * 1000ULL; 14 | 15 | // Get current timestamp in steady clock since epoch in nanoseconds. 16 | inline int64_t GetSteadyNowNanoSecSinceEpoch() { 17 | return std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()) 18 | .count(); 19 | } 20 | 21 | // Get current timestamp in steady clock since epoch in milliseconds. 22 | inline int64_t GetSteadyNowMilliSecSinceEpoch() { 23 | return GetSteadyNowNanoSecSinceEpoch() / kMilliToNanos; 24 | } 25 | 26 | // Get current timestamp in steady clock since epoch in nanoseconds. 27 | inline int64_t GetSystemNowNanoSecSinceEpoch() { 28 | return std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()) 29 | .count(); 30 | } 31 | 32 | // Get current timestamp in steady clock since epoch in milliseconds. 33 | inline int64_t GetSystemNowMilliSecSinceEpoch() { 34 | return GetSystemNowNanoSecSinceEpoch() / kMilliToNanos; 35 | } 36 | 37 | } // namespace duckdb 38 | -------------------------------------------------------------------------------- /src/utils/include/type_traits.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | namespace duckdb { 4 | 5 | template 6 | struct VoidTImpl { 7 | using type = void; 8 | }; 9 | 10 | template 11 | using void_t = typename VoidTImpl::type; 12 | 13 | } // namespace duckdb 14 | -------------------------------------------------------------------------------- /src/utils/mock_filesystem.cpp: -------------------------------------------------------------------------------- 1 | #include "mock_filesystem.hpp" 2 | 3 | #include 4 | #include 5 | 6 | namespace duckdb { 7 | 8 | bool operator<(const MockFileSystem::ReadOper &lhs, const MockFileSystem::ReadOper &rhs) { 9 | return std::tie(lhs.start_offset, lhs.bytes_to_read) < std::tie(rhs.start_offset, rhs.bytes_to_read); 10 | } 11 | bool operator>(const MockFileSystem::ReadOper &lhs, const MockFileSystem::ReadOper &rhs) { 12 | return std::tie(lhs.start_offset, lhs.bytes_to_read) > std::tie(rhs.start_offset, rhs.bytes_to_read); 13 | } 14 | bool operator<=(const MockFileSystem::ReadOper &lhs, const MockFileSystem::ReadOper &rhs) { 15 | return std::tie(lhs.start_offset, lhs.bytes_to_read) <= std::tie(rhs.start_offset, rhs.bytes_to_read); 16 | } 17 | bool operator>=(const MockFileSystem::ReadOper &lhs, const MockFileSystem::ReadOper &rhs) { 18 | return std::tie(lhs.start_offset, lhs.bytes_to_read) >= std::tie(rhs.start_offset, rhs.bytes_to_read); 19 | } 20 | bool operator==(const MockFileSystem::ReadOper &lhs, const MockFileSystem::ReadOper &rhs) { 21 | return std::tie(lhs.start_offset, lhs.bytes_to_read) == std::tie(rhs.start_offset, rhs.bytes_to_read); 22 | } 23 | bool operator!=(const MockFileSystem::ReadOper &lhs, const MockFileSystem::ReadOper &rhs) { 24 | return std::tie(lhs.start_offset, lhs.bytes_to_read) != std::tie(rhs.start_offset, rhs.bytes_to_read); 25 | } 26 | 27 | MockFileHandle::MockFileHandle(FileSystem &file_system, string path, FileOpenFlags flags, 28 | std::function close_callback_p, std::function dtor_callback_p) 29 | : FileHandle(file_system, path, flags), close_callback(std::move(close_callback_p)), 30 | dtor_callback(std::move(dtor_callback_p)) { 31 | // Make sure passed-in filesystem is mock filesystem. 32 | [[maybe_unused]] auto &fs = file_system.Cast(); 33 | } 34 | 35 | unique_ptr MockFileSystem::OpenFile(const string &path, FileOpenFlags flags, 36 | optional_ptr opener) { 37 | std::lock_guard lck(mtx); 38 | ++file_open_invocation; 39 | return make_uniq(*this, path, flags, close_callback, dtor_callback); 40 | } 41 | void MockFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) { 42 | std::lock_guard lck(mtx); 43 | std::memset(buffer, 'a', nr_bytes); 44 | read_operations.emplace_back(ReadOper { 45 | .start_offset = location, 46 | .bytes_to_read = nr_bytes, 47 | }); 48 | } 49 | 50 | vector MockFileSystem::GetSortedReadOperations() { 51 | std::sort(read_operations.begin(), read_operations.end(), 52 | [](const auto &lhs, const auto &rhs) { return lhs < rhs; }); 53 | return read_operations; 54 | } 55 | 56 | } // namespace duckdb 57 | -------------------------------------------------------------------------------- /src/utils/thread_pool.cpp: -------------------------------------------------------------------------------- 1 | #include "thread_pool.hpp" 2 | 3 | #include 4 | 5 | #include "duckdb/common/assert.hpp" 6 | #include "thread_utils.hpp" 7 | 8 | namespace duckdb { 9 | 10 | ThreadPool::ThreadPool() : ThreadPool(GetCpuCoreCount()) { 11 | } 12 | 13 | ThreadPool::ThreadPool(size_t thread_num) : idle_num_(thread_num) { 14 | workers_.reserve(thread_num); 15 | for (size_t ii = 0; ii < thread_num; ++ii) { 16 | workers_.emplace_back([this]() { 17 | for (;;) { 18 | Job cur_job; 19 | { 20 | std::unique_lock lck(mutex_); 21 | new_job_cv_.wait(lck, [this]() { return !jobs_.empty() || stopped_; }); 22 | if (stopped_) { 23 | return; 24 | } 25 | cur_job = std::move(jobs_.front()); 26 | jobs_.pop(); 27 | --idle_num_; 28 | } 29 | 30 | // Execute job out of critical section. 31 | cur_job(); 32 | 33 | { 34 | std::unique_lock lck(mutex_); 35 | ++idle_num_; 36 | job_completion_cv_.notify_one(); 37 | } 38 | } 39 | }); 40 | } 41 | } 42 | 43 | void ThreadPool::Wait() { 44 | std::unique_lock lck(mutex_); 45 | job_completion_cv_.wait(lck, [this]() { 46 | if (stopped_) { 47 | return true; 48 | } 49 | if (idle_num_ == workers_.size() && jobs_.empty()) { 50 | return true; 51 | } 52 | return false; 53 | }); 54 | } 55 | 56 | ThreadPool::~ThreadPool() noexcept { 57 | { 58 | std::lock_guard lck(mutex_); 59 | stopped_ = true; 60 | new_job_cv_.notify_all(); 61 | } 62 | for (auto &cur_worker : workers_) { 63 | D_ASSERT(cur_worker.joinable()); 64 | cur_worker.join(); 65 | } 66 | } 67 | 68 | } // namespace duckdb 69 | -------------------------------------------------------------------------------- /src/utils/thread_utils.cpp: -------------------------------------------------------------------------------- 1 | #include "thread_utils.hpp" 2 | 3 | #include 4 | #include 5 | 6 | #if defined(__linux__) 7 | #include 8 | #endif 9 | 10 | namespace duckdb { 11 | 12 | void SetThreadName(const std::string &thread_name) { 13 | #if defined(__APPLE__) 14 | pthread_setname_np(thread_name.c_str()); 15 | #elif defined(__linux__) 16 | // Restricted to 16 characters, include terminator. 17 | pthread_setname_np(pthread_self(), thread_name.substr(0, 15).c_str()); 18 | #endif 19 | } 20 | 21 | int GetCpuCoreCount() { 22 | #if defined(__APPLE__) 23 | return std::thread::hardware_concurrency(); 24 | #else 25 | cpu_set_t cpuset; 26 | CPU_ZERO(&cpuset); 27 | sched_getaffinity(0, sizeof(cpuset), &cpuset); 28 | const int core_count = CPU_COUNT(&cpuset); 29 | return core_count; 30 | #endif 31 | } 32 | 33 | } // namespace duckdb 34 | -------------------------------------------------------------------------------- /test/data/README.md: -------------------------------------------------------------------------------- 1 | # Test data 2 | 3 | ## Database file 4 | 5 | It's used for `ATTACH` test, generated from the below SQL statements. 6 | ```sql 7 | D CREATE TABLE users ( id INTEGER, name TEXT, email TEXT); 8 | D INSERT INTO users (id, name, email)VALUES (1, 'Alice', 'alice@example.com'); 9 | ``` 10 | -------------------------------------------------------------------------------- /test/data/attach_test.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dentiny/duck-read-cache-fs/39bf7bb325d022ceb4f38c41cab43c48e7dc3793/test/data/attach_test.db -------------------------------------------------------------------------------- /test/sql/cache_access_info.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/cache_access_info.test 2 | # description: test cache access info 3 | # group: [sql] 4 | 5 | # Notice: we don't really test glob cache operation, because HTTP filesystem doesn't support real GLOB. 6 | 7 | require cache_httpfs 8 | 9 | query III 10 | SELECT * FROM cache_httpfs_cache_access_info_query(); 11 | ---- 12 | metadata 0 0 13 | data 0 0 14 | file handle 0 0 15 | glob 0 0 16 | 17 | # Start to record profile. 18 | statement ok 19 | SET cache_httpfs_profile_type='temp'; 20 | 21 | # Test uncached query. 22 | statement ok 23 | SELECT cache_httpfs_clear_cache(); 24 | 25 | statement ok 26 | SELECT COUNT(*) FROM read_csv_auto('https://raw.githubusercontent.com/dentiny/duck-read-cache-fs/refs/heads/main/test/data/stock-exchanges.csv'); 27 | 28 | query III 29 | SELECT * FROM cache_httpfs_cache_access_info_query(); 30 | ---- 31 | metadata 2 1 32 | data 0 1 33 | file handle 0 1 34 | glob 0 0 35 | 36 | # Query second time should show cache hit. 37 | statement ok 38 | SELECT COUNT(*) FROM read_csv_auto('https://raw.githubusercontent.com/dentiny/duck-read-cache-fs/refs/heads/main/test/data/stock-exchanges.csv'); 39 | 40 | query III 41 | SELECT * FROM cache_httpfs_cache_access_info_query(); 42 | ---- 43 | metadata 5 1 44 | data 1 1 45 | file handle 1 1 46 | glob 0 0 47 | 48 | statement ok 49 | SELECT cache_httpfs_clear_profile(); 50 | 51 | query III 52 | SELECT * FROM cache_httpfs_cache_access_info_query(); 53 | ---- 54 | metadata 0 0 55 | data 0 0 56 | file handle 0 0 57 | glob 0 0 58 | 59 | # Disable all non data cache fron now on. 60 | statement ok 61 | SET cache_httpfs_enable_metadata_cache=false; 62 | 63 | statement ok 64 | SET cache_httpfs_enable_glob_cache=false; 65 | 66 | statement ok 67 | SET cache_httpfs_enable_file_handle_cache=false; 68 | 69 | statement ok 70 | SET cache_httpfs_type='noop'; 71 | 72 | # After disabling cache, uncached read first time. 73 | statement ok 74 | SELECT COUNT(*) FROM read_csv_auto('https://raw.githubusercontent.com/dentiny/duck-read-cache-fs/refs/heads/main/test/data/stock-exchanges.csv'); 75 | 76 | query III 77 | SELECT * FROM cache_httpfs_cache_access_info_query(); 78 | ---- 79 | metadata 0 0 80 | data 0 0 81 | file handle 0 0 82 | glob 0 0 83 | 84 | # After disabling cache, uncached read second time. 85 | statement ok 86 | SELECT COUNT(*) FROM read_csv_auto('https://raw.githubusercontent.com/dentiny/duck-read-cache-fs/refs/heads/main/test/data/stock-exchanges.csv'); 87 | 88 | query III 89 | SELECT * FROM cache_httpfs_cache_access_info_query(); 90 | ---- 91 | metadata 0 0 92 | data 0 0 93 | file handle 0 0 94 | glob 0 0 95 | -------------------------------------------------------------------------------- /test/sql/clear_cache.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/clear_cache.test 2 | # description: test SQL queries with cache cleared 3 | # group: [sql] 4 | 5 | require cache_httpfs 6 | 7 | statement ok 8 | SELECT cache_httpfs_clear_cache(); 9 | 10 | # Cannot use on-disk cache, because it doesn't support clear cache by filepath. 11 | statement ok 12 | SET cache_httpfs_type='in_mem'; 13 | 14 | # Start to record profile. 15 | statement ok 16 | SET cache_httpfs_profile_type='temp'; 17 | 18 | # ========================== 19 | # Uncached query 20 | # ========================== 21 | query I 22 | SELECT COUNT(*) FROM read_csv_auto('https://raw.githubusercontent.com/dentiny/duck-read-cache-fs/refs/heads/main/test/data/stock-exchanges.csv'); 23 | ---- 24 | 251 25 | 26 | query III 27 | SELECT * FROM cache_httpfs_cache_access_info_query(); 28 | ---- 29 | metadata 2 1 30 | data 0 1 31 | file handle 0 1 32 | glob 0 0 33 | 34 | # ========================== 35 | # Clear all cache 36 | # ========================== 37 | # Clear all cache and re-execute the query. 38 | statement ok 39 | SELECT cache_httpfs_clear_cache(); 40 | 41 | query I 42 | SELECT COUNT(*) FROM read_csv_auto('https://raw.githubusercontent.com/dentiny/duck-read-cache-fs/refs/heads/main/test/data/stock-exchanges.csv'); 43 | ---- 44 | 251 45 | 46 | query III 47 | SELECT * FROM cache_httpfs_cache_access_info_query(); 48 | ---- 49 | metadata 4 2 50 | data 0 2 51 | file handle 0 2 52 | glob 0 0 53 | 54 | # ========================== 55 | # Clear cache by filepath 56 | # ========================== 57 | # Clear cache key-ed by the filepath and re-execute the query. 58 | statement ok 59 | SELECT cache_httpfs_clear_cache_for_file('https://raw.githubusercontent.com/dentiny/duck-read-cache-fs/refs/heads/main/test/data/stock-exchanges.csv'); 60 | 61 | query I 62 | SELECT COUNT(*) FROM read_csv_auto('https://raw.githubusercontent.com/dentiny/duck-read-cache-fs/refs/heads/main/test/data/stock-exchanges.csv'); 63 | ---- 64 | 251 65 | 66 | query III 67 | SELECT * FROM cache_httpfs_cache_access_info_query(); 68 | ---- 69 | metadata 6 3 70 | data 0 3 71 | file handle 0 3 72 | glob 0 0 73 | -------------------------------------------------------------------------------- /test/sql/extension.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/extension.test 2 | # description: test cached_fs extension loading 3 | # group: [sql] 4 | 5 | statement error 6 | SELECT cache_httpfs_get_ondisk_data_cache_size(); 7 | ---- 8 | Catalog Error: Scalar Function with name cache_httpfs_get_ondisk_data_cache_size does not exist! 9 | 10 | require cache_httpfs 11 | 12 | # Make sure extension description is correctly populated. 13 | query I 14 | SELECT COUNT(*) FROM duckdb_extensions() WHERE extension_name = 'cache_httpfs' AND description IS NOT NULL AND description <> ''; 15 | ---- 16 | 1 17 | 18 | statement ok 19 | SELECT cache_httpfs_clear_cache(); 20 | 21 | query I 22 | SELECT cache_httpfs_get_ondisk_data_cache_size(); 23 | ---- 24 | 0 25 | 26 | # Check if extension works if user sets invalid value. 27 | statement ok 28 | SET cache_httpfs_max_in_mem_cache_block_count=0; 29 | 30 | statement ok 31 | SET cache_httpfs_cache_block_size=0; 32 | 33 | query IIIIII 34 | SELECT * FROM read_csv_auto('https://raw.githubusercontent.com/dentiny/duck-read-cache-fs/refs/heads/main/test/data/stock-exchanges.csv') LIMIT 1; 35 | ---- 36 | 1 Africa Lesotho HYBSE NULL 2019-03-25 37 | -------------------------------------------------------------------------------- /test/sql/glob_read.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/glob_read.test 2 | # description: test cached_fs read multiple files 3 | # group: [sql] 4 | 5 | require cache_httpfs 6 | 7 | require parquet 8 | 9 | statement ok 10 | SET cache_httpfs_type='on_disk'; 11 | 12 | statement ok 13 | SET cache_httpfs_cache_directory='/tmp/duckdb_cache_httpfs_cache'; 14 | 15 | statement ok 16 | SELECT cache_httpfs_clear_cache(); 17 | 18 | # Test uncached query. 19 | query IIII 20 | SELECT * FROM read_csv_auto(['https://github.com/duckdb/duckdb/raw/refs/heads/v1.2-histrionicus/data/csv/union-by-name/ubn1.csv', 'https://github.com/duckdb/duckdb/raw/refs/heads/v1.2-histrionicus/data/csv/union-by-name/ubn2.csv'] , union_by_name = true) ORDER BY a; 21 | ---- 22 | 1 2 3 NULL 23 | 3 4 5 NULL 24 | 34fd321 91 NULL 2020-12-30 02:25:58.745232+00 25 | 4 5 6 NULL 26 | 8cb123cb8 90 NULL 2020-12-30 01:25:58.745232+00 27 | fg5391jn4 92 NULL 2020-12-30 03:25:58.745232+00 28 | test 88 NULL 2020-12-30 00:25:58.745232+00 29 | 30 | query IIIII 31 | SELECT * FROM cache_httpfs_cache_status_query() ORDER BY remote_filename; 32 | ---- 33 | /tmp/duckdb_cache_httpfs_cache/790e86440e87f3fe45cbfab00131ea59fbe90728a8277d2bc0610e9c43dae4cf-ubn1.csv-0-23 ubn1.csv 0 23 on-disk 34 | /tmp/duckdb_cache_httpfs_cache/716e708a8a767ba362ce86783df2a9bdf9f1e867fa38091e09865a3d3bf96a7a-ubn2.csv-0-171 ubn2.csv 0 171 on-disk 35 | 36 | # Test cached query. 37 | query IIII 38 | SELECT * FROM read_csv_auto(['https://github.com/duckdb/duckdb/raw/refs/heads/v1.2-histrionicus/data/csv/union-by-name/ubn1.csv', 'https://github.com/duckdb/duckdb/raw/refs/heads/v1.2-histrionicus/data/csv/union-by-name/ubn2.csv'] , union_by_name = true) ORDER BY a; 39 | ---- 40 | 1 2 3 NULL 41 | 3 4 5 NULL 42 | 34fd321 91 NULL 2020-12-30 02:25:58.745232+00 43 | 4 5 6 NULL 44 | 8cb123cb8 90 NULL 2020-12-30 01:25:58.745232+00 45 | fg5391jn4 92 NULL 2020-12-30 03:25:58.745232+00 46 | test 88 NULL 2020-12-30 00:25:58.745232+00 47 | 48 | # Clear cache after test. 49 | statement ok 50 | SELECT cache_httpfs_clear_cache(); 51 | -------------------------------------------------------------------------------- /test/sql/insufficient_disk_space.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/insufficient_disk_space.test 2 | # description: test cache_httpfs behavior when there's no sufficent disk space for disk cache reader 3 | # group: [sql] 4 | 5 | require cache_httpfs 6 | 7 | statement ok 8 | SET cache_httpfs_type='on_disk'; 9 | 10 | # Set requires disk space to be max value of uint64_t, which means we cannot leverage disk cache 11 | statement ok 12 | SET cache_httpfs_min_disk_bytes_for_cache=18446744073709551615 13 | 14 | statement ok 15 | SET cache_httpfs_cache_directory='/tmp/duckdb_cache_httpfs_cache'; 16 | 17 | statement ok 18 | SELECT cache_httpfs_clear_cache(); 19 | 20 | query I 21 | SELECT COUNT(*) FROM read_csv_auto('https://raw.githubusercontent.com/dentiny/duck-read-cache-fs/refs/heads/main/test/data/stock-exchanges.csv'); 22 | ---- 23 | 251 24 | 25 | query I 26 | SELECT COUNT(*) FROM glob('/tmp/duckdb_cache_httpfs_cache/*'); 27 | ---- 28 | 0 29 | -------------------------------------------------------------------------------- /test/sql/max_subrequest_fanout.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/max_subrequest_fanout.test 2 | # description: test cache_httpfs behavior when max fanout subrequests number configured 3 | # group: [sql] 4 | 5 | require cache_httpfs 6 | 7 | statement ok 8 | SET cache_httpfs_type='noop'; 9 | 10 | statement ok 11 | SELECT cache_httpfs_clear_cache(); 12 | 13 | statement ok 14 | SET cache_httpfs_cache_block_size=1; 15 | 16 | # Test max fanout subrequests size for a few possible values. 17 | # //===--------------------------------------------------------------------===// 18 | # // Unlimited subrequest number 19 | # //===--------------------------------------------------------------------===// 20 | statement ok 21 | SET cache_httpfs_max_fanout_subrequest=0 22 | 23 | query I 24 | SELECT COUNT(*) FROM read_csv_auto('https://raw.githubusercontent.com/dentiny/duck-read-cache-fs/refs/heads/main/test/data/stock-exchanges.csv'); 25 | ---- 26 | 251 27 | 28 | # //===--------------------------------------------------------------------===// 29 | # // Subrequest number 1, which means no parallelism 30 | # //===--------------------------------------------------------------------===// 31 | statement ok 32 | SET cache_httpfs_max_fanout_subrequest=1 33 | 34 | query I 35 | SELECT COUNT(*) FROM read_csv_auto('https://raw.githubusercontent.com/dentiny/duck-read-cache-fs/refs/heads/main/test/data/stock-exchanges.csv'); 36 | ---- 37 | 251 38 | 39 | # //===--------------------------------------------------------------------===// 40 | # // Subrequest number 10, which means little parallelism 41 | # //===--------------------------------------------------------------------===// 42 | statement ok 43 | SET cache_httpfs_max_fanout_subrequest=1 44 | 45 | query I 46 | SELECT COUNT(*) FROM read_csv_auto('https://raw.githubusercontent.com/dentiny/duck-read-cache-fs/refs/heads/main/test/data/stock-exchanges.csv'); 47 | ---- 48 | 251 49 | 50 | # //===--------------------------------------------------------------------===// 51 | # // Subrequest number 200, which means large parallelism 52 | # //===--------------------------------------------------------------------===// 53 | statement ok 54 | SET cache_httpfs_max_fanout_subrequest=200 55 | 56 | query I 57 | SELECT COUNT(*) FROM read_csv_auto('https://raw.githubusercontent.com/dentiny/duck-read-cache-fs/refs/heads/main/test/data/stock-exchanges.csv'); 58 | ---- 59 | 251 60 | -------------------------------------------------------------------------------- /test/sql/wrap_cache_filesystem.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/wrap_cache_filesystem.test 2 | # description: test cache httpfs extension wrap feature for filesystem instances other than those in httpfs. 3 | # group: [sql] 4 | 5 | require cache_httpfs 6 | 7 | statement error 8 | SELECT cache_httpfs_wrap_cache_filesystem('unregistered_filesystem'); 9 | ---- 10 | Invalid Input Error: Filesystem unregistered_filesystem hasn't been registered yet! 11 | 12 | statement ok 13 | SELECT cache_httpfs_wrap_cache_filesystem('cache_httpfs_fake_filesystem'); 14 | 15 | statement ok 16 | SELECT cache_httpfs_clear_cache(); 17 | 18 | # Check read through fake filesystem works fine. 19 | statement ok 20 | COPY (SELECT * FROM read_csv('https://raw.githubusercontent.com/dentiny/duck-read-cache-fs/refs/heads/main/test/data/stock-exchanges.csv')) TO '/tmp/cache_httpfs_fake_filesystem/stock-exchanges.csv'; 21 | 22 | # Clear cache after COPY, because reading from httfs already leads to local cache files. 23 | statement ok 24 | SELECT cache_httpfs_clear_cache(); 25 | 26 | query I 27 | SELECT COUNT(*) FROM read_csv('/tmp/cache_httpfs_fake_filesystem/stock-exchanges.csv'); 28 | ---- 29 | 251 30 | 31 | # Check local cache files. 32 | # File count = 16KiB / 64KiB = 1 33 | query I 34 | SELECT COUNT(*) FROM glob('/tmp/duckdb_cache_httpfs_cache/*'); 35 | ---- 36 | 1 37 | 38 | statement ok 39 | SELECT cache_httpfs_clear_cache(); 40 | -------------------------------------------------------------------------------- /test/sql/wrapped_filesystems_info.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/wrapped_filesystems_info.test 2 | # description: test wrapped filesystems info 3 | # group: [sql] 4 | 5 | require cache_httpfs 6 | 7 | query I 8 | SELECT * FROM cache_httpfs_get_cache_filesystems() ORDER BY wrapped_filesystems; 9 | ---- 10 | HTTPFileSystem 11 | HuggingFaceFileSystem 12 | S3FileSystem 13 | 14 | -------------------------------------------------------------------------------- /unit/test_base_cache_filesystem.cpp: -------------------------------------------------------------------------------- 1 | #define CATCH_CONFIG_RUNNER 2 | #include "catch.hpp" 3 | 4 | #include 5 | 6 | #include "disk_cache_reader.hpp" 7 | #include "duckdb/common/virtual_file_system.hpp" 8 | #include "hffs.hpp" 9 | 10 | using namespace duckdb; // NOLINT 11 | 12 | namespace { 13 | 14 | const std::string TEST_CONTENT = "helloworld"; 15 | const std::string TEST_FILEPATH = "/tmp/testfile"; 16 | void CreateTestFile() { 17 | auto local_filesystem = LocalFileSystem::CreateLocal(); 18 | auto file_handle = local_filesystem->OpenFile(TEST_FILEPATH, FileOpenFlags::FILE_FLAGS_WRITE | 19 | FileOpenFlags::FILE_FLAGS_FILE_CREATE_NEW); 20 | local_filesystem->Write(*file_handle, const_cast(TEST_CONTENT.data()), TEST_CONTENT.length(), 21 | /*location=*/0); 22 | file_handle->Sync(); 23 | } 24 | void DeleteTestFile() { 25 | LocalFileSystem::CreateLocal()->RemoveFile(TEST_FILEPATH); 26 | } 27 | 28 | } // namespace 29 | 30 | // A more ideal unit test would be, we could check hugging face filesystem will 31 | // be used for certains files. 32 | TEST_CASE("Test cached filesystem CanHandle", "[base cache filesystem]") { 33 | unique_ptr vfs = make_uniq(); 34 | vfs->RegisterSubSystem(make_uniq(make_uniq())); 35 | vfs->RegisterSubSystem(make_uniq(make_uniq())); 36 | 37 | // VFS can handle local files with cached local filesystem. 38 | auto file_handle = vfs->OpenFile(TEST_FILEPATH, FileOpenFlags::FILE_FLAGS_READ); 39 | // Check casting success to make sure disk cache filesystem is selected, 40 | // rather than the fallback local filesystem within virtual filesystem. 41 | [[maybe_unused]] auto &cached_file_handle = file_handle->Cast(); 42 | } 43 | 44 | int main(int argc, char **argv) { 45 | CreateTestFile(); 46 | int result = Catch::Session().run(argc, argv); 47 | DeleteTestFile(); 48 | return result; 49 | } 50 | -------------------------------------------------------------------------------- /unit/test_cache_filesystem.cpp: -------------------------------------------------------------------------------- 1 | #define CATCH_CONFIG_RUNNER 2 | #include "catch.hpp" 3 | 4 | #include "cache_filesystem_config.hpp" 5 | #include "disk_cache_reader.hpp" 6 | #include "duckdb/common/local_file_system.hpp" 7 | #include "duckdb/common/string_util.hpp" 8 | #include "duckdb/common/thread.hpp" 9 | #include "duckdb/common/types/uuid.hpp" 10 | #include "filesystem_utils.hpp" 11 | #include "scope_guard.hpp" 12 | 13 | #include 14 | 15 | using namespace duckdb; // NOLINT 16 | 17 | namespace { 18 | constexpr uint64_t TEST_FILE_SIZE = 26; 19 | const auto TEST_FILE_CONTENT = []() { 20 | string content(TEST_FILE_SIZE, '\0'); 21 | for (uint64_t idx = 0; idx < TEST_FILE_SIZE; ++idx) { 22 | content[idx] = 'a' + idx; 23 | } 24 | return content; 25 | }(); 26 | const auto TEST_DIRECTORY = "/tmp/duckdb_test_cache"; 27 | const auto TEST_FILENAME = StringUtil::Format("/tmp/duckdb_test_cache/%s", UUID::ToString(UUID::GenerateRandomUUID())); 28 | 29 | void PerformIoOperation(CacheFileSystem *cache_filesystem) { 30 | // Perform glob operation. 31 | auto open_file_info = cache_filesystem->Glob(TEST_FILENAME); 32 | REQUIRE(open_file_info.size() == 1); 33 | REQUIRE(open_file_info[0].path == TEST_FILENAME); 34 | // Perform open file operation. 35 | auto file_handle = cache_filesystem->OpenFile(TEST_FILENAME, FileOpenFlags::FILE_FLAGS_READ); 36 | // Perform get file size operation. 37 | REQUIRE(cache_filesystem->GetFileSize(*file_handle) == TEST_FILE_SIZE); 38 | } 39 | 40 | } // namespace 41 | 42 | TEST_CASE("Test glob operation", "[cache filesystem test]") { 43 | SCOPE_EXIT { 44 | ResetGlobalConfig(); 45 | }; 46 | g_enable_glob_cache = true; 47 | 48 | auto cache_filesystem = make_uniq(LocalFileSystem::CreateLocal()); 49 | 50 | // Glob by filename. 51 | { 52 | auto open_file_info = cache_filesystem->Glob(TEST_FILENAME); 53 | REQUIRE(open_file_info.size() == 1); 54 | REQUIRE(open_file_info[0].path == TEST_FILENAME); 55 | } 56 | // Glob by pattern. 57 | { 58 | auto open_file_info = cache_filesystem->Glob("/tmp/duckdb_test_cache/*"); 59 | REQUIRE(open_file_info.size() == 1); 60 | REQUIRE(open_file_info[0].path == TEST_FILENAME); 61 | } 62 | } 63 | 64 | TEST_CASE("Test clear cache", "[cache filesystem test]") { 65 | SCOPE_EXIT { 66 | ResetGlobalConfig(); 67 | }; 68 | g_enable_glob_cache = true; 69 | g_enable_file_handle_cache = true; 70 | g_enable_metadata_cache = true; 71 | 72 | auto cache_filesystem = make_uniq(LocalFileSystem::CreateLocal()); 73 | 74 | // Perform a series of IO operations without cache. 75 | PerformIoOperation(cache_filesystem.get()); 76 | 77 | // Clear all cache and perform the same operation again. 78 | cache_filesystem->ClearCache(); 79 | PerformIoOperation(cache_filesystem.get()); 80 | 81 | // Clear cache via filepath and retry the same operations. 82 | cache_filesystem->ClearCache(TEST_FILENAME); 83 | PerformIoOperation(cache_filesystem.get()); 84 | 85 | // Retry the same IO operations again. 86 | PerformIoOperation(cache_filesystem.get()); 87 | } 88 | 89 | int main(int argc, char **argv) { 90 | // Set global cache type for testing. 91 | *g_test_cache_type = *ON_DISK_CACHE_TYPE; 92 | 93 | auto local_filesystem = LocalFileSystem::CreateLocal(); 94 | local_filesystem->CreateDirectory(TEST_DIRECTORY); 95 | auto file_handle = local_filesystem->OpenFile(TEST_FILENAME, FileOpenFlags::FILE_FLAGS_WRITE | 96 | FileOpenFlags::FILE_FLAGS_FILE_CREATE_NEW); 97 | local_filesystem->Write(*file_handle, const_cast(static_cast(TEST_FILE_CONTENT.data())), 98 | TEST_FILE_SIZE, /*location=*/0); 99 | file_handle->Sync(); 100 | file_handle->Close(); 101 | 102 | int result = Catch::Session().run(argc, argv); 103 | local_filesystem->RemoveDirectory(TEST_DIRECTORY); 104 | return result; 105 | } 106 | -------------------------------------------------------------------------------- /unit/test_cache_filesystem_with_mock.cpp: -------------------------------------------------------------------------------- 1 | #define CATCH_CONFIG_RUNNER 2 | #include "catch.hpp" 3 | 4 | #include 5 | 6 | #include "cache_filesystem.hpp" 7 | #include "cache_filesystem_config.hpp" 8 | #include "duckdb/common/local_file_system.hpp" 9 | #include "mock_filesystem.hpp" 10 | 11 | using namespace duckdb; // NOLINT 12 | 13 | namespace { 14 | const std::string TEST_FILENAME = "filename"; 15 | const std::string TEST_GLOB_NAME = "*"; // Need to contain glob characters. 16 | constexpr int64_t TEST_FILESIZE = 26; 17 | constexpr int64_t TEST_CHUNK_SIZE = 5; 18 | 19 | void TestReadWithMockFileSystem() { 20 | uint64_t close_invocation = 0; 21 | uint64_t dtor_invocation = 0; 22 | auto close_callback = [&close_invocation]() { 23 | ++close_invocation; 24 | }; 25 | auto dtor_callback = [&dtor_invocation]() { 26 | ++dtor_invocation; 27 | }; 28 | 29 | auto mock_filesystem = make_uniq(std::move(close_callback), std::move(dtor_callback)); 30 | mock_filesystem->SetFileSize(TEST_FILESIZE); 31 | auto *mock_filesystem_ptr = mock_filesystem.get(); 32 | auto cache_filesystem = make_uniq(std::move(mock_filesystem)); 33 | 34 | // Uncached read. 35 | { 36 | // Make sure it's mock file handle. 37 | auto handle = cache_filesystem->OpenFile(TEST_FILENAME, FileOpenFlags::FILE_FLAGS_READ); 38 | auto &cache_file_handle = handle->Cast(); 39 | auto &mock_file_handle = cache_file_handle.internal_file_handle->Cast(); 40 | 41 | std::string buffer(TEST_FILESIZE, '\0'); 42 | cache_filesystem->Read(*handle, const_cast(buffer.data()), TEST_FILESIZE, /*location=*/0); 43 | REQUIRE(buffer == std::string(TEST_FILESIZE, 'a')); 44 | 45 | auto read_operations = mock_filesystem_ptr->GetSortedReadOperations(); 46 | REQUIRE(read_operations.size() == 6); 47 | for (idx_t idx = 0; idx < 6; ++idx) { 48 | REQUIRE(read_operations[idx].start_offset == idx * TEST_CHUNK_SIZE); 49 | REQUIRE(read_operations[idx].bytes_to_read == 50 | MinValue(TEST_CHUNK_SIZE, TEST_FILESIZE - idx * TEST_CHUNK_SIZE)); 51 | } 52 | 53 | // Glob operation. 54 | REQUIRE(cache_filesystem->Glob(TEST_GLOB_NAME).empty()); 55 | } 56 | 57 | // Cache read. 58 | { 59 | // Make sure it's mock file handle. 60 | auto handle = cache_filesystem->OpenFile(TEST_FILENAME, FileOpenFlags::FILE_FLAGS_READ); 61 | auto &cache_file_handle = handle->Cast(); 62 | [[maybe_unused]] auto &mock_file_handle = cache_file_handle.internal_file_handle->Cast(); 63 | 64 | // Create a new handle, which cannot leverage the cached one. 65 | auto another_handle = cache_filesystem->OpenFile(TEST_FILENAME, FileOpenFlags::FILE_FLAGS_READ); 66 | [[maybe_unused]] auto &another_mock_handle = 67 | another_handle->Cast().internal_file_handle->Cast(); 68 | 69 | std::string buffer(TEST_FILESIZE, '\0'); 70 | cache_filesystem->Read(*handle, const_cast(buffer.data()), TEST_FILESIZE, /*location=*/0); 71 | REQUIRE(buffer == std::string(TEST_FILESIZE, 'a')); 72 | 73 | mock_filesystem_ptr->ClearReadOperations(); 74 | auto read_operations = mock_filesystem_ptr->GetSortedReadOperations(); 75 | REQUIRE(read_operations.empty()); 76 | 77 | // Glob operation. 78 | REQUIRE(cache_filesystem->Glob(TEST_GLOB_NAME).empty()); 79 | 80 | // [handle] and [another_handle] go out of scope and place back to file handle, but due to insufficient capacity 81 | // only one of them will be cached and another one closed and destructed. 82 | } 83 | 84 | // One of the file handles resides in cache, another one gets closed and destructed. 85 | REQUIRE(close_invocation == 1); 86 | REQUIRE(dtor_invocation == 1); 87 | REQUIRE(mock_filesystem_ptr->GetGlobInvocation() == 1); 88 | 89 | // Destructing the cache filesystem cleans file handle cache, which in turns close and destruct all cached file 90 | // handles. 91 | REQUIRE(mock_filesystem_ptr->GetFileOpenInvocation() == 2); 92 | 93 | cache_filesystem = nullptr; 94 | REQUIRE(close_invocation == 2); 95 | REQUIRE(dtor_invocation == 2); 96 | } 97 | 98 | } // namespace 99 | 100 | TEST_CASE("Test disk cache reader with mock filesystem", "[mock filesystem test]") { 101 | *g_test_cache_type = *ON_DISK_CACHE_TYPE; 102 | g_cache_block_size = TEST_CHUNK_SIZE; 103 | g_max_file_handle_cache_entry = 1; 104 | LocalFileSystem::CreateLocal()->RemoveDirectory(*g_on_disk_cache_directory); 105 | TestReadWithMockFileSystem(); 106 | } 107 | 108 | TEST_CASE("Test in-memory cache reader with mock filesystem", "[mock filesystem test]") { 109 | *g_test_cache_type = *IN_MEM_CACHE_TYPE; 110 | g_cache_block_size = TEST_CHUNK_SIZE; 111 | g_max_file_handle_cache_entry = 1; 112 | LocalFileSystem::CreateLocal()->RemoveDirectory(*g_on_disk_cache_directory); 113 | TestReadWithMockFileSystem(); 114 | } 115 | 116 | TEST_CASE("Test clear cache", "[mock filesystem test]") { 117 | g_max_file_handle_cache_entry = 1; 118 | 119 | uint64_t close_invocation = 0; 120 | uint64_t dtor_invocation = 0; 121 | auto close_callback = [&close_invocation]() { 122 | ++close_invocation; 123 | }; 124 | auto dtor_callback = [&dtor_invocation]() { 125 | ++dtor_invocation; 126 | }; 127 | 128 | auto mock_filesystem = make_uniq(std::move(close_callback), std::move(dtor_callback)); 129 | mock_filesystem->SetFileSize(TEST_FILESIZE); 130 | auto *mock_filesystem_ptr = mock_filesystem.get(); 131 | auto cache_filesystem = make_uniq(std::move(mock_filesystem)); 132 | 133 | auto perform_io_operation = [&]() { 134 | auto handle = cache_filesystem->OpenFile(TEST_FILENAME, FileOpenFlags::FILE_FLAGS_READ); 135 | REQUIRE(cache_filesystem->Glob(TEST_GLOB_NAME).empty()); 136 | }; 137 | 138 | // Uncached IO operations. 139 | perform_io_operation(); 140 | REQUIRE(mock_filesystem_ptr->GetGlobInvocation() == 1); 141 | REQUIRE(mock_filesystem_ptr->GetFileOpenInvocation() == 1); 142 | 143 | // Clear cache and perform IO operations. 144 | cache_filesystem->ClearCache(); 145 | perform_io_operation(); 146 | REQUIRE(mock_filesystem_ptr->GetGlobInvocation() == 2); 147 | REQUIRE(mock_filesystem_ptr->GetFileOpenInvocation() == 2); 148 | 149 | // Clear cache by filepath and perform IO operation. 150 | cache_filesystem->ClearCache(TEST_FILENAME); 151 | cache_filesystem->ClearCache(TEST_GLOB_NAME); 152 | perform_io_operation(); 153 | REQUIRE(mock_filesystem_ptr->GetGlobInvocation() == 3); 154 | REQUIRE(mock_filesystem_ptr->GetFileOpenInvocation() == 3); 155 | 156 | // Retry one cached IO operation. 157 | perform_io_operation(); 158 | REQUIRE(mock_filesystem_ptr->GetGlobInvocation() == 3); 159 | REQUIRE(mock_filesystem_ptr->GetFileOpenInvocation() == 3); 160 | } 161 | 162 | int main(int argc, char **argv) { 163 | int result = Catch::Session().run(argc, argv); 164 | return result; 165 | } 166 | -------------------------------------------------------------------------------- /unit/test_copiable_value_lru_cache.cpp: -------------------------------------------------------------------------------- 1 | #define CATCH_CONFIG_RUNNER 2 | #include "catch.hpp" 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "copiable_value_lru_cache.hpp" 12 | 13 | using namespace duckdb; // NOLINT 14 | 15 | namespace { 16 | struct MapKey { 17 | std::string fname; 18 | uint64_t off; 19 | }; 20 | struct MapKeyEqual { 21 | bool operator()(const MapKey &lhs, const MapKey &rhs) const { 22 | return std::tie(lhs.fname, lhs.off) == std::tie(rhs.fname, rhs.off); 23 | } 24 | }; 25 | struct MapKeyHash { 26 | std::size_t operator()(const MapKey &key) const { 27 | return std::hash {}(key.fname) ^ std::hash {}(key.off); 28 | } 29 | }; 30 | } // namespace 31 | 32 | TEST_CASE("PutAndGetSameKey", "[shared lru test]") { 33 | ThreadSafeCopiableValLruCache cache {/*max_entries_p=*/1, /*timeout_millisec_p=*/0}; 34 | 35 | // No value initially. 36 | auto val = cache.Get("1"); 37 | REQUIRE(val.empty()); 38 | 39 | // Check put and get. 40 | cache.Put("1", std::string("1")); 41 | val = cache.Get("1"); 42 | REQUIRE(!val.empty()); 43 | REQUIRE(val == "1"); 44 | 45 | // Check key eviction. 46 | cache.Put("2", std::string("2")); 47 | val = cache.Get("1"); 48 | REQUIRE(val.empty()); 49 | val = cache.Get("2"); 50 | REQUIRE(!val.empty()); 51 | REQUIRE(val == "2"); 52 | 53 | // Check deletion. 54 | REQUIRE(!cache.Delete("1")); 55 | REQUIRE(cache.Delete("2")); 56 | val = cache.Get("2"); 57 | REQUIRE(val.empty()); 58 | } 59 | 60 | TEST_CASE("CustomizedStruct", "[shared lru test]") { 61 | ThreadSafeCopiableValLruCache cache {/*max_entries_p=*/1, 62 | /*timeout_millisec_p=*/0}; 63 | MapKey key; 64 | key.fname = "hello"; 65 | key.off = 10; 66 | cache.Put(key, std::string("world")); 67 | 68 | MapKey lookup_key; 69 | lookup_key.fname = key.fname; 70 | lookup_key.off = key.off; 71 | auto val = cache.Get(lookup_key); 72 | REQUIRE(!val.empty()); 73 | REQUIRE(val == "world"); 74 | } 75 | 76 | TEST_CASE("Clear with filter test", "[shared lru test]") { 77 | ThreadSafeCopiableValLruCache cache {/*max_entries_p=*/3, /*timeout_millisec_p=*/0}; 78 | cache.Put("key1", std::string("val1")); 79 | cache.Put("key2", std::string("val2")); 80 | cache.Put("key3", std::string("val3")); 81 | cache.Clear([](const std::string &key) { return key >= "key2"; }); 82 | 83 | // Still valid keys. 84 | auto val = cache.Get("key1"); 85 | REQUIRE(!val.empty()); 86 | REQUIRE(val == "val1"); 87 | 88 | // Non-existent keys. 89 | val = cache.Get("key2"); 90 | REQUIRE(val.empty()); 91 | val = cache.Get("key3"); 92 | REQUIRE(val.empty()); 93 | } 94 | 95 | TEST_CASE("GetOrCreate test", "[shared lru test]") { 96 | using CacheType = ThreadSafeCopiableValLruCache; 97 | 98 | std::atomic invoked = {false}; // Used to check only invoke once. 99 | auto factory = [&invoked](const std::string &key) -> std::string { 100 | REQUIRE(!invoked.exchange(true)); 101 | // Sleep for a while so multiple threads could kick in and get blocked. 102 | std::this_thread::sleep_for(std::chrono::seconds(3)); 103 | return key; 104 | }; 105 | 106 | CacheType cache {/*max_entries_p=*/1, /*timeout_millisec_p=*/0}; 107 | 108 | constexpr size_t kFutureNum = 100; 109 | std::vector> futures; 110 | futures.reserve(kFutureNum); 111 | 112 | const std::string key = "key"; 113 | for (size_t idx = 0; idx < kFutureNum; ++idx) { 114 | futures.emplace_back( 115 | std::async(std::launch::async, [&cache, &key, &factory]() { return cache.GetOrCreate(key, factory); })); 116 | } 117 | for (auto &fut : futures) { 118 | auto val = fut.get(); 119 | REQUIRE(val == key); 120 | } 121 | 122 | // After we're sure key-value pair exists in cache, make one more call. 123 | auto cached_val = cache.GetOrCreate(key, factory); 124 | REQUIRE(cached_val == key); 125 | } 126 | 127 | TEST_CASE("Put and get with timeout test", "[shared lru test]") { 128 | using CacheType = ThreadSafeCopiableValLruCache; 129 | 130 | CacheType cache {/*max_entries_p=*/1, /*timeout_millisec_p=*/500}; 131 | cache.Put("key", "val"); 132 | 133 | // Getting key-value pair right afterwards is able to get the value. 134 | auto val = cache.Get("key"); 135 | REQUIRE(!val.empty()); 136 | REQUIRE(val == "val"); 137 | 138 | // Sleep for a while which exceeds timeout, re-fetch key-value pair fails to get value. 139 | std::this_thread::sleep_for(std::chrono::milliseconds(1000)); 140 | val = cache.Get("key"); 141 | REQUIRE(val.empty()); 142 | } 143 | 144 | int main(int argc, char **argv) { 145 | int result = Catch::Session().run(argc, argv); 146 | return result; 147 | } 148 | -------------------------------------------------------------------------------- /unit/test_exclusive_lru_cache.cpp: -------------------------------------------------------------------------------- 1 | #define CATCH_CONFIG_RUNNER 2 | #include "catch.hpp" 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "exclusive_lru_cache.hpp" 12 | 13 | using namespace duckdb; // NOLINT 14 | 15 | namespace { 16 | struct MapKey { 17 | std::string fname; 18 | uint64_t off; 19 | }; 20 | struct MapKeyEqual { 21 | bool operator()(const MapKey &lhs, const MapKey &rhs) const { 22 | return std::tie(lhs.fname, lhs.off) == std::tie(rhs.fname, rhs.off); 23 | } 24 | }; 25 | struct MapKeyHash { 26 | std::size_t operator()(const MapKey &key) const { 27 | return std::hash {}(key.fname) ^ std::hash {}(key.off); 28 | } 29 | }; 30 | } // namespace 31 | 32 | TEST_CASE("PutAndGetSameKey", "[exclusive lru test]") { 33 | ThreadSafeExclusiveLruCache cache {/*max_entries_p=*/1, /*timeout_millisec_p=*/0}; 34 | 35 | // No value initially. 36 | auto val = cache.GetAndPop("1"); 37 | REQUIRE(val == nullptr); 38 | 39 | // Check put and get. 40 | cache.Put("1", make_uniq("1")); 41 | val = cache.GetAndPop("1"); 42 | REQUIRE(val != nullptr); 43 | REQUIRE(*val == "1"); 44 | 45 | // Check key eviction. 46 | cache.Put("2", make_uniq("2")); 47 | val = cache.GetAndPop("1"); 48 | REQUIRE(val == nullptr); 49 | val = cache.GetAndPop("2"); 50 | REQUIRE(val != nullptr); 51 | REQUIRE(*val == "2"); 52 | 53 | // Check deletion. 54 | REQUIRE(!cache.Delete("1")); 55 | 56 | cache.Put("2", make_uniq("2")); 57 | REQUIRE(cache.Delete("2")); 58 | } 59 | 60 | TEST_CASE("CustomizedStruct", "[exclusive lru test]") { 61 | ThreadSafeExclusiveLruCache cache {/*max_entries_p=*/1, 62 | /*timeout_millisec_p=*/0}; 63 | MapKey key; 64 | key.fname = "hello"; 65 | key.off = 10; 66 | auto evicted = cache.Put(key, make_uniq("world")); 67 | REQUIRE(evicted == nullptr); 68 | 69 | MapKey lookup_key; 70 | lookup_key.fname = key.fname; 71 | lookup_key.off = key.off; 72 | auto val = cache.GetAndPop(lookup_key); 73 | REQUIRE(val != nullptr); 74 | REQUIRE(*val == "world"); 75 | } 76 | 77 | TEST_CASE("Clear with filter test", "[exclusive lru test]") { 78 | ThreadSafeExclusiveLruCache cache {/*max_entries_p=*/3, /*timeout_millisec_p=*/0}; 79 | auto evicted = cache.Put("key1", make_uniq("val1")); 80 | REQUIRE(evicted == nullptr); 81 | evicted = cache.Put("key2", make_uniq("val2")); 82 | REQUIRE(evicted == nullptr); 83 | evicted = cache.Put("key3", make_uniq("val3")); 84 | REQUIRE(evicted == nullptr); 85 | cache.Clear([](const std::string &key) { return key >= "key2"; }); 86 | 87 | // Still valid keys. 88 | auto val = cache.GetAndPop("key1"); 89 | REQUIRE(val != nullptr); 90 | REQUIRE(*val == "val1"); 91 | 92 | // Non-existent keys. 93 | val = cache.GetAndPop("key2"); 94 | REQUIRE(val == nullptr); 95 | val = cache.GetAndPop("key3"); 96 | REQUIRE(val == nullptr); 97 | } 98 | 99 | TEST_CASE("Put and get with timeout test", "[exclusive lru test]") { 100 | using CacheType = ThreadSafeExclusiveLruCache; 101 | 102 | CacheType cache {/*max_entries_p=*/1, /*timeout_millisec_p=*/500}; 103 | auto evicted = cache.Put("key", make_uniq("val")); 104 | REQUIRE(evicted == nullptr); 105 | 106 | // Getting key-value pair right afterwards is able to get the value. 107 | auto val = cache.GetAndPop("key"); 108 | REQUIRE(val != nullptr); 109 | REQUIRE(*val == "val"); 110 | 111 | // Sleep for a while which exceeds timeout, re-fetch key-value pair fails to get value. 112 | std::this_thread::sleep_for(std::chrono::milliseconds(1000)); 113 | val = cache.GetAndPop("key"); 114 | REQUIRE(val == nullptr); 115 | } 116 | 117 | TEST_CASE("Evicted value test", "[exclusive lru test]") { 118 | using CacheType = ThreadSafeExclusiveLruCache; 119 | 120 | CacheType cache {/*max_entries_p=*/1, /*timeout_millisec_p=*/0}; 121 | auto evicted = cache.Put("key1", make_uniq("val1")); 122 | REQUIRE(evicted == nullptr); 123 | 124 | evicted = cache.Put("key2", make_uniq("val2")); 125 | REQUIRE(*evicted == "val1"); 126 | 127 | evicted = cache.Put("key3", make_uniq("val3")); 128 | REQUIRE(*evicted == "val2"); 129 | 130 | auto values = cache.ClearAndGetValues(); 131 | REQUIRE(values.size() == 1); 132 | REQUIRE(*values[0] == "val3"); 133 | } 134 | 135 | int main(int argc, char **argv) { 136 | int result = Catch::Session().run(argc, argv); 137 | return result; 138 | } 139 | -------------------------------------------------------------------------------- /unit/test_filesystem_config.cpp: -------------------------------------------------------------------------------- 1 | #define CATCH_CONFIG_RUNNER 2 | #include "catch.hpp" 3 | 4 | #include "base_profile_collector.hpp" 5 | #include "cache_filesystem.hpp" 6 | #include "cache_filesystem_config.hpp" 7 | #include "cache_reader_manager.hpp" 8 | #include "disk_cache_reader.hpp" 9 | #include "duckdb/common/local_file_system.hpp" 10 | #include "duckdb/common/types/uuid.hpp" 11 | #include "duckdb/storage/standard_buffer_manager.hpp" 12 | #include "duckdb/main/client_context_file_opener.hpp" 13 | #include "duckdb/main/client_context.hpp" 14 | #include "duckdb/main/database.hpp" 15 | #include "in_memory_cache_reader.hpp" 16 | #include "noop_cache_reader.hpp" 17 | #include "temp_profile_collector.hpp" 18 | 19 | using namespace duckdb; // NOLINT 20 | 21 | namespace { 22 | const auto TEST_FILENAME = StringUtil::Format("/tmp/%s", UUID::ToString(UUID::GenerateRandomUUID())); 23 | } // namespace 24 | 25 | TEST_CASE("Filesystem config test", "[filesystem config]") { 26 | REQUIRE(GetThreadCountForSubrequests(10) == 10); 27 | 28 | g_max_subrequest_count = 5; 29 | REQUIRE(GetThreadCountForSubrequests(10) == 5); 30 | } 31 | 32 | TEST_CASE("Filesystem cache config test", "[filesystem config]") { 33 | DuckDB db {}; 34 | StandardBufferManager buffer_manager {*db.instance, "/tmp/cache_httpfs_fs_benchmark"}; 35 | auto cache_fs = make_uniq(LocalFileSystem::CreateLocal()); 36 | auto client_context = make_shared_ptr(db.instance); 37 | auto &cache_reader_manager = CacheReaderManager::Get(); 38 | 39 | // Check noop cache reader. 40 | { 41 | client_context->config.set_variables["cache_httpfs_type"] = Value(*NOOP_CACHE_TYPE); 42 | ClientContextFileOpener file_opener {*client_context}; 43 | cache_fs->OpenFile(TEST_FILENAME, FileOpenFlags::FILE_FLAGS_READ, &file_opener); 44 | auto *cache_reader = cache_reader_manager.GetCacheReader(); 45 | [[maybe_unused]] auto &noop_handle = cache_reader->Cast(); 46 | } 47 | 48 | // Check in-memory cache reader. 49 | { 50 | client_context->config.set_variables["cache_httpfs_type"] = Value(*IN_MEM_CACHE_TYPE); 51 | ClientContextFileOpener file_opener {*client_context}; 52 | cache_fs->OpenFile(TEST_FILENAME, FileOpenFlags::FILE_FLAGS_READ, &file_opener); 53 | auto *cache_reader = cache_reader_manager.GetCacheReader(); 54 | [[maybe_unused]] auto &noop_handle = cache_reader->Cast(); 55 | } 56 | 57 | // Check on-disk cache reader. 58 | { 59 | client_context->config.set_variables["cache_httpfs_type"] = Value(*ON_DISK_CACHE_TYPE); 60 | ClientContextFileOpener file_opener {*client_context}; 61 | cache_fs->OpenFile(TEST_FILENAME, FileOpenFlags::FILE_FLAGS_READ, &file_opener); 62 | auto *cache_reader = cache_reader_manager.GetCacheReader(); 63 | [[maybe_unused]] auto &noop_handle = cache_reader->Cast(); 64 | } 65 | } 66 | 67 | TEST_CASE("Filesystem profile config test", "[filesystem config]") { 68 | DuckDB db {}; 69 | StandardBufferManager buffer_manager {*db.instance, "/tmp/cache_httpfs_fs_benchmark"}; 70 | auto cache_fs = make_uniq(LocalFileSystem::CreateLocal()); 71 | auto client_context = make_shared_ptr(db.instance); 72 | 73 | // Check noop profiler. 74 | { 75 | client_context->config.set_variables["cache_httpfs_profile_type"] = Value(*NOOP_PROFILE_TYPE); 76 | ClientContextFileOpener file_opener {*client_context}; 77 | cache_fs->OpenFile(TEST_FILENAME, FileOpenFlags::FILE_FLAGS_READ, &file_opener); 78 | auto *profiler = cache_fs->GetProfileCollector(); 79 | [[maybe_unused]] auto &noop_profiler = profiler->Cast(); 80 | } 81 | 82 | // Check temp cache reader. 83 | { 84 | client_context->config.set_variables["cache_httpfs_profile_type"] = Value(*TEMP_PROFILE_TYPE); 85 | ClientContextFileOpener file_opener {*client_context}; 86 | cache_fs->OpenFile(TEST_FILENAME, FileOpenFlags::FILE_FLAGS_READ, &file_opener); 87 | auto *profiler = cache_fs->GetProfileCollector(); 88 | [[maybe_unused]] auto &temp_profiler = profiler->Cast(); 89 | } 90 | } 91 | 92 | int main(int argc, char **argv) { 93 | auto local_filesystem = LocalFileSystem::CreateLocal(); 94 | auto file_handle = local_filesystem->OpenFile(TEST_FILENAME, FileOpenFlags::FILE_FLAGS_WRITE | 95 | FileOpenFlags::FILE_FLAGS_FILE_CREATE_NEW); 96 | int result = Catch::Session().run(argc, argv); 97 | local_filesystem->RemoveFile(TEST_FILENAME); 98 | return result; 99 | } 100 | -------------------------------------------------------------------------------- /unit/test_filesystem_utils.cpp: -------------------------------------------------------------------------------- 1 | // This file tests the stale file deletion. 2 | 3 | #define CATCH_CONFIG_RUNNER 4 | #include "catch.hpp" 5 | 6 | #include "duckdb/common/local_file_system.hpp" 7 | #include "duckdb/common/string_util.hpp" 8 | #include "cache_filesystem_config.hpp" 9 | #include "filesystem_utils.hpp" 10 | 11 | #include 12 | 13 | using namespace duckdb; // NOLINT 14 | 15 | namespace { 16 | const auto TEST_ON_DISK_CACHE_DIRECTORY = "/tmp/duckdb_test_cache_httpfs_cache"; 17 | } // namespace 18 | 19 | TEST_CASE("Stale file deletion", "[utils test]") { 20 | auto local_filesystem = LocalFileSystem::CreateLocal(); 21 | const string fname1 = StringUtil::Format("%s/file1", TEST_ON_DISK_CACHE_DIRECTORY); 22 | const string fname2 = StringUtil::Format("%s/file2", TEST_ON_DISK_CACHE_DIRECTORY); 23 | const std::string CONTENT = "helloworld"; 24 | 25 | { 26 | auto file_handle = local_filesystem->OpenFile(fname1, FileOpenFlags::FILE_FLAGS_WRITE | 27 | FileOpenFlags::FILE_FLAGS_FILE_CREATE_NEW); 28 | } 29 | { 30 | auto file_handle = local_filesystem->OpenFile(fname2, FileOpenFlags::FILE_FLAGS_WRITE | 31 | FileOpenFlags::FILE_FLAGS_FILE_CREATE_NEW); 32 | } 33 | 34 | const time_t now = std::time(nullptr); 35 | const time_t two_day_ago = now - 48 * 60 * 60; 36 | struct utimbuf updated_time; 37 | updated_time.actime = two_day_ago; 38 | updated_time.modtime = two_day_ago; 39 | REQUIRE(utime(fname2.data(), &updated_time) == 0); 40 | 41 | EvictStaleCacheFiles(*local_filesystem, TEST_ON_DISK_CACHE_DIRECTORY); 42 | vector fresh_files; 43 | REQUIRE( 44 | local_filesystem->ListFiles(TEST_ON_DISK_CACHE_DIRECTORY, [&fresh_files](const string &fname, bool /*unused*/) { 45 | fresh_files.emplace_back(StringUtil::Format("%s/%s", TEST_ON_DISK_CACHE_DIRECTORY, fname)); 46 | })); 47 | REQUIRE(fresh_files == vector {fname1}); 48 | } 49 | 50 | int main(int argc, char **argv) { 51 | auto local_filesystem = LocalFileSystem::CreateLocal(); 52 | local_filesystem->RemoveDirectory(TEST_ON_DISK_CACHE_DIRECTORY); 53 | local_filesystem->CreateDirectory(TEST_ON_DISK_CACHE_DIRECTORY); 54 | int result = Catch::Session().run(argc, argv); 55 | local_filesystem->RemoveDirectory(TEST_ON_DISK_CACHE_DIRECTORY); 56 | return result; 57 | } 58 | -------------------------------------------------------------------------------- /unit/test_histogram.cpp: -------------------------------------------------------------------------------- 1 | #define CATCH_CONFIG_RUNNER 2 | #include "catch.hpp" 3 | 4 | #include "histogram.hpp" 5 | 6 | using namespace duckdb; // NOLINT 7 | 8 | TEST_CASE("Histogram test", "[histogram test]") { 9 | Histogram hist {/*min_val=*/0, /*max_val=*/10, /*num_bkt=*/10}; 10 | hist.Add(1); 11 | hist.Add(3); 12 | hist.Add(-3); 13 | REQUIRE(hist.outliers() == std::vector {-3}); 14 | REQUIRE(hist.min() == 1); 15 | REQUIRE(hist.max() == 3); 16 | REQUIRE(hist.counts() == 2); 17 | REQUIRE(hist.mean() == 2); 18 | 19 | // Reset and check again. 20 | hist.Reset(); 21 | hist.Add(1); 22 | REQUIRE(hist.outliers().empty()); 23 | REQUIRE(hist.min() == 1); 24 | REQUIRE(hist.max() == 1); 25 | REQUIRE(hist.counts() == 1); 26 | REQUIRE(hist.mean() == 1); 27 | } 28 | 29 | int main(int argc, char **argv) { 30 | int result = Catch::Session().run(argc, argv); 31 | return result; 32 | } 33 | -------------------------------------------------------------------------------- /unit/test_in_memory_cache_filesystem.cpp: -------------------------------------------------------------------------------- 1 | // Unit test for in-memory cache filesystem. 2 | 3 | #define CATCH_CONFIG_RUNNER 4 | #include "catch.hpp" 5 | 6 | #include "cache_filesystem_config.hpp" 7 | #include "duckdb/common/local_file_system.hpp" 8 | #include "duckdb/common/string_util.hpp" 9 | #include "duckdb/common/thread.hpp" 10 | #include "duckdb/common/types/uuid.hpp" 11 | #include "in_memory_cache_reader.hpp" 12 | #include "scope_guard.hpp" 13 | 14 | using namespace duckdb; // NOLINT 15 | 16 | namespace { 17 | constexpr uint64_t TEST_FILE_SIZE = 26; 18 | const auto TEST_FILE_CONTENT = []() { 19 | string content(TEST_FILE_SIZE, '\0'); 20 | for (uint64_t idx = 0; idx < TEST_FILE_SIZE; ++idx) { 21 | content[idx] = 'a' + idx; 22 | } 23 | return content; 24 | }(); 25 | const auto TEST_FILENAME = StringUtil::Format("/tmp/%s", UUID::ToString(UUID::GenerateRandomUUID())); 26 | } // namespace 27 | 28 | TEST_CASE("Test on in-memory cache filesystem", "[in-memory cache filesystem test]") { 29 | g_cache_block_size = TEST_FILE_SIZE; 30 | SCOPE_EXIT { 31 | ResetGlobalConfig(); 32 | }; 33 | 34 | auto in_mem_cache_fs = make_uniq(LocalFileSystem::CreateLocal()); 35 | 36 | // First uncached read. 37 | { 38 | auto handle = in_mem_cache_fs->OpenFile(TEST_FILENAME, FileOpenFlags::FILE_FLAGS_READ); 39 | const uint64_t start_offset = 1; 40 | const uint64_t bytes_to_read = TEST_FILE_SIZE - 2; 41 | string content(bytes_to_read, '\0'); 42 | in_mem_cache_fs->Read(*handle, const_cast(static_cast(content.data())), bytes_to_read, 43 | start_offset); 44 | REQUIRE(content == TEST_FILE_CONTENT.substr(start_offset, bytes_to_read)); 45 | } 46 | 47 | // Second cached read. 48 | { 49 | auto handle = in_mem_cache_fs->OpenFile(TEST_FILENAME, FileOpenFlags::FILE_FLAGS_READ); 50 | const uint64_t start_offset = 1; 51 | const uint64_t bytes_to_read = TEST_FILE_SIZE - 2; 52 | string content(bytes_to_read, '\0'); 53 | in_mem_cache_fs->Read(*handle, const_cast(static_cast(content.data())), bytes_to_read, 54 | start_offset); 55 | REQUIRE(content == TEST_FILE_CONTENT.substr(start_offset, bytes_to_read)); 56 | } 57 | } 58 | 59 | TEST_CASE("Test on concurrent access", "[in-memory cache filesystem test]") { 60 | g_cache_block_size = 5; 61 | SCOPE_EXIT { 62 | ResetGlobalConfig(); 63 | }; 64 | 65 | auto in_mem_cache_fs = make_uniq(LocalFileSystem::CreateLocal()); 66 | 67 | auto handle = in_mem_cache_fs->OpenFile(TEST_FILENAME, 68 | FileOpenFlags::FILE_FLAGS_READ | FileOpenFlags::FILE_FLAGS_PARALLEL_ACCESS); 69 | const uint64_t start_offset = 0; 70 | const uint64_t bytes_to_read = TEST_FILE_SIZE; 71 | 72 | // Spawn multiple threads to read through in-memory cache filesystem. 73 | constexpr idx_t THREAD_NUM = 200; 74 | vector reader_threads; 75 | reader_threads.reserve(THREAD_NUM); 76 | for (idx_t idx = 0; idx < THREAD_NUM; ++idx) { 77 | reader_threads.emplace_back([&]() { 78 | string content(bytes_to_read, '\0'); 79 | in_mem_cache_fs->Read(*handle, const_cast(static_cast(content.data())), bytes_to_read, 80 | start_offset); 81 | REQUIRE(content == TEST_FILE_CONTENT.substr(start_offset, bytes_to_read)); 82 | }); 83 | } 84 | for (auto &cur_thd : reader_threads) { 85 | D_ASSERT(cur_thd.joinable()); 86 | cur_thd.join(); 87 | } 88 | } 89 | 90 | int main(int argc, char **argv) { 91 | // Set global cache type for testing. 92 | *g_test_cache_type = *IN_MEM_CACHE_TYPE; 93 | 94 | auto local_filesystem = LocalFileSystem::CreateLocal(); 95 | auto file_handle = local_filesystem->OpenFile(TEST_FILENAME, FileOpenFlags::FILE_FLAGS_WRITE | 96 | FileOpenFlags::FILE_FLAGS_FILE_CREATE_NEW); 97 | local_filesystem->Write(*file_handle, const_cast(static_cast(TEST_FILE_CONTENT.data())), 98 | TEST_FILE_SIZE, /*location=*/0); 99 | file_handle->Sync(); 100 | file_handle->Close(); 101 | 102 | int result = Catch::Session().run(argc, argv); 103 | local_filesystem->RemoveFile(TEST_FILENAME); 104 | return result; 105 | } 106 | -------------------------------------------------------------------------------- /unit/test_large_file_disk_reader.cpp: -------------------------------------------------------------------------------- 1 | // Similar to on-disk reader unit test, this unit test also checks disk cache reader; but we write large file so 2 | // threading issues and memory issues are easier to detect. 3 | 4 | #define CATCH_CONFIG_RUNNER 5 | #include "catch.hpp" 6 | 7 | #include "cache_filesystem_config.hpp" 8 | #include "disk_cache_reader.hpp" 9 | #include "duckdb/common/local_file_system.hpp" 10 | #include "duckdb/common/string_util.hpp" 11 | #include "duckdb/common/thread.hpp" 12 | #include "duckdb/common/types/uuid.hpp" 13 | #include "filesystem_utils.hpp" 14 | #include "scope_guard.hpp" 15 | 16 | #include 17 | 18 | using namespace duckdb; // NOLINT 19 | 20 | namespace { 21 | 22 | constexpr uint64_t TEST_ALPHA_ITER = 10000; 23 | constexpr uint64_t TEST_FILE_SIZE = 26 * TEST_ALPHA_ITER; // 260K 24 | const auto TEST_FILE_CONTENT = []() { 25 | string content(TEST_FILE_SIZE, '\0'); 26 | for (uint64_t ii = 0; ii < TEST_ALPHA_ITER; ++ii) { 27 | for (uint64_t jj = 0; jj < 26; ++jj) { 28 | const uint64_t idx = ii * 26 + jj; 29 | content[idx] = 'a' + jj; 30 | } 31 | } 32 | return content; 33 | }(); 34 | const auto TEST_FILENAME = StringUtil::Format("/tmp/%s", UUID::ToString(UUID::GenerateRandomUUID())); 35 | const auto TEST_ON_DISK_CACHE_DIRECTORY = "/tmp/duckdb_test_cache_httpfs_cache"; 36 | } // namespace 37 | 38 | TEST_CASE("Read all bytes in one read operation", "[on-disk cache filesystem test]") { 39 | constexpr uint64_t test_block_size = 22; // Intentionally not a divisor of file size. 40 | *g_on_disk_cache_directory = TEST_ON_DISK_CACHE_DIRECTORY; 41 | g_cache_block_size = test_block_size; 42 | SCOPE_EXIT { 43 | ResetGlobalConfig(); 44 | }; 45 | 46 | LocalFileSystem::CreateLocal()->RemoveDirectory(TEST_ON_DISK_CACHE_DIRECTORY); 47 | auto disk_cache_fs = make_uniq(LocalFileSystem::CreateLocal()); 48 | 49 | // First uncached read. 50 | { 51 | auto handle = disk_cache_fs->OpenFile(TEST_FILENAME, FileOpenFlags::FILE_FLAGS_READ); 52 | const uint64_t start_offset = 1; 53 | const uint64_t bytes_to_read = TEST_FILE_SIZE - 2; 54 | string content(bytes_to_read, '\0'); 55 | disk_cache_fs->Read(*handle, const_cast(static_cast(content.data())), bytes_to_read, 56 | start_offset); 57 | REQUIRE(content == TEST_FILE_CONTENT.substr(start_offset, bytes_to_read)); 58 | } 59 | 60 | // Second cached read. 61 | { 62 | auto handle = disk_cache_fs->OpenFile(TEST_FILENAME, FileOpenFlags::FILE_FLAGS_READ); 63 | const uint64_t start_offset = 1; 64 | const uint64_t bytes_to_read = TEST_FILE_SIZE - 2; 65 | string content(bytes_to_read, '\0'); 66 | disk_cache_fs->Read(*handle, const_cast(static_cast(content.data())), bytes_to_read, 67 | start_offset); 68 | REQUIRE(content == TEST_FILE_CONTENT.substr(start_offset, bytes_to_read)); 69 | } 70 | } 71 | 72 | int main(int argc, char **argv) { 73 | // Set global cache type for testing. 74 | *g_test_cache_type = *ON_DISK_CACHE_TYPE; 75 | 76 | auto local_filesystem = LocalFileSystem::CreateLocal(); 77 | auto file_handle = local_filesystem->OpenFile(TEST_FILENAME, FileOpenFlags::FILE_FLAGS_WRITE | 78 | FileOpenFlags::FILE_FLAGS_FILE_CREATE_NEW); 79 | local_filesystem->Write(*file_handle, const_cast(static_cast(TEST_FILE_CONTENT.data())), 80 | TEST_FILE_SIZE, /*location=*/0); 81 | file_handle->Sync(); 82 | file_handle->Close(); 83 | 84 | int result = Catch::Session().run(argc, argv); 85 | local_filesystem->RemoveFile(TEST_FILENAME); 86 | return result; 87 | } 88 | -------------------------------------------------------------------------------- /unit/test_large_file_inmem_reader.cpp: -------------------------------------------------------------------------------- 1 | // Similar to on-disk reader unit test, this unit test also checks disk cache reader; but we write large file so 2 | // threading issues and memory issues are easier to detect. 3 | 4 | #define CATCH_CONFIG_RUNNER 5 | #include "catch.hpp" 6 | 7 | #include "cache_filesystem_config.hpp" 8 | #include "disk_cache_reader.hpp" 9 | #include "duckdb/common/local_file_system.hpp" 10 | #include "duckdb/common/string_util.hpp" 11 | #include "duckdb/common/thread.hpp" 12 | #include "duckdb/common/types/uuid.hpp" 13 | #include "filesystem_utils.hpp" 14 | #include "scope_guard.hpp" 15 | 16 | #include 17 | 18 | using namespace duckdb; // NOLINT 19 | 20 | namespace { 21 | 22 | constexpr uint64_t TEST_ALPHA_ITER = 10000; 23 | constexpr uint64_t TEST_FILE_SIZE = 26 * TEST_ALPHA_ITER; // 260K 24 | const auto TEST_FILE_CONTENT = []() { 25 | string content(TEST_FILE_SIZE, '\0'); 26 | for (uint64_t ii = 0; ii < TEST_ALPHA_ITER; ++ii) { 27 | for (uint64_t jj = 0; jj < 26; ++jj) { 28 | const uint64_t idx = ii * 26 + jj; 29 | content[idx] = 'a' + jj; 30 | } 31 | } 32 | return content; 33 | }(); 34 | const auto TEST_FILENAME = StringUtil::Format("/tmp/%s", UUID::ToString(UUID::GenerateRandomUUID())); 35 | const auto TEST_ON_DISK_CACHE_DIRECTORY = "/tmp/duckdb_test_cache_httpfs_cache"; 36 | } // namespace 37 | 38 | TEST_CASE("Read all bytes in one read operation", "[on-disk cache filesystem test]") { 39 | constexpr uint64_t test_block_size = 22; // Intentionally not a divisor of file size. 40 | *g_on_disk_cache_directory = TEST_ON_DISK_CACHE_DIRECTORY; 41 | g_cache_block_size = test_block_size; 42 | SCOPE_EXIT { 43 | ResetGlobalConfig(); 44 | }; 45 | 46 | LocalFileSystem::CreateLocal()->RemoveDirectory(TEST_ON_DISK_CACHE_DIRECTORY); 47 | auto disk_cache_fs = make_uniq(LocalFileSystem::CreateLocal()); 48 | 49 | // First uncached read. 50 | { 51 | auto handle = disk_cache_fs->OpenFile(TEST_FILENAME, FileOpenFlags::FILE_FLAGS_READ); 52 | const uint64_t start_offset = 1; 53 | const uint64_t bytes_to_read = TEST_FILE_SIZE - 2; 54 | string content(bytes_to_read, '\0'); 55 | disk_cache_fs->Read(*handle, const_cast(static_cast(content.data())), bytes_to_read, 56 | start_offset); 57 | REQUIRE(content == TEST_FILE_CONTENT.substr(start_offset, bytes_to_read)); 58 | } 59 | 60 | // Second cached read. 61 | { 62 | auto handle = disk_cache_fs->OpenFile(TEST_FILENAME, FileOpenFlags::FILE_FLAGS_READ); 63 | const uint64_t start_offset = 1; 64 | const uint64_t bytes_to_read = TEST_FILE_SIZE - 2; 65 | string content(bytes_to_read, '\0'); 66 | disk_cache_fs->Read(*handle, const_cast(static_cast(content.data())), bytes_to_read, 67 | start_offset); 68 | REQUIRE(content == TEST_FILE_CONTENT.substr(start_offset, bytes_to_read)); 69 | } 70 | } 71 | 72 | int main(int argc, char **argv) { 73 | // Set global cache type for testing. 74 | *g_test_cache_type = *IN_MEM_CACHE_TYPE; 75 | 76 | auto local_filesystem = LocalFileSystem::CreateLocal(); 77 | auto file_handle = local_filesystem->OpenFile(TEST_FILENAME, FileOpenFlags::FILE_FLAGS_WRITE | 78 | FileOpenFlags::FILE_FLAGS_FILE_CREATE_NEW); 79 | local_filesystem->Write(*file_handle, const_cast(static_cast(TEST_FILE_CONTENT.data())), 80 | TEST_FILE_SIZE, /*location=*/0); 81 | file_handle->Sync(); 82 | file_handle->Close(); 83 | 84 | int result = Catch::Session().run(argc, argv); 85 | local_filesystem->RemoveFile(TEST_FILENAME); 86 | return result; 87 | } 88 | -------------------------------------------------------------------------------- /unit/test_no_destructor.cpp: -------------------------------------------------------------------------------- 1 | #define CATCH_CONFIG_RUNNER 2 | #include "catch.hpp" 3 | 4 | #include "no_destructor.hpp" 5 | 6 | #include 7 | 8 | using namespace duckdb; // NOLINT 9 | 10 | TEST_CASE("NoDestructor test", "[no destructor test]") { 11 | const std::string s = "helloworld"; 12 | 13 | // Default constructor. 14 | { 15 | NoDestructor content {}; 16 | REQUIRE(*content == ""); 17 | } 18 | 19 | // Construct by const reference. 20 | { 21 | NoDestructor content {s}; 22 | REQUIRE(*content == s); 23 | } 24 | 25 | // Construct by rvalue reference. 26 | { 27 | std::string another_str = "helloworld"; 28 | NoDestructor content {std::move(another_str)}; 29 | REQUIRE(*content == s); 30 | } 31 | 32 | // Construct by ctor with multiple arguments. 33 | { 34 | NoDestructor content {s.begin(), s.end()}; 35 | REQUIRE(*content == "helloworld"); 36 | } 37 | 38 | // Access internal object. 39 | { 40 | NoDestructor content {s.begin(), s.end()}; 41 | (*content)[0] = 'b'; 42 | (*content)[1] = 'c'; 43 | REQUIRE(*content == "bclloworld"); 44 | } 45 | 46 | // Reassign. 47 | { 48 | NoDestructor content {s.begin(), s.end()}; 49 | *content = "worldhello"; 50 | REQUIRE(*content == "worldhello"); 51 | } 52 | } 53 | 54 | int main(int argc, char **argv) { 55 | int result = Catch::Session().run(argc, argv); 56 | return result; 57 | } 58 | -------------------------------------------------------------------------------- /unit/test_noop_cache_reader.cpp: -------------------------------------------------------------------------------- 1 | // Unit test for no-op cache filesystem. 2 | 3 | #define CATCH_CONFIG_RUNNER 4 | #include "catch.hpp" 5 | 6 | #include "cache_filesystem.hpp" 7 | #include "cache_filesystem_config.hpp" 8 | #include "duckdb/common/local_file_system.hpp" 9 | #include "duckdb/common/string_util.hpp" 10 | #include "duckdb/common/thread.hpp" 11 | #include "duckdb/common/types/uuid.hpp" 12 | #include "noop_cache_reader.hpp" 13 | #include "scope_guard.hpp" 14 | 15 | using namespace duckdb; // NOLINT 16 | 17 | namespace { 18 | constexpr uint64_t TEST_FILE_SIZE = 26; 19 | const auto TEST_FILE_CONTENT = []() { 20 | string content(TEST_FILE_SIZE, '\0'); 21 | for (uint64_t idx = 0; idx < TEST_FILE_SIZE; ++idx) { 22 | content[idx] = 'a' + idx; 23 | } 24 | return content; 25 | }(); 26 | const auto TEST_FILENAME = StringUtil::Format("/tmp/%s", UUID::ToString(UUID::GenerateRandomUUID())); 27 | } // namespace 28 | 29 | TEST_CASE("Test on noop cache filesystem", "[noop cache filesystem test]") { 30 | g_cache_block_size = TEST_FILE_SIZE; 31 | SCOPE_EXIT { 32 | ResetGlobalConfig(); 33 | }; 34 | 35 | auto noop_filesystem = make_uniq(LocalFileSystem::CreateLocal()); 36 | 37 | // First uncached read. 38 | { 39 | auto handle = noop_filesystem->OpenFile(TEST_FILENAME, FileOpenFlags::FILE_FLAGS_READ); 40 | const uint64_t start_offset = 1; 41 | const uint64_t bytes_to_read = TEST_FILE_SIZE - 2; 42 | string content(bytes_to_read, '\0'); 43 | noop_filesystem->Read(*handle, const_cast(static_cast(content.data())), bytes_to_read, 44 | start_offset); 45 | REQUIRE(content == TEST_FILE_CONTENT.substr(start_offset, bytes_to_read)); 46 | } 47 | 48 | // Second uncached read. 49 | { 50 | auto handle = noop_filesystem->OpenFile(TEST_FILENAME, FileOpenFlags::FILE_FLAGS_READ); 51 | const uint64_t start_offset = 1; 52 | const uint64_t bytes_to_read = TEST_FILE_SIZE - 2; 53 | string content(bytes_to_read, '\0'); 54 | noop_filesystem->Read(*handle, const_cast(static_cast(content.data())), bytes_to_read, 55 | start_offset); 56 | REQUIRE(content == TEST_FILE_CONTENT.substr(start_offset, bytes_to_read)); 57 | } 58 | } 59 | 60 | TEST_CASE("Test noop read whole file", "[noop cache filesystem test]") { 61 | g_cache_block_size = TEST_FILE_SIZE; 62 | SCOPE_EXIT { 63 | ResetGlobalConfig(); 64 | }; 65 | 66 | auto noop_filesystem = make_uniq(LocalFileSystem::CreateLocal()); 67 | auto handle = noop_filesystem->OpenFile(TEST_FILENAME, FileOpenFlags::FILE_FLAGS_READ); 68 | const uint64_t start_offset = 0; 69 | const uint64_t bytes_to_read = TEST_FILE_SIZE; 70 | string content(bytes_to_read, '\0'); 71 | noop_filesystem->Read(*handle, const_cast(static_cast(content.data())), bytes_to_read, 72 | start_offset); 73 | REQUIRE(content == TEST_FILE_CONTENT.substr(start_offset, bytes_to_read)); 74 | } 75 | 76 | int main(int argc, char **argv) { 77 | // Set global cache type for testing. 78 | *g_test_cache_type = *NOOP_CACHE_TYPE; 79 | 80 | auto local_filesystem = LocalFileSystem::CreateLocal(); 81 | auto file_handle = local_filesystem->OpenFile(TEST_FILENAME, FileOpenFlags::FILE_FLAGS_WRITE | 82 | FileOpenFlags::FILE_FLAGS_FILE_CREATE_NEW); 83 | local_filesystem->Write(*file_handle, const_cast(static_cast(TEST_FILE_CONTENT.data())), 84 | TEST_FILE_SIZE, /*location=*/0); 85 | file_handle->Sync(); 86 | file_handle->Close(); 87 | 88 | int result = Catch::Session().run(argc, argv); 89 | local_filesystem->RemoveFile(TEST_FILENAME); 90 | return result; 91 | } 92 | -------------------------------------------------------------------------------- /unit/test_set_extension_config.cpp: -------------------------------------------------------------------------------- 1 | // Unit test for setting extension config. 2 | 3 | #define CATCH_CONFIG_RUNNER 4 | #include "catch.hpp" 5 | 6 | #include "cache_filesystem_config.hpp" 7 | #include "disk_cache_reader.hpp" 8 | #include "duckdb/common/local_file_system.hpp" 9 | #include "duckdb/common/string_util.hpp" 10 | #include "duckdb/common/types/uuid.hpp" 11 | #include "duckdb/main/database.hpp" 12 | #include "duckdb/main/connection.hpp" 13 | #include "filesystem_utils.hpp" 14 | #include "in_memory_cache_reader.hpp" 15 | 16 | using namespace duckdb; // NOLINT 17 | 18 | namespace { 19 | const std::string TEST_ON_DISK_CACHE_DIRECTORY = "/tmp/duckdb_test_cache_httpfs_cache"; 20 | const std::string TEST_SECOND_ON_DISK_CACHE_DIRECTORY = "/tmp/duckdb_test_cache_httpfs_cache_second"; 21 | const std::string TEST_ON_DISK_CACHE_FILE = "/tmp/test-config.parquet"; 22 | 23 | void CleanupTestDirectory() { 24 | auto local_filesystem = LocalFileSystem::CreateLocal(); 25 | if (local_filesystem->DirectoryExists(TEST_ON_DISK_CACHE_DIRECTORY)) { 26 | local_filesystem->RemoveDirectory(TEST_ON_DISK_CACHE_DIRECTORY); 27 | } 28 | if (local_filesystem->DirectoryExists(TEST_SECOND_ON_DISK_CACHE_DIRECTORY)) { 29 | local_filesystem->RemoveDirectory(TEST_SECOND_ON_DISK_CACHE_DIRECTORY); 30 | } 31 | if (local_filesystem->FileExists(TEST_ON_DISK_CACHE_FILE)) { 32 | local_filesystem->RemoveFile(TEST_ON_DISK_CACHE_FILE); 33 | } 34 | } 35 | } // namespace 36 | 37 | TEST_CASE("Test on incorrect config", "[extension config test]") { 38 | DuckDB db(nullptr); 39 | Connection con(db); 40 | 41 | // Set non-existent config parameter. 42 | auto result = 43 | con.Query(StringUtil::Format("SET wrong_cache_httpfs_cache_directory ='%s'", TEST_ON_DISK_CACHE_DIRECTORY)); 44 | REQUIRE(result->HasError()); 45 | 46 | // Set existent config parameter to incorrect type. 47 | result = con.Query(StringUtil::Format("SET cache_httpfs_cache_block_size='hello'")); 48 | REQUIRE(result->HasError()); 49 | } 50 | 51 | TEST_CASE("Test on correct config", "[extension config test]") { 52 | DuckDB db(nullptr); 53 | Connection con(db); 54 | 55 | // On-disk cache directory. 56 | auto result = con.Query(StringUtil::Format("SET cache_httpfs_cache_directory='helloworld'")); 57 | REQUIRE(!result->HasError()); 58 | 59 | // Cache block size. 60 | result = con.Query(StringUtil::Format("SET cache_httpfs_cache_block_size=10")); 61 | REQUIRE(!result->HasError()); 62 | 63 | // In-memory cache block count. 64 | result = con.Query(StringUtil::Format("SET cache_httpfs_max_in_mem_cache_block_count=10")); 65 | REQUIRE(!result->HasError()); 66 | } 67 | 68 | TEST_CASE("Test on changing extension config change default cache dir path setting", "[extension config test]") { 69 | DuckDB db(nullptr); 70 | auto &instance = db.instance; 71 | auto &fs = instance->GetFileSystem(); 72 | fs.RegisterSubSystem(make_uniq(LocalFileSystem::CreateLocal())); 73 | 74 | Connection con(db); 75 | con.Query(StringUtil::Format("SET cache_httpfs_cache_directory ='%s'", TEST_ON_DISK_CACHE_DIRECTORY)); 76 | con.Query("CREATE TABLE integers AS SELECT i, i+1 as j FROM range(10) r(i)"); 77 | con.Query(StringUtil::Format("COPY integers TO '%s'", TEST_ON_DISK_CACHE_FILE)); 78 | 79 | // Ensure the cache directory is empty before executing the query. 80 | const int files = GetFileCountUnder(TEST_ON_DISK_CACHE_DIRECTORY); 81 | REQUIRE(files == 0); 82 | 83 | // After executing the query, the cache directory should have one cache file. 84 | auto result = con.Query(StringUtil::Format("SELECT * FROM '%s'", TEST_ON_DISK_CACHE_FILE)); 85 | REQUIRE(!result->HasError()); 86 | 87 | const int files_after_query = GetFileCountUnder(TEST_ON_DISK_CACHE_DIRECTORY); 88 | const auto files_in_cache = GetSortedFilesUnder(TEST_ON_DISK_CACHE_DIRECTORY); 89 | REQUIRE(files_after_query == 1); 90 | 91 | // Change the cache directory path and execute the query again. 92 | con.Query(StringUtil::Format("SET cache_httpfs_cache_directory ='%s'", TEST_SECOND_ON_DISK_CACHE_DIRECTORY)); 93 | result = con.Query(StringUtil::Format("SELECT * FROM '%s'", TEST_ON_DISK_CACHE_FILE)); 94 | REQUIRE(!result->HasError()); 95 | 96 | // After executing the query, the NEW directory should have one cache file. 97 | // Both directories should have the same cache file. 98 | const int files_after_query_second = GetFileCountUnder(TEST_SECOND_ON_DISK_CACHE_DIRECTORY); 99 | const auto files_in_cache_second = GetSortedFilesUnder(TEST_SECOND_ON_DISK_CACHE_DIRECTORY); 100 | REQUIRE(files_after_query_second == 1); 101 | REQUIRE(files_in_cache == files_in_cache_second); 102 | 103 | // Update cache type to in-memory cache type and on-disk cache directory, and check no new cache files created. 104 | // 105 | // Set cache directory to the root directory, which test program doesn't have the permission to write to. 106 | con.Query(StringUtil::Format("SET cache_httpfs_cache_directory ='%s'", "/non_existent_directory")); 107 | con.Query("SET cache_httpfs_type='in_mem'"); 108 | result = con.Query(StringUtil::Format("SELECT * FROM '%s'", TEST_ON_DISK_CACHE_FILE)); 109 | REQUIRE(!result->HasError()); 110 | }; 111 | 112 | int main(int argc, char **argv) { 113 | CleanupTestDirectory(); 114 | int result = Catch::Session().run(argc, argv); 115 | CleanupTestDirectory(); 116 | return result; 117 | } 118 | -------------------------------------------------------------------------------- /unit/test_shared_lru_cache.cpp: -------------------------------------------------------------------------------- 1 | #define CATCH_CONFIG_RUNNER 2 | #include "catch.hpp" 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "shared_lru_cache.hpp" 12 | 13 | using namespace duckdb; // NOLINT 14 | 15 | namespace { 16 | struct MapKey { 17 | std::string fname; 18 | uint64_t off; 19 | }; 20 | struct MapKeyEqual { 21 | bool operator()(const MapKey &lhs, const MapKey &rhs) const { 22 | return std::tie(lhs.fname, lhs.off) == std::tie(rhs.fname, rhs.off); 23 | } 24 | }; 25 | struct MapKeyHash { 26 | std::size_t operator()(const MapKey &key) const { 27 | return std::hash {}(key.fname) ^ std::hash {}(key.off); 28 | } 29 | }; 30 | } // namespace 31 | 32 | TEST_CASE("PutAndGetSameKey", "[shared lru test]") { 33 | ThreadSafeSharedLruCache cache {/*max_entries_p=*/1, /*timeout_millisec_p=*/0}; 34 | 35 | // No value initially. 36 | auto val = cache.Get("1"); 37 | REQUIRE(val == nullptr); 38 | 39 | // Check put and get. 40 | cache.Put("1", make_shared_ptr("1")); 41 | val = cache.Get("1"); 42 | REQUIRE(val != nullptr); 43 | REQUIRE(*val == "1"); 44 | 45 | // Check key eviction. 46 | cache.Put("2", make_shared_ptr("2")); 47 | val = cache.Get("1"); 48 | REQUIRE(val == nullptr); 49 | val = cache.Get("2"); 50 | REQUIRE(val != nullptr); 51 | REQUIRE(*val == "2"); 52 | 53 | // Check deletion. 54 | REQUIRE(!cache.Delete("1")); 55 | REQUIRE(cache.Delete("2")); 56 | val = cache.Get("2"); 57 | REQUIRE(val == nullptr); 58 | } 59 | 60 | TEST_CASE("CustomizedStruct", "[shared lru test]") { 61 | ThreadSafeSharedLruCache cache {/*max_entries_p=*/1, 62 | /*timeout_millisec_p=*/0}; 63 | MapKey key; 64 | key.fname = "hello"; 65 | key.off = 10; 66 | cache.Put(key, make_shared_ptr("world")); 67 | 68 | MapKey lookup_key; 69 | lookup_key.fname = key.fname; 70 | lookup_key.off = key.off; 71 | auto val = cache.Get(lookup_key); 72 | REQUIRE(val != nullptr); 73 | REQUIRE(*val == "world"); 74 | } 75 | 76 | TEST_CASE("Clear with filter test", "[shared lru test]") { 77 | ThreadSafeSharedLruCache cache {/*max_entries_p=*/3, /*timeout_millisec_p=*/0}; 78 | cache.Put("key1", make_shared_ptr("val1")); 79 | cache.Put("key2", make_shared_ptr("val2")); 80 | cache.Put("key3", make_shared_ptr("val3")); 81 | cache.Clear([](const std::string &key) { return key >= "key2"; }); 82 | 83 | // Still valid keys. 84 | auto val = cache.Get("key1"); 85 | REQUIRE(val != nullptr); 86 | REQUIRE(*val == "val1"); 87 | 88 | // Non-existent keys. 89 | val = cache.Get("key2"); 90 | REQUIRE(val == nullptr); 91 | val = cache.Get("key3"); 92 | REQUIRE(val == nullptr); 93 | } 94 | 95 | TEST_CASE("GetOrCreate test", "[shared lru test]") { 96 | using CacheType = ThreadSafeSharedLruCache; 97 | 98 | std::atomic invoked = {false}; // Used to check only invoke once. 99 | auto factory = [&invoked](const std::string &key) -> shared_ptr { 100 | REQUIRE(!invoked.exchange(true)); 101 | // Sleep for a while so multiple threads could kick in and get blocked. 102 | std::this_thread::sleep_for(std::chrono::seconds(3)); 103 | return make_shared_ptr(key); 104 | }; 105 | 106 | CacheType cache {/*max_entries_p=*/1, /*timeout_millisec_p=*/0}; 107 | 108 | constexpr size_t kFutureNum = 100; 109 | std::vector>> futures; 110 | futures.reserve(kFutureNum); 111 | 112 | const std::string key = "key"; 113 | for (size_t idx = 0; idx < kFutureNum; ++idx) { 114 | futures.emplace_back( 115 | std::async(std::launch::async, [&cache, &key, &factory]() { return cache.GetOrCreate(key, factory); })); 116 | } 117 | for (auto &fut : futures) { 118 | auto val = fut.get(); 119 | REQUIRE(val != nullptr); 120 | REQUIRE(*val == key); 121 | } 122 | 123 | // After we're sure key-value pair exists in cache, make one more call. 124 | auto cached_val = cache.GetOrCreate(key, factory); 125 | REQUIRE(cached_val != nullptr); 126 | REQUIRE(*cached_val == key); 127 | } 128 | 129 | TEST_CASE("Put and get with timeout test", "[shared lru test]") { 130 | using CacheType = ThreadSafeSharedLruCache; 131 | 132 | CacheType cache {/*max_entries_p=*/1, /*timeout_millisec_p=*/500}; 133 | cache.Put("key", make_shared_ptr("val")); 134 | 135 | // Getting key-value pair right afterwards is able to get the value. 136 | auto val = cache.Get("key"); 137 | REQUIRE(val != nullptr); 138 | REQUIRE(*val == "val"); 139 | 140 | // Sleep for a while which exceeds timeout, re-fetch key-value pair fails to get value. 141 | std::this_thread::sleep_for(std::chrono::milliseconds(1000)); 142 | val = cache.Get("key"); 143 | REQUIRE(val == nullptr); 144 | } 145 | 146 | int main(int argc, char **argv) { 147 | int result = Catch::Session().run(argc, argv); 148 | return result; 149 | } 150 | -------------------------------------------------------------------------------- /unit/test_size_literals.cpp: -------------------------------------------------------------------------------- 1 | #define CATCH_CONFIG_RUNNER 2 | #include "catch.hpp" 3 | 4 | #include "size_literals.hpp" 5 | 6 | TEST_CASE("Size literals test", "[size literals]") { 7 | REQUIRE(2_MiB == 2 * 1024 * 1024); 8 | REQUIRE(2.5_KB == 2500); 9 | REQUIRE(4_GB == 4000000000); 10 | } 11 | 12 | int main(int argc, char **argv) { 13 | int result = Catch::Session().run(argc, argv); 14 | return result; 15 | } 16 | -------------------------------------------------------------------------------- /unit/test_thread_pool.cpp: -------------------------------------------------------------------------------- 1 | #define CATCH_CONFIG_RUNNER 2 | #include "catch.hpp" 3 | 4 | #include 5 | 6 | #include "thread_pool.hpp" 7 | 8 | namespace { 9 | constexpr int kNumPromise = 10; 10 | void SetPromise(std::promise *promise) { 11 | promise->set_value(); 12 | } 13 | int GetInputValue(int val) { 14 | return val; 15 | } 16 | } // namespace 17 | 18 | using namespace duckdb; // NOLINT 19 | 20 | TEST_CASE("Threadpool test", "[threadpool]") { 21 | // Enqueue lambda. 22 | { 23 | std::vector> promises(kNumPromise); 24 | ThreadPool tp(1); 25 | std::vector> futures; 26 | futures.reserve(kNumPromise); 27 | for (int ii = 0; ii < kNumPromise; ++ii) { 28 | auto func = [ii, &promises]() mutable { 29 | promises[ii].set_value(); 30 | }; 31 | futures.emplace_back(tp.Push(std::move(func))); 32 | } 33 | tp.Wait(); 34 | 35 | for (int ii = 0; ii < kNumPromise; ++ii) { 36 | promises[ii].get_future().get(); 37 | } 38 | } 39 | 40 | // Enqueue function with parameters. 41 | { 42 | std::vector> promises(kNumPromise); 43 | ThreadPool tp(1); 44 | std::vector> futures; 45 | futures.reserve(kNumPromise); 46 | for (int ii = 0; ii < kNumPromise; ++ii) { 47 | futures.emplace_back(tp.Push(SetPromise, &promises[ii])); 48 | } 49 | tp.Wait(); 50 | 51 | for (int ii = 0; ii < kNumPromise; ++ii) { 52 | promises[ii].get_future().get(); 53 | } 54 | } 55 | 56 | // Enqueue function with return value. 57 | { 58 | ThreadPool tp(1); 59 | std::vector> futures; 60 | futures.reserve(kNumPromise); 61 | for (int val = 0; val < kNumPromise; ++val) { 62 | futures.emplace_back(tp.Push(GetInputValue, val)); 63 | } 64 | tp.Wait(); 65 | 66 | for (int val = 0; val < kNumPromise; ++val) { 67 | REQUIRE(futures[val].get() == val); 68 | } 69 | } 70 | } 71 | 72 | int main(int argc, char **argv) { 73 | int result = Catch::Session().run(argc, argv); 74 | return result; 75 | } 76 | -------------------------------------------------------------------------------- /vcpkg.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": [ 3 | "openssl" 4 | ] 5 | } 6 | --------------------------------------------------------------------------------