├── .clang-format ├── .clang-tidy ├── .editorconfig ├── .github └── workflows │ └── MainDistributionPipeline.yml ├── .gitignore ├── .gitmodules ├── CMakeLists.txt ├── LICENSE ├── Makefile ├── benchmark └── tpch │ ├── load.sql │ ├── q01.benchmark │ ├── q02.benchmark │ ├── q03.benchmark │ ├── q04.benchmark │ ├── q05.benchmark │ ├── q06.benchmark │ ├── q07.benchmark │ ├── q08.benchmark │ ├── q09.benchmark │ ├── q10.benchmark │ ├── q11.benchmark │ ├── q12.benchmark │ ├── q13.benchmark │ ├── q14.benchmark │ ├── q15.benchmark │ ├── q16.benchmark │ ├── q17.benchmark │ ├── q18.benchmark │ ├── q19.benchmark │ ├── q20.benchmark │ ├── q21.benchmark │ ├── q22.benchmark │ └── tpch.benchmark.in ├── docs ├── NEXT_README.md ├── README.md └── UPDATING.md ├── examples └── minio-demo-server │ └── README.md ├── extension_config.cmake ├── logo ├── DuckLake_Logo-horizontal-dark.svg └── DuckLake_Logo-horizontal.svg ├── scripts ├── bootstrap-template.py ├── change_db.py ├── extension-upload.sh └── setup-custom-toolchain.sh ├── src ├── CMakeLists.txt ├── common │ ├── CMakeLists.txt │ ├── ducklake_data_file.cpp │ ├── ducklake_types.cpp │ └── ducklake_util.cpp ├── ducklake_extension.cpp ├── functions │ ├── CMakeLists.txt │ ├── base_metadata_function.cpp │ ├── ducklake_cleanup_old_files.cpp │ ├── ducklake_expire_snapshots.cpp │ ├── ducklake_merge_adjacent_files.cpp │ ├── ducklake_set_option.cpp │ ├── ducklake_snapshots.cpp │ ├── ducklake_table_changes.cpp │ ├── ducklake_table_info.cpp │ └── ducklake_table_insertions.cpp ├── include │ ├── common │ │ ├── ducklake_data_file.hpp │ │ ├── ducklake_encryption.hpp │ │ ├── ducklake_options.hpp │ │ ├── ducklake_snapshot.hpp │ │ ├── ducklake_types.hpp │ │ ├── ducklake_util.hpp │ │ ├── index.hpp │ │ └── local_change.hpp │ ├── ducklake_extension.hpp │ ├── functions │ │ └── ducklake_table_functions.hpp │ └── storage │ │ ├── ducklake_catalog.hpp │ │ ├── ducklake_catalog_set.hpp │ │ ├── ducklake_compaction.hpp │ │ ├── ducklake_delete.hpp │ │ ├── ducklake_delete_filter.hpp │ │ ├── ducklake_field_data.hpp │ │ ├── ducklake_initializer.hpp │ │ ├── ducklake_inline_data.hpp │ │ ├── ducklake_inlined_data.hpp │ │ ├── ducklake_inlined_data_reader.hpp │ │ ├── ducklake_insert.hpp │ │ ├── ducklake_metadata_info.hpp │ │ ├── ducklake_metadata_manager.hpp │ │ ├── ducklake_multi_file_list.hpp │ │ ├── ducklake_multi_file_reader.hpp │ │ ├── ducklake_partition_data.hpp │ │ ├── ducklake_scan.hpp │ │ ├── ducklake_schema_entry.hpp │ │ ├── ducklake_stats.hpp │ │ ├── ducklake_storage.hpp │ │ ├── ducklake_table_entry.hpp │ │ ├── ducklake_transaction.hpp │ │ ├── ducklake_transaction_changes.hpp │ │ ├── ducklake_transaction_manager.hpp │ │ ├── ducklake_update.hpp │ │ └── ducklake_view_entry.hpp └── storage │ ├── CMakeLists.txt │ ├── ducklake_autoload_helper.cpp │ ├── ducklake_catalog.cpp │ ├── ducklake_catalog_set.cpp │ ├── ducklake_checkpoint.cpp │ ├── ducklake_default_functions.cpp │ ├── ducklake_delete.cpp │ ├── ducklake_delete_filter.cpp │ ├── ducklake_field_data.cpp │ ├── ducklake_initializer.cpp │ ├── ducklake_inline_data.cpp │ ├── ducklake_inlined_data_reader.cpp │ ├── ducklake_insert.cpp │ ├── ducklake_metadata_manager.cpp │ ├── ducklake_multi_file_list.cpp │ ├── ducklake_multi_file_reader.cpp │ ├── ducklake_scan.cpp │ ├── ducklake_schema_entry.cpp │ ├── ducklake_stats.cpp │ ├── ducklake_storage.cpp │ ├── ducklake_table_entry.cpp │ ├── ducklake_transaction.cpp │ ├── ducklake_transaction_changes.cpp │ ├── ducklake_transaction_manager.cpp │ ├── ducklake_update.cpp │ └── ducklake_view_entry.cpp ├── test ├── README.md └── sql │ ├── alter │ ├── add_column.test │ ├── add_column_nested.test │ ├── add_column_transaction_local.test │ ├── drop_column.test │ ├── drop_column_nested.test │ ├── mixed_alter.test │ ├── mixed_alter2.test │ ├── promote_type.test │ ├── rename_column.test │ ├── rename_table.test │ ├── struct_evolution.test │ ├── struct_evolution_alter.test │ ├── struct_evolution_nested.test │ ├── struct_evolution_nested_alter.test │ └── struct_evolution_reuse.test │ ├── autoloading │ └── autoload_data_path.test │ ├── catalog │ ├── drop_table.test │ ├── quoted_identifiers.test │ └── schema.test │ ├── cleanup │ └── create_drop_cleanup.test │ ├── clickbench │ └── clickbench.test_slow │ ├── comments │ ├── comment_on_column.test │ └── comments.test │ ├── compaction │ ├── compaction_alter_table.test │ ├── compaction_delete_conflict.test │ ├── compaction_encrypted.test │ ├── compaction_partitioned_table.test │ ├── expire_snapshots.test │ ├── expire_snapshots_drop_table.test │ ├── expire_snapshots_schema.test │ ├── multi_compaction.test │ └── small_insert_compaction.test │ ├── concurrent │ └── concurrent_insert_conflict.test │ ├── constraints │ ├── not_null.test │ ├── not_null_drop_column.test │ └── unsupported.test │ ├── data_inlining │ ├── basic_data_inlining.test │ ├── data_inlining_alter.test │ ├── data_inlining_constraints.test │ ├── data_inlining_delete.test │ ├── data_inlining_large.test │ ├── data_inlining_table_changes.test │ ├── data_inlining_transaction_local_alter.test │ ├── data_inlining_transaction_local_delete.test │ ├── data_inlining_types.test │ └── data_inlining_update.test │ ├── default │ ├── add_column_with_default.test │ ├── default_values.test │ └── struct_field_default.test │ ├── delete │ ├── basic_delete.test │ ├── delete_join.test │ ├── delete_rollback_cleanup.test │ ├── delete_same_transaction.test │ ├── empty_delete.test │ ├── multi_deletes.test │ └── truncate_table.test │ ├── ducklake_basic.test │ ├── ducklake_multiple_lakes.test │ ├── encryption │ ├── encryption.test │ └── partitioning_encryption.test │ ├── functions │ ├── ducklake_snapshots.test │ └── ducklake_table_info.test │ ├── general │ ├── attach_at_snapshot.test │ ├── data_path_tag.test │ ├── database_size.test │ ├── detach_ducklake.test │ ├── ducklake_read_only.test │ ├── generated_columns.test │ ├── metadata_cache.test │ ├── metadata_parameters.test │ ├── missing_parquet.test │ ├── prepared_statement.test │ └── recursive_metadata_catalog.test │ ├── insert │ ├── insert_column_list.test │ └── insert_into_self.test │ ├── metadata │ └── ducklake_duckdb_tables.test │ ├── partitioning │ ├── basic_partitioning.test │ └── multi_key_partition.test │ ├── rowid │ └── ducklake_row_id.test │ ├── schema_evolution │ └── field_ids.test │ ├── settings │ └── parquet_compression.test │ ├── stats │ ├── cardinality.test │ ├── filter_pushdown.test │ ├── global_stats.test │ └── global_stats_transactions.test │ ├── table_changes │ ├── ducklake_table_changes.test │ ├── ducklake_table_deletions.test │ └── ducklake_table_insertions.test │ ├── time_travel │ ├── basic_time_travel.test │ └── time_travel_views.test │ ├── tpch │ └── tpch_sf1.test_slow │ ├── transaction │ ├── basic_transaction.test │ ├── create_conflict.test │ ├── transaction_conflict_cleanup.test │ ├── transaction_conflicts.test │ ├── transaction_conflicts_delete.test │ ├── transaction_conflicts_view.test │ ├── transaction_isolation.test │ └── transaction_schema.test │ ├── types │ ├── all_types.test │ ├── floats.test │ ├── json.test │ ├── list.test │ ├── map.test │ ├── null_byte.test │ ├── struct.test │ ├── timestamp.test │ └── unsupported.test │ ├── update │ ├── basic_update.test │ ├── update_join_duplicates.test │ ├── update_not_null.test │ ├── update_partitioning.test │ ├── update_rollback.test │ └── update_same_transaction.test │ ├── view │ ├── ducklake_rename_view.test │ ├── ducklake_view.test │ ├── ducklake_view_schema.test │ └── ducklake_view_table_conflict.test │ └── virtualcolumns │ ├── ducklake_snapshot_id.test │ └── ducklake_virtual_columns.test └── vcpkg.json /.clang-format: -------------------------------------------------------------------------------- 1 | duckdb/.clang-format -------------------------------------------------------------------------------- /.clang-tidy: -------------------------------------------------------------------------------- 1 | duckdb/.clang-tidy -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | duckdb/.editorconfig -------------------------------------------------------------------------------- /.github/workflows/MainDistributionPipeline.yml: -------------------------------------------------------------------------------- 1 | # 2 | # This workflow calls the main distribution pipeline from DuckDB to build, test and (optionally) release the extension 3 | # 4 | name: Main Extension Distribution Pipeline 5 | on: 6 | push: 7 | pull_request: 8 | workflow_dispatch: 9 | 10 | concurrency: 11 | group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} 12 | cancel-in-progress: true 13 | 14 | jobs: 15 | duckdb-next-build: 16 | name: Build extension binaries 17 | uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main 18 | with: 19 | duckdb_version: v1.3-ossivalis 20 | ci_tools_version: main 21 | extension_name: ducklake 22 | 23 | duckdb-stable-build: 24 | name: Build extension binaries 25 | uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main 26 | with: 27 | duckdb_version: v1.3.0 28 | ci_tools_version: main 29 | extension_name: ducklake 30 | 31 | duckdb-stable-deploy: 32 | name: Deploy extension binaries 33 | needs: duckdb-stable-build 34 | uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@main 35 | secrets: inherit 36 | with: 37 | extension_name: ducklake 38 | duckdb_version: v1.3.0 39 | ci_tools_version: main 40 | deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} 41 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | .idea 3 | cmake-build-debug 4 | duckdb_unittest_tempdir/ 5 | .DS_Store 6 | testext 7 | test/python/__pycache__/ 8 | .Rhistory 9 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "duckdb"] 2 | path = duckdb 3 | url = https://github.com/duckdb/duckdb 4 | branch = main 5 | [submodule "extension-ci-tools"] 6 | path = extension-ci-tools 7 | url = https://github.com/duckdb/extension-ci-tools 8 | branch = main -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.5) 2 | 3 | # Set extension name here 4 | set(TARGET_NAME ducklake) 5 | 6 | set(EXTENSION_NAME ${TARGET_NAME}_extension) 7 | set(LOADABLE_EXTENSION_NAME ${TARGET_NAME}_loadable_extension) 8 | 9 | project(${TARGET_NAME}) 10 | include_directories(src/include) 11 | 12 | add_subdirectory(src) 13 | set(EXTENSION_SOURCES ${ALL_OBJECT_FILES}) 14 | 15 | build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES}) 16 | build_loadable_extension(${TARGET_NAME} " " ${EXTENSION_SOURCES}) 17 | 18 | install( 19 | TARGETS ${EXTENSION_NAME} 20 | EXPORT "${DUCKDB_EXPORT_SET}" 21 | LIBRARY DESTINATION "${INSTALL_LIB_DIR}" 22 | ARCHIVE DESTINATION "${INSTALL_LIB_DIR}") 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2018-2025 Stichting DuckDB Foundation 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PROJ_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) 2 | 3 | # Configuration of extension 4 | EXT_NAME=ducklake 5 | EXT_CONFIG=${PROJ_DIR}extension_config.cmake 6 | 7 | # Include the Makefile from extension-ci-tools 8 | include extension-ci-tools/makefiles/duckdb_extension.Makefile -------------------------------------------------------------------------------- /benchmark/tpch/load.sql: -------------------------------------------------------------------------------- 1 | CALL dbgen(sf=${sf}); 2 | ATTACH 'ducklake:${BENCHMARK_DIR}/ducklake.db' AS ducklake (DATA_FILES '${BENCHMARK_DIR}/ducklake_data'); 3 | COPY FROM DATABASE main TO ducklake; 4 | -------------------------------------------------------------------------------- /benchmark/tpch/q01.benchmark: -------------------------------------------------------------------------------- 1 | # name: benchmark/tpch/sf10/q01.benchmark 2 | # description: Run query 01 from the TPC-H benchmark 3 | # group: [sf10] 4 | 5 | template benchmark/tpch/tpch.benchmark.in 6 | QUERY_NUMBER=1 7 | QUERY_NUMBER_PADDED=01 -------------------------------------------------------------------------------- /benchmark/tpch/q02.benchmark: -------------------------------------------------------------------------------- 1 | # name: benchmark/tpch/sf10/q02.benchmark 2 | # description: Run query 02 from the TPC-H benchmark 3 | # group: [sf10] 4 | 5 | template benchmark/tpch/tpch.benchmark.in 6 | QUERY_NUMBER=2 7 | QUERY_NUMBER_PADDED=02 -------------------------------------------------------------------------------- /benchmark/tpch/q03.benchmark: -------------------------------------------------------------------------------- 1 | # name: benchmark/tpch/sf10/q03.benchmark 2 | # description: Run query 03 from the TPC-H benchmark 3 | # group: [sf10] 4 | 5 | template benchmark/tpch/tpch.benchmark.in 6 | QUERY_NUMBER=3 7 | QUERY_NUMBER_PADDED=03 -------------------------------------------------------------------------------- /benchmark/tpch/q04.benchmark: -------------------------------------------------------------------------------- 1 | # name: benchmark/tpch/sf10/q04.benchmark 2 | # description: Run query 04 from the TPC-H benchmark 3 | # group: [sf10] 4 | 5 | template benchmark/tpch/tpch.benchmark.in 6 | QUERY_NUMBER=4 7 | QUERY_NUMBER_PADDED=04 -------------------------------------------------------------------------------- /benchmark/tpch/q05.benchmark: -------------------------------------------------------------------------------- 1 | # name: benchmark/tpch/sf10/q05.benchmark 2 | # description: Run query 05 from the TPC-H benchmark 3 | # group: [sf10] 4 | 5 | template benchmark/tpch/tpch.benchmark.in 6 | QUERY_NUMBER=5 7 | QUERY_NUMBER_PADDED=05 -------------------------------------------------------------------------------- /benchmark/tpch/q06.benchmark: -------------------------------------------------------------------------------- 1 | # name: benchmark/tpch/sf10/q06.benchmark 2 | # description: Run query 06 from the TPC-H benchmark 3 | # group: [sf10] 4 | 5 | template benchmark/tpch/tpch.benchmark.in 6 | QUERY_NUMBER=6 7 | QUERY_NUMBER_PADDED=06 -------------------------------------------------------------------------------- /benchmark/tpch/q07.benchmark: -------------------------------------------------------------------------------- 1 | # name: benchmark/tpch/sf10/q07.benchmark 2 | # description: Run query 07 from the TPC-H benchmark 3 | # group: [sf10] 4 | 5 | template benchmark/tpch/tpch.benchmark.in 6 | QUERY_NUMBER=7 7 | QUERY_NUMBER_PADDED=07 -------------------------------------------------------------------------------- /benchmark/tpch/q08.benchmark: -------------------------------------------------------------------------------- 1 | # name: benchmark/tpch/sf10/q08.benchmark 2 | # description: Run query 08 from the TPC-H benchmark 3 | # group: [sf10] 4 | 5 | template benchmark/tpch/tpch.benchmark.in 6 | QUERY_NUMBER=8 7 | QUERY_NUMBER_PADDED=08 -------------------------------------------------------------------------------- /benchmark/tpch/q09.benchmark: -------------------------------------------------------------------------------- 1 | # name: benchmark/tpch/sf10/q09.benchmark 2 | # description: Run query 09 from the TPC-H benchmark 3 | # group: [sf10] 4 | 5 | template benchmark/tpch/tpch.benchmark.in 6 | QUERY_NUMBER=9 7 | QUERY_NUMBER_PADDED=09 -------------------------------------------------------------------------------- /benchmark/tpch/q10.benchmark: -------------------------------------------------------------------------------- 1 | # name: benchmark/tpch/sf10/q10.benchmark 2 | # description: Run query 10 from the TPC-H benchmark 3 | # group: [sf10] 4 | 5 | template benchmark/tpch/tpch.benchmark.in 6 | QUERY_NUMBER=10 7 | QUERY_NUMBER_PADDED=10 -------------------------------------------------------------------------------- /benchmark/tpch/q11.benchmark: -------------------------------------------------------------------------------- 1 | # name: benchmark/tpch/sf10/q11.benchmark 2 | # description: Run query 11 from the TPC-H benchmark 3 | # group: [sf10] 4 | 5 | template benchmark/tpch/tpch.benchmark.in 6 | QUERY_NUMBER=11 7 | QUERY_NUMBER_PADDED=11 -------------------------------------------------------------------------------- /benchmark/tpch/q12.benchmark: -------------------------------------------------------------------------------- 1 | # name: benchmark/tpch/sf10/q12.benchmark 2 | # description: Run query 12 from the TPC-H benchmark 3 | # group: [sf10] 4 | 5 | template benchmark/tpch/tpch.benchmark.in 6 | QUERY_NUMBER=12 7 | QUERY_NUMBER_PADDED=12 -------------------------------------------------------------------------------- /benchmark/tpch/q13.benchmark: -------------------------------------------------------------------------------- 1 | # name: benchmark/tpch/sf10/q13.benchmark 2 | # description: Run query 13 from the TPC-H benchmark 3 | # group: [sf10] 4 | 5 | template benchmark/tpch/tpch.benchmark.in 6 | QUERY_NUMBER=13 7 | QUERY_NUMBER_PADDED=13 -------------------------------------------------------------------------------- /benchmark/tpch/q14.benchmark: -------------------------------------------------------------------------------- 1 | # name: benchmark/tpch/sf10/q14.benchmark 2 | # description: Run query 14 from the TPC-H benchmark 3 | # group: [sf10] 4 | 5 | template benchmark/tpch/tpch.benchmark.in 6 | QUERY_NUMBER=14 7 | QUERY_NUMBER_PADDED=14 -------------------------------------------------------------------------------- /benchmark/tpch/q15.benchmark: -------------------------------------------------------------------------------- 1 | # name: benchmark/tpch/sf10/q15.benchmark 2 | # description: Run query 15 from the TPC-H benchmark 3 | # group: [sf10] 4 | 5 | template benchmark/tpch/tpch.benchmark.in 6 | QUERY_NUMBER=15 7 | QUERY_NUMBER_PADDED=15 -------------------------------------------------------------------------------- /benchmark/tpch/q16.benchmark: -------------------------------------------------------------------------------- 1 | # name: benchmark/tpch/sf10/q16.benchmark 2 | # description: Run query 16 from the TPC-H benchmark 3 | # group: [sf10] 4 | 5 | template benchmark/tpch/tpch.benchmark.in 6 | QUERY_NUMBER=16 7 | QUERY_NUMBER_PADDED=16 -------------------------------------------------------------------------------- /benchmark/tpch/q17.benchmark: -------------------------------------------------------------------------------- 1 | # name: benchmark/tpch/sf10/q17.benchmark 2 | # description: Run query 17 from the TPC-H benchmark 3 | # group: [sf10] 4 | 5 | template benchmark/tpch/tpch.benchmark.in 6 | QUERY_NUMBER=17 7 | QUERY_NUMBER_PADDED=17 -------------------------------------------------------------------------------- /benchmark/tpch/q18.benchmark: -------------------------------------------------------------------------------- 1 | # name: benchmark/tpch/sf10/q18.benchmark 2 | # description: Run query 18 from the TPC-H benchmark 3 | # group: [sf10] 4 | 5 | template benchmark/tpch/tpch.benchmark.in 6 | QUERY_NUMBER=18 7 | QUERY_NUMBER_PADDED=18 -------------------------------------------------------------------------------- /benchmark/tpch/q19.benchmark: -------------------------------------------------------------------------------- 1 | # name: benchmark/tpch/sf10/q19.benchmark 2 | # description: Run query 19 from the TPC-H benchmark 3 | # group: [sf10] 4 | 5 | template benchmark/tpch/tpch.benchmark.in 6 | QUERY_NUMBER=19 7 | QUERY_NUMBER_PADDED=19 -------------------------------------------------------------------------------- /benchmark/tpch/q20.benchmark: -------------------------------------------------------------------------------- 1 | # name: benchmark/tpch/sf10/q20.benchmark 2 | # description: Run query 20 from the TPC-H benchmark 3 | # group: [sf10] 4 | 5 | template benchmark/tpch/tpch.benchmark.in 6 | QUERY_NUMBER=20 7 | QUERY_NUMBER_PADDED=20 -------------------------------------------------------------------------------- /benchmark/tpch/q21.benchmark: -------------------------------------------------------------------------------- 1 | # name: benchmark/tpch/sf10/q21.benchmark 2 | # description: Run query 21 from the TPC-H benchmark 3 | # group: [sf10] 4 | 5 | template benchmark/tpch/tpch.benchmark.in 6 | QUERY_NUMBER=21 7 | QUERY_NUMBER_PADDED=21 -------------------------------------------------------------------------------- /benchmark/tpch/q22.benchmark: -------------------------------------------------------------------------------- 1 | # name: benchmark/tpch/sf10/q22.benchmark 2 | # description: Run query 22 from the TPC-H benchmark 3 | # group: [sf10] 4 | 5 | template benchmark/tpch/tpch.benchmark.in 6 | QUERY_NUMBER=22 7 | QUERY_NUMBER_PADDED=22 -------------------------------------------------------------------------------- /benchmark/tpch/tpch.benchmark.in: -------------------------------------------------------------------------------- 1 | # name: ${FILE_PATH} 2 | # description: ${DESCRIPTION} 3 | # group: [sf1] 4 | 5 | argument sf 1 6 | 7 | name DuckLake Q${QUERY_NUMBER_PADDED} 8 | group tpch 9 | subgroup sf${sf} 10 | 11 | require ducklake 12 | 13 | require parquet 14 | 15 | require tpch 16 | 17 | cache_file tpch_sf${sf}_ducklake.db 18 | 19 | load 20 | ATTACH ':memory:' AS mem; 21 | USE mem; 22 | CALL dbgen(sf=${sf}); 23 | ATTACH 'ducklake:${BENCHMARK_DIR}/tpch_sf${sf}_ducklake.db' AS ducklake (DATA_PATH '${BENCHMARK_DIR}/tpch_sf${sf}_ducklake_files'); 24 | COPY FROM DATABASE mem TO ducklake; 25 | USE ducklake; 26 | DETACH mem; 27 | 28 | reload 29 | ATTACH 'ducklake:${BENCHMARK_DIR}/tpch_sf${sf}_ducklake.db' AS ducklake; 30 | USE ducklake; 31 | 32 | run duckdb/extension/tpch/dbgen/queries/q${QUERY_NUMBER_PADDED}.sql 33 | 34 | result duckdb/extension/tpch/dbgen/answers/sf1/q${QUERY_NUMBER_PADDED}.csv sf=1 35 | 36 | result duckdb/extension/tpch/dbgen/answers/sf100/q${QUERY_NUMBER_PADDED}.csv sf=100 37 | -------------------------------------------------------------------------------- /docs/UPDATING.md: -------------------------------------------------------------------------------- 1 | # Extension updating 2 | When cloning this template, the target version of DuckDB should be the latest stable release of DuckDB. However, there 3 | will inevitably come a time when a new DuckDB is released and the extension repository needs updating. This process goes 4 | as follows: 5 | 6 | - Bump submodules 7 | - `./duckdb` should be set to latest tagged release 8 | - `./extension-ci-tools` should be set to updated branch corresponding to latest DuckDB release. So if you're building for DuckDB `v1.1.0` there will be a branch in `extension-ci-tools` named `v1.1.0` to which you should check out. 9 | - Bump versions in `./github/workflows` 10 | - `duckdb_version` input in `duckdb-stable-build` job in `MainDistributionPipeline.yml` should be set to latest tagged release 11 | - `duckdb_version` input in `duckdb-stable-deploy` job in `MainDistributionPipeline.yml` should be set to latest tagged release 12 | - the reusable workflow `duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml` for the `duckdb-stable-build` job should be set to latest tagged release 13 | 14 | # API changes 15 | DuckDB extensions built with this extension template are built against the internal C++ API of DuckDB. This API is not guaranteed to be stable. 16 | What this means for extension development is that when updating your extensions DuckDB target version using the above steps, you may run into the fact that your extension no longer builds properly. 17 | 18 | Currently, DuckDB does not (yet) provide a specific change log for these API changes, but it is generally not too hard to figure out what has changed. 19 | 20 | For figuring out how and why the C++ API changed, we recommend using the following resources: 21 | - DuckDB's [Release Notes](https://github.com/duckdb/duckdb/releases) 22 | - DuckDB's history of [Core extension patches](https://github.com/duckdb/duckdb/commits/main/.github/patches/extensions) 23 | - The git history of the relevant C++ Header file of the API that has changed -------------------------------------------------------------------------------- /extension_config.cmake: -------------------------------------------------------------------------------- 1 | # This file is included by DuckDB's build system. It specifies which extension to load 2 | 3 | # Extension from this repo 4 | duckdb_extension_load(ducklake 5 | SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} 6 | LOAD_TESTS 7 | ) 8 | 9 | duckdb_extension_load(icu) 10 | duckdb_extension_load(json) 11 | duckdb_extension_load(tpch) 12 | # Any extra extensions that should be built 13 | # e.g.: duckdb_extension_load(json) -------------------------------------------------------------------------------- /scripts/setup-custom-toolchain.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This is an example script that can be used to install additional toolchain dependencies. Feel free to remove this script 4 | # if no additional toolchains are required 5 | 6 | # To enable this script, set the `custom_toolchain_script` option to true when calling the reusable workflow 7 | # `.github/workflows/_extension_distribution.yml` from `https://github.com/duckdb/extension-ci-tools` 8 | 9 | # note that the $DUCKDB_PLATFORM environment variable can be used to discern between the platforms 10 | echo "This is the sample custom toolchain script running for architecture '$DUCKDB_PLATFORM' for the quack extension." 11 | 12 | -------------------------------------------------------------------------------- /src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_subdirectory(common) 2 | add_subdirectory(functions) 3 | add_subdirectory(storage) 4 | 5 | add_library(ducklake_library OBJECT ducklake_extension.cpp) 6 | set(ALL_OBJECT_FILES 7 | ${ALL_OBJECT_FILES} $ 8 | PARENT_SCOPE) 9 | -------------------------------------------------------------------------------- /src/common/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(ducklake_common OBJECT ducklake_data_file.cpp ducklake_types.cpp 2 | ducklake_util.cpp) 3 | set(ALL_OBJECT_FILES 4 | ${ALL_OBJECT_FILES} $ 5 | PARENT_SCOPE) 6 | -------------------------------------------------------------------------------- /src/common/ducklake_data_file.cpp: -------------------------------------------------------------------------------- 1 | #include "common/ducklake_data_file.hpp" 2 | 3 | namespace duckdb { 4 | 5 | DuckLakeDataFile::DuckLakeDataFile(const DuckLakeDataFile &other) { 6 | file_name = other.file_name; 7 | row_count = other.row_count; 8 | file_size_bytes = other.file_size_bytes; 9 | footer_size = other.footer_size; 10 | partition_id = other.partition_id; 11 | if (other.delete_file) { 12 | delete_file = make_uniq(*other.delete_file); 13 | } 14 | column_stats = other.column_stats; 15 | partition_values = other.partition_values; 16 | encryption_key = other.encryption_key; 17 | } 18 | 19 | DuckLakeDataFile &DuckLakeDataFile::operator=(const DuckLakeDataFile &other) { 20 | file_name = other.file_name; 21 | row_count = other.row_count; 22 | file_size_bytes = other.file_size_bytes; 23 | footer_size = other.footer_size; 24 | partition_id = other.partition_id; 25 | if (other.delete_file) { 26 | delete_file = make_uniq(*other.delete_file); 27 | } 28 | column_stats = other.column_stats; 29 | partition_values = other.partition_values; 30 | encryption_key = other.encryption_key; 31 | return *this; 32 | } 33 | 34 | } // namespace duckdb 35 | -------------------------------------------------------------------------------- /src/ducklake_extension.cpp: -------------------------------------------------------------------------------- 1 | #ifndef DUCKDB_BUILD_LOADABLE_EXTENSION 2 | #define DUCKDB_BUILD_LOADABLE_EXTENSION 3 | #endif 4 | #include "ducklake_extension.hpp" 5 | #include "duckdb.hpp" 6 | #include "duckdb/common/exception.hpp" 7 | #include "duckdb/common/string_util.hpp" 8 | #include "storage/ducklake_storage.hpp" 9 | #include "functions/ducklake_table_functions.hpp" 10 | #include "duckdb/main/extension_util.hpp" 11 | 12 | namespace duckdb { 13 | 14 | static void LoadInternal(DatabaseInstance &instance) { 15 | ExtensionUtil::RegisterExtension(instance, "ducklake", {"Adds support for DuckLake, SQL as a Lakehouse Format"}); 16 | 17 | auto &config = DBConfig::GetConfig(instance); 18 | config.storage_extensions["ducklake"] = make_uniq(); 19 | 20 | DuckLakeSnapshotsFunction snapshots; 21 | ExtensionUtil::RegisterFunction(instance, snapshots); 22 | 23 | DuckLakeTableInfoFunction table_info; 24 | ExtensionUtil::RegisterFunction(instance, table_info); 25 | 26 | auto table_insertions = DuckLakeTableInsertionsFunction::GetFunctions(); 27 | ExtensionUtil::RegisterFunction(instance, table_insertions); 28 | 29 | auto table_deletions = DuckLakeTableDeletionsFunction::GetFunctions(); 30 | ExtensionUtil::RegisterFunction(instance, table_deletions); 31 | 32 | DuckLakeMergeAdjacentFilesFunction merge_adjacent_files; 33 | ExtensionUtil::RegisterFunction(instance, merge_adjacent_files); 34 | 35 | DuckLakeCleanupOldFilesFunction cleanup_old_files; 36 | ExtensionUtil::RegisterFunction(instance, cleanup_old_files); 37 | 38 | DuckLakeExpireSnapshotsFunction expire_snapshots; 39 | ExtensionUtil::RegisterFunction(instance, expire_snapshots); 40 | 41 | DuckLakeSetOptionFunction set_options; 42 | ExtensionUtil::RegisterFunction(instance, set_options); 43 | 44 | auto table_changes = DuckLakeTableInsertionsFunction::GetDuckLakeTableChanges(); 45 | ExtensionUtil::RegisterFunction(instance, *table_changes); 46 | } 47 | 48 | void DucklakeExtension::Load(DuckDB &db) { 49 | LoadInternal(*db.instance); 50 | } 51 | std::string DucklakeExtension::Name() { 52 | return "ducklake"; 53 | } 54 | 55 | std::string DucklakeExtension::Version() const { 56 | #ifdef EXT_VERSION_DUCKLAKE 57 | return EXT_VERSION_DUCKLAKE; 58 | #else 59 | return ""; 60 | #endif 61 | } 62 | 63 | } // namespace duckdb 64 | 65 | extern "C" { 66 | 67 | DUCKDB_EXTENSION_API void ducklake_init(duckdb::DatabaseInstance &db) { 68 | duckdb::DuckDB db_wrapper(db); 69 | db_wrapper.LoadExtension(); 70 | } 71 | 72 | DUCKDB_EXTENSION_API const char *ducklake_version() { 73 | return duckdb::DuckDB::LibraryVersion(); 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/functions/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library( 2 | ducklake_functions OBJECT 3 | base_metadata_function.cpp 4 | ducklake_cleanup_old_files.cpp 5 | ducklake_expire_snapshots.cpp 6 | ducklake_snapshots.cpp 7 | ducklake_merge_adjacent_files.cpp 8 | ducklake_set_option.cpp 9 | ducklake_table_changes.cpp 10 | ducklake_table_info.cpp 11 | ducklake_table_insertions.cpp) 12 | set(ALL_OBJECT_FILES 13 | ${ALL_OBJECT_FILES} $ 14 | PARENT_SCOPE) 15 | -------------------------------------------------------------------------------- /src/functions/base_metadata_function.cpp: -------------------------------------------------------------------------------- 1 | #include "functions/ducklake_table_functions.hpp" 2 | #include "duckdb/main/attached_database.hpp" 3 | #include "duckdb/main/database_manager.hpp" 4 | 5 | namespace duckdb { 6 | 7 | Catalog &BaseMetadataFunction::GetCatalog(ClientContext &context, const Value &input) { 8 | if (input.IsNull()) { 9 | throw BinderException("Catalog cannot be NULL"); 10 | } 11 | // look up the database to query 12 | auto db_name = input.GetValue(); 13 | auto &db_manager = DatabaseManager::Get(context); 14 | auto db = db_manager.GetDatabase(context, db_name); 15 | if (!db) { 16 | throw BinderException("Failed to find attached database \"%s\"", db_name); 17 | } 18 | auto &catalog = db->GetCatalog(); 19 | if (catalog.GetCatalogType() != "ducklake") { 20 | throw BinderException("Attached database \"%s\" does not refer to a DuckLake database", db_name); 21 | } 22 | return catalog; 23 | } 24 | 25 | struct MetadataFunctionData : public GlobalTableFunctionState { 26 | MetadataFunctionData() : offset(0) { 27 | } 28 | 29 | idx_t offset; 30 | }; 31 | 32 | unique_ptr MetadataFunctionInit(ClientContext &context, TableFunctionInitInput &input) { 33 | auto result = make_uniq(); 34 | return std::move(result); 35 | } 36 | 37 | void MetadataFunctionExecute(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { 38 | auto &data = data_p.bind_data->Cast(); 39 | auto &state = data_p.global_state->Cast(); 40 | if (state.offset >= data.rows.size()) { 41 | // finished returning values 42 | return; 43 | } 44 | // start returning values 45 | // either fill up the chunk or return all the remaining columns 46 | idx_t count = 0; 47 | while (state.offset < data.rows.size() && count < STANDARD_VECTOR_SIZE) { 48 | auto &entry = data.rows[state.offset++]; 49 | if (entry.size() != output.ColumnCount()) { 50 | throw InternalException("Unaligned metadata row in result"); 51 | } 52 | 53 | for (idx_t c = 0; c < entry.size(); c++) { 54 | output.SetValue(c, count, entry[c]); 55 | } 56 | count++; 57 | } 58 | output.SetCardinality(count); 59 | } 60 | 61 | BaseMetadataFunction::BaseMetadataFunction(string name_p, table_function_bind_t bind) 62 | : TableFunction(std::move(name_p), {LogicalType::VARCHAR}, MetadataFunctionExecute, bind, MetadataFunctionInit) { 63 | } 64 | 65 | } // namespace duckdb 66 | -------------------------------------------------------------------------------- /src/functions/ducklake_table_changes.cpp: -------------------------------------------------------------------------------- 1 | #include "duckdb/catalog/default/default_table_functions.hpp" 2 | #include "functions/ducklake_table_functions.hpp" 3 | 4 | namespace duckdb { 5 | 6 | // clang-format off 7 | static const DefaultTableMacro ducklake_table_macros[] = { 8 | {DEFAULT_SCHEMA, "ducklake_table_changes", {"catalog", "schema_name", "table_name", "start_snapshot", "end_snapshot", nullptr}, {{nullptr, nullptr}}, R"( 9 | SELECT snapshot_id, rowid, CASE WHEN (snapshot_id, rowid) in (SELECT snapshot_id, rowid FROM ducklake_table_deletions(catalog, schema_name, table_name, start_snapshot, end_snapshot)) 10 | THEN 'update_postimage' 11 | ELSE 'insert' 12 | END AS change_type, * FROM ducklake_table_insertions(catalog, schema_name, table_name, start_snapshot, end_snapshot) 13 | UNION ALL 14 | SELECT snapshot_id, rowid, CASE WHEN (snapshot_id, rowid) in (SELECT snapshot_id, rowid FROM ducklake_table_insertions(catalog, schema_name, table_name, start_snapshot, end_snapshot)) 15 | THEN 'update_preimage' 16 | ELSE 'delete' 17 | END AS change_type, * FROM ducklake_table_deletions(catalog, schema_name, table_name, start_snapshot, end_snapshot) 18 | )"}, 19 | {nullptr, nullptr, {nullptr}, {{nullptr, nullptr}}, nullptr} 20 | }; 21 | // clang-format on 22 | 23 | unique_ptr DuckLakeTableInsertionsFunction::GetDuckLakeTableChanges() { 24 | return DefaultTableFunctionGenerator::CreateTableMacroInfo(ducklake_table_macros[0]); 25 | } 26 | 27 | } // namespace duckdb 28 | -------------------------------------------------------------------------------- /src/functions/ducklake_table_info.cpp: -------------------------------------------------------------------------------- 1 | #include "functions/ducklake_table_functions.hpp" 2 | #include "storage/ducklake_transaction.hpp" 3 | #include "common/ducklake_util.hpp" 4 | #include "storage/ducklake_transaction_changes.hpp" 5 | 6 | namespace duckdb { 7 | 8 | static unique_ptr DuckLakeTableInfoBind(ClientContext &context, TableFunctionBindInput &input, 9 | vector &return_types, vector &names) { 10 | auto &catalog = BaseMetadataFunction::GetCatalog(context, input.inputs[0]); 11 | auto &transaction = DuckLakeTransaction::Get(context, catalog); 12 | 13 | auto &metadata_manager = transaction.GetMetadataManager(); 14 | auto tables = metadata_manager.GetTableSizes(transaction.GetSnapshot()); 15 | auto result = make_uniq(); 16 | for (auto &table_info : tables) { 17 | vector row_values; 18 | row_values.push_back(Value(table_info.table_name)); 19 | row_values.push_back(Value::BIGINT(table_info.schema_id.index)); 20 | row_values.push_back(Value::BIGINT(table_info.table_id.index)); 21 | row_values.push_back(Value::UUID(table_info.table_uuid)); 22 | row_values.push_back(Value::BIGINT(table_info.file_count)); 23 | row_values.push_back(Value::BIGINT(table_info.file_size_bytes)); 24 | row_values.push_back(Value::BIGINT(table_info.delete_file_count)); 25 | row_values.push_back(Value::BIGINT(table_info.delete_file_size_bytes)); 26 | result->rows.push_back(std::move(row_values)); 27 | } 28 | 29 | names.emplace_back("table_name"); 30 | return_types.emplace_back(LogicalType::VARCHAR); 31 | 32 | names.emplace_back("schema_id"); 33 | return_types.emplace_back(LogicalType::BIGINT); 34 | 35 | names.emplace_back("table_id"); 36 | return_types.emplace_back(LogicalType::BIGINT); 37 | 38 | names.emplace_back("table_uuid"); 39 | return_types.emplace_back(LogicalType::UUID); 40 | 41 | names.emplace_back("file_count"); 42 | return_types.emplace_back(LogicalType::BIGINT); 43 | 44 | names.emplace_back("file_size_bytes"); 45 | return_types.emplace_back(LogicalType::BIGINT); 46 | 47 | names.emplace_back("delete_file_count"); 48 | return_types.emplace_back(LogicalType::BIGINT); 49 | 50 | names.emplace_back("delete_file_size_bytes"); 51 | return_types.emplace_back(LogicalType::BIGINT); 52 | return std::move(result); 53 | } 54 | 55 | DuckLakeTableInfoFunction::DuckLakeTableInfoFunction() 56 | : BaseMetadataFunction("ducklake_table_info", DuckLakeTableInfoBind) { 57 | } 58 | 59 | } // namespace duckdb 60 | -------------------------------------------------------------------------------- /src/include/common/ducklake_data_file.hpp: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // DuckDB 3 | // 4 | // common/ducklake_data_file.hpp 5 | // 6 | // 7 | //===----------------------------------------------------------------------===// 8 | 9 | #pragma once 10 | 11 | #include "storage/ducklake_stats.hpp" 12 | #include "duckdb/common/optional_idx.hpp" 13 | #include "common/index.hpp" 14 | 15 | namespace duckdb { 16 | 17 | struct DuckLakeFilePartition { 18 | idx_t partition_column_idx; 19 | string partition_value; 20 | }; 21 | 22 | struct DuckLakeDeleteFile { 23 | DataFileIndex data_file_id; 24 | string data_file_path; 25 | string file_name; 26 | idx_t delete_count; 27 | idx_t file_size_bytes; 28 | idx_t footer_size; 29 | string encryption_key; 30 | bool overwrites_existing_delete = false; 31 | }; 32 | 33 | struct DuckLakeDataFile { 34 | DuckLakeDataFile() = default; 35 | DuckLakeDataFile(const DuckLakeDataFile &other); 36 | DuckLakeDataFile &operator=(const DuckLakeDataFile &); 37 | 38 | string file_name; 39 | idx_t row_count; 40 | idx_t file_size_bytes; 41 | idx_t footer_size; 42 | optional_idx partition_id; 43 | unique_ptr delete_file; 44 | map column_stats; 45 | vector partition_values; 46 | string encryption_key; 47 | }; 48 | 49 | } // namespace duckdb 50 | -------------------------------------------------------------------------------- /src/include/common/ducklake_encryption.hpp: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // DuckDB 3 | // 4 | // common/ducklake_encryption.hpp 5 | // 6 | // 7 | //===----------------------------------------------------------------------===// 8 | 9 | #pragma once 10 | 11 | #include "duckdb/common/common.hpp" 12 | 13 | namespace duckdb { 14 | 15 | enum class DuckLakeEncryption { AUTOMATIC, ENCRYPTED, UNENCRYPTED }; 16 | 17 | } // namespace duckdb 18 | -------------------------------------------------------------------------------- /src/include/common/ducklake_options.hpp: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // DuckDB 3 | // 4 | // common/ducklake_options.hpp 5 | // 6 | // 7 | //===----------------------------------------------------------------------===// 8 | 9 | #pragma once 10 | 11 | #include "duckdb/common/common.hpp" 12 | #include "duckdb/common/types.hpp" 13 | #include "duckdb/common/enums/access_mode.hpp" 14 | #include "common/ducklake_encryption.hpp" 15 | #include "duckdb/planner/tableref/bound_at_clause.hpp" 16 | #include "duckdb/common/optional_idx.hpp" 17 | 18 | namespace duckdb { 19 | 20 | struct DuckLakeOptions { 21 | string metadata_database; 22 | string metadata_path; 23 | string metadata_schema; 24 | string data_path; 25 | AccessMode access_mode = AccessMode::AUTOMATIC; 26 | DuckLakeEncryption encryption = DuckLakeEncryption::AUTOMATIC; 27 | idx_t data_inlining_row_limit = 0; 28 | unique_ptr at_clause; 29 | unordered_map metadata_parameters; 30 | unordered_map config_options; 31 | }; 32 | 33 | } // namespace duckdb 34 | -------------------------------------------------------------------------------- /src/include/common/ducklake_snapshot.hpp: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // DuckDB 3 | // 4 | // common/ducklake_snapshot.hpp 5 | // 6 | // 7 | //===----------------------------------------------------------------------===// 8 | 9 | #pragma once 10 | 11 | #include "duckdb/common/common.hpp" 12 | 13 | namespace duckdb { 14 | 15 | struct DuckLakeSnapshot { 16 | DuckLakeSnapshot(idx_t snapshot_id, idx_t schema_version, idx_t next_catalog_id, idx_t next_file_id) 17 | : snapshot_id(snapshot_id), schema_version(schema_version), next_catalog_id(next_catalog_id), 18 | next_file_id(next_file_id) { 19 | } 20 | 21 | idx_t snapshot_id; 22 | idx_t schema_version; 23 | idx_t next_catalog_id; 24 | idx_t next_file_id; 25 | }; 26 | 27 | } // namespace duckdb 28 | -------------------------------------------------------------------------------- /src/include/common/ducklake_types.hpp: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // DuckDB 3 | // 4 | // common/ducklake_types.hpp 5 | // 6 | // 7 | //===----------------------------------------------------------------------===// 8 | 9 | #pragma once 10 | 11 | #include "duckdb/common/common.hpp" 12 | #include "duckdb/common/types.hpp" 13 | 14 | namespace duckdb { 15 | 16 | class DuckLakeTypes { 17 | public: 18 | static LogicalType FromString(const string &str); 19 | static string ToString(const LogicalType &str); 20 | static void CheckSupportedType(const LogicalType &type); 21 | }; 22 | 23 | } // namespace duckdb 24 | -------------------------------------------------------------------------------- /src/include/common/ducklake_util.hpp: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // DuckDB 3 | // 4 | // common/ducklake_util.hpp 5 | // 6 | // 7 | //===----------------------------------------------------------------------===// 8 | 9 | #pragma once 10 | 11 | #include "duckdb/common/common.hpp" 12 | #include "duckdb/common/unordered_set.hpp" 13 | #include "duckdb/common/types/value.hpp" 14 | 15 | namespace duckdb { 16 | class FileSystem; 17 | 18 | struct ParsedCatalogEntry { 19 | string schema; 20 | string name; 21 | }; 22 | 23 | class DuckLakeUtil { 24 | public: 25 | static string ParseQuotedValue(const string &input, idx_t &pos); 26 | static string ToQuotedList(const vector &input, char list_separator = ','); 27 | static vector ParseQuotedList(const string &input, char list_separator = ','); 28 | static string SQLIdentifierToString(const string &text); 29 | static string SQLLiteralToString(const string &text); 30 | static string StatsToString(const string &text); 31 | static string ValueToSQL(const Value &val); 32 | 33 | static ParsedCatalogEntry ParseCatalogEntry(const string &input); 34 | static string JoinPath(FileSystem &fs, const string &a, const string &b); 35 | }; 36 | 37 | } // namespace duckdb 38 | -------------------------------------------------------------------------------- /src/include/common/local_change.hpp: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // DuckDB 3 | // 4 | // common/local_change.hpp 5 | // 6 | // 7 | //===----------------------------------------------------------------------===// 8 | 9 | #pragma once 10 | 11 | #include "duckdb/common/common.hpp" 12 | #include "common/index.hpp" 13 | 14 | namespace duckdb { 15 | 16 | enum class LocalChangeType { 17 | NONE, 18 | CREATED, 19 | RENAMED, 20 | SET_PARTITION_KEY, 21 | SET_COMMENT, 22 | SET_COLUMN_COMMENT, 23 | SET_NULL, 24 | DROP_NULL, 25 | RENAME_COLUMN, 26 | ADD_COLUMN, 27 | REMOVE_COLUMN, 28 | CHANGE_COLUMN_TYPE, 29 | SET_DEFAULT 30 | }; 31 | 32 | struct LocalChange { 33 | LocalChange(LocalChangeType type) // NOLINT: allow implicit conversion from LocalChangeType 34 | : type(type) { 35 | } 36 | 37 | LocalChangeType type; 38 | //! For operations that alter individual columns 39 | FieldIndex field_index; 40 | 41 | static LocalChange SetColumnComment(FieldIndex field_idx) { 42 | LocalChange result(LocalChangeType::SET_COLUMN_COMMENT); 43 | result.field_index = field_idx; 44 | return result; 45 | } 46 | static LocalChange SetNull(FieldIndex field_idx) { 47 | LocalChange result(LocalChangeType::SET_NULL); 48 | result.field_index = field_idx; 49 | return result; 50 | } 51 | static LocalChange DropNull(FieldIndex field_idx) { 52 | LocalChange result(LocalChangeType::DROP_NULL); 53 | result.field_index = field_idx; 54 | return result; 55 | } 56 | static LocalChange SetDefault(FieldIndex field_idx) { 57 | LocalChange result(LocalChangeType::SET_DEFAULT); 58 | result.field_index = field_idx; 59 | return result; 60 | } 61 | static LocalChange RenameColumn(FieldIndex field_idx) { 62 | LocalChange result(LocalChangeType::RENAME_COLUMN); 63 | result.field_index = field_idx; 64 | return result; 65 | } 66 | static LocalChange RemoveColumn(FieldIndex field_idx) { 67 | LocalChange result(LocalChangeType::REMOVE_COLUMN); 68 | result.field_index = field_idx; 69 | return result; 70 | } 71 | }; 72 | 73 | } // namespace duckdb 74 | -------------------------------------------------------------------------------- /src/include/ducklake_extension.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "duckdb.hpp" 4 | 5 | namespace duckdb { 6 | 7 | class DucklakeExtension : public Extension { 8 | public: 9 | void Load(DuckDB &db) override; 10 | std::string Name() override; 11 | std::string Version() const override; 12 | }; 13 | 14 | } // namespace duckdb 15 | -------------------------------------------------------------------------------- /src/include/functions/ducklake_table_functions.hpp: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // DuckDB 3 | // 4 | // functions/ducklake_table_functions.hpp 5 | // 6 | // 7 | //===----------------------------------------------------------------------===// 8 | 9 | #pragma once 10 | 11 | #include "duckdb/function/table_function.hpp" 12 | #include "duckdb/parser/parsed_data/create_macro_info.hpp" 13 | #include "duckdb/function/function_set.hpp" 14 | 15 | namespace duckdb { 16 | class DuckLakeCatalog; 17 | struct DuckLakeSnapshotInfo; 18 | 19 | struct MetadataBindData : public TableFunctionData { 20 | MetadataBindData() { 21 | } 22 | 23 | vector> rows; 24 | }; 25 | 26 | class BaseMetadataFunction : public TableFunction { 27 | public: 28 | BaseMetadataFunction(string name, table_function_bind_t bind); 29 | 30 | static Catalog &GetCatalog(ClientContext &context, const Value &input); 31 | }; 32 | 33 | class DuckLakeSnapshotsFunction : public BaseMetadataFunction { 34 | public: 35 | DuckLakeSnapshotsFunction(); 36 | 37 | static void GetSnapshotTypes(vector &return_types, vector &names); 38 | static vector GetSnapshotValues(const DuckLakeSnapshotInfo &snapshot); 39 | }; 40 | 41 | class DuckLakeTableInfoFunction : public BaseMetadataFunction { 42 | public: 43 | DuckLakeTableInfoFunction(); 44 | }; 45 | 46 | class DuckLakeTableInsertionsFunction { 47 | public: 48 | static TableFunctionSet GetFunctions(); 49 | static unique_ptr GetDuckLakeTableChanges(); 50 | }; 51 | 52 | class DuckLakeTableDeletionsFunction { 53 | public: 54 | static TableFunctionSet GetFunctions(); 55 | DuckLakeTableDeletionsFunction(); 56 | }; 57 | 58 | class DuckLakeMergeAdjacentFilesFunction : public TableFunction { 59 | public: 60 | DuckLakeMergeAdjacentFilesFunction(); 61 | }; 62 | 63 | class DuckLakeCleanupOldFilesFunction : public TableFunction { 64 | public: 65 | DuckLakeCleanupOldFilesFunction(); 66 | }; 67 | 68 | class DuckLakeExpireSnapshotsFunction : public TableFunction { 69 | public: 70 | DuckLakeExpireSnapshotsFunction(); 71 | }; 72 | 73 | class DuckLakeSetOptionFunction : public TableFunction { 74 | public: 75 | DuckLakeSetOptionFunction(); 76 | }; 77 | 78 | } // namespace duckdb 79 | -------------------------------------------------------------------------------- /src/include/storage/ducklake_catalog_set.hpp: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // DuckDB 3 | // 4 | // storage/ducklake_catalog_set.hpp 5 | // 6 | // 7 | //===----------------------------------------------------------------------===// 8 | 9 | #pragma once 10 | 11 | #include "duckdb/catalog/catalog.hpp" 12 | #include "duckdb/common/case_insensitive_map.hpp" 13 | #include "duckdb/catalog/catalog_entry.hpp" 14 | #include "common/ducklake_snapshot.hpp" 15 | #include "common/index.hpp" 16 | 17 | namespace duckdb { 18 | class DuckLakeTransaction; 19 | class DuckLakeSchemaEntry; 20 | 21 | using ducklake_entries_map_t = case_insensitive_map_t>; 22 | 23 | //! The DuckLakeCatalogSet contains a set of catalog entries for a given schema version of the DuckLake 24 | //! Note that we don't need any locks to access this - the catalog set is constant for a given snapshot 25 | class DuckLakeCatalogSet { 26 | public: 27 | DuckLakeCatalogSet(); 28 | DuckLakeCatalogSet(ducklake_entries_map_t catalog_entries_p); 29 | 30 | void CreateEntry(unique_ptr entry); 31 | optional_ptr GetEntry(const string &name); 32 | unique_ptr DropEntry(const string &name); 33 | optional_ptr GetEntryById(TableIndex index); 34 | void AddEntry(DuckLakeSchemaEntry &schema, TableIndex id, unique_ptr entry); 35 | 36 | template 37 | optional_ptr GetEntry(const string &name) { 38 | auto entry = GetEntry(name); 39 | if (!entry) { 40 | return nullptr; 41 | } 42 | return entry->Cast(); 43 | } 44 | 45 | const ducklake_entries_map_t &GetEntries() { 46 | return catalog_entries; 47 | } 48 | 49 | private: 50 | ducklake_entries_map_t catalog_entries; 51 | map> table_entry_map; 52 | }; 53 | 54 | } // namespace duckdb 55 | -------------------------------------------------------------------------------- /src/include/storage/ducklake_compaction.hpp: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // DuckDB 3 | // 4 | // storage/ducklake_compaction.hpp 5 | // 6 | // 7 | //===----------------------------------------------------------------------===// 8 | 9 | #pragma once 10 | 11 | #include "duckdb/execution/operator/persistent/physical_copy_to_file.hpp" 12 | 13 | #include "duckdb/execution/physical_operator.hpp" 14 | #include "duckdb/common/index_vector.hpp" 15 | #include "storage/ducklake_stats.hpp" 16 | #include "storage/ducklake_metadata_info.hpp" 17 | 18 | namespace duckdb { 19 | class DuckLakeTableEntry; 20 | 21 | class DuckLakeCompaction : public PhysicalOperator { 22 | public: 23 | DuckLakeCompaction(const vector &types, DuckLakeTableEntry &table, 24 | vector source_files_p, string encryption_key, 25 | optional_idx partition_id, vector partition_values, PhysicalOperator &child); 26 | 27 | DuckLakeTableEntry &table; 28 | vector source_files; 29 | string encryption_key; 30 | optional_idx partition_id; 31 | vector partition_values; 32 | 33 | public: 34 | // // Source interface 35 | SourceResultType GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const override; 36 | 37 | bool IsSource() const override { 38 | return true; 39 | } 40 | 41 | public: 42 | // Sink interface 43 | SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override; 44 | SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context, 45 | OperatorSinkFinalizeInput &input) const override; 46 | unique_ptr GetGlobalSinkState(ClientContext &context) const override; 47 | 48 | bool IsSink() const override { 49 | return true; 50 | } 51 | 52 | bool ParallelSink() const override { 53 | return false; 54 | } 55 | 56 | string GetName() const override; 57 | }; 58 | 59 | } // namespace duckdb 60 | -------------------------------------------------------------------------------- /src/include/storage/ducklake_delete_filter.hpp: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // DuckDB 3 | // 4 | // storage/ducklake_delete_filter.hpp 5 | // 6 | // 7 | //===----------------------------------------------------------------------===// 8 | 9 | #pragma once 10 | 11 | #include "duckdb/common/multi_file/multi_file_reader.hpp" 12 | #include "storage/ducklake_metadata_info.hpp" 13 | 14 | namespace duckdb { 15 | 16 | struct DuckLakeDeleteData { 17 | vector deleted_rows; 18 | 19 | idx_t Filter(row_t start_row_index, idx_t count, SelectionVector &result_sel) const; 20 | }; 21 | 22 | class DuckLakeDeleteFilter : public DeleteFilter { 23 | public: 24 | DuckLakeDeleteFilter(); 25 | 26 | shared_ptr delete_data; 27 | optional_idx max_row_count; 28 | 29 | idx_t Filter(row_t start_row_index, idx_t count, SelectionVector &result_sel) override; 30 | void Initialize(ClientContext &context, const DuckLakeFileData &delete_file); 31 | void Initialize(const DuckLakeInlinedDataDeletes &inlined_deletes); 32 | void Initialize(ClientContext &context, const DuckLakeDeleteScanEntry &delete_scan); 33 | void SetMaxRowCount(idx_t max_row_count); 34 | 35 | private: 36 | static vector ScanDeleteFile(ClientContext &context, const DuckLakeFileData &delete_file); 37 | }; 38 | 39 | } // namespace duckdb 40 | -------------------------------------------------------------------------------- /src/include/storage/ducklake_initializer.hpp: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // DuckDB 3 | // 4 | // storage/ducklake_initializer.hpp 5 | // 6 | // 7 | //===----------------------------------------------------------------------===// 8 | 9 | #pragma once 10 | 11 | #include "storage/ducklake_catalog.hpp" 12 | #include "duckdb/common/types/data_chunk.hpp" 13 | #include "duckdb/main/connection.hpp" 14 | 15 | namespace duckdb { 16 | class DuckLakeTransaction; 17 | 18 | class DuckLakeInitializer { 19 | public: 20 | DuckLakeInitializer(ClientContext &context, DuckLakeCatalog &catalog, DuckLakeOptions &options); 21 | 22 | public: 23 | void Initialize(); 24 | 25 | private: 26 | void InitializeNewDuckLake(DuckLakeTransaction &transaction, bool has_explicit_schema); 27 | void LoadExistingDuckLake(DuckLakeTransaction &transaction); 28 | void InitializeDataPath(); 29 | string GetAttachOptions(); 30 | void CheckAndAutoloadedRequiredExtension(const string &pattern); 31 | 32 | private: 33 | ClientContext &context; 34 | DuckLakeCatalog &catalog; 35 | DuckLakeOptions &options; 36 | }; 37 | 38 | } // namespace duckdb 39 | -------------------------------------------------------------------------------- /src/include/storage/ducklake_inline_data.hpp: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // DuckDB 3 | // 4 | // storage/ducklake_inline_data.hpp 5 | // 6 | // 7 | //===----------------------------------------------------------------------===// 8 | 9 | #pragma once 10 | 11 | #include "duckdb/execution/physical_operator.hpp" 12 | #include "duckdb/common/index_vector.hpp" 13 | #include "storage/ducklake_stats.hpp" 14 | #include "common/ducklake_data_file.hpp" 15 | 16 | namespace duckdb { 17 | class DuckLakeInsert; 18 | 19 | class DuckLakeInlineData : public PhysicalOperator { 20 | public: 21 | static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::EXTENSION; 22 | 23 | public: 24 | DuckLakeInlineData(PhysicalOperator &child, idx_t inline_row_limit); 25 | 26 | idx_t inline_row_limit; 27 | optional_ptr insert; 28 | 29 | public: 30 | unique_ptr GetOperatorState(ExecutionContext &context) const override; 31 | unique_ptr GetGlobalOperatorState(ClientContext &context) const override; 32 | OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk, 33 | GlobalOperatorState &gstate, OperatorState &state) const override; 34 | OperatorFinalizeResultType FinalExecute(ExecutionContext &context, DataChunk &chunk, GlobalOperatorState &gstate, 35 | OperatorState &state) const override; 36 | OperatorFinalResultType OperatorFinalize(Pipeline &pipeline, Event &event, ClientContext &context, 37 | OperatorFinalizeInput &input) const override; 38 | 39 | bool RequiresFinalExecute() const override { 40 | return true; 41 | } 42 | bool RequiresOperatorFinalize() const override { 43 | return true; 44 | } 45 | bool ParallelOperator() const override { 46 | return true; 47 | } 48 | string GetName() const override; 49 | }; 50 | 51 | } // namespace duckdb 52 | -------------------------------------------------------------------------------- /src/include/storage/ducklake_inlined_data.hpp: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // DuckDB 3 | // 4 | // storage/ducklake_inlined_data.hpp 5 | // 6 | // 7 | //===----------------------------------------------------------------------===// 8 | 9 | #pragma once 10 | 11 | #include "duckdb/common/types/column/column_data_collection.hpp" 12 | #include "storage/ducklake_stats.hpp" 13 | #include "common/index.hpp" 14 | 15 | namespace duckdb { 16 | 17 | struct DuckLakeInlinedData { 18 | unique_ptr data; 19 | map column_stats; 20 | }; 21 | 22 | struct DuckLakeInlinedDataDeletes { 23 | set rows; 24 | }; 25 | 26 | } // namespace duckdb 27 | -------------------------------------------------------------------------------- /src/include/storage/ducklake_inlined_data_reader.hpp: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // DuckDB 3 | // 4 | // storage/ducklake_inlined_data_reader.hpp 5 | // 6 | // 7 | //===----------------------------------------------------------------------===// 8 | 9 | #pragma once 10 | 11 | #include "duckdb/common/multi_file/base_file_reader.hpp" 12 | #include "storage/ducklake_inlined_data.hpp" 13 | #include "common/ducklake_snapshot.hpp" 14 | 15 | namespace duckdb { 16 | class DuckLakeFieldData; 17 | struct DuckLakeFunctionInfo; 18 | 19 | class DuckLakeInlinedDataReader : public BaseFileReader { 20 | public: 21 | //! Initialize an inlined data reader over a set of data stored within a table in the metadata catalog 22 | DuckLakeInlinedDataReader(DuckLakeFunctionInfo &read_info, const OpenFileInfo &info, string table_name, 23 | vector columns); 24 | //! Initialize an inlined data reader over a set of data 25 | DuckLakeInlinedDataReader(DuckLakeFunctionInfo &read_info, const OpenFileInfo &info, 26 | shared_ptr data, vector columns); 27 | 28 | public: 29 | bool TryInitializeScan(ClientContext &context, GlobalTableFunctionState &gstate, 30 | LocalTableFunctionState &lstate) override; 31 | void Scan(ClientContext &context, GlobalTableFunctionState &global_state, LocalTableFunctionState &local_state, 32 | DataChunk &chunk) override; 33 | 34 | string GetReaderType() const override; 35 | 36 | void AddVirtualColumn(column_t virtual_column_id) override; 37 | 38 | private: 39 | mutex lock; 40 | DuckLakeFunctionInfo &read_info; 41 | string table_name; 42 | shared_ptr data; 43 | bool initialized_scan = false; 44 | vector is_virtual; 45 | int64_t file_row_number = 0; 46 | vector scan_column_ids; 47 | ColumnDataScanState state; 48 | DataChunk scan_chunk; 49 | }; 50 | 51 | } // namespace duckdb 52 | -------------------------------------------------------------------------------- /src/include/storage/ducklake_partition_data.hpp: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // DuckDB 3 | // 4 | // storage/ducklake_partition_data.hpp 5 | // 6 | // 7 | //===----------------------------------------------------------------------===// 8 | 9 | #pragma once 10 | 11 | #include "duckdb/common/case_insensitive_map.hpp" 12 | #include "duckdb/common/common.hpp" 13 | 14 | namespace duckdb { 15 | class BaseStatistics; 16 | 17 | enum class DuckLakeTransformType { 18 | IDENTITY, 19 | }; 20 | 21 | struct DuckLakeTransform { 22 | DuckLakeTransformType type; 23 | }; 24 | 25 | struct DuckLakePartitionField { 26 | idx_t partition_key_index = 0; 27 | idx_t column_id; 28 | DuckLakeTransform transform; 29 | }; 30 | 31 | struct DuckLakePartition { 32 | idx_t partition_id = 0; 33 | vector fields; 34 | }; 35 | 36 | } // namespace duckdb 37 | -------------------------------------------------------------------------------- /src/include/storage/ducklake_scan.hpp: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // DuckDB 3 | // 4 | // storage/ducklake_scan.hpp 5 | // 6 | // 7 | //===----------------------------------------------------------------------===// 8 | 9 | #pragma once 10 | 11 | #include "duckdb/common/common.hpp" 12 | #include "duckdb/function/table_function.hpp" 13 | #include "common/ducklake_snapshot.hpp" 14 | #include "common/index.hpp" 15 | 16 | namespace duckdb { 17 | class DuckLakeMultiFileList; 18 | class DuckLakeTableEntry; 19 | class DuckLakeTransaction; 20 | 21 | class DuckLakeFunctions { 22 | public: 23 | //! Table Functions 24 | static TableFunction GetDuckLakeScanFunction(DatabaseInstance &instance); 25 | 26 | static unique_ptr BindDuckLakeScan(ClientContext &context, TableFunction &function); 27 | 28 | static CopyFunctionCatalogEntry &GetCopyFunction(ClientContext &context, const string &name); 29 | }; 30 | 31 | enum class DuckLakeScanType { SCAN_TABLE, SCAN_INSERTIONS, SCAN_DELETIONS }; 32 | 33 | struct DuckLakeFunctionInfo : public TableFunctionInfo { 34 | DuckLakeFunctionInfo(DuckLakeTableEntry &table, DuckLakeTransaction &transaction, DuckLakeSnapshot snapshot); 35 | 36 | DuckLakeTableEntry &table; 37 | weak_ptr transaction; 38 | string table_name; 39 | vector column_names; 40 | vector column_types; 41 | DuckLakeSnapshot snapshot; 42 | TableIndex table_id; 43 | DuckLakeScanType scan_type = DuckLakeScanType::SCAN_TABLE; 44 | //! Start snapshot - only set for DuckLakeScanType::SCAN_INSERTIONS and DuckLakeScanType::SCAN_DELETIONS 45 | unique_ptr start_snapshot; 46 | 47 | shared_ptr GetTransaction(); 48 | }; 49 | 50 | } // namespace duckdb 51 | -------------------------------------------------------------------------------- /src/include/storage/ducklake_stats.hpp: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // DuckDB 3 | // 4 | // storage/ducklake_stats.hpp 5 | // 6 | // 7 | //===----------------------------------------------------------------------===// 8 | 9 | #pragma once 10 | 11 | #include "duckdb/common/case_insensitive_map.hpp" 12 | #include "duckdb/common/common.hpp" 13 | #include "duckdb/common/optional_idx.hpp" 14 | #include "common/index.hpp" 15 | 16 | namespace duckdb { 17 | class BaseStatistics; 18 | 19 | struct DuckLakeColumnStats { 20 | explicit DuckLakeColumnStats(LogicalType type_p) : type(std::move(type_p)) { 21 | } 22 | 23 | LogicalType type; 24 | string min; 25 | string max; 26 | idx_t null_count = 0; 27 | idx_t column_size_bytes = 0; 28 | bool contains_nan = false; 29 | bool has_null_count = false; 30 | bool has_min = false; 31 | bool has_max = false; 32 | bool any_valid = true; 33 | bool has_contains_nan = false; 34 | 35 | public: 36 | unique_ptr ToStats() const; 37 | void MergeStats(const DuckLakeColumnStats &new_stats); 38 | 39 | private: 40 | unique_ptr CreateNumericStats() const; 41 | unique_ptr CreateStringStats() const; 42 | }; 43 | 44 | //! These are the global, table-wide stats 45 | struct DuckLakeTableStats { 46 | idx_t record_count = 0; 47 | idx_t table_size_bytes = 0; 48 | idx_t next_row_id = 0; 49 | map column_stats; 50 | 51 | void MergeStats(FieldIndex col_id, const DuckLakeColumnStats &file_stats); 52 | }; 53 | 54 | struct DuckLakeStats { 55 | map> table_stats; 56 | }; 57 | 58 | } // namespace duckdb 59 | -------------------------------------------------------------------------------- /src/include/storage/ducklake_storage.hpp: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // DuckDB 3 | // 4 | // storage/ducklake_storage.hpp 5 | // 6 | // 7 | //===----------------------------------------------------------------------===// 8 | 9 | #pragma once 10 | 11 | #include "duckdb/storage/storage_extension.hpp" 12 | 13 | namespace duckdb { 14 | 15 | class DuckLakeStorageExtension : public StorageExtension { 16 | public: 17 | DuckLakeStorageExtension(); 18 | }; 19 | 20 | } // namespace duckdb 21 | -------------------------------------------------------------------------------- /src/include/storage/ducklake_transaction_changes.hpp: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // DuckDB 3 | // 4 | // storage/ducklake_transaction_changes.hpp 5 | // 6 | // 7 | //===----------------------------------------------------------------------===// 8 | 9 | #pragma once 10 | 11 | #include "duckdb/transaction/transaction.hpp" 12 | #include "duckdb/common/case_insensitive_map.hpp" 13 | #include "common/ducklake_snapshot.hpp" 14 | #include "common/index.hpp" 15 | #include "duckdb/common/types/value_map.hpp" 16 | 17 | namespace duckdb { 18 | 19 | struct SnapshotChangeInformation { 20 | case_insensitive_set_t created_schemas; 21 | set dropped_schemas; 22 | case_insensitive_map_t> created_tables; 23 | set altered_tables; 24 | set altered_views; 25 | set dropped_tables; 26 | set dropped_views; 27 | set inserted_tables; 28 | set tables_deleted_from; 29 | set tables_compacted; 30 | 31 | static SnapshotChangeInformation ParseChangesMade(const string &changes_made); 32 | }; 33 | 34 | } // namespace duckdb 35 | -------------------------------------------------------------------------------- /src/include/storage/ducklake_transaction_manager.hpp: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // DuckDB 3 | // 4 | // storageducklake_transaction_manager.hpp 5 | // 6 | // 7 | //===----------------------------------------------------------------------===// 8 | 9 | #pragma once 10 | 11 | #include "duckdb/transaction/transaction_manager.hpp" 12 | #include "storage/ducklake_transaction.hpp" 13 | #include "storage/ducklake_catalog.hpp" 14 | 15 | namespace duckdb { 16 | 17 | class DuckLakeTransactionManager : public TransactionManager { 18 | public: 19 | DuckLakeTransactionManager(AttachedDatabase &db_p, DuckLakeCatalog &ducklake_catalog); 20 | 21 | Transaction &StartTransaction(ClientContext &context) override; 22 | ErrorData CommitTransaction(ClientContext &context, Transaction &transaction) override; 23 | void RollbackTransaction(Transaction &transaction) override; 24 | 25 | void Checkpoint(ClientContext &context, bool force = false) override; 26 | 27 | private: 28 | DuckLakeCatalog &ducklake_catalog; 29 | mutex transaction_lock; 30 | reference_map_t> transactions; 31 | }; 32 | 33 | } // namespace duckdb 34 | -------------------------------------------------------------------------------- /src/include/storage/ducklake_update.hpp: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // DuckDB 3 | // 4 | // storage/ducklake_update.hpp 5 | // 6 | // 7 | //===----------------------------------------------------------------------===// 8 | 9 | #pragma once 10 | 11 | #include "storage/ducklake_insert.hpp" 12 | 13 | namespace duckdb { 14 | 15 | class DuckLakeUpdate : public PhysicalOperator { 16 | public: 17 | DuckLakeUpdate(DuckLakeTableEntry &table, vector columns, PhysicalOperator &child, 18 | PhysicalOperator ©_op, PhysicalOperator &delete_op, PhysicalOperator &insert_op); 19 | 20 | //! The table to update 21 | DuckLakeTableEntry &table; 22 | //! The order of to-be-inserted columns 23 | vector columns; 24 | //! The copy operator for writing new data to files 25 | PhysicalOperator ©_op; 26 | //! The delete operator for deleting the old data 27 | PhysicalOperator &delete_op; 28 | //! The (final) insert operator that registers inserted data 29 | PhysicalOperator &insert_op; 30 | 31 | public: 32 | // // Source interface 33 | SourceResultType GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const override; 34 | 35 | bool IsSource() const override { 36 | return true; 37 | } 38 | 39 | public: 40 | // Sink interface 41 | SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override; 42 | SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override; 43 | SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context, 44 | OperatorSinkFinalizeInput &input) const override; 45 | unique_ptr GetGlobalSinkState(ClientContext &context) const override; 46 | unique_ptr GetLocalSinkState(ExecutionContext &context) const override; 47 | 48 | bool IsSink() const override { 49 | return true; 50 | } 51 | 52 | bool ParallelSink() const override { 53 | return true; 54 | } 55 | 56 | string GetName() const override; 57 | InsertionOrderPreservingMap ParamsToString() const override; 58 | }; 59 | 60 | } // namespace duckdb 61 | -------------------------------------------------------------------------------- /src/include/storage/ducklake_view_entry.hpp: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // DuckDB 3 | // 4 | // storage/ducklake_view_entry.hpp 5 | // 6 | // 7 | //===----------------------------------------------------------------------===// 8 | 9 | #pragma once 10 | 11 | #include "duckdb/catalog/catalog_entry/view_catalog_entry.hpp" 12 | #include "duckdb/common/mutex.hpp" 13 | #include "common/index.hpp" 14 | #include "common/local_change.hpp" 15 | 16 | namespace duckdb { 17 | struct SetCommentInfo; 18 | class DuckLakeTransaction; 19 | 20 | class DuckLakeViewEntry : public ViewCatalogEntry { 21 | public: 22 | DuckLakeViewEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateViewInfo &info, TableIndex view_id, 23 | string view_uuid, string query_sql, LocalChange local_change); 24 | 25 | public: 26 | TableIndex GetViewId() const { 27 | return view_id; 28 | } 29 | const string &GetViewUUID() const { 30 | return view_uuid; 31 | } 32 | void SetViewId(TableIndex new_view_id) { 33 | view_id = new_view_id; 34 | } 35 | bool IsTransactionLocal() const { 36 | return local_change.type != LocalChangeType::NONE; 37 | } 38 | LocalChange GetLocalChange() const { 39 | return local_change; 40 | } 41 | 42 | public: 43 | unique_ptr AlterEntry(ClientContext &context, AlterInfo &info) override; 44 | unique_ptr Copy(ClientContext &context) const override; 45 | 46 | const SelectStatement &GetQuery() override; 47 | bool HasTypes() const override { 48 | return false; 49 | } 50 | unique_ptr GetInfo() const override; 51 | string ToSQL() const override; 52 | 53 | string GetQuerySQL(); 54 | 55 | public: 56 | // ALTER VIEW 57 | DuckLakeViewEntry(DuckLakeViewEntry &parent, CreateViewInfo &info, LocalChange local_change); 58 | 59 | private: 60 | unique_ptr ParseSelectStatement() const; 61 | 62 | private: 63 | mutex parse_lock; 64 | TableIndex view_id; 65 | string view_uuid; 66 | string query_sql; 67 | LocalChange local_change; 68 | }; 69 | 70 | } // namespace duckdb 71 | -------------------------------------------------------------------------------- /src/storage/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library( 2 | ducklake_storage OBJECT 3 | ducklake_catalog.cpp 4 | ducklake_checkpoint.cpp 5 | ducklake_default_functions.cpp 6 | ducklake_delete_filter.cpp 7 | ducklake_field_data.cpp 8 | ducklake_inline_data.cpp 9 | ducklake_inlined_data_reader.cpp 10 | ducklake_insert.cpp 11 | ducklake_schema_entry.cpp 12 | ducklake_transaction_manager.cpp 13 | ducklake_catalog_set.cpp 14 | ducklake_metadata_manager.cpp 15 | ducklake_multi_file_list.cpp 16 | ducklake_storage.cpp 17 | ducklake_delete.cpp 18 | ducklake_multi_file_reader.cpp 19 | ducklake_stats.cpp 20 | ducklake_table_entry.cpp 21 | ducklake_initializer.cpp 22 | ducklake_autoload_helper.cpp 23 | ducklake_update.cpp 24 | ducklake_scan.cpp 25 | ducklake_transaction.cpp 26 | ducklake_view_entry.cpp 27 | ducklake_transaction_changes.cpp) 28 | set(ALL_OBJECT_FILES 29 | ${ALL_OBJECT_FILES} $ 30 | PARENT_SCOPE) 31 | -------------------------------------------------------------------------------- /src/storage/ducklake_autoload_helper.cpp: -------------------------------------------------------------------------------- 1 | #include "duckdb/main/extension_entries.hpp" 2 | #include "duckdb/main/attached_database.hpp" 3 | #include "duckdb/main/database.hpp" 4 | #include "duckdb/main/extension_helper.hpp" 5 | 6 | #include "storage/ducklake_initializer.hpp" 7 | 8 | namespace duckdb { 9 | 10 | static string LookupExtensionForPattern(const string &pattern) { 11 | for (const auto &entry : EXTENSION_FILE_PREFIXES) { 12 | if (StringUtil::StartsWith(pattern, entry.name)) { 13 | return entry.extension; 14 | } 15 | } 16 | return ""; 17 | } 18 | 19 | void DuckLakeInitializer::CheckAndAutoloadedRequiredExtension(const string &pattern) { 20 | // This functions will: 21 | // 1. Check if a known extension pattern matches the start of the data_path 22 | // 2. If so, either load the required extension or throw a relevant error message 23 | 24 | // FIXME: This function is currently a copy of the logic at FileSystem::GlobFiles in duckdb/duckdb (version 1.3.0) 25 | // repository Proper solution would be offer this functionality as part of DuckDB C++ API, so this file can be 26 | // simplified reducing the risk of misalignment between the two codebases 27 | 28 | string required_extension = LookupExtensionForPattern(pattern); 29 | if (!required_extension.empty() && !context.db->ExtensionIsLoaded(required_extension)) { 30 | auto &dbconfig = DBConfig::GetConfig(context); 31 | if (!ExtensionHelper::CanAutoloadExtension(required_extension) || !dbconfig.options.autoload_known_extensions) { 32 | auto error_message = 33 | "Data path " + pattern + " requires the extension " + required_extension + " to be loaded"; 34 | error_message = 35 | ExtensionHelper::AddExtensionInstallHintToErrorMsg(context, error_message, required_extension); 36 | throw MissingExtensionException(error_message); 37 | } 38 | // an extension is required to read this file, but it is not loaded - try to load it 39 | ExtensionHelper::AutoLoadExtension(context, required_extension); 40 | // success! glob again 41 | // check the extension is loaded just in case to prevent an infinite loop here 42 | if (!context.db->ExtensionIsLoaded(required_extension)) { 43 | throw InternalException("Extension load \"%s\" did not throw but somehow the extension was not loaded", 44 | required_extension); 45 | } 46 | } 47 | } 48 | 49 | } // namespace duckdb 50 | -------------------------------------------------------------------------------- /src/storage/ducklake_catalog_set.cpp: -------------------------------------------------------------------------------- 1 | #include "storage/ducklake_catalog_set.hpp" 2 | #include "storage/ducklake_schema_entry.hpp" 3 | #include "storage/ducklake_transaction.hpp" 4 | 5 | namespace duckdb { 6 | 7 | DuckLakeCatalogSet::DuckLakeCatalogSet() { 8 | } 9 | DuckLakeCatalogSet::DuckLakeCatalogSet(ducklake_entries_map_t catalog_entries_p) 10 | : catalog_entries(std::move(catalog_entries_p)) { 11 | } 12 | 13 | void DuckLakeCatalogSet::CreateEntry(unique_ptr catalog_entry) { 14 | auto name = catalog_entry->name; 15 | auto entry = catalog_entries.find(name); 16 | if (entry != catalog_entries.end()) { 17 | catalog_entry->SetChild(std::move(entry->second)); 18 | } 19 | catalog_entries[std::move(name)] = std::move(catalog_entry); 20 | } 21 | 22 | unique_ptr DuckLakeCatalogSet::DropEntry(const string &name) { 23 | auto entry = catalog_entries.find(name); 24 | auto catalog_entry = std::move(entry->second); 25 | catalog_entries.erase(entry); 26 | return catalog_entry; 27 | } 28 | 29 | optional_ptr DuckLakeCatalogSet::GetEntry(const string &name) { 30 | auto entry = catalog_entries.find(name); 31 | if (entry == catalog_entries.end()) { 32 | return nullptr; 33 | } 34 | return entry->second.get(); 35 | } 36 | 37 | optional_ptr DuckLakeCatalogSet::GetEntryById(TableIndex index) { 38 | auto entry = table_entry_map.find(index); 39 | if (entry == table_entry_map.end()) { 40 | return nullptr; 41 | } 42 | D_ASSERT(entry->second.get().type == CatalogType::TABLE_ENTRY); 43 | return entry->second.get(); 44 | } 45 | 46 | void DuckLakeCatalogSet::AddEntry(DuckLakeSchemaEntry &schema, TableIndex id, unique_ptr entry) { 47 | auto catalog_type = entry->type; 48 | table_entry_map.insert(make_pair(id, reference(*entry))); 49 | schema.AddEntry(catalog_type, std::move(entry)); 50 | } 51 | 52 | } // namespace duckdb 53 | -------------------------------------------------------------------------------- /src/storage/ducklake_checkpoint.cpp: -------------------------------------------------------------------------------- 1 | #include "storage/ducklake_transaction_manager.hpp" 2 | #include "storage/ducklake_catalog.hpp" 3 | #include "storage/ducklake_schema_entry.hpp" 4 | #include "storage/ducklake_table_entry.hpp" 5 | 6 | namespace duckdb { 7 | 8 | void DuckLakeTransactionManager::Checkpoint(ClientContext &context, bool force) { 9 | throw NotImplementedException("CHECKPOINT not supported for DuckLake yet"); 10 | } 11 | 12 | } // namespace duckdb 13 | -------------------------------------------------------------------------------- /src/storage/ducklake_default_functions.cpp: -------------------------------------------------------------------------------- 1 | #include "duckdb/catalog/default/default_table_functions.hpp" 2 | #include "storage/ducklake_schema_entry.hpp" 3 | #include "duckdb/catalog/catalog_entry/table_macro_catalog_entry.hpp" 4 | 5 | namespace duckdb { 6 | 7 | // clang-format off 8 | static const DefaultTableMacro ducklake_table_macros[] = { 9 | {DEFAULT_SCHEMA, "merge_adjacent_files", {nullptr}, {{nullptr, nullptr}}, "FROM ducklake_merge_adjacent_files({CATALOG})"}, 10 | {DEFAULT_SCHEMA, "set_option", {"option", "value", nullptr}, {{nullptr, nullptr}}, "FROM ducklake_set_option({CATALOG}, option, value)"}, 11 | {DEFAULT_SCHEMA, "snapshots", {nullptr}, {{nullptr, nullptr}}, "FROM ducklake_snapshots({CATALOG})"}, 12 | {DEFAULT_SCHEMA, "table_info", {nullptr}, {{nullptr, nullptr}}, "FROM ducklake_table_info({CATALOG})"}, 13 | {DEFAULT_SCHEMA, "table_changes", {"table_name", "start_snapshot", "end_snapshot", nullptr}, {{nullptr, nullptr}}, "FROM ducklake_table_changes({CATALOG}, {SCHEMA}, table_name, start_snapshot, end_snapshot)"}, 14 | {nullptr, nullptr, {nullptr}, {{nullptr, nullptr}}, nullptr} 15 | }; 16 | // clang-format on 17 | 18 | optional_ptr DuckLakeSchemaEntry::LoadBuiltInFunction(DefaultTableMacro macro) { 19 | string macro_def = macro.macro; 20 | macro_def = StringUtil::Replace(macro_def, "{CATALOG}", KeywordHelper::WriteQuoted(catalog.GetName(), '\'')); 21 | macro_def = StringUtil::Replace(macro_def, "{SCHEMA}", KeywordHelper::WriteQuoted(name, '\'')); 22 | macro.macro = macro_def.c_str(); 23 | auto info = DefaultTableFunctionGenerator::CreateTableMacroInfo(macro); 24 | auto table_macro = 25 | make_uniq_base(catalog, *this, info->Cast()); 26 | auto result = table_macro.get(); 27 | default_function_map.emplace(macro.name, std::move(table_macro)); 28 | return result; 29 | } 30 | 31 | optional_ptr DuckLakeSchemaEntry::TryLoadBuiltInFunction(const string &entry_name) { 32 | lock_guard guard(default_function_lock); 33 | auto entry = default_function_map.find(entry_name); 34 | if (entry != default_function_map.end()) { 35 | return entry->second.get(); 36 | } 37 | for (idx_t index = 0; ducklake_table_macros[index].name != nullptr; index++) { 38 | if (StringUtil::CIEquals(ducklake_table_macros[index].name, entry_name)) { 39 | return LoadBuiltInFunction(ducklake_table_macros[index]); 40 | } 41 | } 42 | return nullptr; 43 | } 44 | 45 | } // namespace duckdb 46 | -------------------------------------------------------------------------------- /src/storage/ducklake_transaction_manager.cpp: -------------------------------------------------------------------------------- 1 | #include "storage/ducklake_transaction_manager.hpp" 2 | 3 | namespace duckdb { 4 | 5 | DuckLakeTransactionManager::DuckLakeTransactionManager(AttachedDatabase &db_p, DuckLakeCatalog &ducklake_catalog) 6 | : TransactionManager(db_p), ducklake_catalog(ducklake_catalog) { 7 | } 8 | 9 | Transaction &DuckLakeTransactionManager::StartTransaction(ClientContext &context) { 10 | auto transaction = make_shared_ptr(ducklake_catalog, *this, context); 11 | transaction->Start(); 12 | auto &result = *transaction; 13 | lock_guard l(transaction_lock); 14 | transactions[result] = std::move(transaction); 15 | return result; 16 | } 17 | 18 | ErrorData DuckLakeTransactionManager::CommitTransaction(ClientContext &context, Transaction &transaction) { 19 | auto &ducklake_transaction = transaction.Cast(); 20 | try { 21 | ducklake_transaction.Commit(); 22 | } catch (std::exception &ex) { 23 | return ErrorData(ex); 24 | } 25 | lock_guard l(transaction_lock); 26 | transactions.erase(transaction); 27 | return ErrorData(); 28 | } 29 | 30 | void DuckLakeTransactionManager::RollbackTransaction(Transaction &transaction) { 31 | auto &ducklake_transaction = transaction.Cast(); 32 | ducklake_transaction.Rollback(); 33 | lock_guard l(transaction_lock); 34 | transactions.erase(transaction); 35 | } 36 | 37 | } // namespace duckdb 38 | -------------------------------------------------------------------------------- /test/README.md: -------------------------------------------------------------------------------- 1 | # Testing this extension 2 | This directory contains all the tests for this extension. The `sql` directory holds tests that are written as [SQLLogicTests](https://duckdb.org/dev/sqllogictest/intro.html). DuckDB aims to have most its tests in this format as SQL statements, so for the quack extension, this should probably be the goal too. 3 | 4 | The root makefile contains targets to build and run all of these tests. To run the SQLLogicTests: 5 | ```bash 6 | make test 7 | ``` 8 | or 9 | ```bash 10 | make test_debug 11 | ``` -------------------------------------------------------------------------------- /test/sql/alter/add_column.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/alter/add_column.test 2 | # description: test ducklake add columns 3 | # group: [alter] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_add_col.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_add_col_files') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test(col1 INTEGER); 14 | 15 | statement ok 16 | ALTER TABLE ducklake.test ADD COLUMN new_col2 INTEGER 17 | 18 | statement error 19 | ALTER TABLE ducklake.test ADD COLUMN new_col2 INTEGER 20 | ---- 21 | already exists 22 | 23 | statement ok 24 | ALTER TABLE ducklake.test ADD COLUMN IF NOT EXISTS new_col2 INTEGER 25 | 26 | statement ok 27 | INSERT INTO ducklake.test VALUES (1, 2), (NULL, 3); 28 | 29 | query II 30 | SELECT col1, new_col2 FROM ducklake.test 31 | ---- 32 | 1 2 33 | NULL 3 34 | 35 | statement ok 36 | ALTER TABLE ducklake.test ADD COLUMN new_col3 VARCHAR 37 | 38 | query IIIIII 39 | DESCRIBE ducklake.test 40 | ---- 41 | col1 INTEGER YES NULL NULL NULL 42 | new_col2 INTEGER YES NULL NULL NULL 43 | new_col3 VARCHAR YES NULL NULL NULL 44 | 45 | statement ok 46 | INSERT INTO ducklake.test VALUES (1, 2, 'hello'), (NULL, 3, 'world'); 47 | 48 | query III rowsort 49 | SELECT * FROM ducklake.test 50 | ---- 51 | 1 2 NULL 52 | 1 2 hello 53 | NULL 3 NULL 54 | NULL 3 world 55 | 56 | # filters 57 | query III rowsort 58 | SELECT * FROM ducklake.test WHERE new_col3='hello' 59 | ---- 60 | 1 2 hello 61 | -------------------------------------------------------------------------------- /test/sql/alter/add_column_nested.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/alter/add_column_nested.test 2 | # description: test ducklake add nested columns 3 | # group: [alter] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_add_nested_col.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_add_nested_col_files', METADATA_CATALOG 'xx') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test(col1 STRUCT(i INT, j INT)); 14 | 15 | statement ok 16 | INSERT INTO ducklake.test VALUES ({'i': 1, 'j': 2}) 17 | 18 | statement ok 19 | ALTER TABLE ducklake.test ADD COLUMN new_col2 INT[] 20 | 21 | statement ok 22 | INSERT INTO ducklake.test VALUES ({'i': 100, 'j': 200}, []) 23 | 24 | statement ok 25 | ALTER TABLE ducklake.test ADD COLUMN new_col3 STRUCT(k INT, v INT); 26 | 27 | statement ok 28 | INSERT INTO ducklake.test VALUES ({'i': 42, 'j': NULL}, [1, 2, 3], {'k': 1, 'v': 2}) 29 | 30 | query III 31 | SELECT * FROM ducklake.test 32 | ---- 33 | {'i': 1, 'j': 2} NULL NULL 34 | {'i': 100, 'j': 200} [] NULL 35 | {'i': 42, 'j': NULL} [1, 2, 3] {'k': 1, 'v': 2} 36 | -------------------------------------------------------------------------------- /test/sql/alter/add_column_transaction_local.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/alter/add_column_transaction_local.test 2 | # description: test ducklake add columns 3 | # group: [alter] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_add_col_tl.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_add_col_tl_files') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test(col1 INTEGER); 14 | 15 | statement ok 16 | BEGIN 17 | 18 | statement ok 19 | INSERT INTO ducklake.test VALUES (42); 20 | 21 | statement ok 22 | ALTER TABLE ducklake.test ADD COLUMN new_col2 INTEGER 23 | 24 | query II 25 | SELECT * FROM ducklake.test 26 | ---- 27 | 42 NULL 28 | -------------------------------------------------------------------------------- /test/sql/alter/drop_column.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/alter/drop_column.test 2 | # description: test ducklake drop columns 3 | # group: [alter] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_drop_col.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_drop_col_files') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test(col1 INTEGER, col2 INTEGER, col3 INTEGER); 14 | 15 | statement ok 16 | ALTER TABLE ducklake.test DROP COLUMN col3 17 | 18 | statement ok 19 | ALTER TABLE ducklake.test DROP COLUMN col2 20 | 21 | statement error 22 | ALTER TABLE ducklake.test DROP COLUMN col2 23 | ---- 24 | does not exist 25 | 26 | statement ok 27 | ALTER TABLE ducklake.test DROP COLUMN IF EXISTS col2 28 | 29 | statement error 30 | ALTER TABLE ducklake.test DROP COLUMN col1 31 | ---- 32 | only has one column remaining 33 | 34 | statement ok 35 | INSERT INTO ducklake.test VALUES (1), (2), (3); 36 | 37 | query I 38 | FROM ducklake.test 39 | ---- 40 | 1 41 | 2 42 | 3 43 | 44 | statement error 45 | ALTER TABLE ducklake.test DROP COLUMN nonexistent_column 46 | ---- 47 | nonexistent_column 48 | -------------------------------------------------------------------------------- /test/sql/alter/drop_column_nested.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/alter/drop_column_nested.test 2 | # description: test ducklake dropping nested columns 3 | # group: [alter] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_drop_nested_col.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_drop_nested_col_files', METADATA_CATALOG 'xx') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test(col1 STRUCT(i INT, j INT), col2 STRUCT(k INT, v INT), col3 INT[]); 14 | 15 | statement ok 16 | ALTER TABLE ducklake.test DROP COLUMN col2 17 | 18 | statement ok 19 | ALTER TABLE ducklake.test ADD COLUMN new_col2 INT[] 20 | 21 | statement ok 22 | ALTER TABLE ducklake.test DROP COLUMN col3 23 | 24 | statement ok 25 | ALTER TABLE ducklake.test ADD COLUMN new_col3 STRUCT(k INT, v INT); 26 | 27 | statement ok 28 | INSERT INTO ducklake.test VALUES ({'i': 42, 'j': NULL}, [1, 2, 3], {'k': 1, 'v': 2}) 29 | 30 | query III 31 | SELECT col1, new_col2, new_col3 FROM ducklake.test 32 | ---- 33 | {'i': 42, 'j': NULL} [1, 2, 3] {'k': 1, 'v': 2} 34 | -------------------------------------------------------------------------------- /test/sql/alter/mixed_alter.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/alter/mixed_alter.test 2 | # description: test ducklake mixed alter statements 3 | # group: [alter] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_mixed_alter.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_mixed_alter_files') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test(col1 INTEGER, col2 INTEGER, col3 INTEGER); 14 | 15 | statement ok 16 | INSERT INTO ducklake.test (col1, col2, col3) VALUES (1, 2, 3); 17 | 18 | statement ok 19 | ALTER TABLE ducklake.test DROP COLUMN col2 20 | 21 | statement ok 22 | INSERT INTO ducklake.test (col1, col3) VALUES (10, 20); 23 | 24 | statement ok 25 | ALTER TABLE ducklake.test ADD COLUMN col2 VARCHAR 26 | 27 | statement ok 28 | INSERT INTO ducklake.test (col1, col3, col2) VALUES (100, 300, 'hello world'); 29 | 30 | query III 31 | SELECT col1, col2, col3 FROM ducklake.test 32 | ---- 33 | 1 NULL 3 34 | 10 NULL 20 35 | 100 hello world 300 36 | -------------------------------------------------------------------------------- /test/sql/alter/mixed_alter2.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/alter/mixed_alter2.test 2 | # group: [alter] 3 | 4 | require ducklake 5 | 6 | require parquet 7 | 8 | statement ok 9 | ATTACH 'ducklake:__TEST_DIR__/mixed_alter2.db' AS ducklake (DATA_PATH '__TEST_DIR__/mixed_alter2_files'); 10 | 11 | statement ok 12 | USE ducklake; 13 | 14 | statement ok 15 | CREATE TABLE tbl(col1 INTEGER); 16 | 17 | statement ok 18 | INSERT INTO tbl VALUES (42); 19 | 20 | statement ok 21 | ALTER TABLE tbl ADD COLUMN col2 VARCHAR; 22 | 23 | statement ok 24 | ALTER TABLE tbl ADD COLUMN new_column VARCHAR DEFAULT 'my_default'; 25 | 26 | statement ok 27 | ALTER TABLE tbl ADD COLUMN nested_column STRUCT(i INTEGER); 28 | 29 | query IIII 30 | FROM tbl 31 | ---- 32 | 42 NULL my_default NULL 33 | 34 | statement ok 35 | ALTER TABLE tbl DROP COLUMN new_column; 36 | 37 | query III 38 | FROM tbl 39 | ---- 40 | 42 NULL NULL 41 | -------------------------------------------------------------------------------- /test/sql/alter/promote_type.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/alter/promote_type.test 2 | # description: test ducklake promoting integer columns 3 | # group: [alter] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_promote_type.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_promote_type_files', METADATA_CATALOG 'xx') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test(col1 TINYINT); 14 | 15 | statement ok 16 | INSERT INTO ducklake.test VALUES (25) 17 | 18 | statement error 19 | INSERT INTO ducklake.test VALUES (1000) 20 | ---- 21 | out of range 22 | 23 | statement ok 24 | ALTER TABLE ducklake.test ALTER COLUMN col1 SET DATA TYPE INT; 25 | 26 | statement ok 27 | INSERT INTO ducklake.test VALUES (1000) 28 | 29 | query I 30 | FROM ducklake.test 31 | ---- 32 | 25 33 | 1000 34 | 35 | # cannot widen type 36 | statement error 37 | ALTER TABLE ducklake.test ALTER COLUMN col1 SET DATA TYPE SMALLINT; 38 | ---- 39 | only widening 40 | 41 | statement error 42 | ALTER TABLE ducklake.test ALTER COLUMN nonexistent_column SET DATA TYPE SMALLINT; 43 | ---- 44 | nonexistent_column 45 | -------------------------------------------------------------------------------- /test/sql/alter/rename_column.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/alter/rename_column.test 2 | # description: test ducklake renanming columns 3 | # group: [alter] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_rename_col.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_rename_col_files') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test(col1 INTEGER, col2 INTEGER); 14 | 15 | statement ok 16 | ALTER TABLE ducklake.test RENAME COLUMN col1 TO new_col1 17 | 18 | statement ok 19 | INSERT INTO ducklake.test VALUES (1, 2), (NULL, 3); 20 | 21 | query II 22 | SELECT new_col1, col2 FROM ducklake.test 23 | ---- 24 | 1 2 25 | NULL 3 26 | 27 | statement ok 28 | ALTER TABLE ducklake.test RENAME COLUMN col2 TO new_col2 29 | 30 | query IIIIII 31 | DESCRIBE ducklake.test 32 | ---- 33 | new_col1 INTEGER YES NULL NULL NULL 34 | new_col2 INTEGER YES NULL NULL NULL 35 | 36 | query II 37 | SELECT new_col1, new_col2 FROM ducklake.test 38 | ---- 39 | 1 2 40 | NULL 3 41 | 42 | statement error 43 | ALTER TABLE ducklake.test RENAME COLUMN blablabla TO k 44 | ---- 45 | column blablabla does not exist 46 | -------------------------------------------------------------------------------- /test/sql/alter/struct_evolution.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/alter/struct_evolution.test 2 | # description: test ducklake struct field evolution 3 | # group: [alter] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_struct_evolution.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_struct_evolution_files') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test(col1 STRUCT(i INT, j INT)); 14 | 15 | statement ok 16 | INSERT INTO ducklake.test VALUES ({'i': 1, 'j': 2}) 17 | 18 | # add k TINYINT 19 | statement ok 20 | ALTER TABLE ducklake.test ALTER COLUMN col1 SET DATA TYPE STRUCT(i INT, j INT, k TINYINT); 21 | 22 | statement ok 23 | INSERT INTO ducklake.test VALUES ({'i': 10, 'j': 20, 'k': 3}), ({'i': 11, 'j': 21, 'k': 10}) 24 | 25 | query I 26 | FROM ducklake.test 27 | ---- 28 | {'i': 1, 'j': 2, 'k': NULL} 29 | {'i': 10, 'j': 20, 'k': 3} 30 | {'i': 11, 'j': 21, 'k': 10} 31 | 32 | statement error 33 | INSERT INTO ducklake.test VALUES ({'i': 10, 'j': 20, 'k': 1000}) 34 | ---- 35 | out of range 36 | 37 | # promote k to INT 38 | statement ok 39 | ALTER TABLE ducklake.test ALTER COLUMN col1 SET DATA TYPE STRUCT(i INT, j INT, k INTEGER); 40 | 41 | statement ok 42 | INSERT INTO ducklake.test VALUES ({'i': 100, 'j': 200, 'k': 1000}) 43 | 44 | query I 45 | FROM ducklake.test 46 | ---- 47 | {'i': 1, 'j': 2, 'k': NULL} 48 | {'i': 10, 'j': 20, 'k': 3} 49 | {'i': 11, 'j': 21, 'k': 10} 50 | {'i': 100, 'j': 200, 'k': 1000} 51 | 52 | # drop i 53 | statement ok 54 | ALTER TABLE ducklake.test ALTER COLUMN col1 SET DATA TYPE STRUCT(j INT, k INTEGER); 55 | 56 | statement ok 57 | INSERT INTO ducklake.test VALUES ({'j': 150, 'k': 1000}), ({'j': 151, 'k': 1001}) 58 | 59 | query I 60 | FROM ducklake.test 61 | ---- 62 | {'j': 2, 'k': NULL} 63 | {'j': 20, 'k': 3} 64 | {'j': 21, 'k': 10} 65 | {'j': 200, 'k': 1000} 66 | {'j': 150, 'k': 1000} 67 | {'j': 151, 'k': 1001} 68 | 69 | # drop all original columns 70 | statement ok 71 | ALTER TABLE ducklake.test ALTER COLUMN col1 SET DATA TYPE STRUCT(k INTEGER); 72 | 73 | statement ok 74 | INSERT INTO ducklake.test VALUES ({'k': 10000}) 75 | 76 | query I 77 | FROM ducklake.test ORDER BY ALL 78 | ---- 79 | {'k': 3} 80 | {'k': 10} 81 | {'k': 1000} 82 | {'k': 1000} 83 | {'k': 1001} 84 | {'k': 10000} 85 | {'k': NULL} 86 | 87 | query I 88 | SELECT col1.k FROM ducklake.test WHERE col1.k=1000 89 | ---- 90 | 1000 91 | 1000 92 | 93 | query I 94 | SELECT col1.k FROM ducklake.test WHERE col1.k>3 ORDER BY ALL 95 | ---- 96 | 10 97 | 1000 98 | 1000 99 | 1001 100 | 10000 101 | 102 | query I 103 | SELECT col1.k FROM ducklake.test WHERE col1.k IS NULL 104 | ---- 105 | NULL 106 | -------------------------------------------------------------------------------- /test/sql/alter/struct_evolution_nested.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/alter/struct_evolution_nested.test 2 | # description: test ducklake struct nested evolution 3 | # group: [alter] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_struct_evolution_nested.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_struct_evolution_nested_files') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test(col1 STRUCT(i INT, j STRUCT(c1 TINYINT, c2 INT[]), k INT)); 14 | 15 | statement ok 16 | INSERT INTO ducklake.test VALUES ({'i': 1, 'j': {'c1': 2, 'c2': []}, 'k': 1}) 17 | 18 | # add a column to j and promote c1 19 | statement ok 20 | ALTER TABLE ducklake.test ALTER COLUMN col1 SET DATA TYPE STRUCT(i INT, j STRUCT(c1 INT, c2 INT[], c3 TINYINT), k INT); 21 | 22 | statement ok 23 | INSERT INTO ducklake.test VALUES ({'i': 10, 'j': {'c1': 1000, 'c2': [1, 2, 3], 'c3': 25}, 'k': 10}) 24 | 25 | query I 26 | FROM ducklake.test 27 | ---- 28 | {'i': 1, 'j': {'c1': 2, 'c2': [], 'c3': NULL}, 'k': 1} 29 | {'i': 10, 'j': {'c1': 1000, 'c2': [1, 2, 3], 'c3': 25}, 'k': 10} 30 | 31 | # drop struct fields again 32 | statement ok 33 | ALTER TABLE ducklake.test ALTER COLUMN col1 SET DATA TYPE STRUCT(j STRUCT(c2 INT[]), k INT); 34 | 35 | statement ok 36 | INSERT INTO ducklake.test VALUES ({'j': {'c2': [100]}, 'k': 100}) 37 | 38 | query I 39 | FROM ducklake.test 40 | ---- 41 | {'j': {'c2': []}, 'k': 1} 42 | {'j': {'c2': [1, 2, 3]}, 'k': 10} 43 | {'j': {'c2': [100]}, 'k': 100} 44 | 45 | # add a nested column 46 | statement ok 47 | ALTER TABLE ducklake.test ALTER COLUMN col1 SET DATA TYPE STRUCT(j STRUCT(c2 INT[], x STRUCT(a INT, b INT, c INT)), k INT); 48 | 49 | statement ok 50 | INSERT INTO ducklake.test VALUES ({'j': {'c2': NULL, 'x': {'a': 1, 'b': 2, 'c': 3}}, 'k': 1000}) 51 | 52 | query I 53 | FROM ducklake.test 54 | ---- 55 | {'j': {'c2': [], 'x': NULL}, 'k': 1} 56 | {'j': {'c2': [1, 2, 3], 'x': NULL}, 'k': 10} 57 | {'j': {'c2': [100], 'x': NULL}, 'k': 100} 58 | {'j': {'c2': NULL, 'x': {'a': 1, 'b': 2, 'c': 3}}, 'k': 1000} 59 | 60 | # drop the column entirely 61 | statement ok 62 | ALTER TABLE ducklake.test ALTER COLUMN col1 SET DATA TYPE STRUCT(k INT); 63 | 64 | query I 65 | FROM ducklake.test 66 | ---- 67 | {'k': 1} 68 | {'k': 10} 69 | {'k': 100} 70 | {'k': 1000} 71 | 72 | # now add a new deeply nested column 73 | statement ok 74 | ALTER TABLE ducklake.test ADD COLUMN col2 STRUCT(i INT, j STRUCT(c1 TINYINT, c2 INT[]), k INT) 75 | 76 | query II 77 | FROM ducklake.test 78 | ---- 79 | {'k': 1} NULL 80 | {'k': 10} NULL 81 | {'k': 100} NULL 82 | {'k': 1000} NULL 83 | -------------------------------------------------------------------------------- /test/sql/alter/struct_evolution_reuse.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/alter/struct_evolution_reuse.test 2 | # description: test ducklake struct field evolution re-use 3 | # group: [alter] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_struct_evolution_reuse.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_struct_evolution_reuse_files', METADATA_CATALOG 'xx') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test(col1 STRUCT(i INT, j INT)); 14 | 15 | statement ok 16 | INSERT INTO ducklake.test VALUES ({'i': 1, 'j': 2}) 17 | 18 | # drop column i 19 | statement ok 20 | ALTER TABLE ducklake.test ALTER COLUMN col1 SET DATA TYPE STRUCT(j INT); 21 | 22 | statement ok 23 | INSERT INTO ducklake.test VALUES ({'i': 10, 'j': 20}) 24 | 25 | query I 26 | FROM ducklake.test 27 | ---- 28 | {'j': 2} 29 | {'j': 20} 30 | 31 | # re-add column i 32 | statement ok 33 | ALTER TABLE ducklake.test ALTER COLUMN col1 SET DATA TYPE STRUCT(j INT, i INT); 34 | 35 | query I 36 | FROM ducklake.test 37 | ---- 38 | {'j': 2, 'i': NULL} 39 | {'j': 20, 'i': NULL} 40 | -------------------------------------------------------------------------------- /test/sql/autoloading/autoload_data_path.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/autoloading/autoload_data_path.test 2 | # description: Tests for autoloading with filesystems 3 | # group: [autoloading] 4 | 5 | require-env LOCAL_EXTENSION_REPO 6 | 7 | statement ok 8 | LOAD ducklake 9 | 10 | statement ok 11 | set allow_persistent_secrets=false; 12 | 13 | # Ensure we have a clean extension directory without any preinstalled extensions 14 | statement ok 15 | set extension_directory='__TEST_DIR__/autoloading_filesystems' 16 | 17 | ### No autoloading nor installing: throw error with installation hint 18 | statement ok 19 | set autoload_known_extensions=false 20 | 21 | statement ok 22 | set autoinstall_known_extensions=false 23 | 24 | statement error 25 | ATTACH 'ducklake:autoload_problem.ducklake' (DATA_PATH 's3://some-bucket/') 26 | ---- 27 | Missing Extension Error: Data path s3://some-bucket/ requires the extension httpfs to be loaded 28 | 29 | ### With autoloading, install and correct repo 30 | statement ok 31 | set autoload_known_extensions=true 32 | 33 | statement ok 34 | set autoinstall_known_extensions=true 35 | 36 | statement ok 37 | set autoinstall_extension_repository='${LOCAL_EXTENSION_REPO}'; 38 | 39 | # Set an invalid endpoint to ensure we fail in the httpfs extension when trying to connect 40 | statement ok 41 | SET s3_endpoint='false_endpoint'; 42 | 43 | ## Note that attaching Ducklake will NOT actually validate the data path 44 | 45 | statement ok 46 | ATTACH 'ducklake:autoload_problem.ducklake' (DATA_PATH 's3://some-bucket/') 47 | -------------------------------------------------------------------------------- /test/sql/catalog/drop_table.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/catalog/drop_table.test 2 | # description: Test dropping of tables in DuckLake 3 | # group: [catalog] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_drop.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_drop_files') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test(i INTEGER); 14 | 15 | statement ok 16 | BEGIN 17 | 18 | statement ok 19 | DROP TABLE ducklake.test 20 | 21 | statement error 22 | SELECT * FROM ducklake.test 23 | ---- 24 | does not exist 25 | 26 | query I 27 | SELECT table_name FROM duckdb_tables() WHERE database_name = 'ducklake' 28 | ---- 29 | 30 | statement ok 31 | ROLLBACK 32 | 33 | query I 34 | SELECT * FROM ducklake.test 35 | ---- 36 | 37 | statement ok 38 | DROP TABLE ducklake.test 39 | 40 | statement error 41 | SELECT * FROM ducklake.test 42 | ---- 43 | does not exist 44 | 45 | statement ok 46 | DROP TABLE IF EXISTS ducklake.test 47 | 48 | statement error 49 | DROP TABLE ducklake.test 50 | ---- 51 | does not exist 52 | 53 | # test drop of a transaction-local table 54 | statement ok 55 | BEGIN 56 | 57 | statement ok 58 | CREATE TABLE ducklake.test2(i INTEGER); 59 | 60 | query I 61 | SELECT * FROM ducklake.test2 62 | ---- 63 | 64 | statement ok 65 | DROP TABLE ducklake.test2 66 | 67 | statement error 68 | SELECT * FROM ducklake.test2 69 | ---- 70 | does not exist 71 | 72 | statement ok 73 | COMMIT 74 | 75 | statement ok 76 | CREATE TABLE ducklake.test(i INTEGER); 77 | 78 | # drop and create the same table in the same transaction 79 | statement ok 80 | BEGIN 81 | 82 | statement ok 83 | DROP TABLE ducklake.test; 84 | 85 | statement ok 86 | CREATE TABLE ducklake.test(i VARCHAR, j VARCHAR); 87 | 88 | statement ok 89 | INSERT INTO ducklake.test VALUES ('hello', 'world'); 90 | 91 | query II 92 | SELECT * FROM ducklake.test 93 | ---- 94 | hello world 95 | 96 | statement ok 97 | COMMIT 98 | 99 | query II 100 | SELECT * FROM ducklake.test 101 | ---- 102 | hello world 103 | -------------------------------------------------------------------------------- /test/sql/catalog/quoted_identifiers.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/catalog/quoted_identifiers.test 2 | # description: Test quoted identifiers in DuckLake 3 | # group: [catalog] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | # windows doesn't like these paths 10 | require notwindows 11 | 12 | statement ok 13 | ATTACH 'ducklake:__TEST_DIR__/ducklake ''quoted'' "db".db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake ''quoted'' "path"', METADATA_CATALOG '''quoted'' catalog "name"', METADATA_SCHEMA '''quoted'' catalog "schema"') 14 | 15 | statement ok 16 | CREATE TABLE ducklake."quoted 'table' ""name"""("quoted 'column' ""name""" INTEGER); 17 | 18 | query I 19 | SELECT "quoted 'column' ""name""" FROM ducklake."quoted 'table' ""name""" 20 | ---- 21 | 22 | statement ok 23 | DROP TABLE ducklake."quoted 'table' ""name""" 24 | -------------------------------------------------------------------------------- /test/sql/cleanup/create_drop_cleanup.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/cleanup/create_drop_cleanup.test 2 | # description: Cleanup files when creating and dropping a table 3 | # group: [cleanup] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_createq_drop_cleanup.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_createq_drop_cleanup') 11 | 12 | statement ok 13 | BEGIN 14 | 15 | statement ok 16 | CREATE TABLE ducklake.tbl(i INT); 17 | 18 | statement ok 19 | INSERT INTO ducklake.tbl VALUES (42); 20 | 21 | query I 22 | SELECT COUNT(*) FROM glob('__TEST_DIR__/ducklake_createq_drop_cleanup/*.parquet') 23 | ---- 24 | 1 25 | 26 | statement ok 27 | DROP TABLE ducklake.tbl 28 | 29 | query I 30 | SELECT COUNT(*) FROM glob('__TEST_DIR__/ducklake_createq_drop_cleanup/*.parquet') 31 | ---- 32 | 0 33 | 34 | statement ok 35 | COMMIT 36 | 37 | -------------------------------------------------------------------------------- /test/sql/comments/comment_on_column.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/comments/comment_on_column.test 2 | # description: test ducklake comments on a column 3 | # group: [comments] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_column_comments.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_column_comments_files') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test_table as SELECT 1 as test_table_column 14 | 15 | ### Comment on column from table 16 | query I 17 | select comment from duckdb_columns() where column_name='test_table_column'; 18 | ---- 19 | NULL 20 | 21 | statement ok 22 | COMMENT ON COLUMN ducklake.test_table.test_table_column IS 'very gezellige column' 23 | 24 | query I 25 | select comment from duckdb_columns() where column_name='test_table_column'; 26 | ---- 27 | very gezellige column 28 | 29 | statement ok 30 | BEGIN 31 | 32 | statement ok 33 | COMMENT ON COLUMN ducklake.test_table.test_table_column IS 'toch niet zo gezellig' 34 | 35 | query I 36 | select comment from duckdb_columns() where column_name='test_table_column'; 37 | ---- 38 | toch niet zo gezellig 39 | 40 | # take that back! 41 | statement ok 42 | ROLLBACK 43 | 44 | query I 45 | select comment from duckdb_columns() where column_name='test_table_column'; 46 | ---- 47 | very gezellige column 48 | -------------------------------------------------------------------------------- /test/sql/comments/comments.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/comments/comments.test 2 | # description: test ducklake comments 3 | # group: [comments] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_comments.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_comments_files') 11 | 12 | ### Comment on Tables 13 | statement ok 14 | CREATE TABLE ducklake.test(i INTEGER, j INTEGER); 15 | 16 | statement ok 17 | COMMENT ON TABLE ducklake.test IS 'very gezellige table' 18 | 19 | query I 20 | select comment from duckdb_tables() where table_name='test'; 21 | ---- 22 | very gezellige table 23 | 24 | # Reverting to null goes like dis 25 | statement ok 26 | COMMENT ON TABLE ducklake.test IS NULL 27 | 28 | query I 29 | select comment from duckdb_tables() where table_name='test'; 30 | ---- 31 | NULL 32 | 33 | # rollback 34 | statement ok 35 | BEGIN 36 | 37 | statement ok 38 | COMMENT ON TABLE ducklake.test IS 'rolled back comment' 39 | 40 | query I 41 | select comment from duckdb_tables() where table_name='test'; 42 | ---- 43 | rolled back comment 44 | 45 | statement ok 46 | ROLLBACK 47 | 48 | query I 49 | select comment from duckdb_tables() where table_name='test'; 50 | ---- 51 | NULL 52 | 53 | ## Comment on view 54 | statement ok 55 | CREATE VIEW ducklake.test_view as SELECT 1 as test_view_column 56 | 57 | query I 58 | select comment from duckdb_views() where view_name='test_view'; 59 | ---- 60 | NULL 61 | 62 | statement ok 63 | COMMENT ON VIEW ducklake.test_view IS 'very gezellige view' 64 | 65 | query I 66 | select comment from duckdb_views() where view_name='test_view'; 67 | ---- 68 | very gezellige view 69 | 70 | statement error 71 | COMMENT ON VIEW ducklake.test IS '123' 72 | ---- 73 | not a view 74 | 75 | statement error 76 | COMMENT ON TABLE ducklake.test_view IS '123' 77 | ---- 78 | not a table 79 | -------------------------------------------------------------------------------- /test/sql/compaction/compaction_encrypted.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/compaction/compaction_encrypted.test 2 | # description: test compaction on an encrypted database 3 | # group: [compaction] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_compaction_encrypted.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_compaction_encrypted_files', ENCRYPTED) 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test(i INTEGER); 14 | 15 | statement ok 16 | INSERT INTO ducklake.test VALUES (1); 17 | 18 | statement ok 19 | INSERT INTO ducklake.test VALUES (2); 20 | 21 | statement ok 22 | INSERT INTO ducklake.test VALUES (3); 23 | 24 | statement ok 25 | CALL ducklake_merge_adjacent_files('ducklake'); 26 | 27 | # delete the old files 28 | statement ok 29 | CALL ducklake_cleanup_old_files('ducklake', cleanup_all => true); 30 | 31 | # all files have been compacted into one file 32 | query I 33 | SELECT COUNT(*) FROM GLOB('__TEST_DIR__/ducklake_compaction_encrypted_files/*') 34 | ---- 35 | 1 36 | 37 | # verify the file is encrypted 38 | statement error 39 | SELECT * FROM '__TEST_DIR__/ducklake_compaction_encrypted_files/*.parquet' 40 | ---- 41 | encrypted 42 | 43 | # all reading still works 44 | query I 45 | SELECT * FROM ducklake.test AT (VERSION => 2) 46 | ---- 47 | 1 48 | 49 | query III 50 | SELECT snapshot_id, rowid, * FROM ducklake.test ORDER BY ALL 51 | ---- 52 | 2 0 1 53 | 3 1 2 54 | 4 2 3 55 | -------------------------------------------------------------------------------- /test/sql/compaction/compaction_partitioned_table.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/compaction/compaction_partitioned_table.test 2 | # description: test compaction of partitioned tables 3 | # group: [compaction] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_partitioned_compact.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_partitioned_compact_files', METADATA_CATALOG 'ducklake_metadata') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.partitioned(part_key INTEGER, value INTEGER); 14 | 15 | statement ok 16 | ALTER TABLE ducklake.partitioned SET PARTITIONED BY (part_key); 17 | 18 | statement ok 19 | INSERT INTO ducklake.partitioned VALUES (1, 10); 20 | 21 | statement ok 22 | INSERT INTO ducklake.partitioned VALUES (1, 20); 23 | 24 | statement ok 25 | INSERT INTO ducklake.partitioned VALUES (2, 100); 26 | 27 | statement ok 28 | INSERT INTO ducklake.partitioned VALUES (2, 200); 29 | 30 | query I 31 | SELECT COUNT(*) FROM GLOB('__TEST_DIR__/ducklake_partitioned_compact_files/**/*.parquet') 32 | ---- 33 | 4 34 | 35 | query IIII 36 | SELECT snapshot_id, rowid, * FROM ducklake.partitioned ORDER BY ALL 37 | ---- 38 | 3 0 1 10 39 | 4 1 1 20 40 | 5 2 2 100 41 | 6 3 2 200 42 | 43 | statement ok 44 | CALL ducklake_merge_adjacent_files('ducklake'); 45 | 46 | # we gain two files here 47 | query I 48 | SELECT COUNT(*) FROM GLOB('__TEST_DIR__/ducklake_partitioned_compact_files/**/*.parquet') 49 | ---- 50 | 6 51 | 52 | # cleanup 53 | statement ok 54 | CALL ducklake_cleanup_old_files('ducklake', cleanup_all => true); 55 | 56 | # two files left 57 | query I 58 | SELECT COUNT(*) FROM GLOB('__TEST_DIR__/ducklake_partitioned_compact_files/**/*.parquet') 59 | ---- 60 | 2 61 | 62 | # verify we have written partition info correctly 63 | query II 64 | SELECT partition_id, partition_value 65 | FROM ducklake_metadata.ducklake_data_file 66 | JOIN ducklake_metadata.ducklake_file_partition_value USING (data_file_id) 67 | ORDER BY ALL 68 | ---- 69 | 2 1 70 | 2 2 71 | 72 | query II 73 | SELECT * FROM ducklake.partitioned AT (VERSION => 3) 74 | ---- 75 | 1 10 76 | 77 | query II 78 | SELECT * FROM ducklake.partitioned AT (VERSION => 4) 79 | ---- 80 | 1 10 81 | 1 20 82 | 83 | query IIII 84 | SELECT snapshot_id, rowid, * FROM ducklake.partitioned ORDER BY ALL 85 | ---- 86 | 3 0 1 10 87 | 4 1 1 20 88 | 5 2 2 100 89 | 6 3 2 200 90 | -------------------------------------------------------------------------------- /test/sql/compaction/expire_snapshots.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/compaction/expire_snapshots.test 2 | # description: test ducklake expiration of snapshots 3 | # group: [compaction] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_expire_snapshots.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_expire_snapshots_files', METADATA_CATALOG 'metadata') 11 | 12 | # delete all values in a table 13 | # snapshot 1 14 | statement ok 15 | CREATE TABLE ducklake.test(i INTEGER) 16 | 17 | # snapshot 2 18 | statement ok 19 | INSERT INTO ducklake.test FROM range(1000) 20 | 21 | # snapshot 3 22 | statement ok 23 | DELETE FROM ducklake.test 24 | 25 | # snapshot 4 26 | statement ok 27 | DROP TABLE ducklake.test 28 | 29 | # explicitly expire snapshot 2 30 | statement ok 31 | CALL ducklake_expire_snapshots('ducklake', versions => [2]) 32 | 33 | # the snapshot is no longer available 34 | statement error 35 | FROM ducklake.test AT (VERSION => 2) 36 | ---- 37 | No snapshot found at version 2 38 | 39 | # we can query around that though 40 | query I 41 | FROM ducklake.test AT (VERSION => 1) 42 | ---- 43 | 44 | query I 45 | FROM ducklake.test AT (VERSION => 3) 46 | ---- 47 | 48 | # the data file is no longer required (since only snapshot 2 references it) -> we can delete it now 49 | query I 50 | SELECT COUNT(*) FROM ducklake_cleanup_old_files('ducklake', dry_run => true, cleanup_all => true); 51 | ---- 52 | 1 53 | 54 | statement ok 55 | CALL ducklake_cleanup_old_files('ducklake', cleanup_all => true); 56 | 57 | # verify that it is actually gone 58 | query I 59 | SELECT COUNT(*) FROM GLOB('__TEST_DIR__/ducklake_expire_snapshots_files/*') 60 | ---- 61 | 0 62 | 63 | # let's delete snapshots 1/3 64 | statement ok 65 | CALL ducklake_expire_snapshots('ducklake', versions => [1, 3]) 66 | 67 | # all traces of the table are gone 68 | foreach tbl ducklake_table ducklake_column ducklake_table_stats ducklake_table_column_stats 69 | 70 | query I 71 | SELECT COUNT(*) FROM metadata.${tbl} 72 | ---- 73 | 0 74 | 75 | endloop 76 | -------------------------------------------------------------------------------- /test/sql/compaction/expire_snapshots_drop_table.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/compaction/expire_snapshots_drop_table.test 2 | # description: test ducklake expiration of snapshots 3 | # group: [compaction] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_expire_snapshots_drop_table.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_expire_snapshots_drop_table_files', METADATA_CATALOG 'metadata') 11 | 12 | # delete all values in a table 13 | # snapshot 1 14 | statement ok 15 | CREATE TABLE ducklake.test(i INTEGER) 16 | 17 | # snapshot 2 18 | statement ok 19 | INSERT INTO ducklake.test FROM range(1000) 20 | 21 | # snapshot 3 22 | statement ok 23 | DELETE FROM ducklake.test WHERE i%4=0 24 | 25 | # snapshot 4 26 | statement ok 27 | DELETE FROM ducklake.test WHERE i%2=0 28 | 29 | # snapshot 5 30 | statement ok 31 | DROP TABLE ducklake.test 32 | 33 | # we have 3 files now (insert from snapshot 2, delete from snapshot 3, delete from snapshot 4) 34 | query I 35 | SELECT COUNT(*) FROM GLOB('__TEST_DIR__/ducklake_expire_snapshots_drop_table_files/*') 36 | ---- 37 | 3 38 | 39 | # explicitly expire snapshot 3 and cleanup files 40 | statement ok 41 | CALL ducklake_expire_snapshots('ducklake', versions => [3]) 42 | 43 | statement ok 44 | CALL ducklake_cleanup_old_files('ducklake', cleanup_all => true); 45 | 46 | # the deletes from snapshot 3 should now be gone 47 | query I 48 | SELECT COUNT(*) FROM GLOB('__TEST_DIR__/ducklake_expire_snapshots_drop_table_files/*') 49 | ---- 50 | 2 51 | 52 | # now expire snapshots 1, 2 and 4 - this should fully remove all traces of the table 53 | statement ok 54 | CALL ducklake_expire_snapshots('ducklake', versions => [1, 2, 4]) 55 | 56 | statement ok 57 | CALL ducklake_cleanup_old_files('ducklake', cleanup_all => true); 58 | 59 | # verify that all files are actually gone 60 | query I 61 | SELECT COUNT(*) FROM GLOB('__TEST_DIR__/ducklake_expire_snapshots_drop_table_files/*') 62 | ---- 63 | 0 64 | 65 | # all traces of the table are gone 66 | foreach tbl ducklake_table ducklake_column ducklake_table_stats ducklake_table_column_stats ducklake_data_file ducklake_delete_file 67 | 68 | query I 69 | SELECT COUNT(*) FROM metadata.${tbl} 70 | ---- 71 | 0 72 | 73 | endloop 74 | -------------------------------------------------------------------------------- /test/sql/compaction/expire_snapshots_schema.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/compaction/expire_snapshots_schema.test 2 | # description: test ducklake expiration of snapshots with schema changes 3 | # group: [compaction] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_expire_snapshots_schema.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_expire_snapshots_schema_files', METADATA_CATALOG 'metadata') 11 | 12 | # snapshot 1 13 | statement ok 14 | CREATE SCHEMA s1; 15 | 16 | # snapshot 2 17 | statement ok 18 | CREATE VIEW s1.vw AS SELECT 42 19 | 20 | # snapshot 3 21 | statement ok 22 | CREATE TABLE s1.tbl(i INTEGER) 23 | 24 | # snapshot 4 25 | statement ok 26 | DROP TABLE s1.tbl 27 | 28 | # snapshot 5 29 | statement ok 30 | DROP VIEW s1.vw 31 | 32 | # snapshot 6 33 | statement ok 34 | DROP SCHEMA s1 35 | 36 | # expire all snapshots 37 | statement ok 38 | CALL ducklake_expire_snapshots('ducklake', versions => [1, 2, 3, 4, 5]) 39 | 40 | # we have one schema remaining (`main`) 41 | query I 42 | SELECT COUNT(*) FROM metadata.ducklake_schema 43 | ---- 44 | 1 45 | 46 | # all traces of the schema are gone 47 | foreach tbl ducklake_view ducklake_table ducklake_column ducklake_table_stats ducklake_table_column_stats ducklake_data_file ducklake_delete_file 48 | 49 | query I 50 | SELECT COUNT(*) FROM metadata.${tbl} 51 | ---- 52 | 0 53 | 54 | endloop 55 | -------------------------------------------------------------------------------- /test/sql/compaction/multi_compaction.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/compaction/multi_compaction.test 2 | # description: test chain of compaction statements 3 | # group: [compaction] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_multi_compaction.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_multi_compaction_files') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test(i INTEGER); 14 | 15 | foreach BASE 0 3 16 | 17 | statement ok 18 | INSERT INTO ducklake.test VALUES (${BASE} + 1); 19 | 20 | statement ok 21 | INSERT INTO ducklake.test VALUES (${BASE} + 2); 22 | 23 | statement ok 24 | INSERT INTO ducklake.test VALUES (${BASE} + 3); 25 | 26 | statement ok 27 | CALL ducklake_merge_adjacent_files('ducklake'); 28 | 29 | endloop 30 | 31 | # delete the old files 32 | statement ok 33 | CALL ducklake_cleanup_old_files('ducklake', cleanup_all => true); 34 | 35 | # all files have been compacted into one file 36 | query I 37 | SELECT COUNT(*) FROM GLOB('__TEST_DIR__/ducklake_multi_compaction_files/*') 38 | ---- 39 | 1 40 | 41 | # verify that after multi-compaction time travel still works 42 | query I 43 | SELECT * FROM ducklake.test AT (VERSION => 2) 44 | ---- 45 | 1 46 | 47 | # row-ids/snapshot-ids are kept also across multiple compaction runs 48 | query III 49 | SELECT snapshot_id, rowid, * FROM ducklake.test ORDER BY ALL 50 | ---- 51 | 2 0 1 52 | 3 1 2 53 | 4 2 3 54 | 6 3 4 55 | 7 4 5 56 | 8 5 6 57 | -------------------------------------------------------------------------------- /test/sql/concurrent/concurrent_insert_conflict.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/concurrent/concurrent_insert_conflict.test 2 | # description: test concurrent inserts 3 | # group: [concurrent] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_concurrent_insert.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_concurrent_insert_files') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.tbl(key INTEGER); 14 | 15 | concurrentloop i 0 2 16 | 17 | query I 18 | INSERT INTO ducklake.tbl VALUES (${i}) 19 | ---- 20 | 1 21 | 22 | endloop 23 | 24 | query II 25 | SELECT COUNT(*), SUM(key) FROM ducklake.tbl 26 | ---- 27 | 2 1 28 | -------------------------------------------------------------------------------- /test/sql/constraints/not_null.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/constraints/not_null.test 2 | # description: test NOT NULL constraint 3 | # group: [constraints] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_not_null.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_not_null_files') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test(i INTEGER NOT NULL, j INTEGER); 14 | 15 | statement ok 16 | INSERT INTO ducklake.test VALUES (42, NULL); 17 | 18 | statement error 19 | INSERT INTO ducklake.test VALUES (NULL, 84) 20 | ---- 21 | NOT NULL constraint 22 | 23 | # check that NOT NULL shows up in DESCRIBE 24 | query IIIIII 25 | DESCRIBE ducklake.test 26 | ---- 27 | i INTEGER NO NULL NULL NULL 28 | j INTEGER YES NULL NULL NULL 29 | 30 | # we cannot add a null constraint to a column that already has it 31 | statement error 32 | ALTER TABLE ducklake.test ALTER i SET NOT NULL; 33 | ---- 34 | already 35 | 36 | # we cannot drop the null constraint from a column that does not have it 37 | statement error 38 | ALTER TABLE ducklake.test ALTER j DROP NOT NULL; 39 | ---- 40 | no NOT NULL constraint 41 | 42 | # column does not exist 43 | statement error 44 | ALTER TABLE ducklake.test ALTER nonexistent_column SET NOT NULL; 45 | ---- 46 | nonexistent_column 47 | 48 | statement error 49 | ALTER TABLE ducklake.test ALTER nonexistent_column DROP NOT NULL; 50 | ---- 51 | nonexistent_column 52 | 53 | # we can drop the NOT NULL constraint 54 | statement ok 55 | ALTER TABLE ducklake.test ALTER i DROP NOT NULL; 56 | 57 | # we can then re-add it - and roll it back 58 | statement ok 59 | BEGIN 60 | 61 | statement ok 62 | ALTER TABLE ducklake.test ALTER i SET NOT NULL; 63 | 64 | statement ok 65 | ROLLBACK 66 | 67 | # if we re-add, we cannot add NULL values in the same transaction 68 | statement ok 69 | BEGIN 70 | 71 | statement ok 72 | ALTER TABLE ducklake.test ALTER i SET NOT NULL; 73 | 74 | statement error 75 | INSERT INTO ducklake.test VALUES (NULL, 42) 76 | ---- 77 | NOT NULL constraint failed 78 | 79 | statement ok 80 | ROLLBACK 81 | 82 | # if we re-add, we cannot add NULL values in the same transaction 83 | statement ok 84 | BEGIN 85 | 86 | statement ok 87 | INSERT INTO ducklake.test VALUES (NULL, 84) 88 | 89 | statement error 90 | ALTER TABLE ducklake.test ALTER i SET NOT NULL; 91 | ---- 92 | transaction-local 93 | 94 | statement ok 95 | ROLLBACK 96 | 97 | # we cannot SET NOT NULL on a table that has NULL values 98 | statement error 99 | ALTER TABLE ducklake.test ALTER j SET NOT NULL; 100 | ---- 101 | has NULL values 102 | 103 | # after dropping the constraint - we can add NULL rows 104 | statement ok 105 | INSERT INTO ducklake.test VALUES (NULL, 84) 106 | -------------------------------------------------------------------------------- /test/sql/constraints/not_null_drop_column.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/constraints/not_null_drop_column.test 2 | # description: test dropping NOT NULL columns 3 | # group: [constraints] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_not_null_drop.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_not_null_drop_files') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test(i INTEGER NOT NULL, j INTEGER, k INTEGER NOT NULL); 14 | 15 | statement ok 16 | INSERT INTO ducklake.test VALUES (42, NULL, 3); 17 | 18 | statement error 19 | INSERT INTO ducklake.test VALUES (NULL, 84, 3) 20 | ---- 21 | NOT NULL constraint 22 | 23 | statement ok 24 | ALTER TABLE ducklake.test DROP COLUMN j 25 | 26 | statement error 27 | INSERT INTO ducklake.test VALUES (42, NULL) 28 | ---- 29 | NOT NULL constraint 30 | 31 | statement ok 32 | ALTER TABLE ducklake.test DROP COLUMN k 33 | 34 | statement error 35 | INSERT INTO ducklake.test VALUES (NULL) 36 | ---- 37 | NOT NULL constraint 38 | -------------------------------------------------------------------------------- /test/sql/constraints/unsupported.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/constraints/unsupported.test 2 | # description: test unsupported constraints 3 | # group: [constraints] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_unsupported.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_unsupported_files') 11 | 12 | statement error 13 | CREATE TABLE ducklake.test(i INTEGER PRIMARY KEY, j INTEGER); 14 | ---- 15 | not supported 16 | 17 | statement error 18 | CREATE TABLE ducklake.test(i INTEGER, j INTEGER, CHECK (i > j)); 19 | ---- 20 | not supported 21 | -------------------------------------------------------------------------------- /test/sql/data_inlining/basic_data_inlining.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/data_inlining/basic_data_inlining.test 2 | # description: test ducklake extension 3 | # group: [data_inlining] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_inlining.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_inlining_files', METADATA_CATALOG 'ducklake_meta', DATA_INLINING_ROW_LIMIT 10) 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test(i INTEGER, j INTEGER); 14 | 15 | query II 16 | SELECT * FROM ducklake.test 17 | ---- 18 | 19 | statement ok 20 | BEGIN 21 | 22 | query I 23 | INSERT INTO ducklake.test VALUES (1, 2), (NULL, 3); 24 | ---- 25 | 2 26 | 27 | query II 28 | SELECT * FROM ducklake.test 29 | ---- 30 | 1 2 31 | NULL 3 32 | 33 | query II 34 | SELECT j, i FROM ducklake.test 35 | ---- 36 | 2 1 37 | 3 NULL 38 | 39 | query IIII 40 | SELECT rowid, snapshot_id, * FROM ducklake.test 41 | ---- 42 | 1000000000000000000 NULL 1 2 43 | 1000000000000000001 NULL NULL 3 44 | 45 | query III 46 | SELECT filename, file_row_number, file_index FROM ducklake.test 47 | ---- 48 | __ducklake_inlined_transaction_local_data 0 1 49 | __ducklake_inlined_transaction_local_data 1 1 50 | 51 | statement ok 52 | COMMIT 53 | 54 | query II 55 | SELECT * FROM ducklake.test 56 | ---- 57 | 1 2 58 | NULL 3 59 | 60 | query IIII 61 | SELECT rowid, snapshot_id, * FROM ducklake.test 62 | ---- 63 | 0 2 1 2 64 | 1 2 NULL 3 65 | 66 | # virtual columns 67 | query III 68 | SELECT filename, file_row_number, file_index FROM ducklake.test 69 | ---- 70 | ducklake_inlined_data_1_1 0 0 71 | ducklake_inlined_data_1_1 1 0 72 | 73 | query II 74 | SELECT * FROM ducklake.test WHERE i IS NULL 75 | ---- 76 | NULL 3 77 | 78 | query II 79 | SELECT * FROM ducklake.test WHERE i=1 80 | ---- 81 | 1 2 82 | 83 | # all data is inlined - so we have no files 84 | query I 85 | SELECT COUNT(*) FROM GLOB('__TEST_DIR__/ducklake_inlining_files/**') 86 | ---- 87 | 0 88 | 89 | # insert more rows than the inlining limit allows 90 | query I 91 | INSERT INTO ducklake.test SELECT i, 100 + i FROM range(11) t(i); 92 | ---- 93 | 11 94 | 95 | query III 96 | SELECT COUNT(*), SUM(i), SUM(j) FROM ducklake.test 97 | ---- 98 | 13 56 1160 99 | 100 | # now we have a file 101 | query I 102 | SELECT COUNT(*) FROM GLOB('__TEST_DIR__/ducklake_inlining_files/**') 103 | ---- 104 | 1 105 | -------------------------------------------------------------------------------- /test/sql/data_inlining/data_inlining_alter.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/data_inlining/data_inlining_alter.test 2 | # description: test data inlining with ALTER statements 3 | # group: [data_inlining] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_inlining_alter.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_inlining_alter_files', METADATA_CATALOG 'ducklake_meta', DATA_INLINING_ROW_LIMIT 10) 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test AS SELECT 1 i, 2 j 14 | 15 | query II 16 | FROM ducklake.test 17 | ---- 18 | 1 2 19 | 20 | statement ok 21 | ALTER TABLE ducklake.test ADD COLUMN k INTEGER 22 | 23 | statement ok 24 | INSERT INTO ducklake.test VALUES (10, 20, 30); 25 | 26 | query III 27 | FROM ducklake.test 28 | ---- 29 | 1 2 NULL 30 | 10 20 30 31 | 32 | statement ok 33 | ALTER TABLE ducklake.test DROP COLUMN i 34 | 35 | query II 36 | FROM ducklake.test 37 | ---- 38 | 2 NULL 39 | 20 30 40 | 41 | statement ok 42 | ALTER TABLE ducklake.test ALTER j SET TYPE BIGINT 43 | 44 | statement ok 45 | INSERT INTO ducklake.test VALUES (1000000000000, 0) 46 | 47 | query II 48 | FROM ducklake.test 49 | ---- 50 | 2 NULL 51 | 20 30 52 | 1000000000000 0 53 | -------------------------------------------------------------------------------- /test/sql/data_inlining/data_inlining_constraints.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/data_inlining/data_inlining_constraints.test 2 | # description: test data inlining with NOT NULL constraints 3 | # group: [data_inlining] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_inlining_constraint.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_inlining_constraint_files', METADATA_CATALOG 'ducklake_meta', DATA_INLINING_ROW_LIMIT 10) 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test(i INTEGER, j INTEGER NOT NULL); 14 | 15 | statement error 16 | INSERT INTO ducklake.test VALUES (42, NULL) 17 | ---- 18 | NOT NULL constraint failed 19 | -------------------------------------------------------------------------------- /test/sql/data_inlining/data_inlining_delete.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/data_inlining/data_inlining_delete.test 2 | # description: test ducklake extension 3 | # group: [data_inlining] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_inlining_delete.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_inlining_delete_files', METADATA_CATALOG 'ducklake_meta', DATA_INLINING_ROW_LIMIT 10) 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test AS SELECT 1 i, 2 j UNION ALL SELECT NULL, 3 UNION ALL SELECT 10, 20 14 | 15 | statement ok 16 | BEGIN 17 | 18 | # we can delete inlined data 19 | query I 20 | DELETE FROM ducklake.test WHERE i=1 21 | ---- 22 | 1 23 | 24 | query II 25 | SELECT * FROM ducklake.test 26 | ---- 27 | NULL 3 28 | 10 20 29 | 30 | statement ok 31 | COMMIT 32 | 33 | query II 34 | SELECT * FROM ducklake.test 35 | ---- 36 | NULL 3 37 | 10 20 38 | 39 | # no files 40 | query I 41 | SELECT COUNT(*) FROM GLOB('__TEST_DIR__/ducklake_inlining_delete_files/**') 42 | ---- 43 | 0 44 | 45 | # delete the other row 46 | statement ok 47 | BEGIN 48 | 49 | query I 50 | DELETE FROM ducklake.test WHERE i=10 51 | ---- 52 | 1 53 | 54 | query II 55 | SELECT * FROM ducklake.test 56 | ---- 57 | NULL 3 58 | 59 | statement ok 60 | COMMIT 61 | 62 | query II 63 | SELECT * FROM ducklake.test 64 | ---- 65 | NULL 3 66 | 67 | # delete all remaining rows in the table 68 | query I 69 | DELETE FROM ducklake.test 70 | ---- 71 | 1 72 | 73 | query II 74 | SELECT * FROM ducklake.test 75 | ---- 76 | 77 | # no files 78 | query I 79 | SELECT COUNT(*) FROM GLOB('__TEST_DIR__/ducklake_inlining_delete_files/**') 80 | ---- 81 | 0 82 | -------------------------------------------------------------------------------- /test/sql/data_inlining/data_inlining_large.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/data_inlining/data_inlining_large.test 2 | # description: test data inlining with many rows 3 | # group: [data_inlining] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_inlining_large.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_inlining_large_files', METADATA_CATALOG 'ducklake_meta', DATA_INLINING_ROW_LIMIT 999999) 11 | 12 | statement ok 13 | CREATE TABLE bigtbl AS FROM range(1000000) t(i); 14 | 15 | statement ok 16 | CREATE TABLE ducklake.test(i INTEGER) 17 | 18 | query I 19 | INSERT INTO ducklake.test FROM bigtbl 20 | ---- 21 | 1000000 22 | 23 | query IIII 24 | SELECT COUNT(*), SUM(i), MIN(i), MAX(i) FROM bigtbl 25 | ---- 26 | 1000000 499999500000 0 999999 27 | 28 | # this exceeds the inline limit so we have a file 29 | query I 30 | SELECT COUNT(*) FROM GLOB('__TEST_DIR__/ducklake_inlining_large_files/**') 31 | ---- 32 | 1 33 | -------------------------------------------------------------------------------- /test/sql/data_inlining/data_inlining_table_changes.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/data_inlining/data_inlining_table_changes.test 2 | # description: test ducklake_table_changes function with inlined data 3 | # group: [data_inlining] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_inlining_table_changes.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_inlining_table_changes_files', DATA_INLINING_ROW_LIMIT 10) 11 | 12 | # snapshot 1 13 | statement ok 14 | CREATE TABLE ducklake.test(i INTEGER); 15 | 16 | # snapshot 2 17 | statement ok 18 | INSERT INTO ducklake.test FROM range(3); 19 | 20 | # snapshot 3 21 | statement ok 22 | UPDATE ducklake.test SET i=i+100 23 | 24 | # snapshot 4 25 | statement ok 26 | UPDATE ducklake.test SET i=i+100 27 | 28 | # snapshot 5 29 | statement ok 30 | DELETE FROM ducklake.test 31 | 32 | query IIII 33 | FROM ducklake.table_changes('test', 0, 2) ORDER BY ALL 34 | ---- 35 | 2 0 insert 0 36 | 2 1 insert 1 37 | 2 2 insert 2 38 | 39 | query IIII 40 | FROM ducklake.table_changes('test', 3, 3) ORDER BY ALL 41 | ---- 42 | 3 0 update_postimage 100 43 | 3 0 update_preimage 0 44 | 3 1 update_postimage 101 45 | 3 1 update_preimage 1 46 | 3 2 update_postimage 102 47 | 3 2 update_preimage 2 48 | 49 | query IIII 50 | FROM ducklake.table_changes('test', 4, 4) ORDER BY ALL 51 | ---- 52 | 4 0 update_postimage 200 53 | 4 0 update_preimage 100 54 | 4 1 update_postimage 201 55 | 4 1 update_preimage 101 56 | 4 2 update_postimage 202 57 | 4 2 update_preimage 102 58 | 59 | query IIII 60 | FROM ducklake.table_changes('test', 5, 5) ORDER BY ALL 61 | ---- 62 | 5 0 delete 200 63 | 5 1 delete 201 64 | 5 2 delete 202 65 | 66 | # all changes 67 | query IIII 68 | FROM ducklake.table_changes('test', 0, 5) ORDER BY ALL 69 | ---- 70 | 2 0 insert 0 71 | 2 1 insert 1 72 | 2 2 insert 2 73 | 3 0 update_postimage 100 74 | 3 0 update_preimage 0 75 | 3 1 update_postimage 101 76 | 3 1 update_preimage 1 77 | 3 2 update_postimage 102 78 | 3 2 update_preimage 2 79 | 4 0 update_postimage 200 80 | 4 0 update_preimage 100 81 | 4 1 update_postimage 201 82 | 4 1 update_preimage 101 83 | 4 2 update_postimage 202 84 | 4 2 update_preimage 102 85 | 5 0 delete 200 86 | 5 1 delete 201 87 | 5 2 delete 202 88 | -------------------------------------------------------------------------------- /test/sql/data_inlining/data_inlining_transaction_local_alter.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/data_inlining/data_inlining_transaction_local_alter.test 2 | # description: test alter of transaction local data that is inlined 3 | # group: [data_inlining] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_inlining_local_alter.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_inlining_local_alter_files', METADATA_CATALOG 'ducklake_meta', DATA_INLINING_ROW_LIMIT 10) 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test(i INTEGER, j INTEGER) 14 | 15 | # insert -> alter is not supported 16 | statement ok 17 | BEGIN 18 | 19 | statement ok 20 | INSERT INTO ducklake.test VALUES (42, 84); 21 | 22 | statement error 23 | ALTER TABLE ducklake.test ADD COLUMN k INTEGER 24 | ---- 25 | ALTER on a table with transaction-local inlined data is not supported 26 | 27 | statement ok 28 | ROLLBACK 29 | 30 | # alter -> insert works 31 | statement ok 32 | BEGIN 33 | 34 | statement ok 35 | ALTER TABLE ducklake.test ADD COLUMN k INTEGER 36 | 37 | statement ok 38 | INSERT INTO ducklake.test VALUES (42, 84, 100); 39 | 40 | query III 41 | FROM ducklake.test 42 | ---- 43 | 42 84 100 44 | 45 | statement ok 46 | COMMIT 47 | 48 | query III 49 | FROM ducklake.test 50 | ---- 51 | 42 84 100 52 | -------------------------------------------------------------------------------- /test/sql/data_inlining/data_inlining_transaction_local_delete.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/data_inlining/data_inlining_transaction_local_delete.test 2 | # description: test ducklake extension 3 | # group: [data_inlining] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_inlining_delete_local.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_inlining_delete_local_files', METADATA_CATALOG 'ducklake_meta', DATA_INLINING_ROW_LIMIT 10) 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test(i INTEGER, j INTEGER) 14 | 15 | # delete data from transaction-local inlined insertions 16 | statement ok 17 | BEGIN 18 | 19 | statement ok 20 | INSERT INTO ducklake.test VALUES (42, 84), (100, 200), (200, 300), (300, 400), (400, 500); 21 | 22 | query II 23 | SELECT * FROM ducklake.test 24 | ---- 25 | 42 84 26 | 100 200 27 | 200 300 28 | 300 400 29 | 400 500 30 | 31 | query I 32 | DELETE FROM ducklake.test WHERE i=100 33 | ---- 34 | 1 35 | 36 | query II 37 | SELECT * FROM ducklake.test 38 | ---- 39 | 42 84 40 | 200 300 41 | 300 400 42 | 400 500 43 | 44 | statement ok 45 | COMMIT 46 | 47 | query II 48 | SELECT * FROM ducklake.test 49 | ---- 50 | 42 84 51 | 200 300 52 | 300 400 53 | 400 500 54 | 55 | statement ok 56 | DROP TABLE ducklake.test 57 | 58 | # multiple deletes in the same transaction from a fresh table 59 | statement ok 60 | BEGIN 61 | 62 | statement ok 63 | CREATE TABLE ducklake.test AS FROM (VALUES (42, 84), (200, 300), (300, 400), (400, 500)) t(i, j) 64 | 65 | query I 66 | DELETE FROM ducklake.test WHERE i=300; 67 | ---- 68 | 1 69 | 70 | query II 71 | SELECT * FROM ducklake.test 72 | ---- 73 | 42 84 74 | 200 300 75 | 400 500 76 | 77 | query I 78 | DELETE FROM ducklake.test WHERE i=200; 79 | ---- 80 | 1 81 | 82 | query II 83 | SELECT * FROM ducklake.test 84 | ---- 85 | 42 84 86 | 400 500 87 | 88 | query II 89 | SELECT * FROM ducklake.test WHERE i=400 90 | ---- 91 | 400 500 92 | 93 | statement ok 94 | COMMIT 95 | 96 | query II 97 | SELECT * FROM ducklake.test 98 | ---- 99 | 42 84 100 | 400 500 101 | 102 | query II 103 | SELECT * FROM ducklake.test WHERE i=400 104 | ---- 105 | 400 500 106 | -------------------------------------------------------------------------------- /test/sql/data_inlining/data_inlining_types.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/data_inlining/data_inlining_types.test 2 | # description: test data inlining with different data types 3 | # group: [data_inlining] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_inlining_types.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_inlining_types_files', METADATA_CATALOG 'ducklake_meta', DATA_INLINING_ROW_LIMIT 10) 11 | 12 | statement ok 13 | CREATE VIEW all_types AS SELECT * EXCLUDE (VARINT, BIT, small_enum, 14 | medium_enum, 15 | large_enum, 16 | "union", 17 | fixed_int_array, 18 | fixed_varchar_array, 19 | fixed_nested_int_array, 20 | fixed_nested_varchar_array, 21 | fixed_struct_array, 22 | struct_of_fixed_array, 23 | fixed_array_of_int_list, 24 | list_of_fixed_int_array, hugeint, uhugeint, interval, time_tz) FROM test_all_types(); 25 | 26 | query I nosort alltypes 27 | FROM all_types 28 | ---- 29 | 30 | statement ok 31 | BEGIN 32 | 33 | statement ok 34 | CREATE TABLE ducklake.data_types AS FROM all_types 35 | 36 | query I nosort alltypes 37 | FROM ducklake.data_types 38 | ---- 39 | 40 | statement ok 41 | COMMIT 42 | 43 | query I nosort alltypes 44 | FROM ducklake.data_types 45 | ---- 46 | -------------------------------------------------------------------------------- /test/sql/data_inlining/data_inlining_update.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/data_inlining/data_inlining_update.test 2 | # description: test ducklake updating 3 | # group: [data_inlining] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_inlining_update.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_inlining_update_files', METADATA_CATALOG 'ducklake_meta', DATA_INLINING_ROW_LIMIT 10) 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test AS SELECT 1 i, 2 j UNION ALL SELECT NULL, 3 UNION ALL SELECT 10, 20 14 | 15 | statement ok 16 | BEGIN 17 | 18 | # we can update inlined data 19 | query I 20 | UPDATE ducklake.test SET i=i+100 WHERE i=1 21 | ---- 22 | 1 23 | 24 | query IIII 25 | SELECT rowid, snapshot_id, * FROM ducklake.test ORDER BY rowid 26 | ---- 27 | 0 NULL 101 2 28 | 1 1 NULL 3 29 | 2 1 10 20 30 | 31 | statement ok 32 | COMMIT 33 | 34 | query I 35 | SELECT stats(i) FROM ducklake.test LIMIT 1 36 | ---- 37 | [Min: 1, Max: 101][Has Null: true, Has No Null: true] 38 | 39 | query IIII 40 | SELECT rowid, snapshot_id, * FROM ducklake.test ORDER BY rowid 41 | ---- 42 | 0 2 101 2 43 | 1 1 NULL 3 44 | 2 1 10 20 45 | 46 | # update the other row 47 | statement ok 48 | BEGIN 49 | 50 | query I 51 | UPDATE ducklake.test SET i=i+1000 WHERE i=10 52 | ---- 53 | 1 54 | 55 | query IIII 56 | SELECT rowid, snapshot_id, * FROM ducklake.test ORDER BY rowid 57 | ---- 58 | 0 2 101 2 59 | 1 1 NULL 3 60 | 2 NULL 1010 20 61 | 62 | statement ok 63 | COMMIT 64 | 65 | query IIII 66 | SELECT rowid, snapshot_id, * FROM ducklake.test ORDER BY rowid 67 | ---- 68 | 0 2 101 2 69 | 1 1 NULL 3 70 | 2 3 1010 20 71 | -------------------------------------------------------------------------------- /test/sql/default/add_column_with_default.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/default/add_column_with_default.test 2 | # description: Test adding a column with default values with DuckLake 3 | # group: [default] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_add_default.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_add_default', METADATA_CATALOG 'xx') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test(i INTEGER); 14 | 15 | statement ok 16 | INSERT INTO ducklake.test VALUES (1); 17 | 18 | statement ok 19 | INSERT INTO ducklake.test VALUES (2); 20 | 21 | statement ok 22 | BEGIN 23 | 24 | statement ok 25 | ALTER TABLE ducklake.test ADD COLUMN j INTEGER DEFAULT 42 26 | 27 | statement ok 28 | INSERT INTO ducklake.test VALUES (100, 100) 29 | 30 | query II 31 | FROM ducklake.test ORDER BY ALL 32 | ---- 33 | 1 42 34 | 2 42 35 | 100 100 36 | 37 | statement ok 38 | COMMIT 39 | 40 | query II 41 | FROM ducklake.test ORDER BY ALL 42 | ---- 43 | 1 42 44 | 2 42 45 | 100 100 46 | 47 | # alter the default 48 | foreach finaltransaction ROLLBACK COMMIT 49 | 50 | statement ok 51 | BEGIN 52 | 53 | statement ok 54 | INSERT INTO ducklake.test DEFAULT VALUES 55 | 56 | statement ok 57 | ALTER TABLE ducklake.test ALTER i SET DEFAULT 1000 58 | 59 | statement ok 60 | ALTER TABLE ducklake.test ALTER j DROP DEFAULT 61 | 62 | statement ok 63 | INSERT INTO ducklake.test DEFAULT VALUES 64 | 65 | query II 66 | FROM ducklake.test ORDER BY ALL 67 | ---- 68 | 1 42 69 | 2 42 70 | 100 100 71 | 1000 NULL 72 | NULL 42 73 | 74 | statement ok 75 | ${finaltransaction} 76 | 77 | endloop 78 | 79 | statement ok 80 | INSERT INTO ducklake.test DEFAULT VALUES 81 | 82 | query II 83 | FROM ducklake.test ORDER BY ALL 84 | ---- 85 | 1 42 86 | 2 42 87 | 100 100 88 | 1000 NULL 89 | 1000 NULL 90 | NULL 42 91 | 92 | statement error 93 | ALTER TABLE ducklake.test ALTER nonexistent_column SET DEFAULT 1000 94 | ---- 95 | nonexistent_column 96 | 97 | statement error 98 | ALTER TABLE ducklake.test ALTER nonexistent_column DROP DEFAULT 99 | ---- 100 | nonexistent_column 101 | -------------------------------------------------------------------------------- /test/sql/default/default_values.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/default/default_values.test 2 | # description: Test default values with DuckLake 3 | # group: [default] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_default.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_default_files', METADATA_CATALOG 'xx') 11 | 12 | statement ok 13 | BEGIN 14 | 15 | statement ok 16 | CREATE TABLE ducklake.test(i INTEGER DEFAULT 42, j INTEGER); 17 | 18 | statement ok 19 | INSERT INTO ducklake.test (j) VALUES (100) 20 | 21 | statement ok 22 | COMMIT 23 | 24 | statement ok 25 | INSERT INTO ducklake.test (j) VALUES (200) 26 | 27 | query II 28 | SELECT * FROM ducklake.test 29 | ---- 30 | 42 100 31 | 42 200 32 | 33 | statement ok 34 | CREATE TABLE ducklake.test_special_values(i INTEGER, s1 VARCHAR DEFAULT '', s2 VARCHAR DEFAULT 'NULL'); 35 | 36 | statement ok 37 | INSERT INTO ducklake.test_special_values (i) VALUES (100) 38 | 39 | query III 40 | SELECT * FROM ducklake.test_special_values WHERE s2 IS NOT NULL 41 | ---- 42 | 100 (empty) NULL 43 | -------------------------------------------------------------------------------- /test/sql/default/struct_field_default.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/default/struct_field_default.test 2 | # description: Test adding a struct field with default values 3 | # group: [default] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_struct_default.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_struct_default_files', METADATA_CATALOG 'xx') 11 | 12 | 13 | statement ok 14 | CREATE TABLE ducklake.test(col1 STRUCT(i INT, j INT)); 15 | 16 | statement ok 17 | INSERT INTO ducklake.test VALUES ({'i': 1, 'j': 2}) 18 | 19 | # add k INTEGER 20 | statement ok 21 | BEGIN 22 | 23 | statement ok 24 | ALTER TABLE ducklake.test ADD COLUMN col1.k INTEGER DEFAULT 42 25 | 26 | statement ok 27 | INSERT INTO ducklake.test VALUES ({'i': 100, 'j': 200, 'k': 300}) 28 | 29 | query I 30 | FROM ducklake.test 31 | ---- 32 | {'i': 1, 'j': 2, 'k': 42} 33 | {'i': 100, 'j': 200, 'k': 300} 34 | 35 | statement ok 36 | COMMIT 37 | 38 | query I 39 | FROM ducklake.test 40 | ---- 41 | {'i': 1, 'j': 2, 'k': 42} 42 | {'i': 100, 'j': 200, 'k': 300} 43 | -------------------------------------------------------------------------------- /test/sql/delete/basic_delete.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/delete/basic_delete.test 2 | # description: Test ducklake deletes 3 | # group: [delete] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_deletes.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_delete_files') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test AS SELECT i id FROM range(1000) t(i); 14 | 15 | statement ok 16 | INSERT INTO ducklake.test SELECT i id FROM range(15000, 16000) t(i) 17 | 18 | statement ok 19 | BEGIN 20 | 21 | query I 22 | DELETE FROM ducklake.test WHERE id%2=0 23 | ---- 24 | 1000 25 | 26 | query II 27 | SELECT COUNT(*), COUNT(*) FILTER(WHERE id%2=0) FROM ducklake.test 28 | ---- 29 | 1000 0 30 | 31 | statement ok 32 | COMMIT 33 | 34 | query II 35 | SELECT COUNT(*), COUNT(*) FILTER(WHERE id%2=0) FROM ducklake.test 36 | ---- 37 | 1000 0 38 | 39 | # we can time travel to see the state of the table before deletes 40 | query II 41 | SELECT COUNT(*), COUNT(*) FILTER(WHERE id%2=0) FROM ducklake.test AT (VERSION => 2) 42 | ---- 43 | 2000 1000 44 | -------------------------------------------------------------------------------- /test/sql/delete/delete_join.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/delete/delete_join.test 2 | # description: Test ducklake delete using a join 3 | # group: [delete] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_delete_join.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_delete_join_files') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test AS SELECT i id FROM range(500) t(i); 14 | 15 | statement ok 16 | BEGIN 17 | 18 | statement ok 19 | INSERT INTO ducklake.test FROM range(500, 1000) 20 | 21 | statement ok 22 | CREATE TEMPORARY TABLE deleted_rows AS FROM range(0, 1000, 2) t(delete_id); 23 | 24 | query I 25 | DELETE FROM ducklake.test USING deleted_rows WHERE id=deleted_rows.delete_id 26 | ---- 27 | 500 28 | 29 | statement ok 30 | COMMIT 31 | 32 | query I 33 | SELECT COUNT(*) FROM ducklake.test 34 | ---- 35 | 500 36 | -------------------------------------------------------------------------------- /test/sql/delete/delete_rollback_cleanup.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/delete/delete_rollback_cleanup.test 2 | # description: Test ducklake cleaning up files after rollback of delete 3 | # group: [delete] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_delete_rollback_cleanup.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_delete_rollback_cleanup_files') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test AS SELECT i id FROM range(1000) t(i); 14 | 15 | statement ok 16 | BEGIN 17 | 18 | query I 19 | DELETE FROM ducklake.test WHERE id%2=0 20 | ---- 21 | 500 22 | 23 | statement ok 24 | ROLLBACK 25 | 26 | query I 27 | SELECT COUNT(*) FROM ducklake.test 28 | ---- 29 | 1000 30 | 31 | # verify that we don't have the delete file there anymore 32 | query I 33 | SELECT COUNT(*) FROM glob('__TEST_DIR__/ducklake_delete_rollback_cleanup_files/*-delete.parquet') 34 | ---- 35 | 0 36 | -------------------------------------------------------------------------------- /test/sql/delete/delete_same_transaction.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/delete/delete_same_transaction.test 2 | # description: Test ducklake deleting and creating in the same transaction 3 | # group: [delete] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_delete_same_transaction.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_delete_same_transaction_files') 11 | 12 | statement ok 13 | BEGIN 14 | 15 | statement ok 16 | CREATE TABLE ducklake.test AS SELECT i id FROM range(1000) t(i); 17 | 18 | query I 19 | DELETE FROM ducklake.test WHERE id%2=0 20 | ---- 21 | 500 22 | 23 | query I 24 | SELECT COUNT(*) FROM ducklake.test WHERE id<=250 25 | ---- 26 | 125 27 | 28 | query I 29 | DELETE FROM ducklake.test WHERE id<=250 30 | ---- 31 | 125 32 | 33 | query I 34 | SELECT COUNT(*) FROM ducklake.test WHERE id<=250 35 | ---- 36 | 0 37 | 38 | query II 39 | SELECT COUNT(*), COUNT(*) FILTER(WHERE id%2=0) FROM ducklake.test 40 | ---- 41 | 375 0 42 | 43 | statement ok 44 | COMMIT 45 | 46 | # verify that we only have one delete file written after the commit (i.e. we cleaned up the first file 47 | query I 48 | SELECT COUNT(*) FROM glob('__TEST_DIR__/ducklake_delete_same_transaction_files/*-delete.parquet') 49 | ---- 50 | 1 51 | 52 | query II 53 | SELECT COUNT(*), COUNT(*) FILTER(WHERE id%2=0) FROM ducklake.test 54 | ---- 55 | 375 0 56 | -------------------------------------------------------------------------------- /test/sql/delete/empty_delete.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/delete/empty_delete.test 2 | # description: Test ducklake empty delete 3 | # group: [delete] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_empty_delete.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_empty_delete_files') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test AS SELECT i id FROM range(1000) t(i); 14 | 15 | statement ok 16 | BEGIN 17 | 18 | query I 19 | DELETE FROM ducklake.test WHERE id>10000 20 | ---- 21 | 0 22 | 23 | query II 24 | SELECT COUNT(*), COUNT(*) FILTER(WHERE id%2=0) FROM ducklake.test 25 | ---- 26 | 1000 500 27 | 28 | statement ok 29 | COMMIT 30 | 31 | query II 32 | SELECT COUNT(*), COUNT(*) FILTER(WHERE id%2=0) FROM ducklake.test 33 | ---- 34 | 1000 500 35 | -------------------------------------------------------------------------------- /test/sql/delete/multi_deletes.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/delete/multi_deletes.test 2 | # description: Test ducklake deleting multiple batches 3 | # group: [delete] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_multi_deletes.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_multi_deletes_files') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test AS SELECT i id FROM range(10000) t(i); 14 | 15 | # multiple deletes of the same table in the same transaction 16 | statement ok 17 | BEGIN 18 | 19 | query I 20 | DELETE FROM ducklake.test WHERE id%8=0 21 | ---- 22 | 1250 23 | 24 | query II 25 | SELECT COUNT(*), SUM(id) FROM ducklake.test 26 | ---- 27 | 8750 43750000 28 | 29 | query I 30 | DELETE FROM ducklake.test WHERE id%4=0 31 | ---- 32 | 1250 33 | 34 | query II 35 | SELECT COUNT(*), SUM(id) FROM ducklake.test 36 | ---- 37 | 7500 37500000 38 | 39 | statement ok 40 | COMMIT 41 | 42 | query II 43 | SELECT COUNT(*), SUM(id) FROM ducklake.test 44 | ---- 45 | 7500 37500000 46 | 47 | # verify that we only have one delete file written after the commit (i.e. we cleaned up the first file) 48 | query I 49 | SELECT COUNT(*) FROM glob('__TEST_DIR__/ducklake_multi_deletes_files/*-delete.parquet') 50 | ---- 51 | 1 52 | 53 | # deleting from the file again 54 | query I 55 | DELETE FROM ducklake.test WHERE id%2=0 56 | ---- 57 | 2500 58 | 59 | query II 60 | SELECT COUNT(*), SUM(id) FROM ducklake.test 61 | ---- 62 | 5000 25000000 63 | -------------------------------------------------------------------------------- /test/sql/delete/truncate_table.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/delete/truncate_table.test 2 | # description: Test ducklake truncating a table 3 | # group: [delete] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_truncate.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_truncate_files') 11 | 12 | # transaction local truncate 13 | statement ok 14 | BEGIN 15 | 16 | statement ok 17 | CREATE TABLE ducklake.test_local AS SELECT i id FROM range(10000) t(i); 18 | 19 | query I 20 | SELECT COUNT(*) FROM ducklake.test_local 21 | ---- 22 | 10000 23 | 24 | query I 25 | DELETE FROM ducklake.test_local 26 | ---- 27 | 10000 28 | 29 | query I 30 | SELECT COUNT(*) FROM ducklake.test_local 31 | ---- 32 | 0 33 | 34 | statement ok 35 | COMMIT 36 | 37 | # verify that we are not writing any files here 38 | query I 39 | SELECT COUNT(*) FROM glob('__TEST_DIR__/ducklake_truncate_files/*') 40 | ---- 41 | 0 42 | 43 | statement ok 44 | CREATE TABLE ducklake.test AS SELECT i id FROM range(10000) t(i); 45 | 46 | statement ok 47 | BEGIN 48 | 49 | query I 50 | DELETE FROM ducklake.test 51 | ---- 52 | 10000 53 | 54 | query I 55 | SELECT COUNT(*) FROM ducklake.test 56 | ---- 57 | 0 58 | 59 | statement ok 60 | COMMIT 61 | 62 | query I 63 | SELECT COUNT(*) FROM ducklake.test 64 | ---- 65 | 0 66 | 67 | # verify that we are not writing a delete file if we clear the file list 68 | query I 69 | SELECT COUNT(*) FROM glob('__TEST_DIR__/ducklake_truncate_files/*-delete.parquet') 70 | ---- 71 | 0 72 | 73 | query I 74 | DELETE FROM ducklake.test 75 | ---- 76 | 0 77 | -------------------------------------------------------------------------------- /test/sql/ducklake_basic.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/ducklake_basic.test 2 | # description: test ducklake extension 3 | # group: [sql] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_files', METADATA_CATALOG 'ducklake_meta') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test(i INTEGER, j INTEGER); 14 | 15 | query II 16 | SELECT * FROM ducklake.test 17 | ---- 18 | 19 | statement ok 20 | INSERT INTO ducklake.test VALUES (1, 2), (NULL, 3); 21 | 22 | query II 23 | SELECT * FROM ducklake.test 24 | ---- 25 | 1 2 26 | NULL 3 27 | 28 | query I 29 | SELECT COUNT(*) FROM ducklake.test 30 | ---- 31 | 2 32 | 33 | statement ok 34 | INSERT INTO ducklake.test VALUES (4, 5), (6, 7); 35 | 36 | query II 37 | SELECT * FROM ducklake.test 38 | ---- 39 | 1 2 40 | NULL 3 41 | 4 5 42 | 6 7 43 | 44 | statement ok 45 | CREATE TABLE ducklake.test2 AS SELECT 'hello world' AS j, DATE '1992-01-01' date 46 | 47 | query II 48 | SELECT * FROM ducklake.test2 49 | ---- 50 | hello world 1992-01-01 51 | 52 | # re-attach 53 | statement ok 54 | DETACH ducklake 55 | 56 | statement ok 57 | ATTACH 'ducklake:__TEST_DIR__/ducklake.db' AS ducklake 58 | 59 | query II 60 | SELECT * FROM ducklake.test 61 | ---- 62 | 1 2 63 | NULL 3 64 | 4 5 65 | 6 7 66 | 67 | query II 68 | SELECT * FROM ducklake.test2 69 | ---- 70 | hello world 1992-01-01 71 | 72 | statement ok 73 | SHOW ALL TABLES 74 | 75 | statement ok 76 | USE ducklake 77 | 78 | query I 79 | SHOW TABLES 80 | ---- 81 | test 82 | test2 83 | 84 | # data path is not required for DuckDB databases 85 | statement ok 86 | ATTACH 'ducklake:__TEST_DIR__/ducklake_no_data_path.db' AS ducklake_no_data_path 87 | 88 | statement ok 89 | CREATE TABLE ducklake_no_data_path.tbl(i INT); 90 | 91 | statement ok 92 | INSERT INTO ducklake_no_data_path.tbl VALUES (42); 93 | -------------------------------------------------------------------------------- /test/sql/ducklake_multiple_lakes.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/ducklake_multiple_lakes.test 2 | # group: [sql] 3 | 4 | -------------------------------------------------------------------------------- /test/sql/encryption/encryption.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/encryption/encryption.test 2 | # description: Test ducklake encryption support 3 | # group: [encryption] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_encryption.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_encryption_files', ENCRYPTED) 11 | 12 | statement ok 13 | BEGIN 14 | 15 | statement ok 16 | CREATE TABLE ducklake.test AS SELECT i id FROM range(1000) t(i); 17 | 18 | # we can read them through ducklake 19 | query II 20 | SELECT COUNT(*), SUM(id) FROM ducklake.test 21 | ---- 22 | 1000 499500 23 | 24 | statement ok 25 | COMMIT 26 | 27 | # also after committing 28 | query II 29 | SELECT COUNT(*), SUM(id) FROM ducklake.test 30 | ---- 31 | 1000 499500 32 | 33 | # we cannot read the files using a regular parquet read as they have been encrypted - we need the encryption key 34 | statement error 35 | SELECT * FROM '__TEST_DIR__/ducklake_encryption_files/*.parquet' 36 | ---- 37 | encrypted 38 | 39 | # deletes are also encrypted 40 | statement ok 41 | BEGIN 42 | 43 | statement ok 44 | DELETE FROM ducklake.test WHERE id%2=0 45 | 46 | query II 47 | SELECT COUNT(*), SUM(id) FROM ducklake.test 48 | ---- 49 | 500 250000 50 | 51 | statement ok 52 | COMMIT 53 | 54 | query II 55 | SELECT COUNT(*), SUM(id) FROM ducklake.test 56 | ---- 57 | 500 250000 58 | 59 | # similarly we cannot read deletes, as they are encrypted 60 | statement error 61 | SELECT * FROM '__TEST_DIR__/ducklake_encryption_files/*-del*.parquet' 62 | ---- 63 | encrypted 64 | 65 | # restart an encrypted database works 66 | statement ok 67 | DETACH ducklake 68 | 69 | statement ok 70 | ATTACH 'ducklake:__TEST_DIR__/ducklake_encryption.db' AS ducklake 71 | 72 | query II 73 | SELECT COUNT(*), SUM(id) FROM ducklake.test 74 | ---- 75 | 500 250000 76 | -------------------------------------------------------------------------------- /test/sql/encryption/partitioning_encryption.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/encryption/partitioning_encryption.test 2 | # description: Test partitioning with encryption 3 | # group: [encryption] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_partitioning_encryption.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_partitioning_encryption', METADATA_CATALOG 'ducklake_metadata', ENCRYPTED) 11 | 12 | statement ok 13 | USE ducklake 14 | 15 | statement ok 16 | CREATE TABLE partitioned_tbl(part_key INTEGER, values VARCHAR); 17 | 18 | statement ok 19 | ALTER TABLE partitioned_tbl SET PARTITIONED BY (part_key); 20 | 21 | statement ok 22 | INSERT INTO partitioned_tbl SELECT i%2, concat('thisisastring_', i) FROM range(10000) t(i) 23 | 24 | # verify files are partitioned, but the information is not leaked in the filename 25 | query III 26 | SELECT data_file_id, partition_id, regexp_extract(path, '.*(part_key=[0-9])/.*', 1) FROM ducklake_metadata.ducklake_data_file 27 | ORDER BY ALL 28 | ---- 29 | 0 2 (empty) 30 | 1 2 (empty) 31 | -------------------------------------------------------------------------------- /test/sql/functions/ducklake_table_info.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/functions/ducklake_table_info.test 2 | # group: [functions] 3 | 4 | require ducklake 5 | 6 | require parquet 7 | 8 | statement ok 9 | ATTACH 'ducklake:__TEST_DIR__/ducklake_table_info.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_table_info_files') 10 | 11 | statement ok 12 | CREATE TABLE ducklake.test(i INTEGER) 13 | 14 | statement ok 15 | INSERT INTO ducklake.test FROM range(1000) 16 | 17 | statement ok 18 | DELETE FROM ducklake.test WHERE i%2=0 19 | 20 | query IIIIIII 21 | SELECT table_name, schema_id, table_id, file_count, file_size_bytes > 0, delete_file_count, delete_file_size_bytes > 0 FROM ducklake.table_info(); 22 | ---- 23 | test 0 1 1 true 1 true 24 | -------------------------------------------------------------------------------- /test/sql/general/attach_at_snapshot.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/general/attach_at_snapshot.test 2 | # description: test ducklake extension 3 | # group: [general] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_attach_at_snapshot.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_files_attach_at_snapshot') 11 | 12 | # snapshot 1 13 | statement ok 14 | CREATE TABLE ducklake.test(i INTEGER, j INTEGER); 15 | 16 | # snapshot 2 17 | statement ok 18 | INSERT INTO ducklake.test VALUES (1, 2), (NULL, 3); 19 | 20 | statement ok 21 | DETACH ducklake 22 | 23 | # attach at snapshot 24 | statement ok 25 | ATTACH 'ducklake:__TEST_DIR__/ducklake_attach_at_snapshot.db' AS ducklake (SNAPSHOT_VERSION 1) 26 | 27 | query II 28 | FROM ducklake.test 29 | ---- 30 | 31 | # attaching at a specific snapshot means read-only 32 | statement error 33 | INSERT INTO ducklake.test VALUES (10, 100); 34 | ---- 35 | read-only 36 | 37 | statement ok 38 | DETACH ducklake 39 | 40 | # snapshot does not exist 41 | statement error 42 | ATTACH 'ducklake:__TEST_DIR__/ducklake_attach_at_snapshot.db' AS ducklake (SNAPSHOT_VERSION 33) 43 | ---- 44 | No snapshot found at version 33 45 | 46 | # cannot open a database at a specified snapshot in read_write mode 47 | statement error 48 | ATTACH 'ducklake:__TEST_DIR__/ducklake_attach_at_snapshot.db' AS ducklake (SNAPSHOT_VERSION 1, READ_WRITE) 49 | ---- 50 | can only be used in read-only mode 51 | 52 | # cannot combine snapshot_version/snapshot_time 53 | statement error 54 | ATTACH 'ducklake:__TEST_DIR__/ducklake_attach_at_snapshot.db' AS ducklake (SNAPSHOT_VERSION 1, SNAPSHOT_TIME '2020-01-01') 55 | ---- 56 | Cannot specify both VERSION and TIMESTAMP 57 | -------------------------------------------------------------------------------- /test/sql/general/data_path_tag.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/general/data_path_tag.test 2 | # description: test ducklakes database have a tag['data_path'] via duckdb_databases() 3 | # group: [general] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_duckdb_tag_data_path.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_duckdb_tables_files', METADATA_CATALOG 'ducklake_meta') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test(i INTEGER); 14 | 15 | statement ok 16 | INSERT INTO ducklake.test FROM range(100); 17 | 18 | query III 19 | SELECT table_name, estimated_size, column_count FROM duckdb_tables() WHERE database_name='ducklake'; 20 | ---- 21 | test 100 1 22 | 23 | ### Newly created DuckLake with DATA_PATH, returns the one provided 24 | 25 | query I 26 | SELECT count(*) FROM (SELECT tags['data_path'] as data_path FROM duckdb_databases() WHERE tags['data_path'] IS NOT NULL AND data_path ILIKE '%ducklake_duckdb_tables_files%'); 27 | ---- 28 | 1 29 | 30 | statement ok 31 | DETACH ducklake; 32 | 33 | statement ok 34 | ATTACH 'ducklake:__TEST_DIR__/ducklake_duckdb_tag_data_path.db' AS ducklake; 35 | 36 | ### Existing DuckLake with no DATA_PATH, returns the one from the metadata catalog 37 | 38 | query I 39 | SELECT count(*) FROM (SELECT tags['data_path'] as data_path FROM duckdb_databases() WHERE tags['data_path'] IS NOT NULL AND data_path ILIKE '%ducklake_duckdb_tables_files%'); 40 | ---- 41 | 1 42 | 43 | statement ok 44 | DETACH ducklake; 45 | 46 | statement ok 47 | ATTACH 'ducklake:__TEST_DIR__/ducklake_duckdb_tag_data_path.db' AS ducklake (DATA_PATH something_else); 48 | 49 | ### Existing DuckLake with different DATA_PATH, returns the currently provided one 50 | 51 | query I 52 | SELECT count(*) FROM (SELECT tags['data_path'] as data_path FROM duckdb_databases() WHERE tags['data_path'] IS NOT NULL AND data_path ILIKE '%ducklake_duckdb_tables_files%'); 53 | ---- 54 | 0 55 | 56 | query I 57 | SELECT count(*) FROM (SELECT tags['data_path'] as data_path FROM duckdb_databases() WHERE tags['data_path'] IS NOT NULL AND data_path ILIKE '%something_else%'); 58 | ---- 59 | 1 60 | 61 | statement ok 62 | DETACH ducklake; 63 | 64 | statement ok 65 | ATTACH 'ducklake:__TEST_DIR__/ducklake_duckdb_tag_data_path.db' AS ducklake; 66 | 67 | ### Existing DuckLake with no DATA_PATH, returns the one from the metadata catalog (the original one, not the overridden one) 68 | 69 | query I 70 | SELECT count(*) FROM (SELECT tags['data_path'] as data_path FROM duckdb_databases() WHERE tags['data_path'] IS NOT NULL AND data_path ILIKE '%ducklake_duckdb_tables_files%'); 71 | ---- 72 | 1 73 | 74 | -------------------------------------------------------------------------------- /test/sql/general/database_size.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/general/database_size.test 2 | # description: test ducklake extension 3 | # group: [general] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_database_size.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_database_size_files', METADATA_CATALOG 'ducklake_meta') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test(i INTEGER, j INTEGER); 14 | 15 | statement ok 16 | INSERT INTO ducklake.test SELECT i, i + 100 FROm range(1000) t(i); 17 | 18 | statement ok 19 | PRAGMA database_size 20 | 21 | query I 22 | SELECT database_size <> '0 bytes' FROM PRAGMA_database_size() WHERE database_name='ducklake' 23 | ---- 24 | true 25 | -------------------------------------------------------------------------------- /test/sql/general/detach_ducklake.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/general/detach_ducklake.test 2 | # description: Test detaching ducklake 3 | # group: [general] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_detach.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_detach_files', METADATA_CATALOG 'ducklake_metadata') 11 | 12 | statement ok 13 | DETACH ducklake 14 | 15 | statement ok 16 | ATTACH 'ducklake:__TEST_DIR__/ducklake_detach.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_detach_files', METADATA_CATALOG 'ducklake_metadata') 17 | 18 | # what happens if we detach the metadata catalog? 19 | statement ok 20 | DETACH ducklake_metadata 21 | 22 | statement error 23 | CREATE TABLE ducklake.tbl(i INTEGER) 24 | ---- 25 | Catalog "ducklake_metadata" does not exist! 26 | -------------------------------------------------------------------------------- /test/sql/general/ducklake_read_only.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/general/ducklake_read_only.test 2 | # description: test ducklake with read-only 3 | # group: [general] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | # non-existent 10 | statement error 11 | ATTACH 'ducklake:__TEST_DIR__/ducklake_read_only.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_read_only_files', READ_ONLY) 12 | ---- 13 | 14 | statement ok 15 | ATTACH 'ducklake:__TEST_DIR__/ducklake_read_only.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_read_only_files') 16 | 17 | statement ok 18 | CREATE TABLE ducklake.tbl(i INTEGER) 19 | 20 | statement ok 21 | DETACH ducklake 22 | 23 | statement ok 24 | ATTACH 'ducklake:__TEST_DIR__/ducklake_read_only.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_read_only_files', READ_ONLY) 25 | 26 | query I 27 | FROM ducklake.tbl 28 | ---- 29 | 30 | statement error 31 | INSERT INTO ducklake.tbl VALUES (42); 32 | ---- 33 | read-only 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /test/sql/general/generated_columns.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/general/generated_columns.test 2 | # description: Test generated columns in ducklake 3 | # group: [general] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_generated.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_generated') 11 | 12 | statement error 13 | CREATE TABLE ducklake.t0(c0 INT AS (1), c1 INT); 14 | ---- 15 | does not support 16 | -------------------------------------------------------------------------------- /test/sql/general/metadata_cache.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/general/metadata_cache.test 2 | # description: Test COUNT(*) with Parquet metadata cache 3 | # group: [general] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | SET parquet_metadata_cache=true; 11 | 12 | statement ok 13 | ATTACH 'ducklake:__TEST_DIR__/ducklake_pq_metadata_cache.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_pq_metadata_cache_files') 14 | 15 | statement ok 16 | CREATE TABLE ducklake.tbl AS FROM range(1000) t(i); 17 | 18 | statement ok 19 | DELETE FROM ducklake.tbl WHERE i%2=0 20 | 21 | query I 22 | SELECT COUNT(*) FROM ducklake.tbl 23 | ---- 24 | 500 25 | 26 | query I 27 | SELECT COUNT(*) FROM ducklake.tbl 28 | ---- 29 | 500 30 | -------------------------------------------------------------------------------- /test/sql/general/metadata_parameters.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/general/metadata_parameters.test 2 | # description: Test attach with metadata parameters 3 | # group: [general] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_metadata_parameters.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_metadata_parameters', META_TYPE 'DUCKDB') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.tbl AS FROM range(1000) t(i); 14 | 15 | query I 16 | SELECT COUNT(*) FROM ducklake.tbl 17 | ---- 18 | 1000 19 | 20 | statement error 21 | ATTACH 'ducklake:__TEST_DIR__/ducklake_metadata_parametersxx.db' AS s (DATA_PATH '__TEST_DIR__/ducklake_metadata_parameters', META_TYPE 'DUCKDBXX') 22 | ---- 23 | duckdbxx 24 | -------------------------------------------------------------------------------- /test/sql/general/missing_parquet.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/general/missing_parquet.test 2 | # description: Test with missing parquet extension 3 | # group: [general] 4 | 5 | require ducklake 6 | 7 | require no_extension_autoloading "EXPECTED This is meant to test missing parquet extension" 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_missing_parquet.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_missing_parquet') 11 | 12 | statement ok 13 | SELECT snapshot_id, schema_version, changes FROM ducklake_snapshots('ducklake') 14 | 15 | statement error 16 | CREATE TABLE ducklake.tbl AS FROM range(1000) t(i); 17 | ---- 18 | Missing Extension Error: Could not load the copy function for "parquet". Try explicitly loading the "parquet" extension 19 | 20 | require parquet 21 | 22 | statement ok 23 | CREATE TABLE ducklake.tbl AS FROM range(1000) t(i); 24 | -------------------------------------------------------------------------------- /test/sql/general/prepared_statement.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/general/prepared_statement.test 2 | # description: Test prepared statements with DuckLake 3 | # group: [general] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | SET parquet_metadata_cache=true; 11 | 12 | # FIXME: succeeds on v1.3 branch but fails on v1.3.0 release 13 | mode skip 14 | 15 | statement ok 16 | ATTACH 'ducklake:__TEST_DIR__/ducklake_prepared.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_prepared_files') 17 | 18 | statement ok 19 | CREATE TABLE ducklake.tbl AS FROM range(1000) t(i); 20 | 21 | statement ok 22 | PREPARE v1 AS SELECT * FROM ducklake.tbl LIMIT 3 23 | 24 | statement error 25 | EXECUTE v1 26 | ---- 27 | this use case is not yet supported 28 | -------------------------------------------------------------------------------- /test/sql/general/recursive_metadata_catalog.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/general/recursive_metadata_catalog.test 2 | # description: Test recursive metadata catalogs 3 | # group: [general] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement error 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_recursive.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_recursive_files', METADATA_CATALOG 'ducklake') 11 | ---- 12 | 13 | -------------------------------------------------------------------------------- /test/sql/insert/insert_column_list.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/insert/insert_column_list.test 2 | # description: test ducklake insert column list 3 | # group: [insert] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_insert_list.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_insert_list_files') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test(i INTEGER, j VARCHAR); 14 | 15 | statement ok 16 | INSERT INTO ducklake.test (j, i) VALUES ('hello', 84); 17 | 18 | query II 19 | SELECT * FROM ducklake.test 20 | ---- 21 | 84 hello 22 | 23 | # insert default value 24 | statement ok 25 | INSERT INTO ducklake.test (j) VALUES ('world'); 26 | 27 | statement ok 28 | INSERT INTO ducklake.test (i) VALUES (100); 29 | 30 | query II 31 | SELECT * FROM ducklake.test 32 | ---- 33 | 84 hello 34 | NULL world 35 | 100 NULL 36 | 37 | statement ok 38 | INSERT INTO ducklake.test DEFAULT VALUES 39 | 40 | query II 41 | SELECT * FROM ducklake.test 42 | ---- 43 | 84 hello 44 | NULL world 45 | 100 NULL 46 | NULL NULL 47 | 48 | statement ok 49 | INSERT INTO ducklake.test VALUES (1000, DEFAULT), (DEFAULT, 'xxx'); 50 | 51 | query II 52 | SELECT * FROM ducklake.test 53 | ---- 54 | 84 hello 55 | NULL world 56 | 100 NULL 57 | NULL NULL 58 | 1000 NULL 59 | NULL xxx 60 | -------------------------------------------------------------------------------- /test/sql/insert/insert_into_self.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/insert/insert_into_self.test 2 | # description: Test ducklake insert into self 3 | # group: [insert] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_insert_self.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_insert_self_files') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test(i INTEGER, j VARCHAR); 14 | 15 | statement ok 16 | BEGIN 17 | 18 | query I 19 | INSERT INTO ducklake.test VALUES (1, '2'), (NULL, '3'); 20 | ---- 21 | 2 22 | 23 | query I 24 | INSERT INTO ducklake.test FROM ducklake.test 25 | ---- 26 | 2 27 | 28 | query I 29 | INSERT INTO ducklake.test FROM ducklake.test 30 | ---- 31 | 4 32 | 33 | query I 34 | INSERT INTO ducklake.test FROM ducklake.test 35 | ---- 36 | 8 37 | 38 | query I 39 | INSERT INTO ducklake.test SELECT a.i, a.j FROM ducklake.test a, ducklake.test b 40 | ---- 41 | 256 42 | 43 | statement ok 44 | COMMIT 45 | 46 | query III 47 | SELECT SUM(i), SUM(STRLEN(j)), COUNT(*) FROM ducklake.test 48 | ---- 49 | 136 272 272 50 | -------------------------------------------------------------------------------- /test/sql/metadata/ducklake_duckdb_tables.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/metadata/ducklake_duckdb_tables.test 2 | # description: test duckdb_tables with ducklake 3 | # group: [metadata] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_duckdb_tables.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_duckdb_tables_files', METADATA_CATALOG 'ducklake_meta') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test(i INTEGER); 14 | 15 | statement ok 16 | INSERT INTO ducklake.test FROM range(100); 17 | 18 | query III 19 | SELECT table_name, estimated_size, column_count FROM duckdb_tables() WHERE database_name='ducklake'; 20 | ---- 21 | test 100 1 22 | -------------------------------------------------------------------------------- /test/sql/partitioning/multi_key_partition.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/partitioning/multi_key_partition.test 2 | # description: Test multi-key partitioning 3 | # group: [partitioning] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | # partitioning based on a column 10 | statement ok 11 | ATTACH 'ducklake:__TEST_DIR__/ducklake_mk_partitioning.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_mk_partitioning', METADATA_CATALOG 'ducklake_metadata') 12 | 13 | statement ok 14 | USE ducklake 15 | 16 | statement ok 17 | CREATE TABLE partitioned_tbl(a INTEGER, b INTEGER, c INTEGER, values VARCHAR); 18 | 19 | statement ok 20 | ALTER TABLE partitioned_tbl SET PARTITIONED BY (a,b,c); 21 | 22 | statement ok 23 | INSERT INTO partitioned_tbl VALUES (10, 100, 1000, 'data 1'), (20,200,2000, 'data 2') 24 | 25 | query IIIII 26 | SELECT data_file_id, partition_id, regexp_extract(path, '.*(a=[0-9]+)[/\\].*', 1), regexp_extract(path, '.*(b=[0-9]+)[/\\].*', 1), regexp_extract(path, '.*(c=[0-9]+)[/\\].*', 1) FROM ducklake_metadata.ducklake_data_file 27 | ORDER BY ALL 28 | ---- 29 | 0 2 a=10 b=100 c=1000 30 | 1 2 a=20 b=200 c=2000 31 | 32 | # verify files are pruned when querying the partitions 33 | query II 34 | EXPLAIN ANALYZE SELECT COUNT(*) FROM partitioned_tbl WHERE a=10 35 | ---- 36 | analyzed_plan :.*Total Files Read: 1.* 37 | 38 | query II 39 | EXPLAIN ANALYZE SELECT COUNT(*) FROM partitioned_tbl WHERE b=100 40 | ---- 41 | analyzed_plan :.*Total Files Read: 1.* 42 | 43 | query II 44 | EXPLAIN ANALYZE SELECT COUNT(*) FROM partitioned_tbl WHERE c=1000 45 | ---- 46 | analyzed_plan :.*Total Files Read: 1.* 47 | 48 | query IIII 49 | SELECT * FROM ducklake_metadata.ducklake_file_partition_value ORDER BY ALL 50 | ---- 51 | 0 1 0 10 52 | 0 1 1 100 53 | 0 1 2 1000 54 | 1 1 0 20 55 | 1 1 1 200 56 | 1 1 2 2000 57 | -------------------------------------------------------------------------------- /test/sql/schema_evolution/field_ids.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/schema_evolution/field_ids.test 2 | # description: Verify that field ids are correctly written 3 | # group: [schema_evolution] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_field_ids.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_field_ids_files') 11 | 12 | # insert into 13 | statement ok 14 | CREATE TABLE ducklake.t1(t1_c1 INT, t1_c2 VARCHAR, t1_c3 TIMESTAMP); 15 | 16 | statement ok 17 | INSERT INTO ducklake.t1 VALUES (42, 'hello', TIMESTAMP '1992-01-01'); 18 | 19 | # CTAS 20 | statement ok 21 | CREATE TABLE ducklake.t2 AS SELECT 84 t2_c1, 'world' t2_c2 22 | 23 | query III 24 | SELECT name, type, field_id FROM parquet_schema('__TEST_DIR__/ducklake_field_ids_files/*.parquet') 25 | WHERE name <> 'duckdb_schema' 26 | ORDER BY name 27 | ---- 28 | t1_c1 INT32 1 29 | t1_c2 BYTE_ARRAY 2 30 | t1_c3 INT64 3 31 | t2_c1 INT32 1 32 | t2_c2 BYTE_ARRAY 2 33 | 34 | # deeply_nested 35 | statement ok 36 | CREATE TABLE ducklake.t3 AS 37 | SELECT {'c1': 42, 'c2': {'n1': 84, 'n2': 32}} struct_col, 38 | [42, 48] l, 39 | [{'x': 42, 'y': 84}] as list_of_structs, 40 | {'l': [1,2,3], 'l2': 42, 'l3': {'x': 42, 'y': 84}} as struct_of_lists 41 | 42 | query II 43 | SELECT name, field_id FROM parquet_schema('__TEST_DIR__/ducklake_field_ids_files/*.parquet') 44 | WHERE name <> 'duckdb_schema' AND name NOT LIKE 't1%' AND name NOT LIKE 't2%' 45 | ORDER BY name 46 | ---- 47 | c1 2 48 | c2 3 49 | element 7 50 | element 9 51 | element 14 52 | l 6 53 | l 13 54 | l2 15 55 | l3 16 56 | list NULL 57 | list NULL 58 | list NULL 59 | list_of_structs 8 60 | n1 4 61 | n2 5 62 | struct_col 1 63 | struct_of_lists 12 64 | x 10 65 | x 17 66 | y 11 67 | y 18 68 | -------------------------------------------------------------------------------- /test/sql/settings/parquet_compression.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/settings/parquet_compression.test 2 | # description: Test parquet compression 3 | # group: [settings] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_parquet_compression.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_parquet_compression') 11 | 12 | statement ok 13 | CALL ducklake.set_option('parquet_compression', 'zstd') 14 | 15 | statement ok 16 | CALL ducklake.set_option('parquet_version', '2') 17 | 18 | statement ok 19 | CALL ducklake.set_option('parquet_compression_level', '17') 20 | 21 | statement ok 22 | CALL ducklake.set_option('parquet_row_group_size', '64000') 23 | 24 | statement ok 25 | CREATE TABLE ducklake.tbl AS SELECT i, 'hello world' || i str FROM range(100000) t(i) 26 | 27 | # zstd + v2 encodings are used 28 | query II 29 | SELECT DISTINCT compression, encodings FROM parquet_metadata('__TEST_DIR__/ducklake_parquet_compression/**') ORDER BY ALL 30 | ---- 31 | ZSTD DELTA_BINARY_PACKED 32 | ZSTD DELTA_LENGTH_BYTE_ARRAY 33 | 34 | # we have two row groups (of up to 64K rows) 35 | query I 36 | SELECT COUNT(*) FROM parquet_metadata('__TEST_DIR__/ducklake_parquet_compression/**') ORDER BY ALL 37 | ---- 38 | 2 39 | 40 | # ensure setting persists across restarts 41 | statement ok 42 | DETACH ducklake 43 | 44 | statement ok 45 | ATTACH 'ducklake:__TEST_DIR__/ducklake_parquet_compression.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_parquet_compression') 46 | 47 | statement ok 48 | INSERT INTO ducklake.tbl VALUES (42, 'hello world'); 49 | 50 | query II 51 | SELECT DISTINCT compression, encodings FROM parquet_metadata('__TEST_DIR__/ducklake_parquet_compression/**') ORDER BY ALL 52 | ---- 53 | ZSTD DELTA_BINARY_PACKED 54 | ZSTD DELTA_LENGTH_BYTE_ARRAY 55 | ZSTD RLE_DICTIONARY 56 | 57 | # unknown settings 58 | statement error 59 | CALL ducklake.set_option('parquet_compression', 'zstdx') 60 | ---- 61 | Unsupported 62 | 63 | statement error 64 | CALL ducklake.set_option('parquet_version', 'z') 65 | ---- 66 | Could not convert 67 | 68 | statement error 69 | CALL ducklake.set_option('parquet_compression_level', 'z') 70 | ---- 71 | Could not convert 72 | 73 | statement error 74 | CALL ducklake.set_option('parquet_row_group_size', 'z') 75 | ---- 76 | Could not convert 77 | 78 | statement error 79 | CALL ducklake.set_option('parquet_bla', '42') 80 | ---- 81 | Unsupported 82 | -------------------------------------------------------------------------------- /test/sql/stats/cardinality.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/stats/cardinality.test 2 | # description: test ducklake cardinality extension 3 | # group: [stats] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_cardinality.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_cardinality_files') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test(i INTEGER); 14 | 15 | statement ok 16 | INSERT INTO ducklake.test FROM range(1000); 17 | 18 | query II 19 | EXPLAIN SELECT * FROM ducklake.test 20 | ---- 21 | physical_plan :.*~1000.* 22 | -------------------------------------------------------------------------------- /test/sql/stats/global_stats_transactions.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/stats/global_stats_transactions.test 2 | # description: Test global stats in the presence of transaction conflicts 3 | # group: [stats] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_global_stats_transactions.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_global_stats_transactions') 11 | 12 | statement ok 13 | SET immediate_transaction_mode=true 14 | 15 | # test tracking of stats across transactions 16 | statement ok 17 | CREATE TABLE ducklake.test(i INTEGER); 18 | 19 | statement ok con1 20 | BEGIN 21 | 22 | statement ok con2 23 | BEGIN 24 | 25 | statement ok con3 26 | BEGIN 27 | 28 | statement ok con1 29 | INSERT INTO ducklake.test VALUES (42); 30 | 31 | statement ok con2 32 | INSERT INTO ducklake.test VALUES (84); 33 | 34 | statement ok con3 35 | INSERT INTO ducklake.test VALUES (NULL); 36 | 37 | statement ok con1 38 | COMMIT 39 | 40 | statement ok con2 41 | COMMIT 42 | 43 | statement ok con3 44 | COMMIT 45 | 46 | query I 47 | SELECT stats(i) FROM ducklake.test LIMIT 1 48 | ---- 49 | :.*Min.*42.*Max.*84.*Has Null.*true.* 50 | -------------------------------------------------------------------------------- /test/sql/table_changes/ducklake_table_changes.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/table_changes/ducklake_table_changes.test 2 | # description: test ducklake_table_changes function 3 | # group: [table_changes] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_table_changes.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_table_changes_files') 11 | 12 | # snapshot 1 13 | statement ok 14 | CREATE TABLE ducklake.test(i INTEGER); 15 | 16 | # snapshot 2 17 | statement ok 18 | INSERT INTO ducklake.test FROM range(3); 19 | 20 | # snapshot 3 21 | statement ok 22 | UPDATE ducklake.test SET i=i+100 23 | 24 | # snapshot 4 25 | statement ok 26 | UPDATE ducklake.test SET i=i+100 27 | 28 | # snapshot 5 29 | statement ok 30 | DELETE FROM ducklake.test 31 | 32 | query IIII 33 | FROM ducklake.table_changes('test', 0, 2) ORDER BY ALL 34 | ---- 35 | 2 0 insert 0 36 | 2 1 insert 1 37 | 2 2 insert 2 38 | 39 | # with timestamps 40 | statement ok 41 | SET VARIABLE begin_ts = (SELECT snapshot_time FROM ducklake.snapshots() WHERE snapshot_id = 0) 42 | 43 | statement ok 44 | SET VARIABLE end_ts = (SELECT snapshot_time FROM ducklake.snapshots() WHERE snapshot_id = 2) 45 | 46 | query IIII 47 | FROM ducklake.table_changes('test', getvariable('begin_ts'), getvariable('end_ts')) ORDER BY ALL 48 | ---- 49 | 2 0 insert 0 50 | 2 1 insert 1 51 | 2 2 insert 2 52 | 53 | query IIII 54 | FROM ducklake.table_changes('test', 3, 3) ORDER BY ALL 55 | ---- 56 | 3 0 update_postimage 100 57 | 3 0 update_preimage 0 58 | 3 1 update_postimage 101 59 | 3 1 update_preimage 1 60 | 3 2 update_postimage 102 61 | 3 2 update_preimage 2 62 | 63 | query IIII 64 | FROM ducklake.table_changes('test', 4, 4) ORDER BY ALL 65 | ---- 66 | 4 0 update_postimage 200 67 | 4 0 update_preimage 100 68 | 4 1 update_postimage 201 69 | 4 1 update_preimage 101 70 | 4 2 update_postimage 202 71 | 4 2 update_preimage 102 72 | 73 | query IIII 74 | FROM ducklake.table_changes('test', 5, 5) ORDER BY ALL 75 | ---- 76 | 5 0 delete 200 77 | 5 1 delete 201 78 | 5 2 delete 202 79 | 80 | # all changes 81 | query IIII 82 | FROM ducklake.table_changes('test', 0, 5) ORDER BY ALL 83 | ---- 84 | 2 0 insert 0 85 | 2 1 insert 1 86 | 2 2 insert 2 87 | 3 0 update_postimage 100 88 | 3 0 update_preimage 0 89 | 3 1 update_postimage 101 90 | 3 1 update_preimage 1 91 | 3 2 update_postimage 102 92 | 3 2 update_preimage 2 93 | 4 0 update_postimage 200 94 | 4 0 update_preimage 100 95 | 4 1 update_postimage 201 96 | 4 1 update_preimage 101 97 | 4 2 update_postimage 202 98 | 4 2 update_preimage 102 99 | 5 0 delete 200 100 | 5 1 delete 201 101 | 5 2 delete 202 102 | -------------------------------------------------------------------------------- /test/sql/table_changes/ducklake_table_insertions.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/table_changes/ducklake_table_insertions.test 2 | # description: test ducklake_table_insertions function 3 | # group: [table_changes] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_table_insertions.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_table_insertions_files'); 11 | 12 | # snapshot 1 13 | statement ok 14 | CREATE TABLE ducklake.test(i INTEGER); 15 | 16 | # snapshot 2 17 | statement ok 18 | INSERT INTO ducklake.test VALUES (1); 19 | 20 | # snapshot 3 21 | statement ok 22 | INSERT INTO ducklake.test VALUES (2); 23 | 24 | # snapshot 4 25 | statement ok 26 | INSERT INTO ducklake.test VALUES (3); 27 | 28 | # snapshot 5 29 | statement ok 30 | INSERT INTO ducklake.test VALUES (NULL); 31 | 32 | # snapshot 6 33 | statement ok 34 | INSERT INTO ducklake.test FROM range(10, 12); 35 | 36 | query II 37 | SELECT rowid, * FROM ducklake_table_insertions('ducklake', 'main', 'test', 0, 2); 38 | ---- 39 | 0 1 40 | 41 | query II 42 | SELECT rowid, * FROM ducklake_table_insertions('ducklake', 'main', 'test', 0, 3); 43 | ---- 44 | 0 1 45 | 1 2 46 | 47 | query II 48 | SELECT rowid, * FROM ducklake_table_insertions('ducklake', 'main', 'test', 0, 4); 49 | ---- 50 | 0 1 51 | 1 2 52 | 2 3 53 | 54 | query II 55 | SELECT rowid, * FROM ducklake_table_insertions('ducklake', 'main', 'test', 4, 5); 56 | ---- 57 | 2 3 58 | 3 NULL 59 | 60 | query II 61 | SELECT rowid, * FROM ducklake_table_insertions('ducklake', 'main', 'test', 5, 5); 62 | ---- 63 | 3 NULL 64 | 65 | # snapshot 7: update a subset of the rows 66 | statement ok 67 | UPDATE ducklake.test SET i=i+100 WHERE i < 11; 68 | 69 | query II 70 | SELECT rowid, * FROM ducklake_table_insertions('ducklake', 'main', 'test', 7, 7) ORDER BY rowid 71 | ---- 72 | 0 101 73 | 1 102 74 | 2 103 75 | 4 110 76 | 77 | # the change feed has both the original rows and the updated rows 78 | query II 79 | SELECT rowid, * FROM ducklake_table_insertions('ducklake', 'main', 'test', 0, 7) ORDER BY ALL 80 | ---- 81 | 0 1 82 | 0 101 83 | 1 2 84 | 1 102 85 | 2 3 86 | 2 103 87 | 3 NULL 88 | 4 10 89 | 4 110 90 | 5 11 91 | -------------------------------------------------------------------------------- /test/sql/time_travel/basic_time_travel.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/time_travel/basic_time_travel.test 2 | # description: test time travel in ducklake 3 | # group: [time_travel] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | require icu 10 | 11 | statement ok 12 | ATTACH 'ducklake:__TEST_DIR__/ducklake_time_travel.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_time_travel_files') 13 | 14 | statement ok 15 | CREATE TABLE ducklake.test(s STRUCT(i INTEGER, j INTEGER)); 16 | 17 | query I 18 | SELECT * FROM ducklake.test 19 | ---- 20 | 21 | statement ok 22 | INSERT INTO ducklake.test VALUES ({'i': 1, 'j': 2}), ({'i': NULL, 'j': 3}), (NULL); 23 | 24 | # non-existent version 25 | statement error 26 | SELECT * FROM ducklake.test AT (VERSION => 10) 27 | ---- 28 | No snapshot found at version 10 29 | 30 | # table does not exist 31 | statement error 32 | SELECT * FROM ducklake.test AT (VERSION => 0) 33 | ---- 34 | does not exist at version 0 35 | 36 | query I 37 | SELECT * FROM ducklake.test AT (VERSION => 1) 38 | ---- 39 | 40 | query I 41 | SELECT * FROM ducklake.test AT (VERSION => 2) 42 | ---- 43 | {'i': 1, 'j': 2} 44 | {'i': NULL, 'j': 3} 45 | NULL 46 | 47 | # timestamp-based query 48 | query I 49 | SELECT * FROM ducklake.test AT (TIMESTAMP => NOW()) 50 | ---- 51 | {'i': 1, 'j': 2} 52 | {'i': NULL, 'j': 3} 53 | NULL 54 | 55 | statement ok 56 | SET VARIABLE snapshot_time = (SELECT snapshot_time FROM ducklake.snapshots() WHERE snapshot_id=1); 57 | 58 | query I 59 | SELECT * FROM ducklake.test AT (TIMESTAMP => getvariable('snapshot_time')) 60 | ---- 61 | 62 | # read a dropped table 63 | statement ok 64 | DROP TABLE ducklake.test 65 | 66 | statement error 67 | SELECT * FROM ducklake.test 68 | ---- 69 | does not exist 70 | 71 | query I 72 | SELECT * FROM ducklake.test AT (VERSION => 2) 73 | ---- 74 | {'i': 1, 'j': 2} 75 | {'i': NULL, 'j': 3} 76 | NULL 77 | 78 | # get a changeset 79 | query I rowsort 80 | SELECT * FROM ducklake.test AT (version => 2) EXCEPT SELECT * FROM ducklake.test AT (version => 1) 81 | ---- 82 | NULL 83 | {'i': 1, 'j': 2} 84 | {'i': NULL, 'j': 3} 85 | 86 | # time travel with schemas 87 | statement ok 88 | CREATE SCHEMA ducklake.s1 89 | 90 | statement ok 91 | CREATE TABLE ducklake.s1.tbl(i INT); 92 | 93 | statement ok 94 | INSERT INTO ducklake.s1.tbl VALUES (42); 95 | 96 | statement ok 97 | DROP SCHEMA ducklake.s1 CASCADE 98 | 99 | query I 100 | SELECT * FROM ducklake.s1.tbl AT (version => 6) 101 | ---- 102 | 42 103 | 104 | statement error 105 | SELECT * FROM ducklake.s1.tbl AT (version => 7) 106 | ---- 107 | does not exist 108 | -------------------------------------------------------------------------------- /test/sql/time_travel/time_travel_views.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/time_travel/time_travel_views.test 2 | # description: test time travel of views in ducklake 3 | # group: [time_travel] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_time_travel_views.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_time_travel_views_files') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test(i INTEGER, j INTEGER); 14 | 15 | statement ok 16 | CREATE VIEW ducklake.v1 AS SELECT i * 100 AS i, j * 100 AS j FROM test ORDER BY ALL 17 | 18 | statement ok 19 | INSERT INTO ducklake.test VALUES (1, 2), (2, 3); 20 | 21 | statement ok 22 | INSERT INTO ducklake.test VALUES (3, 4), (5, 6); 23 | 24 | query II 25 | SELECT * FROM ducklake.v1 26 | ---- 27 | 100 200 28 | 200 300 29 | 300 400 30 | 500 600 31 | 32 | # view does not exist 33 | statement error 34 | SELECT * FROM ducklake.v1 AT (VERSION => 0) 35 | ---- 36 | does not exist at version 0 37 | 38 | query II 39 | SELECT * FROM ducklake.v1 AT (VERSION => 2) 40 | ---- 41 | 42 | query II 43 | SELECT * FROM ducklake.v1 AT (VERSION => 3) 44 | ---- 45 | 100 200 46 | 200 300 47 | 48 | query II 49 | SELECT * FROM ducklake.v1 AT (VERSION => 4) 50 | ---- 51 | 100 200 52 | 200 300 53 | 300 400 54 | 500 600 55 | 56 | # time travel with schemas 57 | statement ok 58 | BEGIN 59 | 60 | statement ok 61 | CREATE SCHEMA ducklake.s1 62 | 63 | statement ok 64 | CREATE TABLE ducklake.s1.test(i INT) 65 | 66 | statement ok 67 | CREATE VIEW ducklake.s1_view AS SELECT * FROM s1.test 68 | 69 | statement ok 70 | COMMIT 71 | 72 | statement ok 73 | INSERT INTO ducklake.s1.test VALUES (42), (84); 74 | 75 | statement ok 76 | DROP SCHEMA ducklake.s1 CASCADE 77 | 78 | statement error 79 | SELECT * FROM ducklake.s1_view 80 | ---- 81 | Table with name test does not exist 82 | 83 | query I 84 | SELECT * FROM ducklake.s1_view AT (VERSION => 5) 85 | ---- 86 | 87 | query I rowsort 88 | SELECT * FROM ducklake.s1_view AT (VERSION => 6) 89 | ---- 90 | 42 91 | 84 92 | 93 | # explicit time travel clause mentioned in view takes priority over any versioning over the view itself 94 | statement ok 95 | CREATE VIEW ducklake.my_view AS SELECT * FROM ducklake.test AT (VERSION => 3) 96 | 97 | query II rowsort 98 | FROM ducklake.my_view 99 | ---- 100 | 1 2 101 | 2 3 102 | 103 | query II rowsort 104 | FROM ducklake.my_view AT (VERSION => 8) 105 | ---- 106 | 1 2 107 | 2 3 108 | -------------------------------------------------------------------------------- /test/sql/tpch/tpch_sf1.test_slow: -------------------------------------------------------------------------------- 1 | # name: test/sql/tpch/tpch_sf1.test_slow 2 | # description: Test running TPC-H on DuckLake 3 | # group: [tpch] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | require tpch 10 | 11 | mode skip 12 | 13 | statement ok 14 | CALL dbgen(sf=1); 15 | 16 | statement ok 17 | ATTACH 'ducklake:__TEST_DIR__/ducklake_tpch.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_tpch_files') 18 | 19 | statement ok 20 | COPY FROM DATABASE memory TO ducklake 21 | 22 | statement ok 23 | USE ducklake 24 | 25 | loop i 1 9 26 | 27 | query I 28 | PRAGMA tpch(${i}) 29 | ---- 30 | :duckdb/extension/tpch/dbgen/answers/sf1/q0${i}.csv 31 | 32 | endloop 33 | 34 | loop i 10 23 35 | 36 | query I 37 | PRAGMA tpch(${i}) 38 | ---- 39 | :duckdb/extension/tpch/dbgen/answers/sf1/q${i}.csv 40 | 41 | endloop 42 | -------------------------------------------------------------------------------- /test/sql/transaction/basic_transaction.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/transaction/basic_transaction.test 2 | # description: Test basic transaction support of ducklakes 3 | # group: [transaction] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_tl.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_tl_files', METADATA_CATALOG 'ducklake_meta') 11 | 12 | statement ok 13 | USE ducklake 14 | 15 | # table creation 16 | statement ok 17 | BEGIN 18 | 19 | statement ok 20 | CREATE TABLE test(i INTEGER, j INTEGER); 21 | 22 | # we can query the table 23 | query II 24 | SELECT * FROM test 25 | ---- 26 | 27 | query I 28 | SHOW TABLES 29 | ---- 30 | test 31 | 32 | 33 | statement ok 34 | ROLLBACK 35 | 36 | # after rollback the table no longer exists 37 | statement error 38 | SELECT * FROM test 39 | ---- 40 | does not exist 41 | 42 | query I 43 | SHOW TABLES 44 | ---- 45 | 46 | # create the table again 47 | statement ok 48 | CREATE TABLE test(i INTEGER, j INTEGER); 49 | 50 | # we can query transaction local data 51 | statement ok 52 | BEGIN 53 | 54 | statement ok 55 | INSERT INTO test VALUES (42, 84) 56 | 57 | query II 58 | SELECT * FROM test 59 | ---- 60 | 42 84 61 | 62 | # the data exists as files in the data directory 63 | query I 64 | SELECT COUNT(*) FROM glob('__TEST_DIR__/ducklake_tl_files/*.parquet') 65 | ---- 66 | 1 67 | 68 | statement ok 69 | ROLLBACK 70 | 71 | # rolling back deletes the rows from the table 72 | query II 73 | SELECT * FROM test 74 | ---- 75 | 76 | # it also deletes the written files from the data directory 77 | query I 78 | SELECT COUNT(*) FROM glob('__TEST_DIR__/ducklake_tl_files/*.parquet') 79 | ---- 80 | 0 81 | -------------------------------------------------------------------------------- /test/sql/transaction/create_conflict.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/transaction/create_conflict.test 2 | # description: test conflict handling on CREATE 3 | # group: [transaction] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_conflict.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_conflict_files', METADATA_CATALOG 'ducklake_meta') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test(i INTEGER, j INTEGER); 14 | 15 | # default behavior: table already exists (error) 16 | statement error 17 | CREATE TABLE ducklake.test(i VARCHAR); 18 | ---- 19 | already exists 20 | 21 | # ignore if exists 22 | statement ok 23 | CREATE TABLE IF NOT EXISTS ducklake.test(i VARCHAR); 24 | 25 | query II 26 | FROM ducklake.test 27 | ---- 28 | 29 | # replace table 30 | statement ok 31 | CREATE OR REPLACE TABLE ducklake.test(i VARCHAR); 32 | 33 | query I 34 | FROM ducklake.test 35 | ---- 36 | 37 | # views also conflict with tables 38 | statement error 39 | CREATE VIEW ducklake.test AS SELECT 42 40 | ---- 41 | already exists 42 | 43 | statement ok 44 | CREATE VIEW ducklake.v1 AS SELECT 42 45 | 46 | # if not exists is ignored 47 | statement ok 48 | CREATE VIEW IF NOT EXISTS ducklake.v1 AS SELECT 84 49 | 50 | query I 51 | FROM ducklake.v1 52 | ---- 53 | 42 54 | 55 | # or replace replaces the view 56 | statement ok 57 | CREATE OR REPLACE VIEW ducklake.v1 AS SELECT 84 58 | 59 | query I 60 | FROM ducklake.v1 61 | ---- 62 | 84 63 | 64 | # tables also conflict with views 65 | statement error 66 | CREATE TABLE ducklake.v1(i INT) 67 | ---- 68 | already exists 69 | 70 | statement ok 71 | DROP VIEW ducklake.v1 72 | 73 | # transaction-local view conflicts 74 | # conflict creating a view with the same name 75 | statement ok 76 | BEGIN 77 | 78 | statement ok 79 | CREATE VIEW ducklake.v1 AS SELECT 42 80 | 81 | statement error 82 | CREATE VIEW ducklake.v1 AS SELECT 84 83 | ---- 84 | already exists 85 | 86 | statement ok 87 | ROLLBACK 88 | 89 | # conflict creating a view with the same name as a table 90 | statement ok 91 | BEGIN 92 | 93 | statement ok 94 | CREATE TABLE ducklake.t1 AS SELECT 42 95 | 96 | statement error 97 | CREATE VIEW ducklake.t1 AS SELECT 84 98 | ---- 99 | already exists 100 | 101 | statement ok 102 | ROLLBACK 103 | -------------------------------------------------------------------------------- /test/sql/transaction/transaction_conflict_cleanup.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/transaction/transaction_conflict_cleanup.test 2 | # description: Verify files are cleaned up after a transaction conflict 3 | # group: [transaction] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_conflict_cleanup.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_files_conflict_cleanup') 11 | 12 | statement ok 13 | SET immediate_transaction_mode=true 14 | 15 | # con2 creates a conflicting table (with data) 16 | statement ok con1 17 | BEGIN 18 | 19 | statement ok con2 20 | BEGIN 21 | 22 | statement ok con1 23 | CREATE TABLE ducklake.test(i INTEGER); 24 | 25 | statement ok con2 26 | CREATE TABLE ducklake.test(s VARCHAR); 27 | 28 | statement ok con2 29 | INSERT INTO ducklake.test VALUES ('hello'), ('world'); 30 | 31 | statement ok con1 32 | COMMIT 33 | 34 | # con2 has the transaction-local files written in the directory 35 | query I 36 | SELECT COUNT(*) FROM glob('__TEST_DIR__/ducklake_files_conflict_cleanup/*.parquet') 37 | ---- 38 | 1 39 | 40 | statement error con2 41 | COMMIT 42 | ---- 43 | conflict 44 | 45 | # they are cleaned up after the conflict 46 | query I 47 | SELECT COUNT(*) FROM glob('__TEST_DIR__/ducklake_files_conflict_cleanup/*.parquet') 48 | ---- 49 | 0 50 | -------------------------------------------------------------------------------- /test/sql/transaction/transaction_conflicts_delete.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/transaction/transaction_conflicts_delete.test 2 | # description: Test transaction conflicts in DuckLake 3 | # group: [transaction] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_conflicts_deletes.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_conflicts_deletes_files') 11 | 12 | statement ok 13 | SET immediate_transaction_mode=true 14 | 15 | statement ok 16 | CREATE TABLE ducklake.test AS FROM range(1000) t(i) 17 | 18 | # two transactions try to delete the same data: conflict 19 | statement ok con1 20 | BEGIN 21 | 22 | statement ok con2 23 | BEGIN 24 | 25 | query I con1 26 | DELETE FROM ducklake.test WHERE i<200 27 | ---- 28 | 200 29 | 30 | query I con2 31 | DELETE FROM ducklake.test WHERE i<100 32 | ---- 33 | 100 34 | 35 | statement ok con1 36 | COMMIT 37 | 38 | statement error con2 39 | COMMIT 40 | ---- 41 | conflict 42 | 43 | # two transactions try to delete all data of the same table: conflict 44 | statement ok con1 45 | BEGIN 46 | 47 | statement ok con2 48 | BEGIN 49 | 50 | query I con1 51 | DELETE FROM ducklake.test 52 | ---- 53 | 800 54 | 55 | query I con2 56 | DELETE FROM ducklake.test 57 | ---- 58 | 800 59 | 60 | statement ok con1 61 | COMMIT 62 | 63 | statement error con2 64 | COMMIT 65 | ---- 66 | 67 | 68 | statement ok 69 | INSERT INTO ducklake.test FROM range(1000) 70 | 71 | # transaction tries to delete from a table that was dropped 72 | statement ok con1 73 | BEGIN 74 | 75 | statement ok con2 76 | BEGIN 77 | 78 | query I con1 79 | DROP TABLE ducklake.test 80 | 81 | query I con2 82 | DELETE FROM ducklake.test 83 | ---- 84 | 1000 85 | 86 | statement ok con1 87 | COMMIT 88 | 89 | statement error con2 90 | COMMIT 91 | ---- 92 | has been dropped by another transaction 93 | 94 | statement ok 95 | CREATE TABLE ducklake.test AS FROM range(1000) t(i) 96 | 97 | # transaction tries to delete from a table that was altered 98 | statement ok con1 99 | BEGIN 100 | 101 | statement ok con2 102 | BEGIN 103 | 104 | query I con1 105 | ALTER TABLE ducklake.test ADD COLUMN j INTEGER 106 | 107 | query I con2 108 | DELETE FROM ducklake.test 109 | ---- 110 | 1000 111 | 112 | statement ok con1 113 | COMMIT 114 | 115 | statement error con2 116 | COMMIT 117 | ---- 118 | has been altered by another transaction 119 | -------------------------------------------------------------------------------- /test/sql/transaction/transaction_conflicts_view.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/transaction/transaction_conflicts_view.test 2 | # description: Test transaction conflicts in DuckLake 3 | # group: [transaction] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_conflicts_view.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_conflicts_view_files') 11 | 12 | statement ok 13 | SET immediate_transaction_mode=true 14 | 15 | # two transactions try to create a view with different names: no conflict 16 | statement ok con1 17 | BEGIN 18 | 19 | statement ok con2 20 | BEGIN 21 | 22 | statement ok con1 23 | CREATE VIEW ducklake.test2 AS SELECT 42 24 | 25 | statement ok con2 26 | CREATE VIEW ducklake.test3 AS SELECT 84 27 | 28 | statement ok con1 29 | COMMIT 30 | 31 | statement ok con2 32 | COMMIT 33 | 34 | # two transactions try to create a view with the same name: conflict 35 | statement ok con1 36 | BEGIN 37 | 38 | statement ok con2 39 | BEGIN 40 | 41 | statement ok con1 42 | CREATE VIEW ducklake.test AS SELECT 42 43 | 44 | statement ok con2 45 | CREATE VIEW ducklake.test AS SELECT 84 46 | 47 | statement ok con1 48 | COMMIT 49 | 50 | statement error con2 51 | COMMIT 52 | ---- 53 | Transaction conflict 54 | 55 | # two transactions try to create a view and table with the same name: conflict 56 | statement ok con1 57 | BEGIN 58 | 59 | statement ok con2 60 | BEGIN 61 | 62 | statement ok con1 63 | CREATE TABLE ducklake.t_name AS SELECT 42 64 | 65 | statement ok con2 66 | CREATE VIEW ducklake.t_name AS SELECT 84 67 | 68 | statement ok con1 69 | COMMIT 70 | 71 | statement error con2 72 | COMMIT 73 | ---- 74 | Transaction conflict 75 | 76 | # two transactions try to drop the same view: conflict 77 | statement ok con1 78 | BEGIN 79 | 80 | statement ok con2 81 | BEGIN 82 | 83 | statement ok con1 84 | DROP VIEW ducklake.test 85 | 86 | statement ok con2 87 | DROP VIEW ducklake.test 88 | 89 | statement ok con1 90 | COMMIT 91 | 92 | statement error con2 93 | COMMIT 94 | ---- 95 | 96 | statement ok 97 | CREATE VIEW ducklake.comment_view AS SELECT 42; 98 | 99 | # two transactions try to alter the same view: conflict 100 | statement ok con1 101 | BEGIN 102 | 103 | statement ok con2 104 | BEGIN 105 | 106 | statement ok con1 107 | COMMENT ON VIEW ducklake.comment_view IS 'con1 comment' 108 | 109 | statement ok con2 110 | COMMENT ON VIEW ducklake.comment_view IS 'con2 comment' 111 | 112 | statement ok con1 113 | COMMIT 114 | 115 | statement error con2 116 | COMMIT 117 | ---- 118 | altered by another transaction 119 | 120 | query I 121 | SELECT comment FROM duckdb_views WHERE view_name='comment_view' 122 | ---- 123 | con1 comment 124 | -------------------------------------------------------------------------------- /test/sql/transaction/transaction_isolation.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/transaction/transaction_isolation.test 2 | # group: [transaction] 3 | 4 | -------------------------------------------------------------------------------- /test/sql/transaction/transaction_schema.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/transaction/transaction_schema.test 2 | # description: Test multi-schema support in DuckLake 3 | # group: [transaction] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | # store multiple DuckLake catalogs in the same metadata store (with different schemas) 10 | statement ok 11 | ATTACH 'ducklake:__TEST_DIR__/ducklake_schema.db' AS ducklake_1 (DATA_PATH '__TEST_DIR__/ducklake_files_s1', METADATA_SCHEMA 'metadata_s1') 12 | 13 | statement ok 14 | CREATE TABLE ducklake_1.tbl(i INTEGER); 15 | 16 | statement ok 17 | INSERT INTO ducklake_1.tbl VALUES (42); 18 | 19 | statement ok 20 | DETACH ducklake_1 21 | 22 | statement ok 23 | ATTACH 'ducklake:__TEST_DIR__/ducklake_schema.db' AS ducklake_2 (DATA_PATH '__TEST_DIR__/ducklake_files_s2', METADATA_SCHEMA 'metadata_s2') 24 | 25 | statement ok 26 | CREATE TABLE ducklake_2.tbl(s VARCHAR); 27 | 28 | statement ok 29 | INSERT INTO ducklake_2.tbl VALUES ('hello world'); 30 | -------------------------------------------------------------------------------- /test/sql/types/all_types.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/types/all_types.test 2 | # description: test ducklake with different data types 3 | # group: [types] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_all_types.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_all_types_files', METADATA_CATALOG 'ducklake_meta') 11 | 12 | statement ok 13 | CREATE VIEW all_types AS SELECT * EXCLUDE (VARINT, BIT, small_enum, 14 | medium_enum, 15 | large_enum, 16 | "union", 17 | fixed_int_array, 18 | fixed_varchar_array, 19 | fixed_nested_int_array, 20 | fixed_nested_varchar_array, 21 | fixed_struct_array, 22 | struct_of_fixed_array, 23 | fixed_array_of_int_list, 24 | list_of_fixed_int_array, hugeint, uhugeint, interval, time_tz) FROM test_all_types(); 25 | 26 | query I nosort alltypes 27 | FROM all_types 28 | ---- 29 | 30 | statement ok 31 | BEGIN 32 | 33 | statement ok 34 | CREATE TABLE ducklake.data_types AS FROM all_types 35 | 36 | query I nosort alltypes 37 | FROM ducklake.data_types 38 | ---- 39 | 40 | statement ok 41 | COMMIT 42 | 43 | query I nosort alltypes 44 | FROM ducklake.data_types 45 | ---- 46 | -------------------------------------------------------------------------------- /test/sql/types/floats.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/types/floats.test 2 | # description: test ducklake floats 3 | # group: [types] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | PRAGMA enable_verification 11 | 12 | statement ok 13 | ATTACH 'ducklake:__TEST_DIR__/ducklake_float.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_float_files') 14 | 15 | foreach type FLOAT DOUBLE 16 | 17 | statement ok 18 | CREATE OR REPLACE TABLE ducklake.test(f ${type}); 19 | 20 | statement ok 21 | INSERT INTO ducklake.test VALUES (1), (10); 22 | 23 | statement ok 24 | INSERT INTO ducklake.test VALUES ('NaN'), (1); 25 | 26 | # predicate on NaN 27 | query I 28 | SELECT COUNT(*) FROM ducklake.test WHERE f='NaN' 29 | ---- 30 | 1 31 | 32 | query I 33 | SELECT COUNT(*) FROM ducklake.test WHERE f>'NaN' 34 | ---- 35 | 0 36 | 37 | query I 38 | SELECT COUNT(*) FROM ducklake.test WHERE f>='NaN' 39 | ---- 40 | 1 41 | 42 | query I 43 | SELECT COUNT(*) FROM ducklake.test WHERE f<'NaN' 44 | ---- 45 | 3 46 | 47 | query I 48 | SELECT COUNT(*) FROM ducklake.test WHERE f<='NaN' 49 | ---- 50 | 4 51 | 52 | # upper-bound requires NaN checks 53 | query I 54 | SELECT COUNT(*) FROM ducklake.test WHERE f>1 55 | ---- 56 | 2 57 | 58 | # test infinite 59 | statement ok 60 | INSERT INTO ducklake.test VALUES ('inf'); 61 | 62 | statement ok 63 | INSERT INTO ducklake.test VALUES ('-inf'); 64 | 65 | query I 66 | SELECT COUNT(*) FROM ducklake.test WHERE f>'inf' 67 | ---- 68 | 1 69 | 70 | query I 71 | SELECT COUNT(*) FROM ducklake.test WHERE f>='inf' 72 | ---- 73 | 2 74 | 75 | query I 76 | SELECT COUNT(*) FROM ducklake.test WHERE f<'inf' 77 | ---- 78 | 4 79 | 80 | query I 81 | SELECT COUNT(*) FROM ducklake.test WHERE f<='inf' 82 | ---- 83 | 5 84 | 85 | query I 86 | SELECT COUNT(*) FROM ducklake.test WHERE f>'-inf' 87 | ---- 88 | 5 89 | 90 | query I 91 | SELECT COUNT(*) FROM ducklake.test WHERE f>='-inf' 92 | ---- 93 | 6 94 | 95 | endloop 96 | -------------------------------------------------------------------------------- /test/sql/types/json.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/types/json.test 2 | # description: test storing json in ducklake 3 | # group: [types] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | require json 10 | 11 | statement ok 12 | ATTACH 'ducklake:__TEST_DIR__/ducklake_json.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_json_files') 13 | 14 | statement ok 15 | CREATE TABLE ducklake.test(l JSON); 16 | 17 | query I 18 | SELECT * FROM ducklake.test 19 | ---- 20 | 21 | statement ok 22 | INSERT INTO ducklake.test VALUES ('{"key": "value"}'); 23 | 24 | query I 25 | SELECT * FROM ducklake.test 26 | ---- 27 | {"key": "value"} 28 | 29 | query I 30 | SELECT typeof(l) FROM ducklake.test 31 | ---- 32 | JSON 33 | 34 | statement ok 35 | DETACH ducklake 36 | 37 | statement ok 38 | ATTACH 'ducklake:__TEST_DIR__/ducklake_json.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_json_files') 39 | 40 | query I 41 | SELECT * FROM ducklake.test 42 | ---- 43 | {"key": "value"} 44 | 45 | query I 46 | SELECT typeof(l) FROM ducklake.test 47 | ---- 48 | JSON 49 | -------------------------------------------------------------------------------- /test/sql/types/list.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/types/list.test 2 | # description: test storing list types in ducklake 3 | # group: [types] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_list.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_list_files') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test(l INTEGER[]); 14 | 15 | query I 16 | SELECT * FROM ducklake.test 17 | ---- 18 | 19 | statement ok 20 | INSERT INTO ducklake.test VALUES ([1]), ([NULL]), (NULL), ([3]); 21 | 22 | query I 23 | SELECT * FROM ducklake.test 24 | ---- 25 | [1] 26 | [NULL] 27 | NULL 28 | [3] 29 | 30 | query I 31 | SELECT * FROM ducklake.test WHERE l[1]=1 32 | ---- 33 | [1] 34 | 35 | query I 36 | SELECT * FROM ducklake.test WHERE l[1]=100 37 | ---- 38 | 39 | statement ok 40 | INSERT INTO ducklake.test VALUES ([4, 5]), ([6, 7]); 41 | 42 | query I 43 | SELECT * FROM ducklake.test 44 | ---- 45 | [1] 46 | [NULL] 47 | NULL 48 | [3] 49 | [4, 5] 50 | [6, 7] 51 | 52 | # stats 53 | query I 54 | SELECT stats(l[1]) FROM ducklake.test LIMIT 1 55 | ---- 56 | :.*Min.*1.*Max.*7.*Has Null.*true.* 57 | -------------------------------------------------------------------------------- /test/sql/types/map.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/types/map.test 2 | # description: test storing map types in ducklake 3 | # group: [types] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_map.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_map_files') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test(s MAP(VARCHAR, INTEGER)); 14 | 15 | query I 16 | SELECT * FROM ducklake.test 17 | ---- 18 | 19 | statement ok 20 | INSERT INTO ducklake.test VALUES (MAP {'i': 1, 'j': 2}), (MAP {'j': 3}), (NULL); 21 | 22 | query I 23 | SELECT * FROM ducklake.test 24 | ---- 25 | {i=1, j=2} 26 | {j=3} 27 | NULL 28 | 29 | query I 30 | SELECT * FROM ducklake.test WHERE s.i=1 31 | ---- 32 | {i=1, j=2} 33 | 34 | query I 35 | SELECT * FROM ducklake.test WHERE s.i=100 36 | ---- 37 | 38 | statement ok 39 | INSERT INTO ducklake.test VALUES (MAP {'i': 4, 'j': 5}), (MAP {'i': 6}); 40 | 41 | query I 42 | SELECT * FROM ducklake.test 43 | ---- 44 | {i=1, j=2} 45 | {j=3} 46 | NULL 47 | {i=4, j=5} 48 | {i=6} 49 | 50 | # map stats not supported yet 51 | mode skip 52 | 53 | query I 54 | SELECT stats(s.i) FROM ducklake.test LIMIT 1 55 | ---- 56 | :.*Min.*1.*Max.*6.*Has Null.*true.* 57 | -------------------------------------------------------------------------------- /test/sql/types/null_byte.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/types/null_byte.test 2 | # description: Test null bytes in strings 3 | # group: [types] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_null_byte.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_null_byte', METADATA_CATALOG 'ducklake_meta') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.tbl(s VARCHAR); 14 | 15 | statement ok 16 | INSERT INTO ducklake.tbl VALUES ('goo' || chr(0) || 'se'), ('hello'); 17 | 18 | query I 19 | FROM ducklake.tbl 20 | ---- 21 | goo\0se 22 | hello 23 | 24 | query I 25 | FROM ducklake.tbl WHERE s < 'hello' 26 | ---- 27 | goo\0se 28 | -------------------------------------------------------------------------------- /test/sql/types/struct.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/types/struct.test 2 | # description: test storing struct types in ducklake 3 | # group: [types] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_struct.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_struct_files') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test(s STRUCT(i INTEGER, j INTEGER)); 14 | 15 | query I 16 | SELECT * FROM ducklake.test 17 | ---- 18 | 19 | statement ok 20 | INSERT INTO ducklake.test VALUES ({'i': 1, 'j': 2}), ({'i': NULL, 'j': 3}), (NULL); 21 | 22 | query I 23 | SELECT * FROM ducklake.test 24 | ---- 25 | {'i': 1, 'j': 2} 26 | {'i': NULL, 'j': 3} 27 | NULL 28 | 29 | query I 30 | SELECT * FROM ducklake.test WHERE s.i=1 31 | ---- 32 | {'i': 1, 'j': 2} 33 | 34 | query I 35 | SELECT * FROM ducklake.test WHERE s.i=100 36 | ---- 37 | 38 | statement ok 39 | INSERT INTO ducklake.test VALUES ({'i': 4, 'j': 5}), ({'i': 6, 'j': 7}); 40 | 41 | query I 42 | SELECT * FROM ducklake.test 43 | ---- 44 | {'i': 1, 'j': 2} 45 | {'i': NULL, 'j': 3} 46 | NULL 47 | {'i': 4, 'j': 5} 48 | {'i': 6, 'j': 7} 49 | 50 | # stats 51 | query I 52 | SELECT stats(s.i) FROM ducklake.test LIMIT 1 53 | ---- 54 | :.*Min.*1.*Max.*6.*Has Null.*true.* 55 | -------------------------------------------------------------------------------- /test/sql/types/timestamp.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/types/timestamp.test 2 | # description: test ducklake timestamps 3 | # group: [types] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | PRAGMA enable_verification 11 | 12 | statement ok 13 | ATTACH 'ducklake:__TEST_DIR__/ducklake_ts.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_ts_files') 14 | 15 | statement ok 16 | CREATE OR REPLACE TABLE ducklake.test(ts TIMESTAMP); 17 | 18 | statement ok 19 | INSERT INTO ducklake.test VALUES (TIMESTAMP '1992-01-01'), (TIMESTAMP '2020-01-01'); 20 | 21 | statement ok 22 | INSERT INTO ducklake.test VALUES ('infinity'), (TIMESTAMP '2022-01-01'); 23 | 24 | # predicate on infinity 25 | query I 26 | SELECT COUNT(*) FROM ducklake.test WHERE ts='infinity' 27 | ---- 28 | 1 29 | 30 | query I 31 | SELECT COUNT(*) FROM ducklake.test WHERE ts<='infinity' 32 | ---- 33 | 4 34 | 35 | query I 36 | SELECT COUNT(*) FROM ducklake.test WHERE ts>'-infinity' 37 | ---- 38 | 4 39 | -------------------------------------------------------------------------------- /test/sql/types/unsupported.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/types/unsupported.test 2 | # description: test unsupported types 3 | # group: [types] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_union.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_union_files') 11 | 12 | # unsupported types 13 | statement error 14 | CREATE TABLE ducklake.test(ts INT[3]); 15 | ---- 16 | unsupported type 17 | 18 | statement error 19 | CREATE TABLE ducklake.test(ts UNION(i INT, j INT)); 20 | ---- 21 | unsupported type 22 | 23 | # enum 24 | statement error 25 | CREATE TABLE ducklake.test AS SELECT 'hello'::ENUM('world', 'hello') AS h; 26 | ---- 27 | unsupported type 28 | 29 | # varchar with collation 30 | statement error 31 | CREATE TABLE ducklake.test(s VARCHAR COLLATE NOACCENT); 32 | ---- 33 | Collations are not supported 34 | 35 | statement error 36 | CREATE TABLE ducklake.test(s VARCHAR USING COMPRESSION ZSTD); 37 | ---- 38 | compression type for a column is not supported in DuckLake 39 | 40 | # unsupported type in a struct 41 | statement error 42 | CREATE TABLE ducklake.test(ts STRUCT(x INT[3])); 43 | ---- 44 | unsupported type 45 | -------------------------------------------------------------------------------- /test/sql/update/basic_update.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/update/basic_update.test 2 | # description: Test ducklake updates 3 | # group: [update] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_update.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_update_files'); 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test AS SELECT 1000 + i id, i % 10 as val FROM range(1000) t(i); 14 | 15 | statement ok 16 | BEGIN 17 | 18 | query III 19 | SELECT COUNT(*), SUM(id), SUM(val) FROM ducklake.test 20 | ---- 21 | 1000 1499500 4500 22 | 23 | query I 24 | UPDATE ducklake.test SET id=id+2 WHERE id%2=0 25 | ---- 26 | 500 27 | 28 | query III 29 | SELECT COUNT(*), SUM(id), SUM(val) FROM ducklake.test 30 | ---- 31 | 1000 1500500 4500 32 | 33 | statement ok 34 | COMMIT 35 | 36 | query III 37 | SELECT COUNT(*), SUM(id), SUM(val) FROM ducklake.test 38 | ---- 39 | 1000 1500500 4500 40 | 41 | query III 42 | SELECT COUNT(*), SUM(id), SUM(val) FROM ducklake.test AT (VERSION => 1) 43 | ---- 44 | 1000 1499500 4500 45 | -------------------------------------------------------------------------------- /test/sql/update/update_join_duplicates.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/update/update_join_duplicates.test 2 | # description: Test ducklake update using a join with duplicates 3 | # group: [update] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_update_join.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_update_join_files') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test AS SELECT i id FROM range(5) t(i); 14 | 15 | statement ok 16 | BEGIN 17 | 18 | statement ok 19 | INSERT INTO ducklake.test FROM range(5, 10) 20 | 21 | statement ok 22 | CREATE TEMPORARY TABLE updated_rows AS FROM range(0, 10, 2) t(update_id) UNION ALL FROM range(0, 10, 2); 23 | 24 | # not supported yet 25 | # FIXME: should be fixed 26 | statement error 27 | UPDATE ducklake.test SET id=id+1000 FROM updated_rows WHERE id=updated_rows.update_id 28 | ---- 29 | not yet supported 30 | 31 | mode skip 32 | 33 | query III 34 | SELECT COUNT(*), SUM(id), AVG(id) FROM ducklake.test 35 | ---- 36 | 10 5045 504.5 37 | 38 | statement ok 39 | COMMIT 40 | 41 | query III 42 | SELECT COUNT(*), SUM(id), AVG(id) FROM ducklake.test 43 | ---- 44 | 10 5045 504.5 45 | 46 | 47 | -------------------------------------------------------------------------------- /test/sql/update/update_not_null.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/update/update_not_null.test 2 | # description: test updating a table with a NOT NULL constraint 3 | # group: [update] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_update_not_null.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_update_not_null_files') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test(i INTEGER NOT NULL, j INTEGER); 14 | 15 | statement ok 16 | INSERT INTO ducklake.test VALUES (42, NULL); 17 | 18 | statement ok 19 | BEGIN 20 | 21 | statement error 22 | UPDATE ducklake.test SET i=NULL 23 | ---- 24 | NOT NULL constraint failed 25 | 26 | statement error 27 | UPDATE ducklake.test SET i=100 28 | ---- 29 | Current transaction is aborted 30 | 31 | statement ok 32 | ROLLBACK 33 | 34 | query II 35 | FROM ducklake.test 36 | ---- 37 | 42 NULL 38 | -------------------------------------------------------------------------------- /test/sql/update/update_partitioning.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/update/update_partitioning.test 2 | # description: Update a partitioned table 3 | # group: [update] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | # partitioning based on a column 10 | statement ok 11 | ATTACH 'ducklake:__TEST_DIR__/ducklake_update_partitioning.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_update_partitioning', METADATA_CATALOG 'ducklake_metadata') 12 | 13 | statement ok 14 | USE ducklake 15 | 16 | statement ok 17 | CREATE TABLE partitioned_tbl(part_key INTEGER, values VARCHAR); 18 | 19 | statement ok 20 | ALTER TABLE partitioned_tbl SET PARTITIONED BY (part_key); 21 | 22 | statement ok 23 | INSERT INTO partitioned_tbl SELECT i%2, concat('thisisastring_', i) FROM range(10000) t(i) 24 | 25 | statement ok 26 | UPDATE partitioned_tbl SET part_key=2 WHERE part_key=0 27 | 28 | # verify files are partitioned 29 | query III 30 | SELECT data_file_id, partition_id, regexp_extract(path, '.*(part_key=[0-9])[/\\].*', 1) FROM ducklake_metadata.ducklake_data_file 31 | ORDER BY ALL 32 | ---- 33 | 0 2 part_key=0 34 | 1 2 part_key=1 35 | 2 2 part_key=2 36 | 37 | query I 38 | SELECT COUNT(*) FROM partitioned_tbl 39 | ---- 40 | 10000 41 | 42 | # query the new partition 43 | query I 44 | SELECT COUNT(*) FROM partitioned_tbl WHERE part_key=2 45 | ---- 46 | 5000 47 | 48 | query II 49 | EXPLAIN ANALYZE SELECT COUNT(*) FROM partitioned_tbl WHERE part_key=2 50 | ---- 51 | analyzed_plan :.*Total Files Read: 1.* 52 | 53 | # query the old partition with time travel 54 | query I 55 | SELECT COUNT(*) FROM partitioned_tbl AT (VERSION => 3) WHERE part_key=0 56 | ---- 57 | 5000 58 | 59 | query II 60 | EXPLAIN ANALYZE SELECT COUNT(*) FROM partitioned_tbl AT (VERSION => 3) WHERE part_key=0 61 | ---- 62 | analyzed_plan :.*Total Files Read: 1.* 63 | -------------------------------------------------------------------------------- /test/sql/update/update_rollback.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/update/update_rollback.test 2 | # description: Test ducklake update rollback 3 | # group: [update] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_update_rollback.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_update_rollback_files'); 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test AS SELECT 1000 + i id, i % 10 as val FROM range(1000) t(i); 14 | 15 | statement ok 16 | BEGIN 17 | 18 | statement ok 19 | UPDATE ducklake.test SET id=id+1 20 | 21 | query III 22 | SELECT COUNT(*), SUM(id), SUM(val) FROM ducklake.test 23 | ---- 24 | 1000 1500500 4500 25 | 26 | statement ok 27 | ROLLBACK 28 | 29 | query III 30 | SELECT COUNT(*), SUM(id), SUM(val) FROM ducklake.test 31 | ---- 32 | 1000 1499500 4500 33 | 34 | # verify any additional files were deleted 35 | query I 36 | SELECT COUNT(*) FROM glob('__TEST_DIR__/ducklake_update_rollback_files/*.parquet') 37 | ---- 38 | 1 -------------------------------------------------------------------------------- /test/sql/update/update_same_transaction.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/update/update_same_transaction.test 2 | # description: Test running updates the same transaction 3 | # group: [update] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/update_same_transaction.db' AS ducklake (DATA_PATH '__TEST_DIR__/update_same_transaction', METADATA_CATALOG 'ducklake_metadata') 11 | 12 | statement ok 13 | BEGIN TRANSACTION; 14 | 15 | statement ok 16 | CREATE TABLE ducklake.test (id INTEGER, name TEXT); 17 | 18 | statement ok 19 | INSERT INTO ducklake.test VALUES (1, 'Bob'); 20 | 21 | statement ok 22 | UPDATE ducklake.test SET name = 'Alice' Where id = 1; 23 | 24 | statement ok 25 | COMMIT; 26 | 27 | query T 28 | select name from ducklake.test where id = 1; 29 | ---- 30 | Alice 31 | -------------------------------------------------------------------------------- /test/sql/view/ducklake_rename_view.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/view/ducklake_rename_view.test 2 | # description: Test renaming views in DuckLake 3 | # group: [view] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_rename_view.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_rename_view_files') 11 | 12 | statement ok 13 | CREATE VIEW ducklake.v1 AS SELECT 42 14 | 15 | statement ok 16 | ALTER VIEW ducklake.v1 RENAME TO v2 17 | 18 | query I 19 | SELECT * FROM ducklake.v2 20 | ---- 21 | 42 22 | 23 | # rename in a transaction 24 | statement ok 25 | BEGIN 26 | 27 | statement ok 28 | ALTER VIEW ducklake.v2 RENAME TO v3 29 | 30 | statement error 31 | SELECT * FROM ducklake.v2 32 | ---- 33 | does not exist 34 | 35 | query I 36 | SELECT * FROM ducklake.v3 37 | ---- 38 | 42 39 | 40 | statement ok 41 | ROLLBACK 42 | 43 | query I 44 | SELECT * FROM ducklake.v2 45 | ---- 46 | 42 47 | 48 | # rename a transaction-local view 49 | statement ok 50 | BEGIN 51 | 52 | statement ok 53 | CREATE VIEW ducklake.local_view AS SELECT 100 54 | 55 | statement ok 56 | ALTER VIEW ducklake.local_view RENAME TO local_view2 57 | 58 | statement error 59 | SELECT * FROM ducklake.local_view 60 | ---- 61 | does not exist 62 | 63 | query I 64 | SELECT * FROM ducklake.local_view2 65 | ---- 66 | 100 67 | 68 | # and I'll do it again! 69 | statement ok 70 | ALTER VIEW ducklake.local_view2 RENAME TO local_view3 71 | 72 | query I 73 | SELECT * FROM ducklake.local_view3 74 | ---- 75 | 100 76 | 77 | statement ok 78 | COMMIT 79 | 80 | query I 81 | SELECT * FROM ducklake.local_view3 82 | ---- 83 | 100 84 | 85 | statement error 86 | ALTER TABLE ducklake.local_view3 RENAME TO local_view4 87 | ---- 88 | not a table 89 | -------------------------------------------------------------------------------- /test/sql/view/ducklake_view.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/view/ducklake_view.test 2 | # description: test ducklake view creation 3 | # group: [view] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_view.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_view_files') 11 | 12 | # create a view 13 | statement ok 14 | CREATE VIEW ducklake.v1 AS SELECT 42 15 | 16 | query I 17 | SELECT * FROM ducklake.v1 18 | ---- 19 | 42 20 | 21 | query I 22 | SELECT sql FROM duckdb_views() WHERE database_name='ducklake' 23 | ---- 24 | CREATE VIEW v1 AS SELECT 42; 25 | 26 | # we can drop the view 27 | statement ok 28 | DROP VIEW ducklake.v1 29 | 30 | # aaaand it's gone 31 | statement error 32 | SELECT * FROM ducklake.v1 33 | ---- 34 | does not exist 35 | 36 | # transaction-local view drop and re-create 37 | statement ok 38 | CREATE VIEW ducklake.v1 AS SELECT 42 39 | 40 | statement ok 41 | BEGIN 42 | 43 | statement ok 44 | DROP VIEW ducklake.v1 45 | 46 | statement error 47 | FROM ducklake.v1 48 | ---- 49 | does not exist 50 | 51 | statement ok 52 | CREATE VIEW ducklake.v1 AS SELECT 84 53 | 54 | query I 55 | FROM ducklake.v1 56 | ---- 57 | 84 58 | 59 | statement ok 60 | COMMIT 61 | 62 | query I 63 | FROM ducklake.v1 64 | ---- 65 | 84 66 | 67 | # view with explicit column aliases 68 | statement ok 69 | CREATE VIEW ducklake.aliased_view(a) AS SELECT 42 AS X, 84 as Y 70 | 71 | query I 72 | SELECT a FROM ducklake.aliased_view 73 | ---- 74 | 42 75 | -------------------------------------------------------------------------------- /test/sql/view/ducklake_view_table_conflict.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/view/ducklake_view_table_conflict.test 2 | # description: test ducklake view/table conflicts 3 | # group: [view] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_view_conflict.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_view_conflict_files') 11 | 12 | # create a view 13 | statement ok 14 | CREATE VIEW ducklake.v1 AS SELECT 42 15 | 16 | statement error 17 | DROP TABLE IF EXISTS ducklake.v1 18 | ---- 19 | trying to drop type Table 20 | 21 | statement error 22 | CREATE OR REPLACE TABLE ducklake.v1(i INTEGER) 23 | ---- 24 | trying to replace with type Table 25 | -------------------------------------------------------------------------------- /test/sql/virtualcolumns/ducklake_snapshot_id.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/virtualcolumns/ducklake_snapshot_id.test 2 | # description: test snapshot_id virtual column 3 | # group: [virtualcolumns] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_snapshot_id.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_snapshot_id_files') 11 | 12 | # snapshot 1 13 | statement ok 14 | CREATE TABLE ducklake.test(i INTEGER); 15 | 16 | # snapshot 2 17 | statement ok 18 | INSERT INTO ducklake.test VALUES (1); 19 | 20 | # snapshot 3 21 | statement ok 22 | INSERT INTO ducklake.test VALUES (2); 23 | 24 | # snapshot 4 25 | statement ok 26 | INSERT INTO ducklake.test VALUES (3); 27 | 28 | # snapshot 5 29 | statement ok 30 | INSERT INTO ducklake.test VALUES (NULL); 31 | 32 | statement ok 33 | BEGIN 34 | 35 | # snapshot 6 36 | statement ok 37 | INSERT INTO ducklake.test FROM range(10, 12); 38 | 39 | query II 40 | SELECT snapshot_id, * FROM ducklake.test ORDER BY ALL 41 | ---- 42 | 2 1 43 | 3 2 44 | 4 3 45 | 5 NULL 46 | NULL 10 47 | NULL 11 48 | 49 | statement ok 50 | COMMIT 51 | 52 | query II 53 | SELECT snapshot_id, * FROM ducklake.test ORDER BY ALL 54 | ---- 55 | 2 1 56 | 3 2 57 | 4 3 58 | 5 NULL 59 | 6 10 60 | 6 11 61 | 62 | query II 63 | SELECT snapshot_id, * FROM ducklake.test WHERE snapshot_id=4 64 | ---- 65 | 4 3 66 | -------------------------------------------------------------------------------- /test/sql/virtualcolumns/ducklake_virtual_columns.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/virtualcolumns/ducklake_virtual_columns.test 2 | # description: test ducklake virtual columns 3 | # group: [virtualcolumns] 4 | 5 | require ducklake 6 | 7 | require parquet 8 | 9 | statement ok 10 | ATTACH 'ducklake:__TEST_DIR__/ducklake_virtual.db' AS ducklake (DATA_PATH '__TEST_DIR__/ducklake_virtual_files') 11 | 12 | statement ok 13 | CREATE TABLE ducklake.test(i INTEGER); 14 | 15 | statement ok 16 | INSERT INTO ducklake.test VALUES (1), (2), (3); 17 | 18 | query I 19 | SELECT file_row_number FROM ducklake.test 20 | ---- 21 | 0 22 | 1 23 | 2 24 | 25 | query I 26 | SELECT file_row_number FROM ducklake.test WHERE file_row_number=1 27 | ---- 28 | 1 29 | 30 | query I 31 | SELECT COUNT(DISTINCT filename) FROM ducklake.test WHERE contains(filename, 'ducklake_virtual_files') 32 | ---- 33 | 1 34 | -------------------------------------------------------------------------------- /vcpkg.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": [ 3 | ] 4 | } --------------------------------------------------------------------------------