├── .editorconfig ├── .github └── workflows │ ├── LocalTesting.yml │ ├── MainDistributionPipeline.yml │ └── _extension_deploy.yml ├── .gitignore ├── .gitmodules ├── CMakeLists.txt ├── LICENSE ├── Makefile ├── README.md ├── docs └── UPDATING.md ├── extension_config.cmake ├── src ├── CMakeLists.txt ├── include │ ├── storage │ │ ├── uc_catalog.hpp │ │ ├── uc_catalog_set.hpp │ │ ├── uc_schema_entry.hpp │ │ ├── uc_schema_set.hpp │ │ ├── uc_table_entry.hpp │ │ ├── uc_table_set.hpp │ │ ├── uc_transaction.hpp │ │ └── uc_transaction_manager.hpp │ ├── uc_api.hpp │ ├── uc_catalog_extension.hpp │ └── uc_utils.hpp ├── storage │ ├── CMakeLists.txt │ ├── uc_catalog.cpp │ ├── uc_catalog_set.cpp │ ├── uc_clear_cache.cpp │ ├── uc_schema_entry.cpp │ ├── uc_schema_set.cpp │ ├── uc_table_entry.cpp │ ├── uc_table_set.cpp │ ├── uc_transaction.cpp │ └── uc_transaction_manager.cpp ├── uc_api.cpp ├── uc_catalog_extension.cpp └── uc_utils.cpp ├── test.json ├── test ├── README.md └── sql │ ├── uc_catalog.test │ └── uc_cert.test └── vcpkg.json /.editorconfig: -------------------------------------------------------------------------------- 1 | duckdb/.editorconfig -------------------------------------------------------------------------------- /.github/workflows/LocalTesting.yml: -------------------------------------------------------------------------------- 1 | name: Local functional tests 2 | on: [push, pull_request,repository_dispatch] 3 | concurrency: 4 | group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} 5 | cancel-in-progress: true 6 | defaults: 7 | run: 8 | shell: bash 9 | 10 | env: 11 | BASE_BRANCH: ${{ github.base_ref || (endsWith(github.ref, '_feature') && 'feature' || 'main') }} 12 | 13 | jobs: 14 | unity-catalog-local-linux: 15 | name: Local UC test server tests (Linux) 16 | runs-on: ubuntu-latest 17 | env: 18 | VCPKG_TARGET_TRIPLET: 'x64-linux' 19 | GEN: ninja 20 | VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake 21 | UC_TEST_SERVER_RUNNING: 1 22 | 23 | steps: 24 | - uses: actions/checkout@v4 25 | with: 26 | fetch-depth: 0 27 | submodules: 'true' 28 | 29 | - name: Configure OpenSSL for Rust 30 | run: | 31 | echo "OPENSSL_ROOT_DIR=`pwd`/build/release/vcpkg_installed/x64-linux" >> $GITHUB_ENV 32 | echo "OPENSSL_DIR=`pwd`/build/release/vcpkg_installed/x64-linux" >> $GITHUB_ENV 33 | echo "OPENSSL_USE_STATIC_LIBS=true" >> $GITHUB_ENV 34 | 35 | - name: Install Ninja 36 | shell: bash 37 | run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build 38 | 39 | - name: Setup Ccache 40 | uses: hendrikmuhs/ccache-action@main 41 | with: 42 | key: ${{ github.job }} 43 | 44 | - name: Setup vcpkg 45 | uses: lukka/run-vcpkg@v11.1 46 | with: 47 | vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6 48 | 49 | - uses: actions/setup-node@v4 50 | 51 | - name: Setup Unity Catalog test server 52 | run: | 53 | sudo apt install default-jre 54 | git clone https://github.com/unitycatalog/unitycatalog.git 55 | ./bin/start-uc-server & 56 | sleep 10 57 | 58 | - name: Build extension 59 | run: | 60 | make release 61 | 62 | - name: Test extension 63 | run: | 64 | make test_release 65 | 66 | - name: Move ssl certificate to ensure we still find it 67 | run: | 68 | sudo mv /etc/ssl/certs/ca-certificates.crt /etc/ssl/cert.pem 69 | 70 | - name: Test extension 71 | run: | 72 | make test_release 73 | -------------------------------------------------------------------------------- /.github/workflows/MainDistributionPipeline.yml: -------------------------------------------------------------------------------- 1 | # 2 | # This workflow calls the main distribution pipeline from DuckDB to build, test and (optionally) release the extension 3 | # 4 | name: Main Extension Distribution Pipeline 5 | on: 6 | push: 7 | pull_request: 8 | workflow_dispatch: 9 | 10 | concurrency: 11 | group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} 12 | cancel-in-progress: true 13 | 14 | jobs: 15 | duckdb-stable-build: 16 | name: Build extension binaries 17 | uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.3.0 18 | with: 19 | duckdb_version: v1.3.0 20 | ci_tools_version: v1.3.0 21 | extension_name: uc_catalog 22 | exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools;windows_amd64;linux_arm64;' 23 | 24 | duckdb-stable-deploy: 25 | name: Deploy extension binaries 26 | needs: duckdb-stable-build 27 | uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@v1.3.0 28 | secrets: inherit 29 | with: 30 | duckdb_version: v1.3.0 31 | ci_tools_version: v1.3.0 32 | extension_name: uc_catalog 33 | exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools;windows_amd64;linux_arm64;' 34 | deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} 35 | -------------------------------------------------------------------------------- /.github/workflows/_extension_deploy.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Reusable workflow that deploys the artifacts produced by github.com/duckdb/duckdb/.github/workflows/_extension_distribution.yml 3 | # 4 | # note: this workflow needs to be located in the extension repository, as it requires secrets to be passed to the 5 | # deploy script. However, it should generally not be necessary to modify this workflow in your extension repository, as 6 | # this workflow can be configured to use a custom deploy script. 7 | 8 | 9 | name: Extension Deployment 10 | on: 11 | workflow_call: 12 | inputs: 13 | # The name of the extension 14 | extension_name: 15 | required: true 16 | type: string 17 | # DuckDB version to build against 18 | duckdb_version: 19 | required: true 20 | type: string 21 | # ';' separated list of architectures to exclude, for example: 'linux_amd64;osx_arm64' 22 | exclude_archs: 23 | required: false 24 | type: string 25 | default: "" 26 | # Whether to upload this deployment as the latest. This may overwrite a previous deployment. 27 | deploy_latest: 28 | required: false 29 | type: boolean 30 | default: false 31 | # Whether to upload this deployment under a versioned path. These will not be deleted automatically 32 | deploy_versioned: 33 | required: false 34 | type: boolean 35 | default: false 36 | # Postfix added to artifact names. Can be used to guarantee unique names when this workflow is called multiple times 37 | artifact_postfix: 38 | required: false 39 | type: string 40 | default: "" 41 | # Override the default deploy script with a custom script 42 | deploy_script: 43 | required: false 44 | type: string 45 | default: "./scripts/extension-upload.sh" 46 | # Override the default matrix parse script with a custom script 47 | matrix_parse_script: 48 | required: false 49 | type: string 50 | default: "./duckdb/scripts/modify_distribution_matrix.py" 51 | 52 | jobs: 53 | generate_matrix: 54 | name: Generate matrix 55 | runs-on: ubuntu-latest 56 | outputs: 57 | deploy_matrix: ${{ steps.parse-matrices.outputs.deploy_matrix }} 58 | steps: 59 | - uses: actions/checkout@v3 60 | with: 61 | fetch-depth: 0 62 | submodules: 'true' 63 | 64 | - name: Checkout DuckDB to version 65 | run: | 66 | cd duckdb 67 | git checkout ${{ inputs.duckdb_version }} 68 | 69 | - id: parse-matrices 70 | run: | 71 | python3 ${{ inputs.matrix_parse_script }} --input ./duckdb/.github/config/distribution_matrix.json --deploy_matrix --output deploy_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty 72 | deploy_matrix="`cat deploy_matrix.json`" 73 | echo deploy_matrix=$deploy_matrix >> $GITHUB_OUTPUT 74 | echo `cat $GITHUB_OUTPUT` 75 | 76 | deploy: 77 | name: Deploy 78 | runs-on: ubuntu-latest 79 | needs: generate_matrix 80 | if: ${{ needs.generate_matrix.outputs.deploy_matrix != '{}' && needs.generate_matrix.outputs.deploy_matrix != '' }} 81 | strategy: 82 | matrix: ${{fromJson(needs.generate_matrix.outputs.deploy_matrix)}} 83 | 84 | steps: 85 | - uses: actions/checkout@v3 86 | with: 87 | fetch-depth: 0 88 | submodules: 'true' 89 | 90 | - name: Checkout DuckDB to version 91 | run: | 92 | cd duckdb 93 | git checkout ${{ inputs.duckdb_version }} 94 | 95 | - uses: actions/download-artifact@v2 96 | with: 97 | name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}}${{startsWith(matrix.duckdb, 'wasm') && '.wasm' || ''}} 98 | path: | 99 | /tmp/extension 100 | 101 | - name: Deploy 102 | shell: bash 103 | env: 104 | AWS_ACCESS_KEY_ID: ${{ secrets.S3_DEPLOY_ID }} 105 | AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DEPLOY_KEY }} 106 | AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }} 107 | BUCKET_NAME: ${{ secrets.S3_BUCKET }} 108 | DUCKDB_EXTENSION_SIGNING_PK: ${{ secrets.S3_DUCKDB_ORG_EXTENSION_SIGNING_PK }} 109 | run: | 110 | pwd 111 | python3 -m pip install pip awscli 112 | git config --global --add safe.directory '*' 113 | cd duckdb 114 | git fetch --tags 115 | export DUCKDB_VERSION=`git tag --points-at HEAD` 116 | export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`} 117 | cd .. 118 | git fetch --tags 119 | export EXT_VERSION=`git tag --points-at HEAD` 120 | export EXT_VERSION=${EXT_VERSION:=`git log -1 --format=%h`} 121 | ${{ inputs.deploy_script }} ${{ inputs.extension_name }} $EXT_VERSION $DUCKDB_VERSION ${{ matrix.duckdb_arch }} $BUCKET_NAME ${{inputs.deploy_latest || 'true' && 'false'}} ${{inputs.deploy_versioned || 'true' && 'false'}} 122 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | .idea 3 | cmake-build-debug 4 | duckdb_unittest_tempdir/ 5 | .DS_Store 6 | testext 7 | test/python/__pycache__/ 8 | .Rhistory 9 | scripts/uc-script.sh 10 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "duckdb"] 2 | path = duckdb 3 | url = https://github.com/duckdb/duckdb 4 | branch = main 5 | [submodule "extension-ci-tools"] 6 | path = extension-ci-tools 7 | url = https://github.com/duckdb/extension-ci-tools 8 | branch = main -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.5) 2 | 3 | # Set extension name here 4 | set(TARGET_NAME uc_catalog) 5 | 6 | set(EXTENSION_NAME ${TARGET_NAME}_extension) 7 | set(LOADABLE_EXTENSION_NAME ${TARGET_NAME}_loadable_extension) 8 | 9 | project(${TARGET_NAME}) 10 | include_directories(src/include) 11 | 12 | find_package(CURL REQUIRED) 13 | 14 | add_subdirectory(src) 15 | 16 | build_static_extension(${TARGET_NAME} ${ALL_OBJECT_FILES}) 17 | build_loadable_extension(${TARGET_NAME} " " ${ALL_OBJECT_FILES}) 18 | 19 | target_link_libraries(${EXTENSION_NAME} CURL::libcurl) 20 | target_link_libraries(${LOADABLE_EXTENSION_NAME} CURL::libcurl) 21 | 22 | install( 23 | TARGETS ${EXTENSION_NAME} 24 | EXPORT "${DUCKDB_EXPORT_SET}" 25 | LIBRARY DESTINATION "${INSTALL_LIB_DIR}" 26 | ARCHIVE DESTINATION "${INSTALL_LIB_DIR}") 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2018-2025 Stichting DuckDB Foundation 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PROJ_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) 2 | 3 | # Configuration of extension 4 | EXT_NAME=uc_catalog 5 | EXT_CONFIG=${PROJ_DIR}extension_config.cmake 6 | 7 | CORE_EXTENSIONS='parquet;httpfs' 8 | 9 | # Include the Makefile from extension-ci-tools 10 | include extension-ci-tools/makefiles/duckdb_extension.Makefile -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Unity Catalog Extension 2 | Warning: this extension is an experimental, proof-of-concept for an extension, feel free to try it out, but no guarantees are given whatsoever. 3 | This extension could be renamed, moved or removed at any point. 4 | 5 | This is a proof-of-concept extension demonstrating DuckDB connecting to the Unity Catalog to scan Delta Table using 6 | the [delta extension](https://duckdb.org/docs/extensions/delta). 7 | 8 | You can try it out using DuckDB (>= v1.0.0) on the platforms: `linux_amd64`, `linux_amd64_gcc4`, `osx_amd64` and `osx_arm64` by running: 9 | 10 | ```SQL 11 | INSTALL uc_catalog; 12 | INSTALL delta; 13 | LOAD delta; 14 | LOAD uc_catalog; 15 | CREATE SECRET ( 16 | TYPE UC, 17 | TOKEN '${UC_TOKEN}', 18 | ENDPOINT '${UC_ENDPOINT}', 19 | AWS_REGION '${UC_AWS_REGION}' 20 | ) 21 | ATTACH 'test_catalog' AS test_catalog (TYPE UC_CATALOG) 22 | SHOW ALL TABLES; 23 | SELECT * FROM test_catalog.test_schema.test_table; 24 | ``` 25 | -------------------------------------------------------------------------------- /docs/UPDATING.md: -------------------------------------------------------------------------------- 1 | # Extension updating 2 | When cloning this template, the target version of DuckDB should be the latest stable release of DuckDB. However, there 3 | will inevitably come a time when a new DuckDB is released and the extension repository needs updating. This process goes 4 | as follows: 5 | 6 | - Bump submodules 7 | - `./duckdb` should be set to latest tagged release 8 | - `./extension-ci-tools` should be set to updated branch corresponding to latest DuckDB release 9 | - Bump versions in `./github/workflows` 10 | - `duckdb_version` input in `MainDistributionPipeline.yml` should be set to latest tagged release 11 | - reusable workflow `_extension_distribution.yml` should be set to updated branch corresponding to latest DuckDB release 12 | 13 | -------------------------------------------------------------------------------- /extension_config.cmake: -------------------------------------------------------------------------------- 1 | # This file is included by DuckDB's build system. It specifies which extension to load 2 | 3 | # Extension from this repo 4 | duckdb_extension_load(uc_catalog 5 | SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} 6 | LOAD_TESTS 7 | ) 8 | 9 | # TODO enable this to test with delta 10 | #duckdb_extension_load(httpfs) 11 | #duckdb_extension_load(json) 12 | #duckdb_extension_load(delta 13 | # GIT_URL https://github.com/duckdb/duckdb_delta 14 | # GIT_TAG main 15 | #) -------------------------------------------------------------------------------- /src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | include_directories(include) 2 | 3 | add_subdirectory(storage) 4 | 5 | add_library( 6 | uc_ext_library OBJECT uc_catalog_extension.cpp uc_api.cpp uc_utils.cpp) 7 | 8 | target_link_libraries(uc_ext_library CURL::libcurl) 9 | 10 | set(ALL_OBJECT_FILES 11 | ${ALL_OBJECT_FILES} $ 12 | PARENT_SCOPE) 13 | -------------------------------------------------------------------------------- /src/include/storage/uc_catalog.hpp: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // DuckDB 3 | // 4 | // storage/uc_catalog.hpp 5 | // 6 | // 7 | //===----------------------------------------------------------------------===// 8 | 9 | #pragma once 10 | 11 | #include "duckdb/catalog/catalog.hpp" 12 | #include "duckdb/function/table_function.hpp" 13 | #include "duckdb/common/enums/access_mode.hpp" 14 | #include "storage/uc_schema_set.hpp" 15 | 16 | namespace duckdb { 17 | class UCSchemaEntry; 18 | 19 | struct UCCredentials { 20 | string endpoint; 21 | string token; 22 | 23 | // Not really part of the credentials, but required to query s3 tables 24 | string aws_region; 25 | }; 26 | 27 | class UCClearCacheFunction : public TableFunction { 28 | public: 29 | UCClearCacheFunction(); 30 | 31 | static void ClearCacheOnSetting(ClientContext &context, SetScope scope, Value ¶meter); 32 | }; 33 | 34 | class UCCatalog : public Catalog { 35 | public: 36 | explicit UCCatalog(AttachedDatabase &db_p, const string &internal_name, AccessMode access_mode, 37 | UCCredentials credentials); 38 | ~UCCatalog(); 39 | 40 | string internal_name; 41 | AccessMode access_mode; 42 | UCCredentials credentials; 43 | 44 | public: 45 | void Initialize(bool load_builtin) override; 46 | string GetCatalogType() override { 47 | return "uc"; 48 | } 49 | 50 | optional_ptr CreateSchema(CatalogTransaction transaction, CreateSchemaInfo &info) override; 51 | 52 | void ScanSchemas(ClientContext &context, std::function callback) override; 53 | 54 | optional_ptr LookupSchema(CatalogTransaction transaction, 55 | const EntryLookupInfo &schema_lookup, 56 | OnEntryNotFound if_not_found) override; 57 | 58 | 59 | PhysicalOperator &PlanCreateTableAs(ClientContext &context, PhysicalPlanGenerator &planner, 60 | LogicalCreateTable &op, PhysicalOperator &plan) override; 61 | PhysicalOperator &PlanInsert(ClientContext &context, PhysicalPlanGenerator &planner, LogicalInsert &op, 62 | optional_ptr plan) override; 63 | PhysicalOperator &PlanDelete(ClientContext &context, PhysicalPlanGenerator &planner, LogicalDelete &op, 64 | PhysicalOperator &plan) override; 65 | PhysicalOperator &PlanDelete(ClientContext &context, PhysicalPlanGenerator &planner, LogicalDelete &op) override ; 66 | PhysicalOperator &PlanUpdate(ClientContext &context, PhysicalPlanGenerator &planner, LogicalUpdate &op, 67 | PhysicalOperator &plan) override; 68 | unique_ptr BindCreateIndex(Binder &binder, CreateStatement &stmt, TableCatalogEntry &table, 69 | unique_ptr plan) override; 70 | 71 | DatabaseSize GetDatabaseSize(ClientContext &context) override; 72 | 73 | //! Whether or not this is an in-memory UC database 74 | bool InMemory() override; 75 | string GetDBPath() override; 76 | 77 | void ClearCache(); 78 | 79 | private: 80 | void DropSchema(ClientContext &context, DropInfo &info) override; 81 | 82 | private: 83 | UCSchemaSet schemas; 84 | string default_schema; 85 | }; 86 | 87 | } // namespace duckdb 88 | -------------------------------------------------------------------------------- /src/include/storage/uc_catalog_set.hpp: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // DuckDB 3 | // 4 | // storage/uc_catalog_set.hpp 5 | // 6 | // 7 | //===----------------------------------------------------------------------===// 8 | 9 | #pragma once 10 | 11 | #include "duckdb/transaction/transaction.hpp" 12 | #include "duckdb/common/case_insensitive_map.hpp" 13 | #include "duckdb/common/mutex.hpp" 14 | 15 | namespace duckdb { 16 | struct DropInfo; 17 | class UCSchemaEntry; 18 | class UCTransaction; 19 | 20 | class UCCatalogSet { 21 | public: 22 | UCCatalogSet(Catalog &catalog); 23 | 24 | optional_ptr GetEntry(ClientContext &context, const string &name); 25 | virtual void DropEntry(ClientContext &context, DropInfo &info); 26 | void Scan(ClientContext &context, const std::function &callback); 27 | virtual optional_ptr CreateEntry(unique_ptr entry); 28 | void ClearEntries(); 29 | 30 | protected: 31 | virtual void LoadEntries(ClientContext &context) = 0; 32 | 33 | void EraseEntryInternal(const string &name); 34 | 35 | protected: 36 | Catalog &catalog; 37 | 38 | private: 39 | mutex entry_lock; 40 | case_insensitive_map_t> entries; 41 | bool is_loaded; 42 | }; 43 | 44 | class UCInSchemaSet : public UCCatalogSet { 45 | public: 46 | UCInSchemaSet(UCSchemaEntry &schema); 47 | 48 | optional_ptr CreateEntry(unique_ptr entry) override; 49 | 50 | protected: 51 | UCSchemaEntry &schema; 52 | }; 53 | 54 | } // namespace duckdb 55 | -------------------------------------------------------------------------------- /src/include/storage/uc_schema_entry.hpp: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // DuckDB 3 | // 4 | // storage/uc_schema_entry.hpp 5 | // 6 | // 7 | //===----------------------------------------------------------------------===// 8 | 9 | #pragma once 10 | 11 | #include "uc_api.hpp" 12 | #include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp" 13 | #include "storage/uc_table_set.hpp" 14 | 15 | namespace duckdb { 16 | class UCTransaction; 17 | 18 | class UCSchemaEntry : public SchemaCatalogEntry { 19 | public: 20 | UCSchemaEntry(Catalog &catalog, CreateSchemaInfo &info); 21 | ~UCSchemaEntry() override; 22 | 23 | unique_ptr schema_data; 24 | 25 | public: 26 | optional_ptr CreateTable(CatalogTransaction transaction, BoundCreateTableInfo &info) override; 27 | optional_ptr CreateFunction(CatalogTransaction transaction, CreateFunctionInfo &info) override; 28 | optional_ptr CreateIndex(CatalogTransaction transaction, CreateIndexInfo &info, 29 | TableCatalogEntry &table) override; 30 | optional_ptr CreateView(CatalogTransaction transaction, CreateViewInfo &info) override; 31 | optional_ptr CreateSequence(CatalogTransaction transaction, CreateSequenceInfo &info) override; 32 | optional_ptr CreateTableFunction(CatalogTransaction transaction, 33 | CreateTableFunctionInfo &info) override; 34 | optional_ptr CreateCopyFunction(CatalogTransaction transaction, 35 | CreateCopyFunctionInfo &info) override; 36 | optional_ptr CreatePragmaFunction(CatalogTransaction transaction, 37 | CreatePragmaFunctionInfo &info) override; 38 | optional_ptr CreateCollation(CatalogTransaction transaction, CreateCollationInfo &info) override; 39 | optional_ptr CreateType(CatalogTransaction transaction, CreateTypeInfo &info) override; 40 | void Alter(CatalogTransaction transaction, AlterInfo &info) override; 41 | void Scan(ClientContext &context, CatalogType type, const std::function &callback) override; 42 | void Scan(CatalogType type, const std::function &callback) override; 43 | void DropEntry(ClientContext &context, DropInfo &info) override; 44 | optional_ptr LookupEntry(CatalogTransaction transaction, 45 | const EntryLookupInfo &lookup_info) override; 46 | 47 | private: 48 | UCCatalogSet &GetCatalogSet(CatalogType type); 49 | 50 | private: 51 | UCTableSet tables; 52 | }; 53 | 54 | } // namespace duckdb 55 | -------------------------------------------------------------------------------- /src/include/storage/uc_schema_set.hpp: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // DuckDB 3 | // 4 | // storage/uc_schema_set.hpp 5 | // 6 | // 7 | //===----------------------------------------------------------------------===// 8 | 9 | #pragma once 10 | 11 | #include "storage/uc_catalog_set.hpp" 12 | #include "storage/uc_schema_entry.hpp" 13 | 14 | namespace duckdb { 15 | struct CreateSchemaInfo; 16 | 17 | class UCSchemaSet : public UCCatalogSet { 18 | public: 19 | explicit UCSchemaSet(Catalog &catalog); 20 | 21 | public: 22 | optional_ptr CreateSchema(ClientContext &context, CreateSchemaInfo &info); 23 | 24 | protected: 25 | void LoadEntries(ClientContext &context) override; 26 | }; 27 | 28 | } // namespace duckdb 29 | -------------------------------------------------------------------------------- /src/include/storage/uc_table_entry.hpp: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // DuckDB 3 | // 4 | // storage/uc_table_entry.hpp 5 | // 6 | // 7 | //===----------------------------------------------------------------------===// 8 | 9 | #pragma once 10 | 11 | #include "uc_api.hpp" 12 | #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" 13 | #include "duckdb/parser/parsed_data/create_table_info.hpp" 14 | 15 | namespace duckdb { 16 | 17 | struct UCTableInfo { 18 | UCTableInfo() { 19 | create_info = make_uniq(); 20 | } 21 | UCTableInfo(const string &schema, const string &table) { 22 | create_info = make_uniq(string(), schema, table); 23 | } 24 | UCTableInfo(const SchemaCatalogEntry &schema, const string &table) { 25 | create_info = make_uniq((SchemaCatalogEntry &)schema, table); 26 | } 27 | 28 | const string &GetTableName() const { 29 | return create_info->table; 30 | } 31 | 32 | unique_ptr create_info; 33 | }; 34 | 35 | class UCTableEntry : public TableCatalogEntry { 36 | public: 37 | UCTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateTableInfo &info); 38 | UCTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, UCTableInfo &info); 39 | 40 | unique_ptr table_data; 41 | 42 | public: 43 | unique_ptr GetStatistics(ClientContext &context, column_t column_id) override; 44 | 45 | TableFunction GetScanFunction(ClientContext &context, unique_ptr &bind_data) override; 46 | 47 | TableStorageInfo GetStorageInfo(ClientContext &context) override; 48 | 49 | void BindUpdateConstraints(Binder &binder, LogicalGet &get, LogicalProjection &proj, LogicalUpdate &update, 50 | ClientContext &context) override; 51 | }; 52 | 53 | } // namespace duckdb 54 | -------------------------------------------------------------------------------- /src/include/storage/uc_table_set.hpp: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // DuckDB 3 | // 4 | // storage/uc_table_set.hpp 5 | // 6 | // 7 | //===----------------------------------------------------------------------===// 8 | 9 | #pragma once 10 | 11 | #include "storage/uc_catalog_set.hpp" 12 | #include "storage/uc_table_entry.hpp" 13 | 14 | namespace duckdb { 15 | struct CreateTableInfo; 16 | class UCResult; 17 | class UCSchemaEntry; 18 | 19 | class UCTableSet : public UCInSchemaSet { 20 | public: 21 | explicit UCTableSet(UCSchemaEntry &schema); 22 | 23 | public: 24 | optional_ptr CreateTable(ClientContext &context, BoundCreateTableInfo &info); 25 | 26 | static unique_ptr GetTableInfo(ClientContext &context, UCSchemaEntry &schema, 27 | const string &table_name); 28 | optional_ptr RefreshTable(ClientContext &context, const string &table_name); 29 | 30 | void AlterTable(ClientContext &context, AlterTableInfo &info); 31 | 32 | protected: 33 | void LoadEntries(ClientContext &context) override; 34 | 35 | void AlterTable(ClientContext &context, RenameTableInfo &info); 36 | void AlterTable(ClientContext &context, RenameColumnInfo &info); 37 | void AlterTable(ClientContext &context, AddColumnInfo &info); 38 | void AlterTable(ClientContext &context, RemoveColumnInfo &info); 39 | 40 | static void AddColumn(ClientContext &context, UCResult &result, UCTableInfo &table_info, idx_t column_offset = 0); 41 | }; 42 | 43 | } // namespace duckdb 44 | -------------------------------------------------------------------------------- /src/include/storage/uc_transaction.hpp: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // DuckDB 3 | // 4 | // storage/uc_transaction.hpp 5 | // 6 | // 7 | //===----------------------------------------------------------------------===// 8 | 9 | #pragma once 10 | 11 | #include "duckdb/transaction/transaction.hpp" 12 | 13 | namespace duckdb { 14 | class UCCatalog; 15 | class UCSchemaEntry; 16 | class UCTableEntry; 17 | 18 | enum class UCTransactionState { TRANSACTION_NOT_YET_STARTED, TRANSACTION_STARTED, TRANSACTION_FINISHED }; 19 | 20 | class UCTransaction : public Transaction { 21 | public: 22 | UCTransaction(UCCatalog &uc_catalog, TransactionManager &manager, ClientContext &context); 23 | ~UCTransaction() override; 24 | 25 | void Start(); 26 | void Commit(); 27 | void Rollback(); 28 | 29 | // UCConnection &GetConnection(); 30 | // unique_ptr Query(const string &query); 31 | static UCTransaction &Get(ClientContext &context, Catalog &catalog); 32 | AccessMode GetAccessMode() const { 33 | return access_mode; 34 | } 35 | 36 | private: 37 | // UCConnection connection; 38 | UCTransactionState transaction_state; 39 | AccessMode access_mode; 40 | }; 41 | 42 | } // namespace duckdb 43 | -------------------------------------------------------------------------------- /src/include/storage/uc_transaction_manager.hpp: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // DuckDB 3 | // 4 | // storage/uc_transaction_manager.hpp 5 | // 6 | // 7 | //===----------------------------------------------------------------------===// 8 | 9 | #pragma once 10 | 11 | #include "duckdb/transaction/transaction_manager.hpp" 12 | #include "storage/uc_catalog.hpp" 13 | #include "storage/uc_transaction.hpp" 14 | 15 | namespace duckdb { 16 | 17 | class UCTransactionManager : public TransactionManager { 18 | public: 19 | UCTransactionManager(AttachedDatabase &db_p, UCCatalog &uc_catalog); 20 | 21 | Transaction &StartTransaction(ClientContext &context) override; 22 | ErrorData CommitTransaction(ClientContext &context, Transaction &transaction) override; 23 | void RollbackTransaction(Transaction &transaction) override; 24 | 25 | void Checkpoint(ClientContext &context, bool force = false) override; 26 | 27 | private: 28 | UCCatalog &uc_catalog; 29 | mutex transaction_lock; 30 | reference_map_t> transactions; 31 | }; 32 | 33 | } // namespace duckdb 34 | -------------------------------------------------------------------------------- /src/include/uc_api.hpp: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // DuckDB 3 | // 4 | // src/include/uc_api.hpp 5 | // 6 | // 7 | //===----------------------------------------------------------------------===// 8 | 9 | #pragma once 10 | 11 | #include "duckdb/common/types.hpp" 12 | 13 | namespace duckdb { 14 | struct UCCredentials; 15 | 16 | struct UCAPIColumnDefinition { 17 | string name; 18 | string type_text; 19 | idx_t precision; 20 | idx_t scale; 21 | idx_t position; 22 | }; 23 | 24 | struct UCAPITable { 25 | string table_id; 26 | 27 | string name; 28 | string catalog_name; 29 | string schema_name; 30 | string table_type; 31 | string data_source_format; 32 | 33 | string storage_location; 34 | string delta_last_commit_timestamp; 35 | string delta_last_update_version; 36 | 37 | vector columns; 38 | }; 39 | 40 | struct UCAPISchema { 41 | string schema_name; 42 | string catalog_name; 43 | }; 44 | 45 | struct UCAPITableCredentials { 46 | string key_id; 47 | string secret; 48 | string session_token; 49 | }; 50 | 51 | class UCAPI { 52 | public: 53 | //! WARNING: not thread-safe. To be called once on extension initialization 54 | static void InitializeCurl(); 55 | 56 | static UCAPITableCredentials GetTableCredentials(const string &table_id, UCCredentials credentials); 57 | static vector GetCatalogs(const string &catalog, UCCredentials credentials); 58 | static vector GetTables(const string &catalog, const string &schema, UCCredentials credentials); 59 | static vector GetSchemas(const string &catalog, UCCredentials credentials); 60 | static vector GetTablesInSchema(const string &catalog, const string &schema, UCCredentials credentials); 61 | }; 62 | } // namespace duckdb 63 | -------------------------------------------------------------------------------- /src/include/uc_catalog_extension.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "duckdb.hpp" 4 | 5 | namespace duckdb { 6 | 7 | class UcCatalogExtension : public Extension { 8 | public: 9 | void Load(DuckDB &db) override; 10 | std::string Name() override; 11 | std::string Version() const override; 12 | }; 13 | 14 | } // namespace duckdb 15 | -------------------------------------------------------------------------------- /src/include/uc_utils.hpp: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // DuckDB 3 | // 4 | // mysql_utils.hpp 5 | // 6 | // 7 | //===----------------------------------------------------------------------===// 8 | 9 | #pragma once 10 | 11 | #include "duckdb.hpp" 12 | #include "uc_api.hpp" 13 | 14 | namespace duckdb { 15 | class UCSchemaEntry; 16 | class UCTransaction; 17 | 18 | enum class UCTypeAnnotation { STANDARD, CAST_TO_VARCHAR, NUMERIC_AS_DOUBLE, CTID, JSONB, FIXED_LENGTH_CHAR }; 19 | 20 | struct UCType { 21 | idx_t oid = 0; 22 | UCTypeAnnotation info = UCTypeAnnotation::STANDARD; 23 | vector children; 24 | }; 25 | 26 | class UCUtils { 27 | public: 28 | static LogicalType ToUCType(const LogicalType &input); 29 | static LogicalType TypeToLogicalType(ClientContext &context, const string &columnDefinition); 30 | static string TypeToString(const LogicalType &input); 31 | }; 32 | 33 | } // namespace duckdb 34 | -------------------------------------------------------------------------------- /src/storage/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library( 2 | uc_ext_storage OBJECT 3 | uc_catalog.cpp 4 | uc_catalog_set.cpp 5 | uc_clear_cache.cpp 6 | uc_schema_entry.cpp 7 | uc_schema_set.cpp 8 | uc_table_entry.cpp 9 | uc_table_set.cpp 10 | uc_transaction.cpp 11 | uc_transaction_manager.cpp) 12 | set(ALL_OBJECT_FILES 13 | ${ALL_OBJECT_FILES} $ 14 | PARENT_SCOPE) 15 | -------------------------------------------------------------------------------- /src/storage/uc_catalog.cpp: -------------------------------------------------------------------------------- 1 | #include "storage/uc_catalog.hpp" 2 | #include "storage/uc_schema_entry.hpp" 3 | #include "storage/uc_transaction.hpp" 4 | #include "duckdb/storage/database_size.hpp" 5 | #include "duckdb/parser/parsed_data/drop_info.hpp" 6 | #include "duckdb/parser/parsed_data/create_schema_info.hpp" 7 | #include "duckdb/main/attached_database.hpp" 8 | 9 | namespace duckdb { 10 | 11 | UCCatalog::UCCatalog(AttachedDatabase &db_p, const string &internal_name, AccessMode access_mode, 12 | UCCredentials credentials) 13 | : Catalog(db_p), internal_name(internal_name), access_mode(access_mode), credentials(std::move(credentials)), 14 | schemas(*this) { 15 | } 16 | 17 | UCCatalog::~UCCatalog() = default; 18 | 19 | void UCCatalog::Initialize(bool load_builtin) { 20 | } 21 | 22 | optional_ptr UCCatalog::CreateSchema(CatalogTransaction transaction, CreateSchemaInfo &info) { 23 | if (info.on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT) { 24 | DropInfo try_drop; 25 | try_drop.type = CatalogType::SCHEMA_ENTRY; 26 | try_drop.name = info.schema; 27 | try_drop.if_not_found = OnEntryNotFound::RETURN_NULL; 28 | try_drop.cascade = false; 29 | schemas.DropEntry(transaction.GetContext(), try_drop); 30 | } 31 | return schemas.CreateSchema(transaction.GetContext(), info); 32 | } 33 | 34 | void UCCatalog::DropSchema(ClientContext &context, DropInfo &info) { 35 | return schemas.DropEntry(context, info); 36 | } 37 | 38 | void UCCatalog::ScanSchemas(ClientContext &context, std::function callback) { 39 | schemas.Scan(context, [&](CatalogEntry &schema) { callback(schema.Cast()); }); 40 | } 41 | 42 | optional_ptr UCCatalog::LookupSchema(CatalogTransaction transaction, 43 | const EntryLookupInfo &schema_lookup, 44 | OnEntryNotFound if_not_found) { 45 | if (schema_lookup.GetEntryName() == DEFAULT_SCHEMA) { 46 | if (default_schema.empty()) { 47 | throw InvalidInputException("Attempting to fetch the default schema - but no database was " 48 | "provided in the connection string"); 49 | } 50 | return GetSchema(transaction, default_schema, if_not_found); 51 | } 52 | auto entry = schemas.GetEntry(transaction.GetContext(), schema_lookup.GetEntryName()); 53 | if (!entry && if_not_found != OnEntryNotFound::RETURN_NULL) { 54 | throw BinderException("Schema with name \"%s\" not found", schema_lookup.GetEntryName()); 55 | } 56 | return reinterpret_cast(entry.get()); 57 | } 58 | 59 | bool UCCatalog::InMemory() { 60 | return false; 61 | } 62 | 63 | string UCCatalog::GetDBPath() { 64 | return internal_name; 65 | } 66 | 67 | DatabaseSize UCCatalog::GetDatabaseSize(ClientContext &context) { 68 | if (default_schema.empty()) { 69 | throw InvalidInputException("Attempting to fetch the database size - but no database was provided " 70 | "in the connection string"); 71 | } 72 | DatabaseSize size; 73 | return size; 74 | } 75 | 76 | void UCCatalog::ClearCache() { 77 | schemas.ClearEntries(); 78 | } 79 | 80 | PhysicalOperator &UCCatalog::PlanCreateTableAs(ClientContext &context, PhysicalPlanGenerator &planner, 81 | LogicalCreateTable &op, PhysicalOperator &plan) { 82 | throw NotImplementedException("UCCatalog PlanInsert"); 83 | } 84 | 85 | PhysicalOperator &UCCatalog::PlanInsert(ClientContext &context, PhysicalPlanGenerator &planner, LogicalInsert &op, 86 | optional_ptr plan) { 87 | throw NotImplementedException("UCCatalog PlanCreateTableAs"); 88 | } 89 | 90 | PhysicalOperator &UCCatalog::PlanDelete(ClientContext &context, PhysicalPlanGenerator &planner, LogicalDelete &op, 91 | PhysicalOperator &plan) { 92 | throw NotImplementedException("UCCatalog PlanDelete"); 93 | } 94 | 95 | PhysicalOperator &UCCatalog::PlanDelete(ClientContext &context, PhysicalPlanGenerator &planner, LogicalDelete &op) { 96 | throw NotImplementedException("UCCatalog PlanDelete"); 97 | } 98 | 99 | PhysicalOperator &UCCatalog::PlanUpdate(ClientContext &context, PhysicalPlanGenerator &planner, LogicalUpdate &op, 100 | PhysicalOperator &plan) { 101 | throw NotImplementedException("UCCatalog PlanUpdate"); 102 | } 103 | 104 | unique_ptr UCCatalog::BindCreateIndex(Binder &binder, CreateStatement &stmt, TableCatalogEntry &table, 105 | unique_ptr plan) { 106 | throw NotImplementedException("UCCatalog BindCreateIndex"); 107 | } 108 | 109 | } // namespace duckdb 110 | -------------------------------------------------------------------------------- /src/storage/uc_catalog_set.cpp: -------------------------------------------------------------------------------- 1 | #include "storage/uc_catalog_set.hpp" 2 | #include "storage/uc_transaction.hpp" 3 | #include "duckdb/parser/parsed_data/drop_info.hpp" 4 | #include "storage/uc_schema_entry.hpp" 5 | 6 | namespace duckdb { 7 | 8 | UCCatalogSet::UCCatalogSet(Catalog &catalog) : catalog(catalog), is_loaded(false) { 9 | } 10 | 11 | optional_ptr UCCatalogSet::GetEntry(ClientContext &context, const string &name) { 12 | if (!is_loaded) { 13 | is_loaded = true; 14 | LoadEntries(context); 15 | } 16 | lock_guard l(entry_lock); 17 | auto entry = entries.find(name); 18 | if (entry == entries.end()) { 19 | return nullptr; 20 | } 21 | return entry->second.get(); 22 | } 23 | 24 | void UCCatalogSet::DropEntry(ClientContext &context, DropInfo &info) { 25 | throw NotImplementedException("UCCatalogSet::DropEntry"); 26 | } 27 | 28 | void UCCatalogSet::EraseEntryInternal(const string &name) { 29 | lock_guard l(entry_lock); 30 | entries.erase(name); 31 | } 32 | 33 | void UCCatalogSet::Scan(ClientContext &context, const std::function &callback) { 34 | if (!is_loaded) { 35 | is_loaded = true; 36 | LoadEntries(context); 37 | } 38 | lock_guard l(entry_lock); 39 | for (auto &entry : entries) { 40 | callback(*entry.second); 41 | } 42 | } 43 | 44 | optional_ptr UCCatalogSet::CreateEntry(unique_ptr entry) { 45 | lock_guard l(entry_lock); 46 | auto result = entry.get(); 47 | if (result->name.empty()) { 48 | throw InternalException("UCCatalogSet::CreateEntry called with empty name"); 49 | } 50 | entries.insert(make_pair(result->name, std::move(entry))); 51 | return result; 52 | } 53 | 54 | void UCCatalogSet::ClearEntries() { 55 | entries.clear(); 56 | is_loaded = false; 57 | } 58 | 59 | UCInSchemaSet::UCInSchemaSet(UCSchemaEntry &schema) : UCCatalogSet(schema.ParentCatalog()), schema(schema) { 60 | } 61 | 62 | optional_ptr UCInSchemaSet::CreateEntry(unique_ptr entry) { 63 | if (!entry->internal) { 64 | entry->internal = schema.internal; 65 | } 66 | return UCCatalogSet::CreateEntry(std::move(entry)); 67 | } 68 | 69 | } // namespace duckdb 70 | -------------------------------------------------------------------------------- /src/storage/uc_clear_cache.cpp: -------------------------------------------------------------------------------- 1 | #include "duckdb.hpp" 2 | 3 | #include "duckdb/parser/parsed_data/create_table_function_info.hpp" 4 | #include "duckdb/main/database_manager.hpp" 5 | #include "duckdb/main/attached_database.hpp" 6 | #include "storage/uc_catalog.hpp" 7 | 8 | namespace duckdb { 9 | 10 | struct ClearCacheFunctionData : public TableFunctionData { 11 | bool finished = false; 12 | }; 13 | 14 | static unique_ptr ClearCacheBind(ClientContext &context, TableFunctionBindInput &input, 15 | vector &return_types, vector &names) { 16 | 17 | auto result = make_uniq(); 18 | return_types.push_back(LogicalType::BOOLEAN); 19 | names.emplace_back("Success"); 20 | return std::move(result); 21 | } 22 | 23 | static void ClearUCCaches(ClientContext &context) { 24 | auto databases = DatabaseManager::Get(context).GetDatabases(context); 25 | for (auto &db_ref : databases) { 26 | auto &db = db_ref.get(); 27 | auto &catalog = db.GetCatalog(); 28 | if (catalog.GetCatalogType() != "uc") { 29 | continue; 30 | } 31 | catalog.Cast().ClearCache(); 32 | } 33 | } 34 | 35 | static void ClearCacheFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { 36 | auto &data = data_p.bind_data->CastNoConst(); 37 | if (data.finished) { 38 | return; 39 | } 40 | ClearUCCaches(context); 41 | data.finished = true; 42 | } 43 | 44 | void UCClearCacheFunction::ClearCacheOnSetting(ClientContext &context, SetScope scope, Value ¶meter) { 45 | ClearUCCaches(context); 46 | } 47 | 48 | UCClearCacheFunction::UCClearCacheFunction() : TableFunction("uc_clear_cache", {}, ClearCacheFunction, ClearCacheBind) { 49 | } 50 | } // namespace duckdb 51 | -------------------------------------------------------------------------------- /src/storage/uc_schema_entry.cpp: -------------------------------------------------------------------------------- 1 | #include "storage/uc_schema_entry.hpp" 2 | #include "storage/uc_table_entry.hpp" 3 | #include "storage/uc_transaction.hpp" 4 | #include "duckdb/parser/parsed_data/create_view_info.hpp" 5 | #include "duckdb/parser/parsed_data/create_index_info.hpp" 6 | #include "duckdb/planner/parsed_data/bound_create_table_info.hpp" 7 | #include "duckdb/parser/parsed_data/drop_info.hpp" 8 | #include "duckdb/parser/constraints/list.hpp" 9 | #include "duckdb/common/unordered_set.hpp" 10 | #include "duckdb/parser/parsed_data/alter_info.hpp" 11 | #include "duckdb/parser/parsed_data/alter_table_info.hpp" 12 | #include "duckdb/parser/parsed_expression_iterator.hpp" 13 | 14 | namespace duckdb { 15 | 16 | UCSchemaEntry::UCSchemaEntry(Catalog &catalog, CreateSchemaInfo &info) 17 | : SchemaCatalogEntry(catalog, info), tables(*this) { 18 | } 19 | 20 | UCSchemaEntry::~UCSchemaEntry() { 21 | } 22 | 23 | UCTransaction &GetUCTransaction(CatalogTransaction transaction) { 24 | if (!transaction.transaction) { 25 | throw InternalException("No transaction!?"); 26 | } 27 | return transaction.transaction->Cast(); 28 | } 29 | 30 | optional_ptr UCSchemaEntry::CreateTable(CatalogTransaction transaction, BoundCreateTableInfo &info) { 31 | auto &base_info = info.Base(); 32 | auto table_name = base_info.table; 33 | if (base_info.on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT) { 34 | throw NotImplementedException("REPLACE ON CONFLICT in CreateTable"); 35 | } 36 | return tables.CreateTable(transaction.GetContext(), info); 37 | } 38 | 39 | optional_ptr UCSchemaEntry::CreateFunction(CatalogTransaction transaction, CreateFunctionInfo &info) { 40 | throw BinderException("UC databases do not support creating functions"); 41 | } 42 | 43 | void UCUnqualifyColumnRef(ParsedExpression &expr) { 44 | if (expr.type == ExpressionType::COLUMN_REF) { 45 | auto &colref = expr.Cast(); 46 | auto name = std::move(colref.column_names.back()); 47 | colref.column_names = {std::move(name)}; 48 | return; 49 | } 50 | ParsedExpressionIterator::EnumerateChildren(expr, UCUnqualifyColumnRef); 51 | } 52 | 53 | optional_ptr UCSchemaEntry::CreateIndex(CatalogTransaction transaction, CreateIndexInfo &info, 54 | TableCatalogEntry &table) { 55 | throw NotImplementedException("CreateIndex"); 56 | } 57 | 58 | string GetUCCreateView(CreateViewInfo &info) { 59 | throw NotImplementedException("GetCreateView"); 60 | } 61 | 62 | optional_ptr UCSchemaEntry::CreateView(CatalogTransaction transaction, CreateViewInfo &info) { 63 | if (info.sql.empty()) { 64 | throw BinderException("Cannot create view in UC that originated from an " 65 | "empty SQL statement"); 66 | } 67 | if (info.on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT || 68 | info.on_conflict == OnCreateConflict::IGNORE_ON_CONFLICT) { 69 | auto current_entry = GetEntry(transaction, CatalogType::VIEW_ENTRY, info.view_name); 70 | if (current_entry) { 71 | if (info.on_conflict == OnCreateConflict::IGNORE_ON_CONFLICT) { 72 | return current_entry; 73 | } 74 | throw NotImplementedException("REPLACE ON CONFLICT in CreateView"); 75 | } 76 | } 77 | auto &uc_transaction = GetUCTransaction(transaction); 78 | // uc_transaction.Query(GetUCCreateView(info)); 79 | return tables.RefreshTable(transaction.GetContext(), info.view_name); 80 | } 81 | 82 | optional_ptr UCSchemaEntry::CreateType(CatalogTransaction transaction, CreateTypeInfo &info) { 83 | throw BinderException("UC databases do not support creating types"); 84 | } 85 | 86 | optional_ptr UCSchemaEntry::CreateSequence(CatalogTransaction transaction, CreateSequenceInfo &info) { 87 | throw BinderException("UC databases do not support creating sequences"); 88 | } 89 | 90 | optional_ptr UCSchemaEntry::CreateTableFunction(CatalogTransaction transaction, 91 | CreateTableFunctionInfo &info) { 92 | throw BinderException("UC databases do not support creating table functions"); 93 | } 94 | 95 | optional_ptr UCSchemaEntry::CreateCopyFunction(CatalogTransaction transaction, 96 | CreateCopyFunctionInfo &info) { 97 | throw BinderException("UC databases do not support creating copy functions"); 98 | } 99 | 100 | optional_ptr UCSchemaEntry::CreatePragmaFunction(CatalogTransaction transaction, 101 | CreatePragmaFunctionInfo &info) { 102 | throw BinderException("UC databases do not support creating pragma functions"); 103 | } 104 | 105 | optional_ptr UCSchemaEntry::CreateCollation(CatalogTransaction transaction, CreateCollationInfo &info) { 106 | throw BinderException("UC databases do not support creating collations"); 107 | } 108 | 109 | void UCSchemaEntry::Alter(CatalogTransaction transaction, AlterInfo &info) { 110 | if (info.type != AlterType::ALTER_TABLE) { 111 | throw BinderException("Only altering tables is supported for now"); 112 | } 113 | auto &alter = info.Cast(); 114 | tables.AlterTable(transaction.GetContext(), alter); 115 | } 116 | 117 | bool CatalogTypeIsSupported(CatalogType type) { 118 | switch (type) { 119 | case CatalogType::INDEX_ENTRY: 120 | case CatalogType::TABLE_ENTRY: 121 | case CatalogType::VIEW_ENTRY: 122 | return true; 123 | default: 124 | return false; 125 | } 126 | } 127 | 128 | void UCSchemaEntry::Scan(ClientContext &context, CatalogType type, 129 | const std::function &callback) { 130 | if (!CatalogTypeIsSupported(type)) { 131 | return; 132 | } 133 | GetCatalogSet(type).Scan(context, callback); 134 | } 135 | void UCSchemaEntry::Scan(CatalogType type, const std::function &callback) { 136 | throw NotImplementedException("Scan without context not supported"); 137 | } 138 | 139 | void UCSchemaEntry::DropEntry(ClientContext &context, DropInfo &info) { 140 | GetCatalogSet(info.type).DropEntry(context, info); 141 | } 142 | 143 | optional_ptr 144 | UCSchemaEntry::LookupEntry(CatalogTransaction transaction, const EntryLookupInfo &lookup_info) { 145 | if (!CatalogTypeIsSupported(lookup_info.GetCatalogType())) { 146 | return nullptr; 147 | } 148 | return GetCatalogSet(lookup_info.GetCatalogType()).GetEntry(transaction.GetContext(), lookup_info.GetEntryName()); 149 | } 150 | 151 | UCCatalogSet &UCSchemaEntry::GetCatalogSet(CatalogType type) { 152 | switch (type) { 153 | case CatalogType::TABLE_ENTRY: 154 | case CatalogType::VIEW_ENTRY: 155 | return tables; 156 | default: 157 | throw InternalException("Type not supported for GetCatalogSet"); 158 | } 159 | } 160 | 161 | } // namespace duckdb 162 | -------------------------------------------------------------------------------- /src/storage/uc_schema_set.cpp: -------------------------------------------------------------------------------- 1 | #include "storage/uc_schema_set.hpp" 2 | #include "storage/uc_catalog.hpp" 3 | #include "uc_api.hpp" 4 | #include "storage/uc_transaction.hpp" 5 | #include "duckdb/parser/parsed_data/create_schema_info.hpp" 6 | #include "duckdb/catalog/catalog.hpp" 7 | 8 | namespace duckdb { 9 | 10 | UCSchemaSet::UCSchemaSet(Catalog &catalog) : UCCatalogSet(catalog) { 11 | } 12 | 13 | static bool IsInternalTable(const string &catalog, const string &schema) { 14 | if (schema == "information_schema") { 15 | return true; 16 | } 17 | return false; 18 | } 19 | void UCSchemaSet::LoadEntries(ClientContext &context) { 20 | 21 | auto &uc_catalog = catalog.Cast(); 22 | auto tables = UCAPI::GetSchemas(catalog.GetName(), uc_catalog.credentials); 23 | 24 | for (const auto &schema : tables) { 25 | CreateSchemaInfo info; 26 | info.schema = schema.schema_name; 27 | info.internal = IsInternalTable(schema.catalog_name, schema.schema_name); 28 | auto schema_entry = make_uniq(catalog, info); 29 | schema_entry->schema_data = make_uniq(schema); 30 | CreateEntry(std::move(schema_entry)); 31 | } 32 | } 33 | 34 | optional_ptr UCSchemaSet::CreateSchema(ClientContext &context, CreateSchemaInfo &info) { 35 | throw NotImplementedException("Schema creation"); 36 | } 37 | 38 | } // namespace duckdb 39 | -------------------------------------------------------------------------------- /src/storage/uc_table_entry.cpp: -------------------------------------------------------------------------------- 1 | #include "storage/uc_catalog.hpp" 2 | #include "storage/uc_schema_entry.hpp" 3 | #include "storage/uc_table_entry.hpp" 4 | #include "storage/uc_transaction.hpp" 5 | #include "duckdb/storage/statistics/base_statistics.hpp" 6 | #include "duckdb/storage/table_storage_info.hpp" 7 | #include "duckdb/main/extension_util.hpp" 8 | #include "duckdb/main/database.hpp" 9 | #include "duckdb/main/secret/secret_manager.hpp" 10 | #include "duckdb/catalog/catalog_entry/table_function_catalog_entry.hpp" 11 | #include "duckdb/parser/tableref/table_function_ref.hpp" 12 | #include "uc_api.hpp" 13 | #include "../../duckdb/third_party/catch/catch.hpp" 14 | 15 | namespace duckdb { 16 | 17 | UCTableEntry::UCTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateTableInfo &info) 18 | : TableCatalogEntry(catalog, schema, info) { 19 | this->internal = false; 20 | } 21 | 22 | UCTableEntry::UCTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, UCTableInfo &info) 23 | : TableCatalogEntry(catalog, schema, *info.create_info) { 24 | this->internal = false; 25 | } 26 | 27 | unique_ptr UCTableEntry::GetStatistics(ClientContext &context, column_t column_id) { 28 | return nullptr; 29 | } 30 | 31 | void UCTableEntry::BindUpdateConstraints(Binder &binder, LogicalGet &, LogicalProjection &, LogicalUpdate &, 32 | ClientContext &) { 33 | throw NotImplementedException("BindUpdateConstraints"); 34 | } 35 | 36 | TableFunction UCTableEntry::GetScanFunction(ClientContext &context, unique_ptr &bind_data) { 37 | auto &db = DatabaseInstance::GetDatabase(context); 38 | auto &delta_function_set = ExtensionUtil::GetTableFunction(db, "delta_scan"); 39 | auto delta_scan_function = delta_function_set.functions.GetFunctionByArguments(context, {LogicalType::VARCHAR}); 40 | auto &uc_catalog = catalog.Cast(); 41 | 42 | D_ASSERT(table_data); 43 | 44 | if (table_data->data_source_format != "DELTA") { 45 | throw NotImplementedException("Table '%s' is of unsupported format '%s', ", table_data->name, 46 | table_data->data_source_format); 47 | } 48 | 49 | // Set the S3 path as input to table function 50 | vector inputs = {table_data->storage_location}; 51 | 52 | if (table_data->storage_location.find("file://") != 0) { 53 | auto &secret_manager = SecretManager::Get(context); 54 | // Get Credentials from UCAPI 55 | auto table_credentials = UCAPI::GetTableCredentials(table_data->table_id, uc_catalog.credentials); 56 | 57 | // Inject secret into secret manager scoped to this path 58 | CreateSecretInput input; 59 | input.on_conflict = OnCreateConflict::REPLACE_ON_CONFLICT; 60 | input.persist_type = SecretPersistType::TEMPORARY; 61 | input.name = "__internal_uc_" + table_data->table_id; 62 | input.type = "s3"; 63 | input.provider = "config"; 64 | input.options = { 65 | {"key_id", table_credentials.key_id}, 66 | {"secret", table_credentials.secret}, 67 | {"session_token", table_credentials.session_token}, 68 | {"region", uc_catalog.credentials.aws_region}, 69 | }; 70 | input.scope = {table_data->storage_location}; 71 | 72 | secret_manager.CreateSecret(context, input); 73 | } 74 | named_parameter_map_t param_map; 75 | vector return_types; 76 | vector names; 77 | TableFunctionRef empty_ref; 78 | 79 | TableFunctionBindInput bind_input(inputs, param_map, return_types, names, nullptr, nullptr, delta_scan_function, 80 | empty_ref); 81 | 82 | auto result = delta_scan_function.bind(context, bind_input, return_types, names); 83 | bind_data = std::move(result); 84 | 85 | return delta_scan_function; 86 | } 87 | 88 | TableStorageInfo UCTableEntry::GetStorageInfo(ClientContext &context) { 89 | TableStorageInfo result; 90 | // TODO fill info 91 | return result; 92 | } 93 | 94 | } // namespace duckdb 95 | -------------------------------------------------------------------------------- /src/storage/uc_table_set.cpp: -------------------------------------------------------------------------------- 1 | #include "uc_api.hpp" 2 | #include "uc_utils.hpp" 3 | 4 | #include "storage/uc_catalog.hpp" 5 | #include "storage/uc_table_set.hpp" 6 | #include "storage/uc_transaction.hpp" 7 | #include "duckdb/parser/parsed_data/create_table_info.hpp" 8 | #include "duckdb/parser/constraints/not_null_constraint.hpp" 9 | #include "duckdb/parser/constraints/unique_constraint.hpp" 10 | #include "duckdb/parser/expression/constant_expression.hpp" 11 | #include "duckdb/planner/parsed_data/bound_create_table_info.hpp" 12 | #include "duckdb/catalog/dependency_list.hpp" 13 | #include "duckdb/parser/parsed_data/create_table_info.hpp" 14 | #include "duckdb/parser/constraints/list.hpp" 15 | #include "storage/uc_schema_entry.hpp" 16 | #include "duckdb/parser/parser.hpp" 17 | 18 | namespace duckdb { 19 | 20 | UCTableSet::UCTableSet(UCSchemaEntry &schema) : UCInSchemaSet(schema) { 21 | } 22 | 23 | static ColumnDefinition CreateColumnDefinition(ClientContext &context, UCAPIColumnDefinition &coldef) { 24 | return {coldef.name, UCUtils::TypeToLogicalType(context, coldef.type_text)}; 25 | } 26 | 27 | void UCTableSet::LoadEntries(ClientContext &context) { 28 | auto &transaction = UCTransaction::Get(context, catalog); 29 | 30 | auto &uc_catalog = catalog.Cast(); 31 | 32 | // TODO: handle out-of-order columns using position property 33 | auto tables = UCAPI::GetTables(catalog.GetDBPath(), schema.name, uc_catalog.credentials); 34 | 35 | for (auto &table : tables) { 36 | D_ASSERT(schema.name == table.schema_name); 37 | CreateTableInfo info; 38 | for (auto &col : table.columns) { 39 | info.columns.AddColumn(CreateColumnDefinition(context, col)); 40 | } 41 | 42 | info.table = table.name; 43 | auto table_entry = make_uniq(catalog, schema, info); 44 | table_entry->table_data = make_uniq(table); 45 | 46 | CreateEntry(std::move(table_entry)); 47 | } 48 | } 49 | 50 | optional_ptr UCTableSet::RefreshTable(ClientContext &context, const string &table_name) { 51 | auto table_info = GetTableInfo(context, schema, table_name); 52 | auto table_entry = make_uniq(catalog, schema, *table_info); 53 | auto table_ptr = table_entry.get(); 54 | CreateEntry(std::move(table_entry)); 55 | return table_ptr; 56 | } 57 | 58 | unique_ptr UCTableSet::GetTableInfo(ClientContext &context, UCSchemaEntry &schema, 59 | const string &table_name) { 60 | throw NotImplementedException("UCTableSet::CreateTable"); 61 | } 62 | 63 | optional_ptr UCTableSet::CreateTable(ClientContext &context, BoundCreateTableInfo &info) { 64 | throw NotImplementedException("UCTableSet::CreateTable"); 65 | } 66 | 67 | void UCTableSet::AlterTable(ClientContext &context, RenameTableInfo &info) { 68 | throw NotImplementedException("UCTableSet::AlterTable"); 69 | } 70 | 71 | void UCTableSet::AlterTable(ClientContext &context, RenameColumnInfo &info) { 72 | throw NotImplementedException("UCTableSet::AlterTable"); 73 | } 74 | 75 | void UCTableSet::AlterTable(ClientContext &context, AddColumnInfo &info) { 76 | throw NotImplementedException("UCTableSet::AlterTable"); 77 | } 78 | 79 | void UCTableSet::AlterTable(ClientContext &context, RemoveColumnInfo &info) { 80 | throw NotImplementedException("UCTableSet::AlterTable"); 81 | } 82 | 83 | void UCTableSet::AlterTable(ClientContext &context, AlterTableInfo &alter) { 84 | throw NotImplementedException("UCTableSet::AlterTable"); 85 | } 86 | 87 | } // namespace duckdb 88 | -------------------------------------------------------------------------------- /src/storage/uc_transaction.cpp: -------------------------------------------------------------------------------- 1 | #include "storage/uc_transaction.hpp" 2 | #include "storage/uc_catalog.hpp" 3 | #include "duckdb/parser/parsed_data/create_view_info.hpp" 4 | #include "duckdb/catalog/catalog_entry/index_catalog_entry.hpp" 5 | #include "duckdb/catalog/catalog_entry/view_catalog_entry.hpp" 6 | 7 | namespace duckdb { 8 | 9 | UCTransaction::UCTransaction(UCCatalog &uc_catalog, TransactionManager &manager, ClientContext &context) 10 | : Transaction(manager, context), access_mode(uc_catalog.access_mode) { 11 | // connection = UCConnection::Open(uc_catalog.path); 12 | } 13 | 14 | UCTransaction::~UCTransaction() = default; 15 | 16 | void UCTransaction::Start() { 17 | transaction_state = UCTransactionState::TRANSACTION_NOT_YET_STARTED; 18 | } 19 | void UCTransaction::Commit() { 20 | if (transaction_state == UCTransactionState::TRANSACTION_STARTED) { 21 | transaction_state = UCTransactionState::TRANSACTION_FINISHED; 22 | // connection.Execute("COMMIT"); 23 | } 24 | } 25 | void UCTransaction::Rollback() { 26 | if (transaction_state == UCTransactionState::TRANSACTION_STARTED) { 27 | transaction_state = UCTransactionState::TRANSACTION_FINISHED; 28 | // connection.Execute("ROLLBACK"); 29 | } 30 | } 31 | 32 | // UCConnection &UCTransaction::GetConnection() { 33 | // if (transaction_state == UCTransactionState::TRANSACTION_NOT_YET_STARTED) { 34 | // transaction_state = UCTransactionState::TRANSACTION_STARTED; 35 | // string query = "START TRANSACTION"; 36 | // if (access_mode == AccessMode::READ_ONLY) { 37 | // query += " READ ONLY"; 38 | // } 39 | //// conne/**/ction.Execute(query); 40 | // } 41 | // return connection; 42 | //} 43 | 44 | // unique_ptr UCTransaction::Query(const string &query) { 45 | // if (transaction_state == UCTransactionState::TRANSACTION_NOT_YET_STARTED) { 46 | // transaction_state = UCTransactionState::TRANSACTION_STARTED; 47 | // string transaction_start = "START TRANSACTION"; 48 | // if (access_mode == AccessMode::READ_ONLY) { 49 | // transaction_start += " READ ONLY"; 50 | // } 51 | // connection.Query(transaction_start); 52 | // return connection.Query(query); 53 | // } 54 | // return connection.Query(query); 55 | //} 56 | 57 | UCTransaction &UCTransaction::Get(ClientContext &context, Catalog &catalog) { 58 | return Transaction::Get(context, catalog).Cast(); 59 | } 60 | 61 | } // namespace duckdb 62 | -------------------------------------------------------------------------------- /src/storage/uc_transaction_manager.cpp: -------------------------------------------------------------------------------- 1 | #include "storage/uc_transaction_manager.hpp" 2 | #include "duckdb/main/attached_database.hpp" 3 | 4 | namespace duckdb { 5 | 6 | UCTransactionManager::UCTransactionManager(AttachedDatabase &db_p, UCCatalog &uc_catalog) 7 | : TransactionManager(db_p), uc_catalog(uc_catalog) { 8 | } 9 | 10 | Transaction &UCTransactionManager::StartTransaction(ClientContext &context) { 11 | auto transaction = make_uniq(uc_catalog, *this, context); 12 | transaction->Start(); 13 | auto &result = *transaction; 14 | lock_guard l(transaction_lock); 15 | transactions[result] = std::move(transaction); 16 | return result; 17 | } 18 | 19 | ErrorData UCTransactionManager::CommitTransaction(ClientContext &context, Transaction &transaction) { 20 | auto &uc_transaction = transaction.Cast(); 21 | uc_transaction.Commit(); 22 | lock_guard l(transaction_lock); 23 | transactions.erase(transaction); 24 | return ErrorData(); 25 | } 26 | 27 | void UCTransactionManager::RollbackTransaction(Transaction &transaction) { 28 | auto &uc_transaction = transaction.Cast(); 29 | uc_transaction.Rollback(); 30 | lock_guard l(transaction_lock); 31 | transactions.erase(transaction); 32 | } 33 | 34 | void UCTransactionManager::Checkpoint(ClientContext &context, bool force) { 35 | // auto &transaction = UCTransaction::Get(context, db.GetCatalog()); 36 | // auto &db = transaction.GetConnection(); 37 | // db.Execute("CHECKPOINT"); 38 | } 39 | 40 | } // namespace duckdb 41 | -------------------------------------------------------------------------------- /src/uc_api.cpp: -------------------------------------------------------------------------------- 1 | #include "uc_api.hpp" 2 | #include "storage/uc_catalog.hpp" 3 | #include "yyjson.hpp" 4 | #include 5 | #include 6 | 7 | namespace duckdb { 8 | 9 | //! We use a global here to store the path that is selected on the UCAPI::InitializeCurl call 10 | static string SELECTED_CURL_CERT_PATH = ""; 11 | 12 | static size_t GetRequestWriteCallback(void *contents, size_t size, size_t nmemb, void *userp) { 13 | ((std::string *)userp)->append((char *)contents, size * nmemb); 14 | return size * nmemb; 15 | } 16 | 17 | // we statically compile in libcurl, which means the cert file location of the build machine is the 18 | // place curl will look. But not every distro has this file in the same location, so we search a 19 | // number of common locations and use the first one we find. 20 | static string certFileLocations[] = { 21 | // Arch, Debian-based, Gentoo 22 | "/etc/ssl/certs/ca-certificates.crt", 23 | // RedHat 7 based 24 | "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", 25 | // Redhat 6 based 26 | "/etc/pki/tls/certs/ca-bundle.crt", 27 | // OpenSUSE 28 | "/etc/ssl/ca-bundle.pem", 29 | // Alpine 30 | "/etc/ssl/cert.pem" 31 | }; 32 | 33 | // Look through the the above locations and if one of the files exists, set that as the location curl should use. 34 | static bool SelectCurlCertPath() { 35 | for (string& caFile : certFileLocations) { 36 | struct stat buf; 37 | if (stat(caFile.c_str(), &buf) == 0) { 38 | SELECTED_CURL_CERT_PATH = caFile; 39 | } 40 | } 41 | return false; 42 | } 43 | 44 | static bool SetCurlCAFileInfo(CURL* curl) { 45 | if (!SELECTED_CURL_CERT_PATH.empty()) { 46 | curl_easy_setopt(curl, CURLOPT_CAINFO, SELECTED_CURL_CERT_PATH.c_str()); 47 | return true; 48 | } 49 | return false; 50 | } 51 | 52 | // Note: every curl object we use should set this, because without it some linux distro's may not find the CA certificate. 53 | static void InitializeCurlObject(CURL * curl, const string &token) { 54 | if (!token.empty()) { 55 | curl_easy_setopt(curl, CURLOPT_XOAUTH2_BEARER, token.c_str()); 56 | curl_easy_setopt(curl, CURLOPT_HTTPAUTH, CURLAUTH_BEARER); 57 | } 58 | SetCurlCAFileInfo(curl); 59 | } 60 | 61 | static string GetRequest(const string &url, const string &token = "") { 62 | CURL *curl; 63 | CURLcode res; 64 | string readBuffer; 65 | 66 | curl = curl_easy_init(); 67 | if (curl) { 68 | curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); 69 | curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, GetRequestWriteCallback); 70 | curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer); 71 | InitializeCurlObject(curl, token); 72 | res = curl_easy_perform(curl); 73 | curl_easy_cleanup(curl); 74 | 75 | if (res != CURLcode::CURLE_OK) { 76 | string error = curl_easy_strerror(res); 77 | throw IOException("Curl Request to '%s' failed with error: '%s'", url, error); 78 | } 79 | return readBuffer; 80 | } 81 | throw InternalException("Failed to initialize curl"); 82 | } 83 | 84 | template 85 | static TYPE TemplatedTryGetYYJson(duckdb_yyjson::yyjson_val *obj, const string &field, TYPE default_val, 86 | bool fail_on_missing = true) { 87 | auto val = yyjson_obj_get(obj, field.c_str()); 88 | if (val && yyjson_get_type(val) == TYPE_NUM) { 89 | return get_function(val); 90 | } else if (!fail_on_missing) { 91 | return default_val; 92 | } 93 | throw IOException("Invalid field found while parsing field: " + field); 94 | } 95 | 96 | static uint64_t TryGetNumFromObject(duckdb_yyjson::yyjson_val *obj, const string &field, bool fail_on_missing = true, 97 | uint64_t default_val = 0) { 98 | return TemplatedTryGetYYJson(obj, field, default_val, 99 | fail_on_missing); 100 | } 101 | static bool TryGetBoolFromObject(duckdb_yyjson::yyjson_val *obj, const string &field, bool fail_on_missing = false, 102 | bool default_val = false) { 103 | return TemplatedTryGetYYJson(obj, field, default_val, 104 | fail_on_missing); 105 | } 106 | static string TryGetStrFromObject(duckdb_yyjson::yyjson_val *obj, const string &field, bool fail_on_missing = true, 107 | const char *default_val = "") { 108 | return TemplatedTryGetYYJson(obj, field, default_val, 109 | fail_on_missing); 110 | } 111 | 112 | static string GetCredentialsRequest(const string &url, const string &table_id, const string &token = "") { 113 | CURL *curl; 114 | CURLcode res; 115 | string readBuffer; 116 | 117 | string body = StringUtil::Format(R"({"table_id" : "%s", "operation" : "READ"})", table_id); 118 | 119 | curl = curl_easy_init(); 120 | if (curl) { 121 | curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); 122 | curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, GetRequestWriteCallback); 123 | curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer); 124 | 125 | // Set headers 126 | struct curl_slist *headers = curl_slist_append(nullptr, "Content-Type: application/json"); 127 | curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); 128 | 129 | // Set request body 130 | curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str()); 131 | curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, body.length()); 132 | 133 | InitializeCurlObject(curl, token); 134 | 135 | res = curl_easy_perform(curl); 136 | curl_easy_cleanup(curl); 137 | 138 | if (res != CURLcode::CURLE_OK) { 139 | string error = curl_easy_strerror(res); 140 | throw IOException("Curl Request to '%s' failed with error: '%s'", url, error); 141 | } 142 | return readBuffer; 143 | } 144 | throw InternalException("Failed to initialize curl"); 145 | } 146 | 147 | void UCAPI::InitializeCurl() { 148 | SelectCurlCertPath(); 149 | } 150 | 151 | //# list catalogs 152 | // echo "List of catalogs" 153 | // curl --request GET "https://${DATABRICKS_HOST}/api/2.1/unity-catalog/catalogs" \ 154 | // --header "Authorization: Bearer ${TOKEN}" | jq . 155 | // 156 | //# list short version of all tables 157 | // echo "Table Summaries" 158 | // curl --request GET "https://${DATABRICKS_HOST}/api/2.1/unity-catalog/table-summaries?catalog_name=workspace" \ 159 | // --header "Authorization: Bearer ${TOKEN}" | jq . 160 | // 161 | //# list tables in `default` schema 162 | // echo "Tables in default schema" 163 | // curl --request GET 164 | // "https://${DATABRICKS_HOST}/api/2.1/unity-catalog/tables?catalog_name=workspace&schema_name=default" \ 165 | // --header "Authorization: Bearer ${TOKEN}" | jq . 166 | 167 | UCAPITableCredentials UCAPI::GetTableCredentials(const string &table_id, UCCredentials credentials) { 168 | UCAPITableCredentials result; 169 | 170 | auto api_result = GetCredentialsRequest(credentials.endpoint + "/api/2.1/unity-catalog/temporary-table-credentials", 171 | table_id, credentials.token); 172 | 173 | // Read JSON and get root 174 | duckdb_yyjson::yyjson_doc *doc = duckdb_yyjson::yyjson_read(api_result.c_str(), api_result.size(), 0); 175 | duckdb_yyjson::yyjson_val *root = yyjson_doc_get_root(doc); 176 | 177 | auto *aws_temp_credentials = yyjson_obj_get(root, "aws_temp_credentials"); 178 | if (aws_temp_credentials) { 179 | result.key_id = TryGetStrFromObject(aws_temp_credentials, "access_key_id"); 180 | result.secret = TryGetStrFromObject(aws_temp_credentials, "secret_access_key"); 181 | result.session_token = TryGetStrFromObject(aws_temp_credentials, "session_token"); 182 | } 183 | 184 | return result; 185 | } 186 | 187 | vector UCAPI::GetCatalogs(const string &catalog, UCCredentials credentials) { 188 | throw NotImplementedException("UCAPI::GetCatalogs"); 189 | } 190 | 191 | static UCAPIColumnDefinition ParseColumnDefinition(duckdb_yyjson::yyjson_val *column_def) { 192 | UCAPIColumnDefinition result; 193 | 194 | result.name = TryGetStrFromObject(column_def, "name"); 195 | result.type_text = TryGetStrFromObject(column_def, "type_text"); 196 | result.precision = TryGetNumFromObject(column_def, "type_precision"); 197 | result.scale = TryGetNumFromObject(column_def, "type_scale"); 198 | result.position = TryGetNumFromObject(column_def, "position"); 199 | 200 | return result; 201 | } 202 | 203 | vector UCAPI::GetTables(const string &catalog, const string &schema, UCCredentials credentials) { 204 | vector result; 205 | auto api_result = GetRequest(credentials.endpoint + "/api/2.1/unity-catalog/tables?catalog_name=" + catalog + 206 | "&schema_name=" + schema, 207 | credentials.token); 208 | 209 | // Read JSON and get root 210 | duckdb_yyjson::yyjson_doc *doc = duckdb_yyjson::yyjson_read(api_result.c_str(), api_result.size(), 0); 211 | duckdb_yyjson::yyjson_val *root = yyjson_doc_get_root(doc); 212 | 213 | // Get root["hits"], iterate over the array 214 | auto *tables = yyjson_obj_get(root, "tables"); 215 | size_t idx, max; 216 | duckdb_yyjson::yyjson_val *table; 217 | yyjson_arr_foreach(tables, idx, max, table) { 218 | UCAPITable table_result; 219 | table_result.catalog_name = catalog; 220 | table_result.schema_name = schema; 221 | 222 | table_result.name = TryGetStrFromObject(table, "name"); 223 | table_result.table_type = TryGetStrFromObject(table, "table_type"); 224 | table_result.data_source_format = TryGetStrFromObject(table, "data_source_format", false); 225 | table_result.storage_location = TryGetStrFromObject(table, "storage_location", false); 226 | table_result.table_id = TryGetStrFromObject(table, "table_id"); 227 | 228 | auto *columns = yyjson_obj_get(table, "columns"); 229 | duckdb_yyjson::yyjson_val *col; 230 | size_t col_idx, col_max; 231 | yyjson_arr_foreach(columns, col_idx, col_max, col) { 232 | auto column_definition = ParseColumnDefinition(col); 233 | table_result.columns.push_back(column_definition); 234 | } 235 | 236 | result.push_back(table_result); 237 | } 238 | 239 | return result; 240 | } 241 | 242 | vector UCAPI::GetSchemas(const string &catalog, UCCredentials credentials) { 243 | vector result; 244 | 245 | auto api_result = 246 | GetRequest(credentials.endpoint + "/api/2.1/unity-catalog/schemas?catalog_name=" + catalog, credentials.token); 247 | 248 | // Read JSON and get root 249 | duckdb_yyjson::yyjson_doc *doc = duckdb_yyjson::yyjson_read(api_result.c_str(), api_result.size(), 0); 250 | duckdb_yyjson::yyjson_val *root = yyjson_doc_get_root(doc); 251 | 252 | // Get root["hits"], iterate over the array 253 | auto *schemas = yyjson_obj_get(root, "schemas"); 254 | size_t idx, max; 255 | duckdb_yyjson::yyjson_val *schema; 256 | yyjson_arr_foreach(schemas, idx, max, schema) { 257 | UCAPISchema schema_result; 258 | 259 | auto *name = yyjson_obj_get(schema, "name"); 260 | if (name) { 261 | schema_result.schema_name = yyjson_get_str(name); 262 | } 263 | schema_result.catalog_name = catalog; 264 | 265 | result.push_back(schema_result); 266 | } 267 | 268 | return result; 269 | } 270 | 271 | } // namespace duckdb 272 | -------------------------------------------------------------------------------- /src/uc_catalog_extension.cpp: -------------------------------------------------------------------------------- 1 | #define DUCKDB_EXTENSION_MAIN 2 | 3 | #include "uc_catalog_extension.hpp" 4 | #include "storage/uc_catalog.hpp" 5 | #include "storage/uc_transaction_manager.hpp" 6 | 7 | #include "duckdb.hpp" 8 | #include "duckdb/main/secret/secret_manager.hpp" 9 | #include "duckdb/common/exception.hpp" 10 | #include "duckdb/common/string_util.hpp" 11 | #include "duckdb/function/scalar_function.hpp" 12 | #include "duckdb/main/extension_util.hpp" 13 | #include "duckdb/parser/parsed_data/create_scalar_function_info.hpp" 14 | #include "duckdb/parser/parsed_data/attach_info.hpp" 15 | #include "duckdb/storage/storage_extension.hpp" 16 | #include "uc_api.hpp" 17 | 18 | namespace duckdb { 19 | 20 | static unique_ptr CreateUCSecretFunction(ClientContext &, CreateSecretInput &input) { 21 | // apply any overridden settings 22 | vector prefix_paths; 23 | auto result = make_uniq(prefix_paths, "uc", "config", input.name); 24 | for (const auto &named_param : input.options) { 25 | auto lower_name = StringUtil::Lower(named_param.first); 26 | 27 | if (lower_name == "token") { 28 | result->secret_map["token"] = named_param.second.ToString(); 29 | } else if (lower_name == "endpoint") { 30 | result->secret_map["endpoint"] = named_param.second.ToString(); 31 | } else if (lower_name == "aws_region") { 32 | result->secret_map["aws_region"] = named_param.second.ToString(); 33 | } else { 34 | throw InternalException("Unknown named parameter passed to CreateUCSecretFunction: " + lower_name); 35 | } 36 | } 37 | 38 | //! Set redact keys 39 | result->redact_keys = {"token"}; 40 | 41 | return std::move(result); 42 | } 43 | 44 | static void SetUCSecretParameters(CreateSecretFunction &function) { 45 | function.named_parameters["token"] = LogicalType::VARCHAR; 46 | function.named_parameters["endpoint"] = LogicalType::VARCHAR; 47 | function.named_parameters["aws_region"] = LogicalType::VARCHAR; 48 | } 49 | 50 | unique_ptr GetSecret(ClientContext &context, const string &secret_name) { 51 | auto &secret_manager = SecretManager::Get(context); 52 | auto transaction = CatalogTransaction::GetSystemCatalogTransaction(context); 53 | // FIXME: this should be adjusted once the `GetSecretByName` API supports this 54 | // use case 55 | auto secret_entry = secret_manager.GetSecretByName(transaction, secret_name, "memory"); 56 | if (secret_entry) { 57 | return secret_entry; 58 | } 59 | secret_entry = secret_manager.GetSecretByName(transaction, secret_name, "local_file"); 60 | if (secret_entry) { 61 | return secret_entry; 62 | } 63 | return nullptr; 64 | } 65 | 66 | static unique_ptr UCCatalogAttach(StorageExtensionInfo *storage_info, ClientContext &context, 67 | AttachedDatabase &db, const string &name, AttachInfo &info, 68 | AccessMode access_mode) { 69 | UCCredentials credentials; 70 | 71 | // check if we have a secret provided 72 | string secret_name; 73 | for (auto &entry : info.options) { 74 | auto lower_name = StringUtil::Lower(entry.first); 75 | if (lower_name == "type" || lower_name == "read_only") { 76 | // already handled 77 | } else if (lower_name == "secret") { 78 | secret_name = entry.second.ToString(); 79 | } else { 80 | throw BinderException("Unrecognized option for UC attach: %s", entry.first); 81 | } 82 | } 83 | 84 | // if no secret is specified we default to the unnamed mysql secret, if it 85 | // exists 86 | bool explicit_secret = !secret_name.empty(); 87 | if (!explicit_secret) { 88 | // look up settings from the default unnamed mysql secret if none is 89 | // provided 90 | secret_name = "__default_uc"; 91 | } 92 | 93 | string connection_string = info.path; 94 | auto secret_entry = GetSecret(context, secret_name); 95 | if (secret_entry) { 96 | // secret found - read data 97 | const auto &kv_secret = dynamic_cast(*secret_entry->secret); 98 | string new_connection_info; 99 | 100 | Value input_val = kv_secret.TryGetValue("token"); 101 | credentials.token = input_val.IsNull() ? "" : input_val.ToString(); 102 | 103 | Value endpoint_val = kv_secret.TryGetValue("endpoint"); 104 | credentials.endpoint = endpoint_val.IsNull() ? "" : endpoint_val.ToString(); 105 | StringUtil::RTrim(credentials.endpoint, "/"); 106 | 107 | Value aws_region_val = kv_secret.TryGetValue("aws_region"); 108 | credentials.aws_region = endpoint_val.IsNull() ? "" : aws_region_val.ToString(); 109 | 110 | } else if (explicit_secret) { 111 | // secret not found and one was explicitly provided - throw an error 112 | throw BinderException("Secret with name \"%s\" not found", secret_name); 113 | } 114 | 115 | return make_uniq(db, info.path, access_mode, credentials); 116 | } 117 | 118 | static unique_ptr CreateTransactionManager(StorageExtensionInfo *storage_info, AttachedDatabase &db, 119 | Catalog &catalog) { 120 | auto &uc_catalog = catalog.Cast(); 121 | return make_uniq(db, uc_catalog); 122 | } 123 | 124 | class UCCatalogStorageExtension : public StorageExtension { 125 | public: 126 | UCCatalogStorageExtension() { 127 | attach = UCCatalogAttach; 128 | create_transaction_manager = CreateTransactionManager; 129 | } 130 | }; 131 | 132 | static void LoadInternal(DatabaseInstance &instance) { 133 | UCAPI::InitializeCurl(); 134 | 135 | SecretType secret_type; 136 | secret_type.name = "uc"; 137 | secret_type.deserializer = KeyValueSecret::Deserialize; 138 | secret_type.default_provider = "config"; 139 | 140 | ExtensionUtil::RegisterSecretType(instance, secret_type); 141 | 142 | CreateSecretFunction mysql_secret_function = {"uc", "config", CreateUCSecretFunction}; 143 | SetUCSecretParameters(mysql_secret_function); 144 | ExtensionUtil::RegisterFunction(instance, mysql_secret_function); 145 | 146 | auto &config = DBConfig::GetConfig(instance); 147 | config.storage_extensions["uc_catalog"] = make_uniq(); 148 | } 149 | 150 | void UcCatalogExtension::Load(DuckDB &db) { 151 | LoadInternal(*db.instance); 152 | } 153 | std::string UcCatalogExtension::Name() { 154 | return "uc_catalog"; 155 | } 156 | 157 | std::string UcCatalogExtension::Version() const { 158 | #ifdef EXT_VERSION_UC_CATALOG 159 | return EXT_VERSION_UC_CATALOG; 160 | #else 161 | return ""; 162 | #endif 163 | } 164 | 165 | } // namespace duckdb 166 | 167 | extern "C" { 168 | 169 | DUCKDB_EXTENSION_API void uc_catalog_init(duckdb::DatabaseInstance &db) { 170 | duckdb::DuckDB db_wrapper(db); 171 | db_wrapper.LoadExtension(); 172 | } 173 | 174 | DUCKDB_EXTENSION_API const char *uc_catalog_version() { 175 | return duckdb::DuckDB::LibraryVersion(); 176 | } 177 | } 178 | 179 | #ifndef DUCKDB_EXTENSION_MAIN 180 | #error DUCKDB_EXTENSION_MAIN not defined 181 | #endif 182 | -------------------------------------------------------------------------------- /src/uc_utils.cpp: -------------------------------------------------------------------------------- 1 | #include "uc_utils.hpp" 2 | #include "duckdb/common/operator/cast_operators.hpp" 3 | #include "storage/uc_schema_entry.hpp" 4 | #include "storage/uc_transaction.hpp" 5 | 6 | #include 7 | 8 | namespace duckdb { 9 | 10 | string UCUtils::TypeToString(const LogicalType &input) { 11 | switch (input.id()) { 12 | case LogicalType::VARCHAR: 13 | return "TEXT"; 14 | case LogicalType::UTINYINT: 15 | return "TINYINT UNSIGNED"; 16 | case LogicalType::USMALLINT: 17 | return "SMALLINT UNSIGNED"; 18 | case LogicalType::UINTEGER: 19 | return "INTEGER UNSIGNED"; 20 | case LogicalType::UBIGINT: 21 | return "BIGINT UNSIGNED"; 22 | case LogicalType::TIMESTAMP: 23 | return "DATETIME"; 24 | case LogicalType::TIMESTAMP_TZ: 25 | return "TIMESTAMP"; 26 | default: 27 | return input.ToString(); 28 | } 29 | } 30 | 31 | LogicalType UCUtils::TypeToLogicalType(ClientContext &context, const string &type_text) { 32 | if (type_text == "tinyint") { 33 | return LogicalType::TINYINT; 34 | } else if (type_text == "smallint") { 35 | return LogicalType::SMALLINT; 36 | } else if (type_text == "bigint") { 37 | return LogicalType::BIGINT; 38 | } else if (type_text == "int") { 39 | return LogicalType::INTEGER; 40 | } else if (type_text == "long") { 41 | return LogicalType::BIGINT; 42 | } else if (type_text == "string" || 43 | type_text.find("varchar(") == 0 || 44 | type_text == "char" || 45 | type_text.find("char(") == 0) { 46 | return LogicalType::VARCHAR; 47 | } else if (type_text == "double") { 48 | return LogicalType::DOUBLE; 49 | } else if (type_text == "float") { 50 | return LogicalType::FLOAT; 51 | } else if (type_text == "boolean") { 52 | return LogicalType::BOOLEAN; 53 | } else if (type_text == "timestamp") { 54 | return LogicalType::TIMESTAMP_TZ; 55 | } else if (type_text == "binary") { 56 | return LogicalType::BLOB; 57 | } else if (type_text == "date") { 58 | return LogicalType::DATE; 59 | } else if (type_text == "void") { 60 | return LogicalType::SQLNULL; // TODO: This seems to be the closest match 61 | } else if (type_text.find("decimal(") == 0) { 62 | size_t spec_end = type_text.find(')'); 63 | if (spec_end != string::npos) { 64 | size_t sep = type_text.find(','); 65 | auto prec_str = type_text.substr(8, sep - 8); 66 | auto scale_str = type_text.substr(sep + 1, spec_end - sep - 1); 67 | uint8_t prec = Cast::Operation(prec_str); 68 | uint8_t scale = Cast::Operation(scale_str); 69 | return LogicalType::DECIMAL(prec, scale); 70 | } 71 | } else if (type_text.find("array<") == 0) { 72 | size_t type_end = type_text.rfind('>'); // find last, to deal with nested 73 | if (type_end != string::npos) { 74 | auto child_type_str = type_text.substr(6, type_end - 6); 75 | auto child_type = UCUtils::TypeToLogicalType(context, child_type_str); 76 | return LogicalType::LIST(child_type); 77 | } 78 | } else if (type_text.find("map<") == 0) { 79 | size_t type_end = type_text.rfind('>'); // find last, to deal with nested 80 | if (type_end != string::npos) { 81 | // TODO: Factor this and struct parsing into an iterator over ',' separated values 82 | vector key_val; 83 | size_t cur = 4; 84 | auto nested_opens = 0; 85 | for (;;) { 86 | size_t next_sep = cur; 87 | // find the location of the next ',' ignoring nested commas 88 | while (type_text[next_sep] != ',' || nested_opens > 0) { 89 | if (type_text[next_sep] == '<') { 90 | nested_opens++; 91 | } else if (type_text[next_sep] == '>') { 92 | nested_opens--; 93 | } 94 | next_sep++; 95 | if (next_sep == type_end) { 96 | break; 97 | } 98 | } 99 | auto child_str = type_text.substr(cur, next_sep - cur); 100 | auto child_type = UCUtils::TypeToLogicalType(context, child_str); 101 | key_val.push_back(child_type); 102 | if (next_sep == type_end) { 103 | break; 104 | } 105 | cur = next_sep + 1; 106 | } 107 | if (key_val.size() != 2) { 108 | throw NotImplementedException("Invalid map specification with %i types", key_val.size()); 109 | } 110 | return LogicalType::MAP(key_val[0], key_val[1]); 111 | } 112 | } else if (type_text.find("struct<") == 0) { 113 | size_t type_end = type_text.rfind('>'); // find last, to deal with nested 114 | if (type_end != string::npos) { 115 | child_list_t children; 116 | size_t cur = 7; 117 | auto nested_opens = 0; 118 | for (;;) { 119 | size_t next_sep = cur; 120 | // find the location of the next ',' ignoring nested commas 121 | while (type_text[next_sep] != ',' || nested_opens > 0) { 122 | if (type_text[next_sep] == '<') { 123 | nested_opens++; 124 | } else if (type_text[next_sep] == '>') { 125 | nested_opens--; 126 | } 127 | next_sep++; 128 | if (next_sep == type_end) { 129 | break; 130 | } 131 | } 132 | auto child_str = type_text.substr(cur, next_sep - cur); 133 | size_t type_sep = child_str.find(':'); 134 | if (type_sep == string::npos) { 135 | throw NotImplementedException("Invalid struct child type specifier: %s", child_str); 136 | } 137 | auto child_name = child_str.substr(0, type_sep); 138 | auto child_type = UCUtils::TypeToLogicalType(context, child_str.substr(type_sep + 1, string::npos)); 139 | children.push_back({child_name, child_type}); 140 | if (next_sep == type_end) { 141 | break; 142 | } 143 | cur = next_sep + 1; 144 | } 145 | return LogicalType::STRUCT(children); 146 | } 147 | } 148 | 149 | throw NotImplementedException("Tried to fallback to unknown type for '%s'", type_text); 150 | // fallback for unknown types 151 | return LogicalType::VARCHAR; 152 | } 153 | 154 | LogicalType UCUtils::ToUCType(const LogicalType &input) { 155 | // todo do we need this mapping? 156 | throw NotImplementedException("ToUCType not yet implemented"); 157 | switch (input.id()) { 158 | case LogicalTypeId::BOOLEAN: 159 | case LogicalTypeId::SMALLINT: 160 | case LogicalTypeId::INTEGER: 161 | case LogicalTypeId::BIGINT: 162 | case LogicalTypeId::TINYINT: 163 | case LogicalTypeId::UTINYINT: 164 | case LogicalTypeId::USMALLINT: 165 | case LogicalTypeId::UINTEGER: 166 | case LogicalTypeId::UBIGINT: 167 | case LogicalTypeId::FLOAT: 168 | case LogicalTypeId::DOUBLE: 169 | case LogicalTypeId::BLOB: 170 | case LogicalTypeId::DATE: 171 | case LogicalTypeId::DECIMAL: 172 | case LogicalTypeId::TIMESTAMP: 173 | case LogicalTypeId::TIMESTAMP_TZ: 174 | case LogicalTypeId::VARCHAR: 175 | return input; 176 | case LogicalTypeId::LIST: 177 | throw NotImplementedException("UC does not support arrays - unsupported type \"%s\"", input.ToString()); 178 | case LogicalTypeId::STRUCT: 179 | case LogicalTypeId::MAP: 180 | case LogicalTypeId::UNION: 181 | throw NotImplementedException("UC does not support composite types - unsupported type \"%s\"", 182 | input.ToString()); 183 | case LogicalTypeId::TIMESTAMP_SEC: 184 | case LogicalTypeId::TIMESTAMP_MS: 185 | case LogicalTypeId::TIMESTAMP_NS: 186 | return LogicalType::TIMESTAMP; 187 | case LogicalTypeId::HUGEINT: 188 | return LogicalType::DOUBLE; 189 | default: 190 | return LogicalType::VARCHAR; 191 | } 192 | } 193 | 194 | } // namespace duckdb 195 | -------------------------------------------------------------------------------- /test/README.md: -------------------------------------------------------------------------------- 1 | # Testing this extension 2 | This directory contains all the tests for this extension. The `sql` directory holds tests that are written as [SQLLogicTests](https://duckdb.org/dev/sqllogictest/intro.html). DuckDB aims to have most its tests in this format as SQL statements, so for the uc_catalog extension, this should probably be the goal too. 3 | 4 | The root makefile contains targets to build and run all of these tests. To run the SQLLogicTests: 5 | ```bash 6 | make test 7 | ``` 8 | or 9 | ```bash 10 | make test_debug 11 | ``` -------------------------------------------------------------------------------- /test/sql/uc_catalog.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/uc_catalog.test 2 | # description: test uc_catalog extension 3 | # group: [uc_catalog] 4 | 5 | # Require statement will ensure this test is run with this extension loaded 6 | require parquet 7 | 8 | require uc_catalog 9 | 10 | require delta 11 | 12 | require httpfs 13 | 14 | require-env UC_TEST_SERVER_RUNNING 15 | 16 | statement ok 17 | CREATE SECRET ( 18 | TYPE UC, 19 | TOKEN 'not-used', 20 | ENDPOINT 'http://127.0.0.1:8080', 21 | AWS_REGION 'us-east-2' 22 | ); 23 | 24 | # Catalog Secret 25 | statement ok 26 | ATTACH 'unity' AS unity (TYPE UC_CATALOG); 27 | 28 | query III 29 | SELECT database_name, schema_name, table_name FROM duckdb_tables() where database_name='unity' order by table_name; 30 | ---- 31 | unity default marksheet 32 | unity default marksheet_uniform 33 | unity default numbers 34 | unity default user_countries 35 | 36 | #mode output_result 37 | 38 | query III 39 | FROM unity.default.marksheet; 40 | ---- 41 | 1 nWYHawtqUw 930 42 | 2 uvOzzthsLV 166 43 | 3 WIAehuXWkv 170 44 | 4 wYCSvnJKTo 709 45 | 5 VsslXsUIDZ 993 46 | 6 ZLsACYYTFy 813 47 | 7 BtDDvLeBpK 52 48 | 8 YISVtrPfGr 8 49 | 9 PBPJHDFjjC 45 50 | 10 qbDuUJzJMO 756 51 | 11 EjqqWoaLJn 712 52 | 12 jpZLMdKXpn 847 53 | 13 acpjQXpJCp 649 54 | 14 nOKqHhRwao 133 55 | 15 kxUUZEUoKv 398 56 | 57 | # FIXME: requires fix for file:// urls 58 | #statement ok 59 | #FROM unity.default.marksheet_uniform; 60 | 61 | query II 62 | FROM unity.default.numbers; 63 | ---- 64 | 564 188.75535598441473 65 | 755 883.6105633023361 66 | 644 203.4395591086936 67 | 75 277.8802190765611 68 | 42 403.857969425109 69 | 680 797.6912200731077 70 | 821 767.7998537403159 71 | 484 344.00373976089304 72 | 477 380.6785614543262 73 | 131 35.44373222835895 74 | 294 209.32243623208947 75 | 150 329.19730274053694 76 | 539 425.66102859000944 77 | 247 477.742227230588 78 | 958 509.3712727285101 79 | 80 | query III 81 | FROM unity.default.user_countries; 82 | ---- 83 | Logan Johnson 25 Austria 84 | Mr. Jay Russell 33 Austria 85 | Amber White 64 Austria 86 | Jessica Sherman 74 Austria 87 | Christopher Anderson 61 Austria 88 | Jeffrey Booker 64 Austria 89 | Emily Best 74 Austria 90 | Stephanie Downs MD 48 Austria 91 | Katie Montgomery 47 Belgia 92 | Derrick Gonzalez 55 Belgia 93 | Robert Johnson 26 Belgia 94 | Katherine Reyes 26 Belgia 95 | Andrew Cantrell 59 Belgia 96 | Adrienne Morgan 45 Belgia 97 | Crystal Jones 73 Belgia 98 | Jesse Austin 21 Belgia 99 | Cynthia Lawson 30 Belgia 100 | Crystal Bruce 24 Serbia 101 | Anthony Gonzales 71 Serbia 102 | Christina Hall 69 Serbia 103 | Colleen Spencer 34 Serbia 104 | Brian Smith 24 Serbia 105 | Scott Mcpherson III 74 Serbia 106 | Michael Wolfe 30 Serbia 107 | Virginia Burton 38 Serbia 108 | Rodney Pope 74 Serbia 109 | Gary Wright 74 Serbia 110 | Matthew Jefferson 42 Serbia 111 | Beth Cherry 71 Serbia 112 | Dawn Mendoza 46 Serbia -------------------------------------------------------------------------------- /test/sql/uc_cert.test: -------------------------------------------------------------------------------- 1 | # name: test/sql/uc_cert.test 2 | # description: test uc_catalog ssl cert with a correctly placed cert 3 | # group: [uc_catalog] 4 | 5 | # Require statement will ensure this test is run with this extension loaded 6 | require parquet 7 | 8 | require uc_catalog 9 | 10 | require httpfs 11 | 12 | statement ok 13 | CREATE SECRET ( 14 | TYPE UC, 15 | TOKEN 'not-used', 16 | ENDPOINT 'https://duckdb.org', 17 | AWS_REGION 'us-east-2' 18 | ); 19 | 20 | # Catalog Secret 21 | statement ok 22 | ATTACH 'unity' AS unity (TYPE UC_CATALOG); 23 | 24 | # This would throw SSL cert error if ssl cert is not properly found 25 | statement ok 26 | SELECT database_name, schema_name, table_name FROM duckdb_tables() where database_name='unity' order by table_name; -------------------------------------------------------------------------------- /vcpkg.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": [ 3 | "curl", 4 | "openssl" 5 | ] 6 | } --------------------------------------------------------------------------------